From c5b3597b46037517008b61b02b6c4597695d2d3f Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 16:14:57 +0900 Subject: [PATCH 01/32] feat(pointer): store, serve and pay for mutable references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pointers (ADR-0015) are mutable, owner-signed references addressed at BLAKE3(domain || owner_key). The record and its wire messages live in ant-protocol; this is the node's half. Storage. The chunk store answers "already have it" and stops, which is right for an immutable record and drops every update for a mutable one, and it requires BLAKE3(content) == address, which no pointer satisfies. Pointers get their own store: merge-on-put under one lock, the rename as the commit point, an exclusive directory lock, and an index dropped for any address whose file stops validating so a repair is accepted rather than answered "unchanged". Payment. A quote is paid against the record's state_id and must be issued by the close group around its address — two different values with two different jobs, carried as PaymentTarget { routing, content }. Chunks pass the same address for both, so their behaviour and cache entries are unchanged. The paid cache is keyed by a typed Chunk vs PointerState: a raw 32-byte key would let a client store a chunk crafted to sit on a pointer's entry and buy its update at chunk price. One payment buys one increment. A create is counter 0 and a client update is exactly one past what the node holds, so an owner cannot pay once and jump the counter. Replication keeps the counter order instead, so a replica behind a gap can still catch up. Admission runs before the signature check — capacity, cross-kind refusal, and responsibility for the pointer's address — so a forged record for someone else's address buys no ML-DSA verification. Audit. The commitment leaf gains a record kind, bound by hashing pointer leaves under their own domain, so a chunk leaf cannot be relabelled to escape the bytes_hash == key guard. Chunk-only roots are bit-identical. Pointer leaves are refused at round 1 until round 2 serves a whole record: a peer signs its own commitment, so it could otherwise name any key with the hash of cheap bytes it holds. --- Cargo.lock | 7 +- Cargo.toml | 7 + docs/adr/ADR-0015-pointers-immutable-owner.md | 149 ++ src/devnet.rs | 18 +- src/error.rs | 17 + src/lib.rs | 9 +- src/node.rs | 16 +- src/payment/cache.rs | 131 +- src/payment/verifier.rs | 246 ++- src/pointer/mod.rs | 62 + src/pointer/service.rs | 642 +++++++ src/pointer/store.rs | 1475 +++++++++++++++++ src/replication/commitment.rs | 56 +- src/replication/commitment_state.rs | 22 + src/replication/config.rs | 10 +- src/replication/protocol.rs | 5 +- src/replication/storage_commitment_audit.rs | 66 +- src/replication/subtree.rs | 223 ++- src/storage/handler.rs | 68 +- src/storage/mod.rs | 1 + tests/poc_commitment_audit_attacks.rs | 4 +- tests/pointer_convergence.rs | 657 ++++++++ 22 files changed, 3801 insertions(+), 90 deletions(-) create mode 100644 docs/adr/ADR-0015-pointers-immutable-owner.md create mode 100644 src/pointer/mod.rs create mode 100644 src/pointer/service.rs create mode 100644 src/pointer/store.rs create mode 100644 tests/pointer_convergence.rs diff --git a/Cargo.lock b/Cargo.lock index 2b190ce3..0010ecbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -882,9 +882,8 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7b4fbfd0a8b397d0dc1a0dc7308308242911147224c4e1a56130168e7ad48f" +version = "3.1.0" +source = "git+https://github.com/WithAutonomi/ant-protocol?rev=34b7832a001df8b80b18555ca058734baae43496#34b7832a001df8b80b18555ca058734baae43496" dependencies = [ "blake3", "bytes", @@ -7104,7 +7103,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2e8359d3..046a9d5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,6 +228,13 @@ webrtc-direct = [ "dep:self_encryption", ] +[patch.crates-io] +# Pointers (ADR-0016) add the `Pointer` record and its wire messages on top of +# the published 3.0.0, so this is the only entry that has to leave the release +# baseline. A rev, not a branch, so the pin is immutable. Drop it once the +# pointer PR lands and 3.1.0 is published. +ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", rev = "34b7832a001df8b80b18555ca058734baae43496" } + [profile.release] lto = true codegen-units = 1 diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md new file mode 100644 index 00000000..99367c49 --- /dev/null +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -0,0 +1,149 @@ +# ADR-0015: Pointers — paid mutable references with an immutable owner + +- **Status:** Accepted +- **Date:** 2026-09-18 +- **Decision owners:** Anselme (@grumbach) +- **Related:** ADR-0002 (audit), ADR-0008 (per-record pricing), ADR-0009 (audit families), ADR-0014 (file store) + +## Context + +2.0 stores only immutable chunks. 1.0's pointer had three defects: updates +after the first were free, the merge rule diverged permanently on equal +counters, and the address *was* the owner's key, so ownership could never +change. + +We fix the first two and keep the third deliberately. A former owner keeps its +key forever, so transferable ownership cannot be made fork-proof by any local +rule: hash tie-breaks are grindable in ~2 keygens and payment-order ties fall to +a pre-buy. Declining transfer is what lets this design be small enough to trust. + +## Decision + +One record. No genesis object, no certificates, no lineage. + +```rust +pub struct Pointer { // 5,303 bytes + format_version: u8, + owner: MlDsa65PublicKey, // 1,952 — the pointer's identity + counter: u64, + target: PointerTarget, // 1-byte kind tag + 32-byte address + signature: MlDsa65Signature, // 3,309, over everything above +} +``` + +Fixed-width, big-endian, hand-encoded; no serde in the signed bytes. + +### Three identities + +```text +A = BLAKE3("autonomi.pointer.address.v1" || owner) routes +state_id = BLAKE3("autonomi.pointer.state.v1" || body) authorizes payment +bytes_hash = BLAKE3(record) this node's commitment +``` + +**Public-key addressed and self-verifying.** `A` is a pure function of the owner +key, and the key is in the record, so a node validates a pointer from its own +bytes: one hash, one signature check, no fetch. + +`A` and `state_id` are separate because `A` is stable for the pointer's life +while the paid identifier must change with every update — paying against `A` +would make every update after the first free. + +### Pay to create, pay to update + +Creation is `counter = 0`. Each update is `counter + 1`. Both are paid against +their own `state_id`, so **one payment buys exactly one increment**. + +The client path enforces `+1`. Replication accepts any strictly greater counter, +because a replica that missed an update must be able to catch up; refusing the +gap would leave it permanently stale instead. + +### Merge + +```text +1. larger counter +2. smaller target bytes +``` + +A total order on the states of **one address**. Records of different owners are +not comparable and never contend. **Equal state never replaces**: ML-DSA signing +is randomized, so one state has unboundedly many valid encodings, and ordering +record *bytes* would let an owner sign one paid state repeatedly and have every +submission win. + +### Validation order + +```text +length → version → structure → compare with held → admission → signature → payment → commit +``` + +Cheap first. A resubmission of what is held is refused before any signature +check. Admission (capacity, responsibility for `A`) precedes the signature, so a +forged record for someone else's address buys no cryptography. The commit +re-checks under its lock, because a newer state can land while payment verifies. + +## What this defends against + +| Attack | Defence | +|---|---| +| Tamper with any byte | Signature over the whole body | +| Swap the owner key | `A` is derived from it; the record no longer belongs at its address | +| Store at someone else's address | Same | +| Fork / equivocate at one counter | Total order on `(counter, target)` — every node picks the same one | +| Replay an older record | Loses on counter | +| Re-sign one paid state N times | Equal state never replaces; nothing is written | +| Pay once, jump the counter | Client updates must be `+1` | +| Pay for a chunk to fund a pointer | Paid cache keyed by a typed `Chunk` vs `PointerState`, never by a raw 32-byte value a crafted chunk could occupy | +| Merkle proof with no issuer check | Refused for pointers; single-node proofs only | +| Downgrade the format | `format_version` is signed and inside `state_id`; unknown versions are refused | +| Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | +| Collide a pointer and a chunk address | Refused in both directions rather than resolved | +| Mint an audit leaf for someone's key | Pointer leaves are refused at round 1 (see below) | + +## Consequences + +- Validating a pointer needs nothing but the pointer — no quorum, no lineage, no + Sybil exposure in ownership. +- Creation is one record and one payment, with no retention dependency. +- **Ownership cannot change.** Handover is indirection: point at a new pointer + the recipient owns. The old owner keeps write access forever, so it is a + revocable forwarding state, not a sale. +- **Key compromise is permanent.** No rotation, no recovery. +- The inlined key costs 1,920 bytes on every read, forever — a deliberate trade + for self-contained validation. +- Determinism is not freshness: an eclipsed reader can be handed an older, + correctly signed value and cannot tell. +- Replicas may hold different valid signatures of one state; nothing compares + record bytes across replicas. +- The audit commitment leaf gains a record kind, bound by hashing pointer leaves + under their own domain, so a chunk leaf cannot be relabelled to escape the + `bytes_hash == key` guard. **Pointer leaves are refused at round 1** until + round 2 serves and validates a whole record: a peer signs its own commitment, + so it could otherwise name any key with the hash of cheap bytes it holds. + Refusing costs nothing today — commitment rotation reads the chunk store only. + +## Implementation status + +Built: the record and wire messages (`ant-protocol`), the store with +merge-on-put, request dispatch, payment routed at `state_id` with the close +group of `A`, admission gates, cross-kind refusal, the kind-tagged audit leaf, +and the client API. + +Not built: replication does not yet carry `(A, state_id)` through fresh offers, +sync hints, presence, repair and paid-list, so a pointer is not replicated +version-aware; and audit round 2 does not yet serve a whole record, which is why +pointer leaves are refused rather than trusted. + +## Validation + +- Every delivery order of a record set converges to one value, exhaustively over + all permutations, including on a node started empty and one restarted. +- 64 valid signatures over one paid state yield one stored record and one file. +- A resubmission and a stale arrival are both refused before any signature check. +- Creation is counter 0; an update is `+1`; every jump — including to `u64::MAX` + and a wrap back to 0 — is refused as a non-successor. +- All 256 `format_version` values give distinct paid identifiers. +- Golden vectors pin the encoding, both identities and the signing context. +- A relabelled audit leaf fails structural verification; a chunk-only commitment + root is bit-identical to before. +- A crafted chunk cannot satisfy a pointer's paid-cache entry. diff --git a/src/devnet.rs b/src/devnet.rs index 8a32e088..b6f75bed 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -973,11 +973,19 @@ impl Devnet { let storage = Arc::new(storage); let payment_verifier = Arc::new(payment_verifier); - Ok(AntProtocol::new( - storage, - payment_verifier, - Arc::new(quote_generator), - )) + // Same pointer wiring as a production node, so a devnet exercises the + // real path rather than a node that silently refuses every pointer. + let pointer_store = crate::pointer::PointerStore::new(storage.root_dir()) + .await + .map_err(|e| DevnetError::Startup(format!("Failed to open pointer store: {e}")))?; + let pointers = crate::pointer::PointerService::new(pointer_store) + .with_chunk_store(Arc::clone(&storage)) + .with_payments(Arc::clone(&payment_verifier)); + + Ok( + AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)) + .with_pointer_service(pointers), + ) } #[allow(clippy::too_many_lines)] diff --git a/src/error.rs b/src/error.rs index f71ed7ed..34a5e002 100644 --- a/src/error.rs +++ b/src/error.rs @@ -60,3 +60,20 @@ pub enum Error { #[error("node is shutting down")] ShuttingDown, } + +impl From for Error { + /// Map a wire-level pointer rejection onto the node's error type. + /// + /// Signature failures become [`Error::Crypto`] and everything else becomes + /// [`Error::Protocol`], so a caller can still tell "these bytes are not a + /// pointer" from "these bytes are not signed by the key they carry". + fn from(error: ant_protocol::pointer::PointerError) -> Self { + use ant_protocol::pointer::PointerError; + match error { + PointerError::SignatureInvalid | PointerError::SigningFailed(_) => { + Self::Crypto(error.to_string()) + } + other => Self::Protocol(other.to_string()), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 2af24b0c..42ac7322 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,8 +17,10 @@ //! //! ## Data Types //! -//! Currently supports a single data type: +//! Two data types: //! - **Chunk**: Immutable content-addressed data (hash(value) == key) +//! - **Pointer**: A paid mutable reference signed by an immutable owner, stored +//! at `BLAKE3(domain || owner_key)` (see [`mod@pointer`] and ADR-0015) //! //! ## Example //! @@ -52,6 +54,7 @@ pub mod event; pub mod logging; pub mod node; pub mod payment; +pub mod pointer; pub mod replication; pub mod storage; pub mod upgrade; @@ -77,6 +80,10 @@ pub use error::{Error, Result}; pub use event::{NodeEvent, NodeEventsChannel}; pub use node::{NodeBuilder, RunningNode}; pub use payment::{PaymentStatus, PaymentVerifier, PaymentVerifierConfig}; +pub use pointer::{ + Pointer, PointerState, PointerStore, PointerTarget, PointerTargetKind, Prepared, PreparedPut, + PutOutcome, +}; pub use replication::{config::ReplicationConfig, ReplicationEngine}; pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; diff --git a/src/node.rs b/src/node.rs index 1c640cb5..1d24d427 100644 --- a/src/node.rs +++ b/src/node.rs @@ -576,7 +576,21 @@ impl NodeBuilder { let storage = Arc::new(storage); let payment_verifier = Arc::new(payment_verifier); - let protocol = AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)); + // Pointers live beside the chunks, under the same root. Opening the + // store here is what makes pointer PUT/GET answerable at all: without + // it every pointer request is refused, which is the right answer for a + // node that keeps none but the wrong one for a node that should. + let pointer_store = crate::pointer::PointerStore::new(&config.root_dir).await?; + let pointers = crate::pointer::PointerService::new(pointer_store) + // Refuse a pointer whose address a chunk already occupies, rather + // than letting one kind silently overwrite the other. + .with_chunk_store(Arc::clone(&storage)) + // Payment is verified against each record's state, so every update + // is paid for rather than riding the first one. + .with_payments(Arc::clone(&payment_verifier)); + + let protocol = AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)) + .with_pointer_service(pointers); info!( "ANT protocol handler initialized with ML-DSA-65 signing (protocol={CHUNK_PROTOCOL_ID})" diff --git a/src/payment/cache.rs b/src/payment/cache.rs index 174c45b8..c148a19c 100644 --- a/src/payment/cache.rs +++ b/src/payment/cache.rs @@ -14,6 +14,27 @@ pub use super::quote::XorName; /// Default cache capacity (100,000 entries = 3.2MB memory). const DEFAULT_CACHE_CAPACITY: usize = 100_000; +/// What a cache entry is about. +/// +/// A typed key, not a hashed one. Hashing a pointer's two addresses back into +/// 32 bytes would not create a separate namespace: a chunk's address is +/// `BLAKE3(content)`, so a client could store a chunk whose *content* is +/// exactly that preimage and land on the same key — paying chunk price for a +/// pointer update, and skipping issuer proximity, the price floor and the +/// proof-shape rule with it. Distinct variants cannot collide at all. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum PaidKey { + /// A chunk, paid for and stored at one address. + Chunk(XorName), + /// A pointer state: routed at the pointer's address, paid at its `state_id`. + PointerState { + /// The pointer's address, stable for its life. + routing: XorName, + /// The state paid for, which changes with every update. + state_id: XorName, + }, +} + /// LRU cache for verified `XorName` values. /// /// This cache stores `XorName` values that have been verified to exist on the @@ -25,7 +46,7 @@ const DEFAULT_CACHE_CAPACITY: usize = 100_000; /// entries satisfy weaker lookups. #[derive(Clone)] pub struct VerifiedCache { - inner: Arc>>, + inner: Arc>>, hits: Arc, misses: Arc, additions: Arc, @@ -101,8 +122,8 @@ impl VerifiedCache { /// Returns `true` if the `XorName` is cached (verified to exist on autonomi). /// Paid-list and client-PUT lookups must use their stricter helpers. #[must_use] - pub fn contains(&self, xorname: &XorName) -> bool { - let found = self.inner.lock().get(xorname).is_some(); + pub fn contains_key(&self, key: &PaidKey) -> bool { + let found = self.inner.lock().get(key).is_some(); if found { self.hits.fetch_add(1, Ordering::Relaxed); @@ -119,11 +140,11 @@ impl VerifiedCache { /// A client-PUT entry returns `true` here because it passed the stricter /// store-admission path at the caller. #[must_use] - pub fn contains_paid_list_verified(&self, xorname: &XorName) -> bool { + pub fn contains_paid_list_verified_key(&self, key: &PaidKey) -> bool { let found = self .inner .lock() - .get(xorname) + .get(key) .copied() .is_some_and(|level| level.satisfies(VerificationLevel::PaidList)); @@ -142,11 +163,11 @@ impl VerifiedCache { /// Paid-list entries return `false` here because they did not pass the /// client-PUT store-admission path. #[must_use] - pub fn contains_client_put_verified(&self, xorname: &XorName) -> bool { + pub fn contains_client_put_verified_key(&self, key: &PaidKey) -> bool { let found = self .inner .lock() - .get(xorname) + .get(key) .copied() .is_some_and(|level| level.satisfies(VerificationLevel::ClientPut)); @@ -163,30 +184,30 @@ impl VerifiedCache { /// /// This should be called after verifying that data exists on the autonomi network. /// Also upgrades an existing paid-list-verified entry. - pub fn insert(&self, xorname: XorName) { - self.insert_with_level(xorname, VerificationLevel::ClientPut); + pub fn insert_key(&self, key: PaidKey) { + self.insert_with_level(key, VerificationLevel::ClientPut); } /// Add a `XorName` verified under paid-list admission checks. /// /// Never downgrades an existing client-PUT-verified entry. - pub fn insert_paid_list_verified(&self, xorname: XorName) { - self.insert_with_level(xorname, VerificationLevel::PaidList); + pub fn insert_paid_list_verified_key(&self, key: PaidKey) { + self.insert_with_level(key, VerificationLevel::PaidList); } - fn insert_with_level(&self, xorname: XorName, level: VerificationLevel) { + fn insert_with_level(&self, key: PaidKey, level: VerificationLevel) { let added = { let mut inner = self.inner.lock(); // `get_mut` refreshes LRU recency for existing entries of either kind. - if inner.get(&xorname).is_some() { - if let Some(existing) = inner.get_mut(&xorname) { + if inner.get(&key).is_some() { + if let Some(existing) = inner.get_mut(&key) { if !existing.satisfies(level) { *existing = level; } } false } else { - inner.put(xorname, level); + inner.put(key, level); true } }; @@ -195,6 +216,34 @@ impl VerifiedCache { } } + /// As [`Self::contains_key`], for a chunk at `address`. + #[must_use] + pub fn contains(&self, address: &XorName) -> bool { + self.contains_key(&PaidKey::Chunk(*address)) + } + + /// As [`Self::contains_paid_list_verified_key`], for a chunk at `address`. + #[must_use] + pub fn contains_paid_list_verified(&self, address: &XorName) -> bool { + self.contains_paid_list_verified_key(&PaidKey::Chunk(*address)) + } + + /// As [`Self::contains_client_put_verified_key`], for a chunk at `address`. + #[must_use] + pub fn contains_client_put_verified(&self, address: &XorName) -> bool { + self.contains_client_put_verified_key(&PaidKey::Chunk(*address)) + } + + /// As [`Self::insert_key`], for a chunk at `address`. + pub fn insert(&self, address: XorName) { + self.insert_key(PaidKey::Chunk(address)); + } + + /// As [`Self::insert_paid_list_verified_key`], for a chunk at `address`. + pub fn insert_paid_list_verified(&self, address: XorName) { + self.insert_paid_list_verified_key(PaidKey::Chunk(address)); + } + /// Get current cache statistics. #[must_use] pub fn stats(&self) -> CacheStats { @@ -238,23 +287,23 @@ mod tests { fn test_cache_basic_operations() { let cache = VerifiedCache::new(); - let xorname1 = [1u8; 32]; - let xorname2 = [2u8; 32]; + let key1 = [1u8; 32]; + let key2 = [2u8; 32]; // Initially empty assert!(cache.is_empty()); - assert!(!cache.contains(&xorname1)); + assert!(!cache.contains(&key1)); // Insert and check - cache.insert(xorname1); - assert!(cache.contains(&xorname1)); - assert!(!cache.contains(&xorname2)); + cache.insert(key1); + assert!(cache.contains(&key1)); + assert!(!cache.contains(&key2)); assert_eq!(cache.len(), 1); // Insert another - cache.insert(xorname2); - assert!(cache.contains(&xorname1)); - assert!(cache.contains(&xorname2)); + cache.insert(key2); + assert!(cache.contains(&key1)); + assert!(cache.contains(&key2)); assert_eq!(cache.len(), 2); } @@ -284,21 +333,21 @@ mod tests { #[test] fn test_cache_stats() { let cache = VerifiedCache::new(); - let xorname = [1u8; 32]; + let key = [1u8; 32]; // Miss - assert!(!cache.contains(&xorname)); + assert!(!cache.contains(&key)); let stats = cache.stats(); assert_eq!(stats.misses, 1); assert_eq!(stats.hits, 0); // Add - cache.insert(xorname); + cache.insert(key); let stats = cache.stats(); assert_eq!(stats.additions, 1); // Hit - assert!(cache.contains(&xorname)); + assert!(cache.contains(&key)); let stats = cache.stats(); assert_eq!(stats.hits, 1); assert_eq!(stats.misses, 1); @@ -312,19 +361,19 @@ mod tests { // Small cache for testing eviction let cache = VerifiedCache::with_capacity(2); - let xorname1 = [1u8; 32]; - let xorname2 = [2u8; 32]; - let xorname3 = [3u8; 32]; + let key1 = [1u8; 32]; + let key2 = [2u8; 32]; + let key3 = [3u8; 32]; - cache.insert(xorname1); - cache.insert(xorname2); + cache.insert(key1); + cache.insert(key2); assert_eq!(cache.len(), 2); - // Insert third, should evict xorname1 (least recently used) - cache.insert(xorname3); + // Insert third, should evict key1 (least recently used) + cache.insert(key3); assert_eq!(cache.len(), 2); - assert!(!cache.contains(&xorname1)); // evicted - // Note: after contains call on evicted item, stats will show a miss + assert!(!cache.contains(&key1)); // evicted + // Note: after contains call on evicted item, stats will show a miss } #[test] @@ -409,8 +458,8 @@ mod tests { for i in 0..10u8 { let c = cache.clone(); handles.push(thread::spawn(move || { - let xorname = [i; 32]; - c.insert(xorname); + let key = [i; 32]; + c.insert(key); })); } @@ -418,8 +467,8 @@ mod tests { for i in 0..10u8 { let c = cache.clone(); handles.push(thread::spawn(move || { - let xorname = [i; 32]; - let _ = c.contains(&xorname); + let key = [i; 32]; + let _ = c.contains(&key); })); } diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 7252af43..9561bd87 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -6,7 +6,7 @@ use crate::ant_protocol::CLOSE_GROUP_SIZE; use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; -use crate::payment::cache::{CacheStats, VerifiedCache, XorName}; +use crate::payment::cache::{CacheStats, PaidKey, VerifiedCache, XorName}; use crate::payment::pricing::{calculate_price, derive_records_stored_from_price}; use crate::payment::proof::{ deserialize_merkle_proof, deserialize_single_node_proof, detect_proof_type, ProofType, @@ -480,6 +480,64 @@ pub struct PaymentVerifierConfig { /// majority storage among the configured close group, or majority paid-list /// membership among the closest K. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PaymentTarget { + /// The address whose close group is responsible, and whose members' quotes + /// therefore count. For a chunk this is its content address; for a pointer + /// it is the pointer's address, which never changes. + pub routing: XorName, + /// What the quote must actually name. For a chunk this equals + /// [`Self::routing`]; for a pointer it is the record's `state_id`, so each + /// update is paid for separately rather than riding the first payment. + pub content: XorName, +} + +impl PaymentTarget { + /// A target where one address does both jobs, as every chunk does. + #[must_use] + pub const fn same(address: XorName) -> Self { + Self { + routing: address, + content: address, + } + } + + /// A target that routes at one address and is paid at another. + #[must_use] + pub const fn split(routing: XorName, content: XorName) -> Self { + Self { routing, content } + } + + /// Whether this target's two jobs fall to one address, as a chunk's do. + #[must_use] + pub fn is_single_address(&self) -> bool { + self.routing == self.content + } + + /// The key this target's "already paid" entry is filed under. + /// + /// A **typed** key, not a hashed one. Hashing the two halves back into 32 + /// bytes would not separate the namespaces: a chunk's address is + /// `BLAKE3(content)`, so a client could store a chunk whose *content* is + /// exactly that preimage and land on the same key. Paying for that chunk + /// would then file an entry the pointer path reads as its own — buying a + /// pointer update at chunk prices and skipping the issuer-proximity, + /// price-floor and proof-shape checks along with it. Distinct enum variants + /// cannot collide however the bytes are chosen. + #[must_use] + pub fn cache_key(&self) -> PaidKey { + if self.is_single_address() { + PaidKey::Chunk(self.content) + } else { + PaidKey::PointerState { + routing: self.routing, + state_id: self.content, + } + } + } +} + +/// What a payment verification is admitting. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VerificationContext { /// The node is admitting a chunk store from a direct client PUT, with /// store-strength cache semantics. @@ -1045,11 +1103,29 @@ impl PaymentVerifier { xorname: &XorName, context: VerificationContext, ) -> PaymentStatus { + self.check_payment_required_keyed(PaidKey::Chunk(*xorname), context) + } + + /// As [`Self::check_payment_required`], for an already-typed cache key. + /// + /// A pointer's entry is filed under both its routing address and the state + /// paid for, so it can never be satisfied by a chunk that happens to sit at + /// either value. + #[must_use] + pub fn check_payment_required_keyed( + &self, + key: PaidKey, + context: VerificationContext, + ) -> PaymentStatus { + let xorname = match &key { + PaidKey::Chunk(address) => address, + PaidKey::PointerState { state_id, .. } => state_id, + }; // Check LRU cache (fast path) let cached = if context.is_store_admission() { - self.cache.contains_client_put_verified(xorname) + self.cache.contains_client_put_verified_key(&key) } else { - self.cache.contains_paid_list_verified(xorname) + self.cache.contains_paid_list_verified_key(&key) }; if cached { if crate::logging::enabled!(crate::logging::Level::DEBUG) { @@ -1099,7 +1175,7 @@ impl PaymentVerifier { let proof_type = payment_proof_type_label(payment_proof); let proof_bytes = payment_proof.map_or(0, <[u8]>::len); let result = self - .verify_payment_inner(xorname, payment_proof, context) + .verify_payment_inner(&PaymentTarget::same(*xorname), payment_proof, context) .await; let elapsed_ms = started.elapsed().as_millis(); @@ -1135,12 +1211,19 @@ impl PaymentVerifier { async fn verify_payment_inner( &self, - xorname: &XorName, + target: &PaymentTarget, payment_proof: Option<&[u8]>, context: VerificationContext, ) -> Result { + // What the proof must name: for a chunk its address, for a pointer the + // state being paid for. Caching under the *address* would mark every + // future update of a pointer as already paid — the 1.0 free-update + // defect — so the cache is keyed separately, and never shares a key + // with the chunk whose address happens to equal this state. + let xorname = &target.content; + let cache_key = target.cache_key(); // First check if payment is required - let status = self.check_payment_required(xorname, context); + let status = self.check_payment_required_keyed(cache_key, context); match status { PaymentStatus::CachedAsVerified => { @@ -1165,7 +1248,7 @@ impl PaymentVerifier { // Detect proof type from version tag byte match detect_proof_type(proof) { Some(ProofType::Merkle) => { - self.verify_merkle_payment(xorname, proof, context).await?; + self.verify_merkle_payment(target, proof, context).await?; } Some(ProofType::SingleNode) => { let parsed = deserialize_single_node_proof(proof).map_err(|e| { @@ -1180,7 +1263,7 @@ impl PaymentVerifier { } self.verify_evm_payment( - xorname, + target, &parsed.proof_of_payment, &parsed.commitment_sidecars, context, @@ -1208,9 +1291,9 @@ impl PaymentVerifier { // strength. Stronger entries satisfy weaker future lookups, // but not the reverse. if context.is_store_admission() { - self.cache.insert(*xorname); + self.cache.insert_key(cache_key); } else { - self.cache.insert_paid_list_verified(*xorname); + self.cache.insert_paid_list_verified_key(cache_key); } Ok(PaymentStatus::PaymentVerified) @@ -1228,6 +1311,39 @@ impl PaymentVerifier { } } + /// Verify that a pointer state was paid for. + /// + /// `routing_address` is the pointer's address: it selects the close group + /// whose quotes count, and it never changes for the life of the pointer. + /// `paid_content` is the record's `state_id`: it is what the quote must + /// name, and it changes with every update. Passing one address for both — + /// which is all the chunk path can express — would either check the wrong + /// close group or mark every future update of the pointer as already paid. + /// + /// # Errors + /// + /// Returns [`Error::Payment`] if the proof is missing, malformed, names a + /// different state, was issued by a peer outside the routing address's + /// close group, or did not settle on chain. + pub async fn verify_pointer_payment( + &self, + routing_address: &XorName, + paid_content: &XorName, + payment_proof: &[u8], + ) -> Result<()> { + let target = PaymentTarget::split(*routing_address, *paid_content); + match self + .verify_payment_inner(&target, Some(payment_proof), VerificationContext::ClientPut) + .await? + { + PaymentStatus::CachedAsVerified | PaymentStatus::PaymentVerified => Ok(()), + PaymentStatus::PaymentRequired => Err(Error::Payment(format!( + "no settled payment for pointer state {}", + hex::encode(paid_content) + ))), + } + } + /// Get cache statistics. #[must_use] pub fn cache_stats(&self) -> CacheStats { @@ -1291,11 +1407,15 @@ impl PaymentVerifier { /// was paid 3x. async fn verify_evm_payment( &self, - xorname: &XorName, + target: &PaymentTarget, payment: &ProofOfPayment, commitment_sidecars: &[Vec], context: VerificationContext, ) -> Result<()> { + // `content` is what a quote must name; `routing` is whose close group + // may issue it. For a chunk they are one address, for a pointer they + // are not. + let xorname = &target.content; if crate::logging::enabled!(crate::logging::Level::DEBUG) { let xorname_hex = hex::encode(xorname); let quote_count = payment.peer_quotes.len(); @@ -1332,10 +1452,7 @@ impl PaymentVerifier { for candidate in candidates { let paid_price = candidate.quote.price; let candidate_peer = *candidate.encoded_peer_id.as_bytes(); - match self - .verify_legacy_median_candidate(xorname, candidate) - .await - { + match self.verify_legacy_median_candidate(target, candidate).await { Ok(settled_amount) => { verified_paid_quote = Some((paid_price, settled_amount)); // First settlement-verified median candidate wins the paid @@ -1368,7 +1485,11 @@ impl PaymentVerifier { // unauthenticated bundles can never poison floor telemetry. Shadow // mode logs; enforcement rejects — an economic admission decision // only, never trust/misbehaviour evidence. - self.enforce_price_floor(xorname, paid_price, settled_amount, context) + // The floor compares against the median commitment price of the close + // group that is RESPONSIBLE for the data, so it takes the routing + // address. For a chunk that is the same value; for a pointer the paid + // content is its state, which names no close group at all. + self.enforce_price_floor(&target.routing, paid_price, settled_amount, context) .await?; // ADR-0004 observe-only telemetry: log off-curve quotes only AFTER the @@ -1445,14 +1566,17 @@ impl PaymentVerifier { /// honest client may overpay a cheap quote to clear stricter receivers). async fn verify_legacy_median_candidate( &self, - xorname: &XorName, + target: &PaymentTarget, candidate: LegacyMedianCandidate<'_>, ) -> Result { - Self::validate_paid_quote_content(xorname, candidate)?; + // The two checks take different addresses. The quote must name what was + // paid for; the issuer must be close to what the network routes. A + // chunk supplies one address for both, a pointer two. + Self::validate_paid_quote_content(&target.content, candidate)?; let issuer_peer_id = Self::validate_paid_quote_peer_binding(candidate.encoded_peer_id, candidate.quote)?; - self.validate_paid_quote_issuer_k_closest(xorname, &issuer_peer_id) + self.validate_paid_quote_issuer_k_closest(&target.routing, &issuer_peer_id) .await?; Self::validate_paid_quote_signature(candidate).await?; @@ -3186,10 +3310,28 @@ impl PaymentVerifier { #[allow(clippy::too_many_lines)] async fn verify_merkle_payment( &self, - xorname: &XorName, + target: &PaymentTarget, proof_bytes: &[u8], context: VerificationContext, ) -> Result<()> { + // The proof names what was paid for, which for a pointer is its state + // rather than its address. + let xorname = &target.content; + + // A merkle proof binds the paid address but carries no issuer-proximity + // check, so it cannot express "paid at the state, quoted by the group + // around the address". Accepting one for a pointer would check the + // close group of a state identifier, which names no group at all. + // Single-node proofs do carry that check, so pointers use those until + // the merkle proof shape can say which group issued it. + if target.routing != target.content { + return Err(Error::Payment(format!( + "a pointer update must be paid with a single-node proof: a merkle \ + proof cannot bind the issuing close group of {} to the paid state {}", + hex::encode(target.routing), + hex::encode(target.content) + ))); + } if crate::logging::enabled!(crate::logging::Level::DEBUG) { debug!( "Verifying merkle payment for {} ({context:?})", @@ -3596,6 +3738,70 @@ mod tests { /// Create a verifier for unit tests. EVM is always on, but tests can /// pre-populate the cache to bypass on-chain verification. + /// A chunk whose address equals a pointer's `state_id` must not be able to + /// pay for that pointer's update. + /// + /// `state_id` is `BLAKE3(domain || body)`, so a client can store a chunk + /// whose *content* is exactly `domain || body`; that chunk's address is the + /// pointer's state identifier. If both filed their "already paid" entry + /// under that one value, paying chunk price for the chunk would buy the + /// pointer update — and skip issuer proximity, the price floor and the + /// proof-shape rule with it. + #[test] + fn a_chunk_cannot_pay_for_a_pointer_that_shares_its_address() { + let state_id: XorName = [0x5Au8; 32]; + let pointer_address: XorName = [0xA5u8; 32]; + + let chunk = PaymentTarget::same(state_id); + let pointer = PaymentTarget::split(pointer_address, state_id); + + assert_eq!( + chunk.cache_key(), + PaidKey::Chunk(state_id), + "a chunk keeps its bare address, so existing entries are untouched" + ); + assert_ne!( + pointer.cache_key(), + chunk.cache_key(), + "the pointer must not read the chunk's paid entry as its own" + ); + + // And two different pointers sharing a state cannot borrow either. + let other = PaymentTarget::split([0x11u8; 32], state_id); + assert_ne!(pointer.cache_key(), other.cache_key()); + + // The key is typed, not hashed: there is no 32-byte preimage a client + // could put in a chunk's *content* to land on the pointer's entry, + // because no chunk key is ever a `PointerState` variant. + let cache = VerifiedCache::with_capacity(8); + cache.insert_key(chunk.cache_key()); + assert!( + cache.contains_key(&chunk.cache_key()), + "the chunk's own entry is there" + ); + assert!( + !cache.contains_key(&pointer.cache_key()), + "and it does not satisfy the pointer" + ); + for crafted in [state_id, pointer_address, [0u8; 32], [0xFFu8; 32]] { + assert!( + !cache.contains_key(&PaymentTarget::split(pointer_address, state_id).cache_key()), + "no chunk address {crafted:?} can stand in for the pointer's entry" + ); + cache.insert_key(PaidKey::Chunk(crafted)); + } + assert!( + !cache.contains_key(&pointer.cache_key()), + "still no chunk address satisfies the pointer's typed entry" + ); + } + + #[test] + fn a_single_address_target_is_reported_as_one() { + assert!(PaymentTarget::same([1u8; 32]).is_single_address()); + assert!(!PaymentTarget::split([1u8; 32], [2u8; 32]).is_single_address()); + } + fn create_test_verifier() -> PaymentVerifier { let config = PaymentVerifierConfig { evm: EvmVerifierConfig::default(), diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs new file mode 100644 index 00000000..45c021dc --- /dev/null +++ b/src/pointer/mod.rs @@ -0,0 +1,62 @@ +//! Pointers — paid mutable references with an immutable owner. +//! +//! Implements `docs/adr/ADR-0015-pointers-immutable-owner.md`. +//! +//! A pointer is a mutable, owner-signed reference stored at an address derived +//! from the owner's public key. Ownership is fixed at creation: there is no +//! transfer, no lineage, no certificates and no key rotation. That choice is +//! what lets the design be this small — the owner key is inlined in the +//! record, so validating a pointer needs nothing but the pointer. +//! +//! # What lives here +//! +//! The record itself is a **wire type** and lives in +//! [`ant_protocol::pointer`]: its encoding, identifiers and merge rule are +//! things the client and the node must agree on byte for byte. This module is +//! the node's half. +//! +//! - [`store`] — durable storage with merge-on-put, which is the part the +//! immutable chunk store cannot do. +//! - [`service`] — the request handler: validate, check payment, merge. +//! +//! # The three identifiers +//! +//! | Name | Derivation | Job | +//! |---|---|---| +//! | `A` | `BLAKE3(domain \|\| owner)` | routes, and decides which nodes are responsible | +//! | `state_id` | `BLAKE3(domain \|\| body)` | names the authenticated state: sync hints, and what a quote is paid against | +//! | `bytes_hash` | `BLAKE3(record)` | what *this* node's storage commitment binds | +//! +//! `A` and `state_id` are separate because `A` must be stable for the pointer's +//! life while the paid identifier must change with every update, or updates +//! after the first would be free — which is exactly the 1.0 defect this design +//! exists to fix. There is deliberately no fourth name hashed from `state_id`: +//! it is already a domain-separated, owner-bound identifier for exactly one +//! signed state. +//! +//! `bytes_hash` is per-storer rather than per-state, because two replicas may +//! hold one state under different signatures. That is fine: a storage +//! commitment is built and signed by one node and audited against that node's +//! own bytes, so it never has to agree with a peer's. +//! +//! # Why the merge rule ignores signature bytes +//! +//! ML-DSA signing in `saorsa-pqc` is randomized and exposes no deterministic +//! mode, so one authenticated state has unboundedly many valid encodings. A +//! merge rule that ordered record *bytes* would let an owner sign one paid +//! state repeatedly, sort worst-first, and have every submission win — +//! unbounded storage, replication and Merkle-rebuild work for a single +//! payment. So equal state never replaces, and replicas may legitimately hold +//! different encodings of one state. Nothing compares record bytes across +//! replicas; `state_id` is compared instead. + +pub mod service; +pub mod store; + +pub use ant_protocol::pointer::{ + cmp_merge, merge, pointer_address, state_id_for_body, MergeRank, ParsedPointer, Pointer, + PointerError, PointerState, PointerTarget, PointerTargetKind, DATA_TYPE_POINTER, + POINTER_BODY_LEN, POINTER_FORMAT_VERSION, POINTER_WIRE_LEN, TARGET_WIRE_LEN, +}; +pub use service::PointerService; +pub use store::{PointerStore, Prepared, PreparedPut, PutOutcome}; diff --git a/src/pointer/service.rs b/src/pointer/service.rs new file mode 100644 index 00000000..d72501bc --- /dev/null +++ b/src/pointer/service.rs @@ -0,0 +1,642 @@ +//! The node's pointer request handler. +//! +//! Sits between the wire messages in [`ant_protocol::chunk`] and the +//! [`PointerStore`], and owns the three things a pointer PUT needs that a chunk +//! PUT does not: +//! +//! 1. **Payment at `state_id`, not at the address.** A pointer's address is +//! stable for its life, so quoting against it would make every update after +//! the first free — 1.0's defect. The quote's content is the state +//! identifier; the close group that answers is still the one around the +//! address. +//! 2. **Merge instead of "already exists".** The chunk path answers +//! `AlreadyExists` and stops, which for a mutable record silently drops +//! every update. +//! 3. **Cross-kind collision refusal.** A pointer address and a chunk address +//! are both 32 bytes from the same range; the domain separator makes a +//! collision infeasible, not impossible. A node that holds one kind at an +//! address refuses the other rather than silently choosing. +//! +//! # Order of work +//! +//! ```text +//! parse → compare with what is held → verify signature → check payment → commit +//! ``` +//! +//! The comparison precedes the signature check, so re-submitting a state the +//! node already holds costs a parse and a map lookup rather than an ML-DSA +//! verification. The payment check sits between validation and commit, and the +//! commit re-checks, because a newer state can land while payment is verified. + +use std::sync::Arc; + +use ant_protocol::chunk::{ + PointerGetRequest, PointerGetResponse, PointerPutRequest, PointerPutResponse, ProtocolError, + XorName, +}; +use bytes::Bytes; +use parking_lot::RwLock; +use saorsa_core::P2PNode; + +use crate::error::{Error, Result}; +use crate::logging::{debug, warn}; +use crate::payment::PaymentVerifier; +use crate::pointer::store::{Inspected, PointerStore, PutOutcome}; +use crate::replication::admission; +use crate::storage::{ChunkStore, SELF_CLOSENESS_GATE_WIDTH}; + +/// Handles pointer requests against a [`PointerStore`]. +#[derive(Clone)] +pub struct PointerService { + /// Where pointers are kept. + store: PointerStore, + /// Consulted so an address held as a chunk is never overwritten by a + /// pointer. + chunks: Option>, + /// Confirms a state was paid for. `None` in tests that exercise the merge + /// rather than the payment. + payments: Option>, + /// The node's P2P handle, for the self-closeness gate. + /// + /// Attached after construction, because the node builds its protocol + /// handler before it has a running P2P node. `None` in unit tests that + /// never attach one, exactly as the chunk path does. + p2p_node: Arc>>>, +} + +impl std::fmt::Debug for PointerService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PointerService") + .field("records", &self.store.len()) + .field("checks_chunk_collisions", &self.chunks.is_some()) + .field("verifies_payment", &self.payments.is_some()) + .finish_non_exhaustive() + } +} + +impl PointerService { + /// Build a service over `store`. + #[must_use] + pub fn new(store: PointerStore) -> Self { + Self { + store, + chunks: None, + payments: None, + p2p_node: Arc::new(RwLock::new(None)), + } + } + + /// Apply the self-closeness gate, so a pointer PUT is admitted only where + /// the node is actually responsible. + /// + /// Without it one valid proof can be replayed to every node on the network, + /// each of which would store the record and trigger replication for an + /// address it has no business holding. + pub fn attach_p2p_node(&self, p2p_node: Arc) { + *self.p2p_node.write() = Some(p2p_node); + } + + /// Consult `chunks` before storing, so a cross-kind address collision is + /// refused rather than resolved by whichever kind arrived last. + #[must_use] + pub fn with_chunk_store(mut self, chunks: Arc) -> Self { + self.chunks = Some(chunks); + self + } + + /// Require payment, verified against each record's `state_id`. + #[must_use] + pub fn with_payments(mut self, payments: Arc) -> Self { + self.payments = Some(payments); + self + } + + /// The store this service fronts. + #[must_use] + pub const fn store(&self) -> &PointerStore { + &self.store + } + + /// Handle a pointer PUT. + /// + /// Never returns `Err`: a rejection is a response the peer should see, so + /// every failure is mapped onto [`PointerPutResponse`]. + pub async fn handle_put(&self, request: PointerPutRequest) -> PointerPutResponse { + // Cheap first: parse and compare against what is held. No signature is + // checked yet, so a forged record for an address this node does not + // serve is rejected by the gates below without buying an ML-DSA + // verification. + let parsed = match self.store.inspect(&request.record) { + Ok(Inspected::Noop(PutOutcome::Unchanged)) => { + return match Self::state_of(&request.record) { + Some((address, state_id)) => { + PointerPutResponse::Unchanged { address, state_id } + } + None => PointerPutResponse::Error(ProtocolError::Internal( + "pointer parsed then failed to re-parse".to_string(), + )), + }; + } + Ok(Inspected::Noop(PutOutcome::Stale)) => { + let Some((address, _)) = Self::state_of(&request.record) else { + return PointerPutResponse::Error(ProtocolError::Internal( + "pointer parsed then failed to re-parse".to_string(), + )); + }; + let held = self.store.state_id(&address).unwrap_or_default(); + return PointerPutResponse::Stale { + address, + state_id: held, + }; + } + Ok(Inspected::Noop(other)) => { + return PointerPutResponse::Error(ProtocolError::Internal(format!( + "unexpected no-op outcome {other:?}" + ))); + } + Ok(Inspected::Candidate(parsed)) => parsed, + Err(e) => { + debug!("Pointer PUT refused: {e}"); + return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); + } + }; + + let address = parsed.state().address; + let state_id = parsed.state().state_id; + + if let Some(refusal) = self.admit(parsed.state()).await { + return refusal; + } + + // Only now is the record worth a signature check. + let prepared = match self.store.verify(parsed).await { + Ok(prepared) => prepared, + Err(e) => { + debug!("Pointer PUT refused: {e}"); + return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); + } + }; + + // Payment is checked against the state, not the address: the address + // never changes, so paying against it would buy every future update. + if let Some(payments) = &self.payments { + if let Err(e) = + Self::verify_payment(payments, address, state_id, request.payment_proof.as_ref()) + .await + { + return PointerPutResponse::PaymentRequired { + message: e.to_string(), + }; + } + } + + match self.store.commit(prepared).await { + Ok(PutOutcome::Stored | PutOutcome::Replaced) => { + PointerPutResponse::Success { address, state_id } + } + // The re-check under the commit lock found a newer state. The + // client paid for a state that lost a race; say so plainly. + Ok(PutOutcome::Unchanged) => PointerPutResponse::Unchanged { address, state_id }, + Ok(PutOutcome::Stale) => PointerPutResponse::Stale { + address, + state_id: self.store.state_id(&address).unwrap_or(state_id), + }, + Err(e) => { + warn!("Pointer commit failed for {}: {e}", hex::encode(address)); + PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())) + } + } + } + + /// Everything that must hold before a record is worth verifying. + /// + /// `Some(response)` means refuse. Ordered cheapest first, and all of it + /// ahead of the signature check, so a forged record for an address this + /// node does not serve buys no cryptography. + async fn admit( + &self, + state: &ant_protocol::pointer::PointerState, + ) -> Option { + let address = state.address; + + // One payment buys one increment. A create is counter 0 and an update + // is exactly one past what this node holds; anything else would let an + // owner pay once and jump the counter, skipping every intermediate + // payment. Replication does not come through here — it merges on the + // counter order, so a replica behind a gap can still catch up. + if !self.store.accepts_as_paid_update(state) { + debug!( + "Rejecting pointer PUT for {}: counter {} is not the paid successor", + hex::encode(address), + state.counter + ); + return Some(PointerPutResponse::PaymentRequired { + message: format!( + "a pointer is created at counter 0 and updated by exactly one \ + increment; counter {} does not follow what this node holds", + state.counter + ), + }); + } + + if let Some(chunks) = &self.chunks { + // A chunk already here means the two kinds collided. Refuse rather + // than pick: whichever we chose, someone's data would vanish. + if let Some(refusal) = cross_kind_refusal(address, chunks.exists(&address)) { + return Some(refusal); + } + // Capacity before payment, as the chunk path does. + if let Err(e) = chunks.check_capacity() { + debug!("Rejecting pointer PUT for {}: {e}", hex::encode(address)); + return Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + e.to_string(), + ))); + } + } + + // Self-closeness gate (ADR-0003), judged at the pointer's address + // because that is what the network routes on. Bind the handle out of + // the lock first: no guard may be held across an await. + let attached = self.p2p_node.read().as_ref().map(Arc::clone); + if let Some(p2p) = attached { + let self_id = *p2p.peer_id(); + if !admission::is_responsible(&self_id, &address, &p2p, SELF_CLOSENESS_GATE_WIDTH).await + { + debug!( + "Rejecting pointer PUT for {}: not within local closest peers", + hex::encode(address) + ); + return Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + "node is not within its local closest peers for this address".to_string(), + ))); + } + } + None + } + + /// Handle a pointer GET. + pub async fn handle_get(&self, request: PointerGetRequest) -> PointerGetResponse { + // A replica asking "anything newer than this?" gets a cheap index + // lookup first, but the answer is confirmed against the file before it + // is sent: an index entry for a record whose file has since gone or + // stopped validating would otherwise answer "unchanged" forever, and + // the peer would never fetch the copy that would repair it. `get` + // re-validates and drops such an entry, so the confirmation costs a + // read exactly once, on the way to telling the truth. + let known = request.known_state_id; + if known.is_some_and(|known| self.store.holds_state(&request.address, &known)) { + match self.store.get(&request.address).await { + Ok(Some(record)) if Some(record.state_id()) == known => { + return PointerGetResponse::Unchanged { + state_id: record.state_id(), + }; + } + Ok(_) => {} + Err(e) => { + return PointerGetResponse::Error(ProtocolError::StorageFailed(e.to_string())); + } + } + } + + match self.store.get(&request.address).await { + Ok(Some(record)) => PointerGetResponse::Success { + record: Bytes::copy_from_slice(record.as_bytes()), + }, + Ok(None) => PointerGetResponse::NotFound { + address: request.address, + }, + Err(e) => PointerGetResponse::Error(ProtocolError::StorageFailed(e.to_string())), + } + } + + /// Confirm the submitted state was paid for. + /// + /// `routing_address` selects the close group whose quotes count; + /// `paid_content` is what the quote must name. They are different values + /// here, which is the whole point — the existing chunk path passes one + /// address for both jobs. + async fn verify_payment( + payments: &PaymentVerifier, + routing_address: XorName, + paid_content: XorName, + proof: Option<&Vec>, + ) -> Result<()> { + let Some(proof) = proof else { + return Err(Error::Payment(format!( + "a pointer update must be paid for; no proof supplied for state {}", + hex::encode(paid_content) + ))); + }; + payments + .verify_pointer_payment(&routing_address, &paid_content, proof) + .await + } + + /// Re-read the address and state a record claims, for a response. + fn state_of(record: &[u8]) -> Option<(XorName, XorName)> { + ant_protocol::pointer::PointerState::parse(record) + .ok() + .map(|state| (state.address, state.state_id)) + } +} + +/// Decide whether a chunk already at `address` blocks this pointer. +/// +/// Separated from the handler because the branch cannot be reached in a test +/// any other way: a pointer address is `BLAKE3(domain || owner)` and a chunk +/// address is `BLAKE3(content)`, so occupying both with real data would take an +/// actual hash collision. The decision is what matters, so the decision is what +/// is tested. +/// +/// `Some(response)` means refuse. Refusing is the only safe answer: whichever +/// kind were chosen, the other's data would be destroyed, and the node cannot +/// know which one the network expects. +fn cross_kind_refusal(address: XorName, chunk_present: Result) -> Option { + match chunk_present { + Ok(true) => { + warn!( + "Refusing pointer at {}: a chunk already occupies that address", + hex::encode(address) + ); + Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + format!( + "address {} is already occupied by a chunk; refusing to store a \ + pointer over it", + hex::encode(address) + ), + ))) + } + Ok(false) => None, + Err(e) => Some(PointerPutResponse::Error(ProtocolError::StorageFailed( + format!("cannot check the chunk store for a collision: {e}"), + ))), + } +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + reason = "test assertions" +)] +mod tests { + use super::*; + use ant_protocol::pointer::{Pointer, PointerTarget, PointerTargetKind}; + use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; + + fn keypair(seed: u8) -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair_from_seed(&[seed; 32]) + } + + fn signed(seed: u8, counter: u64, target_byte: u8) -> Pointer { + let (pk, sk) = keypair(seed); + let target = PointerTarget::new(PointerTargetKind::Chunk, [target_byte; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("sign") + } + + async fn service() -> (PointerService, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + (PointerService::new(store), dir) + } + + fn put(record: &Pointer) -> PointerPutRequest { + PointerPutRequest::new(Bytes::copy_from_slice(record.as_bytes())) + } + + #[tokio::test] + async fn a_put_then_a_get_round_trips_the_record() { + let (service, _dir) = service().await; + let record = signed(1, 0, 1); + + match service.handle_put(put(&record)).await { + PointerPutResponse::Success { address, state_id } => { + assert_eq!(address, record.address()); + assert_eq!(state_id, record.state_id()); + } + other => panic!("expected Success, got {other:?}"), + } + + match service + .handle_get(PointerGetRequest::new(record.address())) + .await + { + PointerGetResponse::Success { record: bytes } => { + assert_eq!(bytes.as_ref(), record.as_bytes()); + } + other => panic!("expected Success, got {other:?}"), + } + } + + #[tokio::test] + async fn an_update_replaces_and_a_stale_one_is_told_so() { + let (service, _dir) = service().await; + let first = signed(1, 0, 1); + let second = signed(1, 1, 1); + service.handle_put(put(&first)).await; + + assert!(matches!( + service.handle_put(put(&second)).await, + PointerPutResponse::Success { .. } + )); + + match service.handle_put(put(&first)).await { + PointerPutResponse::Stale { address, state_id } => { + assert_eq!(address, first.address()); + assert_eq!(state_id, second.state_id(), "the newer state is reported"); + } + other => panic!("expected Stale, got {other:?}"), + } + } + + #[tokio::test] + async fn re_submitting_a_held_state_is_unchanged_not_success() { + // A client that retries must be able to tell "your update landed" from + // "you paid for something already stored". + let (service, _dir) = service().await; + let record = signed(1, 0, 4); + service.handle_put(put(&record)).await; + + let variant = signed(1, 0, 4); + assert_ne!(variant.as_bytes(), record.as_bytes()); + match service.handle_put(put(&variant)).await { + PointerPutResponse::Unchanged { address, state_id } => { + assert_eq!(address, record.address()); + assert_eq!(state_id, record.state_id()); + } + other => panic!("expected Unchanged, got {other:?}"), + } + } + + #[tokio::test] + async fn a_conditional_get_skips_the_transfer_when_nothing_changed() { + let (service, _dir) = service().await; + let record = signed(1, 0, 3); + service.handle_put(put(&record)).await; + + match service + .handle_get(PointerGetRequest::if_changed( + record.address(), + record.state_id(), + )) + .await + { + PointerGetResponse::Unchanged { state_id } => { + assert_eq!(state_id, record.state_id()); + } + other => panic!("expected Unchanged, got {other:?}"), + } + + // A stale known state still gets the bytes. + match service + .handle_get(PointerGetRequest::if_changed(record.address(), [0u8; 32])) + .await + { + PointerGetResponse::Success { .. } => {} + other => panic!("expected Success, got {other:?}"), + } + } + + #[tokio::test] + async fn an_absent_pointer_is_not_found() { + let (service, _dir) = service().await; + match service.handle_get(PointerGetRequest::new([7u8; 32])).await { + PointerGetResponse::NotFound { address } => assert_eq!(address, [7u8; 32]), + other => panic!("expected NotFound, got {other:?}"), + } + } + + #[tokio::test] + async fn junk_is_refused_rather_than_stored() { + let (service, _dir) = service().await; + assert!(matches!( + service + .handle_put(PointerPutRequest::new(Bytes::from_static(b"not a pointer"))) + .await, + PointerPutResponse::Error(_) + )); + assert!(service.store().is_empty()); + } + + #[tokio::test] + async fn a_forged_signature_is_refused() { + let (service, _dir) = service().await; + let record = signed(1, 0, 1); + let mut bytes = record.as_bytes().to_vec(); + if let Some(byte) = bytes.get_mut(ant_protocol::pointer::POINTER_BODY_LEN + 3) { + *byte ^= 0xff; + } + assert!(matches!( + service + .handle_put(PointerPutRequest::new(Bytes::from(bytes))) + .await, + PointerPutResponse::Error(_) + )); + assert!(service.store().is_empty()); + } + + #[tokio::test] + async fn payment_is_required_when_a_verifier_is_attached() { + use crate::payment::{PaymentVerifier, PaymentVerifierConfig}; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + let verifier = Arc::new(PaymentVerifier::new(PaymentVerifierConfig { + evm: Default::default(), + cache_capacity: 16, + close_group_size: ant_protocol::chunk::CLOSE_GROUP_SIZE, + local_rewards_address: evmlib::common::Address::new([1u8; 20]), + price_floor: Default::default(), + })); + let service = PointerService::new(store).with_payments(verifier); + + // No proof at all: refused as PaymentRequired, not stored. + match service.handle_put(put(&signed(1, 0, 1))).await { + PointerPutResponse::PaymentRequired { message } => { + assert!(message.contains("must be paid for"), "got: {message}"); + } + other => panic!("expected PaymentRequired, got {other:?}"), + } + assert!( + service.store().is_empty(), + "an unpaid update must not be stored" + ); + } + + #[tokio::test] + async fn a_pointer_is_created_at_zero_and_updated_one_step_at_a_time() { + let (service, _dir) = service().await; + + // A create must be counter 0. + assert!(matches!( + service.handle_put(put(&signed(1, 5, 1))).await, + PointerPutResponse::PaymentRequired { .. } + )); + assert!(service.store().is_empty(), "nothing was stored"); + + let created = signed(1, 0, 1); + assert!(matches!( + service.handle_put(put(&created)).await, + PointerPutResponse::Success { .. } + )); + + // A jump is refused however large, including the terminal counter. + for jump in [0u64, 2, 3, 99, u64::MAX] { + assert!( + matches!( + service.handle_put(put(&signed(1, jump, 2))).await, + PointerPutResponse::PaymentRequired { .. } | PointerPutResponse::Stale { .. } + ), + "counter {jump} must not be accepted after 0" + ); + } + + // Exactly one increment lands. + assert!(matches!( + service.handle_put(put(&signed(1, 1, 2))).await, + PointerPutResponse::Success { .. } + )); + let held = service + .store() + .get(&created.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), 1); + } + + #[test] + fn a_pointer_is_refused_where_a_chunk_already_sits() { + // The two kinds share one 32-byte address space. A collision is + // infeasible, not impossible, and silently picking one would destroy + // the other's data. + let address = [3u8; 32]; + match cross_kind_refusal(address, Ok(true)) { + Some(PointerPutResponse::Error(ProtocolError::StorageFailed(message))) => { + assert!( + message.contains("already occupied by a chunk"), + "got: {message}" + ); + } + other => panic!("a collision must be refused, got {other:?}"), + } + } + + #[test] + fn a_free_address_is_not_refused() { + assert!(cross_kind_refusal([3u8; 32], Ok(false)).is_none()); + } + + #[test] + fn an_unreadable_chunk_store_refuses_rather_than_assumes() { + // "I could not check" must not be treated as "nothing is there". + let refusal = cross_kind_refusal([3u8; 32], Err(Error::Storage("disk gone".into()))); + match refusal { + Some(PointerPutResponse::Error(ProtocolError::StorageFailed(message))) => { + assert!(message.contains("cannot check"), "got: {message}"); + } + other => panic!("expected a refusal, got {other:?}"), + } + } +} diff --git a/src/pointer/store.rs b/src/pointer/store.rs new file mode 100644 index 00000000..a9438a1d --- /dev/null +++ b/src/pointer/store.rs @@ -0,0 +1,1475 @@ +//! Durable storage for pointer records, with merge-on-put. +//! +//! The chunk store answers "already have it" and stops, which is right for an +//! immutable record and wrong for a mutable one — it would silently drop every +//! update. It also requires `BLAKE3(content) == address`, which no pointer can +//! satisfy. So pointers get their own store rather than an exemption carved +//! into that one. +//! +//! # The shape of a write +//! +//! A put is two steps, because the payment check sits between them: +//! +//! 1. [`PointerStore::prepare`] parses, compares against what is held and, only +//! if the arrival could win, verifies its signature. It returns either a +//! no-op outcome or a [`PreparedPut`]. +//! 2. The caller verifies payment against the candidate's address and state +//! identifier, then calls [`PointerStore::commit`]. +//! +//! Signature verification happens in step 1 **outside** any store lock, so a +//! flood of unpaid candidates cannot block every other address behind one +//! ML-DSA check. Step 2 re-checks under the lock, because the world may have +//! moved while payment was being verified. +//! +//! # Atomicity +//! +//! The commit — compare, write, re-index — runs inside a single blocking task +//! holding a synchronous lock. Splitting it across an `await` would let a +//! cancelled caller release the lock while its write was still in flight, +//! leaving the index describing a record the disk no longer holds. Because the +//! whole transaction lives in one task, dropping the caller's future cannot +//! tear it. +//! +//! On disk each record is one file named by its hex address under +//! `{root}/pointers/`. Writes go to a uniquely named temporary file and are +//! renamed into place, so a crash leaves either the old record or the new one, +//! never a torn one. A lock file under the same directory keeps two processes +//! from keeping two indexes over one set of files. + +use std::collections::HashMap; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; + +use fs2::FileExt; +use parking_lot::Mutex; +use tokio::task::spawn_blocking; + +use crate::ant_protocol::XorName; +use crate::error::{Error, Result}; +use crate::logging::{debug, warn}; +use ant_protocol::pointer::{MergeRank, ParsedPointer, Pointer, PointerState, POINTER_WIRE_LEN}; + +/// Directory under the store root that holds pointer records. +const POINTERS_DIR_NAME: &str = "pointers"; + +/// Prefix for the temporary file a write lands in before being renamed. +const TEMP_PREFIX: &str = ".tmp-"; + +/// Name of the file whose lock grants exclusive use of the directory. +const LOCK_FILE_NAME: &str = ".pointer-store-lock"; + +/// What a put did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PutOutcome { + /// No record was held for this address; the incoming one was stored. + Stored, + /// The incoming record won under the merge rule and replaced the held one. + Replaced, + /// The held record is the same authenticated state. Nothing was written. + /// + /// This is the case that keeps one payment from funding many writes: an + /// owner can produce unboundedly many valid signatures over one state, and + /// every one of them lands here. + Unchanged, + /// The incoming record lost under the merge rule. Nothing was written. + Stale, +} + +impl PutOutcome { + /// Whether this outcome changed what the node holds. + /// + /// Only a change is worth announcing to replication or counting towards a + /// commitment rebuild. + #[must_use] + pub const fn changed(self) -> bool { + matches!(self, Self::Stored | Self::Replaced) + } +} + +/// The result of [`PointerStore::inspect`]: what an arrival claims, before any +/// signature has been checked. +pub enum Inspected { + /// The arrival cannot change what is held. No signature check is owed, and + /// none was done. + Noop(PutOutcome), + /// The arrival would win as it stands. Its signature has **not** been + /// checked yet — pass it to [`PointerStore::verify`] once admission gates + /// have had their say. + Candidate(ParsedPointer), +} + +impl Inspected { + /// What this arrival claims, whether or not it is a candidate. + #[must_use] + pub fn state(&self) -> Option<&ant_protocol::pointer::PointerState> { + match self { + Self::Noop(_) => None, + Self::Candidate(parsed) => Some(parsed.state()), + } + } +} + +/// The result of [`PointerStore::prepare`]. +#[derive(Debug)] +pub enum Prepared { + /// The arrival cannot change what is held, and was rejected without a + /// signature check. There is nothing to pay for and nothing to commit. + Noop(PutOutcome), + /// A validated record that would win as of the moment it was prepared. + Candidate(PreparedPut), +} + +/// A record that parsed, out-ranked what was held, and verified. +/// +/// Carries what a payment check needs — [`Self::address`] to route and +/// [`Self::state_id`] to authorize — and holds the validated record so nothing +/// can change between validation and commit. +#[derive(Debug)] +pub struct PreparedPut { + /// The validated record. + record: Pointer, +} + +impl PreparedPut { + /// The address this record belongs at, which is what routing uses. + #[must_use] + pub fn address(&self) -> XorName { + self.record.address() + } + + /// The authenticated-state identifier, which is what a quote is paid against. + #[must_use] + pub fn state_id(&self) -> XorName { + self.record.state_id() + } + + /// The validated record. + #[must_use] + pub const fn record(&self) -> &Pointer { + &self.record + } +} + +/// What the store knows about a held record without reading it back. +#[derive(Debug, Clone, Copy)] +struct IndexEntry { + /// The held record's authenticated-state identifier. + state_id: XorName, + /// The held record's place in the merge order. + /// + /// Carried as the record's own [`MergeRank`] rather than as the fields it + /// is built from, so the store cannot drift from the rule in + /// `PointerState::replaces`. + rank: MergeRank, + /// The held record's counter, for the paid-increment check. + counter: u64, + /// `BLAKE3` over the exact stored bytes, which a storage commitment binds. + bytes_hash: XorName, + /// Which insertion this entry is. + /// + /// Monotonic for the life of this store, so an entry can be told apart + /// from a later one carrying the same `state_id`. Without it a repair that + /// restores exactly the state a reader found corrupt would be + /// indistinguishable from the corrupt entry that reader set out to disown, + /// and a second reader would erase the repair. Reuse would take 2^64 + /// commits without a restart, and an observation never outlives the store + /// that produced it, so a restart resetting the counter is harmless. + generation: u64, +} + +impl IndexEntry { + /// Describe a validated record. + fn of(record: &Pointer, generation: u64) -> Self { + Self { + state_id: record.state_id(), + rank: record.state().rank(), + counter: record.counter(), + bytes_hash: record.bytes_hash(), + generation, + } + } +} + +/// A store of pointer records. +#[derive(Debug, Clone)] +pub struct PointerStore { + inner: Arc, +} + +/// Shared state behind [`PointerStore`]. +#[derive(Debug)] +struct Inner { + /// Directory holding one file per record. + dir: PathBuf, + /// What is held, by address. + /// + /// Guarded by a synchronous lock held across the disk write, so the index + /// and the directory can never disagree about which version is current. + /// The critical section is one ~5 KB write, and it is only ever taken + /// inside a blocking task. + index: Mutex>, + /// Distinguishes concurrent temporary files so two writers to one address + /// cannot land in the same partial file. + write_seq: AtomicU64, + /// Source of index generations, monotonic for this store's lifetime. + generation: AtomicU64, + /// Set if a directory sync ever failed after a commit. + /// + /// Those writes are stored and visible; what is uncertain is whether they + /// survive a power loss. Reporting them as failures would be wrong, and + /// saying nothing would overstate the guarantee, so the store records it. + durability_degraded: AtomicBool, + /// Held for the store's lifetime; releasing it releases the directory. + _lock_file: File, +} + +impl PointerStore { + /// Open a store under `root_dir`, rebuilding its index from disk. + /// + /// Takes an exclusive lock on the directory. Two stores over one directory + /// would each keep their own index, and the one with the staler view would + /// happily overwrite the other's newer record. + /// + /// A file that does not parse as a valid pointer, or that is not at the + /// address its owner derives, is skipped and logged rather than deleted: a + /// store that prunes what it cannot read destroys the evidence of its own + /// bug. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the directory cannot be created, locked or + /// read. + pub async fn new(root_dir: &Path) -> Result { + let dir = root_dir.join(POINTERS_DIR_NAME); + let scan_dir = dir.clone(); + let (lock_file, index) = spawn_blocking(move || { + std::fs::create_dir_all(&scan_dir).map_err(|e| { + Error::Storage(format!("cannot create {}: {e}", scan_dir.display())) + })?; + let lock_file = acquire_lock(&scan_dir)?; + let index = scan(&scan_dir)?; + Ok::<_, Error>((lock_file, index)) + }) + .await + .map_err(|e| Error::Storage(format!("pointer store scan panicked: {e}")))??; + + let next_generation = index + .values() + .map(|entry| entry.generation) + .max() + .map_or(0, |highest| highest.saturating_add(1)); + debug!( + "Pointer store opened at {} with {} records", + dir.display(), + index.len() + ); + Ok(Self { + inner: Arc::new(Inner { + dir, + index: Mutex::new(index), + write_seq: AtomicU64::new(0), + generation: AtomicU64::new(next_generation), + durability_degraded: AtomicBool::new(false), + _lock_file: lock_file, + }), + }) + } + + /// Validate an arriving record against what is held. + /// + /// Cheap checks first: an arrival that cannot change anything is refused + /// before its signature is looked at, because it is a no-op whether or not + /// it is correctly signed. That ordering is what stops repeated submission + /// of one paid state from buying ML-DSA verifications. A record that would + /// win is always verified — outside the store lock, so one slow + /// verification cannot stall every other address. + /// + /// # Errors + /// + /// Returns [`Error::Protocol`] if the bytes are not a well-formed record + /// and [`Error::Crypto`] if a would-be winner's signature does not verify. + pub async fn prepare(&self, bytes: &[u8]) -> Result { + match self.inspect(bytes)? { + Inspected::Noop(outcome) => Ok(Prepared::Noop(outcome)), + Inspected::Candidate(parsed) => Ok(Prepared::Candidate(self.verify(parsed).await?)), + } + } + + /// Parse an arrival and decide whether it could change anything, without + /// verifying its signature. + /// + /// The cheap half of [`Self::prepare`]. Callers that gate on admission — + /// capacity, responsibility for the address — run this first, apply their + /// gates, and only then pay for [`Self::verify`]. Otherwise a forged record + /// for an address the node is not responsible for still buys an ML-DSA + /// verification before anything rejects it. + /// + /// # Errors + /// + /// Returns [`Error::Protocol`] if the bytes are not a well-formed record. + pub fn inspect(&self, bytes: &[u8]) -> Result { + // Parsing decodes the owner key once; `verify` reuses that parse rather + // than decoding again. The bytes travel with it, so the two cannot be + // mismatched. + let parsed = ParsedPointer::parse(bytes.to_vec())?; + let state = *parsed.state(); + + if let Some(entry) = self.snapshot(&state.address) { + if entry.state_id == state.state_id { + return Ok(Inspected::Noop(PutOutcome::Unchanged)); + } + if state.rank() <= entry.rank { + return Ok(Inspected::Noop(PutOutcome::Stale)); + } + } + Ok(Inspected::Candidate(parsed)) + } + + /// Verify a candidate's signature. + /// + /// Runs with no lock held and off the async executor: an ML-DSA + /// verification is milliseconds of CPU, and an attacker can ask for one per + /// forged record, so it must not run on a thread other work needs. + /// + /// # Errors + /// + /// Returns [`Error::Crypto`] if the signature does not verify. + pub async fn verify(&self, parsed: ParsedPointer) -> Result { + let record = spawn_blocking(move || Pointer::verify_parsed(parsed)) + .await + .map_err(|e| Error::Storage(format!("pointer verification panicked: {e}")))??; + Ok(PreparedPut { record }) + } + + /// Commit a prepared record. + /// + /// Re-checks against what is held before writing: `prepare` may have run + /// before a payment check that took long enough for a newer state to + /// arrive, and the re-check is what keeps that newer state from being + /// overwritten. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the write fails. + pub async fn commit(&self, prepared: PreparedPut) -> Result { + let inner = Arc::clone(&self.inner); + // The whole transaction runs in one task, so dropping this future + // cannot leave the write done and the index un-updated. + spawn_blocking(move || inner.commit_blocking(&prepared.record)) + .await + .map_err(|e| Error::Storage(format!("pointer commit panicked: {e}")))? + } + + /// Validate and store in one step, with no payment gate. + /// + /// For callers that have already settled payment, and for tests. The + /// request path should use [`Self::prepare`] and [`Self::commit`] so the + /// payment check can sit between them. + /// + /// # Errors + /// + /// As [`Self::prepare`] and [`Self::commit`]. + pub async fn put_bytes(&self, bytes: &[u8]) -> Result { + match self.prepare(bytes).await? { + Prepared::Noop(outcome) => Ok(outcome), + Prepared::Candidate(prepared) => self.commit(prepared).await, + } + } + + /// Read the record held at `address`, if any. + /// + /// Re-validates on the way out, so a file corrupted under the node is + /// reported as missing rather than served as authentic — and the index + /// entry for it is dropped, so the node will accept a fresh copy of that + /// state instead of answering "unchanged" to its own repair. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file cannot be read for a reason other + /// than its absence. + pub async fn get(&self, address: &XorName) -> Result> { + // What the index claimed before the read. A commit can land while the + // read is in flight, and its entry must not then be mistaken for the + // stale one this read is about to disown. + let claimed = self.snapshot(address).map(|entry| entry.generation); + + let path = self.path_for(address); + // Reading and verifying happen in one blocking task: an ML-DSA check is + // milliseconds of CPU and must not run on an async worker thread. + let read = spawn_blocking(move || { + read_record_file(&path).map(|bytes| bytes.map(|bytes| Pointer::from_bytes(&bytes))) + }) + .await + .map_err(|e| Error::Storage(format!("pointer read panicked: {e}")))?; + + let validated = match read { + Ok(Some(validated)) => validated, + Ok(None) => { + self.forget_if_unchanged(address, claimed); + return Ok(None); + } + Err(e) => return Err(e), + }; + + match validated { + Ok(record) if record.address() == *address => Ok(Some(record)), + Ok(_) => { + warn!( + "Pointer file at {} holds a record for another address; dropping it \ + from the index", + hex::encode(address) + ); + self.forget_if_unchanged(address, claimed); + Ok(None) + } + Err(e) => { + warn!( + "Pointer file at {} does not validate ({e}); dropping it from the index", + hex::encode(address) + ); + self.forget_if_unchanged(address, claimed); + Ok(None) + } + } + } + + /// The authenticated-state identifier held at `address`, if any. + /// + /// This is what a sync hint carries and what a fetch decision compares. It + /// names the state, not the encoding, so two replicas holding one state + /// under different signatures agree and do not refetch each other forever. + #[must_use] + pub fn state_id(&self, address: &XorName) -> Option { + self.snapshot(address).map(|e| e.state_id) + } + + /// `BLAKE3` over the exact bytes held at `address`, if any. + /// + /// What a storage commitment binds for this record. Per-storer by design: + /// each node commits and is audited against the encoding it actually holds. + #[must_use] + pub fn bytes_hash(&self, address: &XorName) -> Option { + self.snapshot(address).map(|e| e.bytes_hash) + } + + /// Whether `state` is the paid successor of what is held. + /// + /// One payment buys one increment: a new pointer starts at counter 0, and + /// an update must be exactly one past what this node holds. Without it an + /// owner pays once, jumps the counter, skips every intermediate payment and + /// strands the pointer at a counter nothing can advance. + /// + /// Only the client path asks this. Replication uses the merge rule instead, + /// so a replica that missed an update can still catch up rather than being + /// stuck behind a gap it can never fill. + #[must_use] + pub fn accepts_as_paid_update(&self, state: &PointerState) -> bool { + self.snapshot(&state.address).map_or_else( + || state.is_genesis(), + |entry| state.counter == entry.counter.wrapping_add(1) && entry.counter != u64::MAX, + ) + } + + /// Whether a record is held at `address`. + #[must_use] + pub fn contains(&self, address: &XorName) -> bool { + self.snapshot(address).is_some() + } + + /// Whether the record held at `address` is already this exact state. + /// + /// The question a fetch decision asks: holding *the key* is not enough for + /// a mutable record, holding *the state* is. + #[must_use] + pub fn holds_state(&self, address: &XorName, state_id: &XorName) -> bool { + self.snapshot(address) + .is_some_and(|e| e.state_id == *state_id) + } + + /// Every address the store holds. + #[must_use] + pub fn all_keys(&self) -> Vec { + self.inner.index.lock().keys().copied().collect() + } + + /// Every address with the state identifier and committed bytes hash held + /// for it. + /// + /// The input a commitment build or a sync round needs in one pass, which is + /// why the index exists rather than each of those re-reading every file. + #[must_use] + pub fn all_states(&self) -> Vec<(XorName, XorName, XorName)> { + self.inner + .index + .lock() + .iter() + .map(|(address, entry)| (*address, entry.state_id, entry.bytes_hash)) + .collect() + } + + /// Which of `wanted` this node does not already hold at the given state. + /// + /// The fetch decision a mutable record needs. The chunk path asks "do I + /// hold this key?" and stops, which for a pointer means a replica on + /// version N never fetches N+1 and the two diverge permanently. This asks + /// "do I hold this *state*?" instead. + #[must_use] + pub fn missing_states(&self, wanted: &[(XorName, XorName)]) -> Vec<(XorName, XorName)> { + let index = self.inner.index.lock(); + wanted + .iter() + .filter(|(address, state_id)| { + // `map_or` rather than `is_none_or`: the latter is stable only + // from 1.82 and this crate's MSRV is 1.75. + index + .get(address) + .map_or(true, |entry| entry.state_id != *state_id) + }) + .copied() + .collect() + } + + /// How many records the store holds. + #[must_use] + pub fn len(&self) -> usize { + self.inner.index.lock().len() + } + + /// Whether the store holds nothing. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Whether any committed write could not have its directory entry flushed. + /// + /// Those records are stored and readable now; what is uncertain is whether + /// they survive a power loss. A put still reports success, because the + /// write did happen — this is how an operator learns the filesystem is not + /// giving the store what it asks for. + #[must_use] + pub fn durability_degraded(&self) -> bool { + self.inner.durability_degraded.load(Ordering::Relaxed) + } + + /// Directory holding the records. + #[must_use] + pub fn dir(&self) -> &Path { + &self.inner.dir + } + + /// Copy out what is held for `address`, releasing the index lock at once. + fn snapshot(&self, address: &XorName) -> Option { + self.inner.index.lock().get(address).copied() + } + + /// Drop the index entry for `address`, but only if it is still the exact + /// entry the caller found unreadable. + /// + /// A read is not atomic with a write. Removing unconditionally would let a + /// slow read of a corrupt file erase the entry for a record committed while + /// it was reading, leaving the node holding a record it no longer + /// announces, commits to, or can be audited for. Matching on the generation + /// rather than the state identifier also covers the case where the record + /// written meanwhile is a *repair of the same state*, which a state + /// comparison could not tell apart from the entry being disowned. + fn forget_if_unchanged(&self, address: &XorName, claimed: Option) { + let Some(claimed) = claimed else { + // Nothing was claimed when the read began, so there is nothing this + // read is entitled to remove. + return; + }; + let mut index = self.inner.index.lock(); + if index.get(address).map(|entry| entry.generation) == Some(claimed) { + index.remove(address); + } + } + + /// Path of the file backing `address`. + fn path_for(&self, address: &XorName) -> PathBuf { + self.inner.dir.join(hex::encode(address)) + } +} + +impl Inner { + /// Compare, write and re-index under one lock. + /// + /// Runs entirely inside a blocking task: the lock is synchronous and is + /// never released between the decision and the index update, so neither + /// caller cancellation nor a concurrent writer can separate them. + // Holding the guard across the write is the point of this function, so the + // lint's advice to drop it earlier would reintroduce exactly the window + // this closes: a second writer deciding against an index that no longer + // describes the disk. + #[allow( + clippy::significant_drop_tightening, + reason = "the write must happen under the same guard as the decision" + )] + fn commit_blocking(&self, record: &Pointer) -> Result { + let address = record.address(); + let path = self.dir.join(hex::encode(address)); + + // A cheap look before doing any work. The authoritative check is the + // one under the lock below; this only avoids staging a file for an + // arrival that is already obviously a no-op. + if let Some(entry) = self.index.lock().get(&address) { + if entry.state_id == record.state_id() { + return Ok(PutOutcome::Unchanged); + } + if record.state().rank() <= entry.rank { + return Ok(PutOutcome::Stale); + } + } + + // Stage and fsync the new bytes *before* taking the lock. This is the + // slow part — a 5 KB write plus an fsync — and holding the index lock + // across it would block every other address, including the cheap + // lookups async callers make. + let seq = self.write_seq.fetch_add(1, Ordering::Relaxed); + let temp = self + .dir + .join(format!("{TEMP_PREFIX}{}-{seq}", hex::encode(address))); + stage(&temp, record.as_bytes())?; + + let outcome = { + let mut index = self.index.lock(); + + // Re-check: staging is not instantaneous and a newer state may + // have committed while it ran. + let outcome = match index.get(&address) { + None => PutOutcome::Stored, + Some(entry) if entry.state_id == record.state_id() => PutOutcome::Unchanged, + Some(entry) if record.state().rank() > entry.rank => PutOutcome::Replaced, + Some(_) => PutOutcome::Stale, + }; + if !outcome.changed() { + let _ = std::fs::remove_file(&temp); + return Ok(outcome); + } + + // The rename is the commit point: nothing fallible happens between + // it and the index update, and both are under this one lock. + if let Err(e) = std::fs::rename(&temp, &path) { + let _ = std::fs::remove_file(&temp); + return Err(Error::Storage(format!( + "cannot rename {} onto {}: {e}", + temp.display(), + path.display() + ))); + } + let generation = self.generation.fetch_add(1, Ordering::Relaxed); + index.insert(address, IndexEntry::of(record, generation)); + outcome + }; + + // Durability of the directory entry, after the commit and outside the + // lock. The record is already stored and indexed, so a failure here + // cannot be reported as "nothing happened"; it is recorded instead, and + // `durability_degraded` reports it. + if let Err(e) = sync_directory(&self.dir) { + self.durability_degraded.store(true, Ordering::Relaxed); + warn!( + "{} is stored and indexed, but {} could not be synced: {e}. It is \ + visible now; its survival across a power loss depends on the \ + filesystem", + path.display(), + self.dir.display() + ); + } + Ok(outcome) + } +} + +/// Take the exclusive lock that grants use of `dir`. +fn acquire_lock(dir: &Path) -> Result { + let path = dir.join(LOCK_FILE_NAME); + let file = OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&path) + .map_err(|e| { + Error::Storage(format!( + "cannot create the pointer store lock {}: {e}", + path.display() + )) + })?; + file.try_lock_exclusive().map_err(|e| { + Error::Storage(format!( + "another pointer store already has {} open ({e}). Two stores over one \ + directory each keep their own index, and the staler one would overwrite \ + the other's newer record", + dir.display() + )) + })?; + Ok(file) +} + +/// Read one record file, refusing anything that is not exactly a record long. +/// +/// The length is taken from the directory entry before any bytes are read, so a +/// corrupt oversized file cannot be pulled into memory. +fn read_record_file(path: &Path) -> Result>> { + let file = match File::open(path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(Error::Storage(format!("pointer read failed: {e}"))), + }; + let len = file + .metadata() + .map_err(|e| Error::Storage(format!("pointer stat failed: {e}")))? + .len(); + if len != POINTER_WIRE_LEN as u64 { + warn!( + "Pointer file {} is {len} bytes, not {POINTER_WIRE_LEN}", + path.display() + ); + return Ok(None); + } + + let mut bytes = Vec::with_capacity(POINTER_WIRE_LEN); + // One byte past the record: a file that grew between the stat and the read + // is refused rather than silently truncated into something that parses. + file.take(POINTER_WIRE_LEN as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|e| Error::Storage(format!("pointer read failed: {e}")))?; + if bytes.len() != POINTER_WIRE_LEN { + warn!( + "Pointer file {} changed size while being read", + path.display() + ); + return Ok(None); + } + Ok(Some(bytes)) +} + +/// Rebuild the index by reading every record in `dir`. +fn scan(dir: &Path) -> Result> { + let entries = std::fs::read_dir(dir) + .map_err(|e| Error::Storage(format!("cannot read {}: {e}", dir.display())))?; + + let mut index = HashMap::new(); + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + warn!("Skipping unreadable pointer directory entry: {e}"); + continue; + } + }; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if name == LOCK_FILE_NAME { + continue; + } + if name.starts_with(TEMP_PREFIX) { + // A temporary file means a crash mid-write: the rename never + // happened, so the old record (if any) is intact and the partial + // file has no claim. Sweep it, or the next write to that address + // collides with it — `create_new` would refuse, and the write + // sequence restarts at zero on every opening. + if let Err(e) = std::fs::remove_file(&path) { + warn!( + "Could not sweep the partial pointer write {}: {e}", + path.display() + ); + } + continue; + } + let bytes = match read_record_file(&path) { + Ok(Some(bytes)) => bytes, + Ok(None) => continue, + Err(e) => { + warn!("Skipping unreadable pointer file {}: {e}", path.display()); + continue; + } + }; + let record = match Pointer::from_bytes(&bytes) { + Ok(record) => record, + Err(e) => { + warn!("Skipping invalid pointer file {}: {e}", path.display()); + continue; + } + }; + let address = record.address(); + if hex::encode(address) != name { + warn!( + "Skipping pointer file {} that is not at its own address", + path.display() + ); + continue; + } + let generation = u64::try_from(index.len()).unwrap_or(u64::MAX); + index.insert(address, IndexEntry::of(&record, generation)); + } + Ok(index) +} + +/// Write `bytes` to `temp` and fsync it, ready to be renamed into place. +/// +/// Leaves nothing half written: the bytes are durable in the temporary file +/// before any rename can make them visible, and a failure at any step removes +/// it. The caller performs the rename, which is the commit point. +fn stage(temp: &Path, bytes: &[u8]) -> Result<()> { + // `create_new` so a leftover temporary file from a crashed write is never + // silently appended to or shared with a concurrent writer. + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(temp) + .map_err(|e| Error::Storage(format!("cannot create {}: {e}", temp.display())))?; + let written = file + .write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|e| Error::Storage(format!("cannot write {}: {e}", temp.display()))); + drop(file); + if let Err(e) = written { + let _ = std::fs::remove_file(temp); + return Err(e); + } + Ok(()) +} + +/// Flush the directory entry a rename created. +/// +/// Without it a crash can leave the entry unflushed and the record invisible on +/// restart. Opening a directory is not portable, so a directory that cannot be +/// opened is reported as success with a note: there is nothing to sync and +/// nothing went wrong with the write. +fn sync_directory(dir: &Path) -> Result<()> { + match File::open(dir) { + Ok(handle) => handle + .sync_all() + .map_err(|e| Error::Storage(format!("cannot sync {}: {e}", dir.display()))), + Err(e) => Err(Error::Storage(format!( + "cannot open {} to flush it: {e}", + dir.display() + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ant_protocol::pointer::{PointerTarget, PointerTargetKind, POINTER_BODY_LEN}; + use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; + + fn keypair(seed: u8) -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair_from_seed(&[seed; 32]) + } + + fn signed(seed: u8, counter: u64, target_byte: u8) -> Pointer { + let (pk, sk) = keypair(seed); + let target = PointerTarget::new(PointerTargetKind::Chunk, [target_byte; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("sign") + } + + /// A record with its signature destroyed: the body still parses, the + /// record does not verify. + fn forged(record: &Pointer) -> Vec { + let mut bytes = record.as_bytes().to_vec(); + if let Some(byte) = bytes.get_mut(POINTER_BODY_LEN + 3) { + *byte ^= 0xff; + } + assert!( + Pointer::from_bytes(&bytes).is_err(), + "the forged record must not verify" + ); + bytes + } + + async fn store() -> (PointerStore, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("open store"); + (store, dir) + } + + #[tokio::test] + async fn stores_then_reads_back_the_same_bytes() { + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + assert_eq!( + store.put_bytes(record.as_bytes()).await.expect("put"), + PutOutcome::Stored + ); + + let read = store + .get(&record.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(read.as_bytes(), record.as_bytes()); + assert_eq!(store.state_id(&record.address()), Some(record.state_id())); + assert_eq!( + store.bytes_hash(&record.address()), + Some(record.bytes_hash()) + ); + assert!(store.contains(&record.address())); + assert!(store.holds_state(&record.address(), &record.state_id())); + assert_eq!(store.len(), 1); + } + + #[tokio::test] + async fn a_higher_counter_replaces_and_a_lower_one_does_not() { + let (store, _dir) = store().await; + let first = signed(1, 1, 1); + let second = signed(1, 2, 1); + assert_eq!( + store.put_bytes(first.as_bytes()).await.expect("put"), + PutOutcome::Stored + ); + assert_eq!( + store.put_bytes(second.as_bytes()).await.expect("put"), + PutOutcome::Replaced + ); + assert_eq!( + store.put_bytes(first.as_bytes()).await.expect("put"), + PutOutcome::Stale + ); + + let held = store + .get(&first.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), 2); + assert_eq!(store.len(), 1, "an update replaces in place"); + } + + #[tokio::test] + async fn re_signing_one_state_never_reaches_the_disk() { + // One payment must not fund many writes. Every re-signature of a stored + // state is a no-op, so storage, replication and commitment work is paid + // for once. + let (store, _dir) = store().await; + let first = signed(1, 5, 5); + store.put_bytes(first.as_bytes()).await.expect("put"); + + let path = store.dir().join(hex::encode(first.address())); + let held_bytes = std::fs::read(&path).expect("read"); + + for _ in 0..16 { + let variant = signed(1, 5, 5); + assert_ne!( + variant.as_bytes(), + first.as_bytes(), + "signing is randomized" + ); + assert_eq!(variant.state_id(), first.state_id()); + assert_eq!( + store.put_bytes(variant.as_bytes()).await.expect("put"), + PutOutcome::Unchanged + ); + } + + assert_eq!( + std::fs::read(&path).expect("read"), + held_bytes, + "not one of the 16 re-signatures reached the disk" + ); + assert_eq!(store.bytes_hash(&first.address()), Some(first.bytes_hash())); + } + + #[tokio::test] + async fn a_resubmission_is_refused_before_its_signature_is_checked() { + // An arrival that cannot change anything is a no-op whether or not it + // is correctly signed, so it must not buy an ML-DSA verification. A + // record whose signature is destroyed proves the check was skipped: + // had it run, this would have been an error rather than Unchanged. + let (store, _dir) = store().await; + let held = signed(1, 4, 4); + store.put_bytes(held.as_bytes()).await.expect("put"); + + assert_eq!( + store.put_bytes(&forged(&held)).await.expect("put"), + PutOutcome::Unchanged + ); + + let after = store + .get(&held.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(after.as_bytes(), held.as_bytes(), "nothing was written"); + } + + #[tokio::test] + async fn a_stale_arrival_is_refused_before_its_signature_is_checked() { + let (store, _dir) = store().await; + store + .put_bytes(signed(1, 9, 1).as_bytes()) + .await + .expect("put"); + + // Unsigned *and* stale: refused as Stale, which can only happen if the + // signature was never checked. + let older = signed(1, 2, 1); + assert_eq!( + store.put_bytes(&forged(&older)).await.expect("put"), + PutOutcome::Stale + ); + } + + #[tokio::test] + async fn a_would_be_winner_is_always_verified() { + // Losing records skip verification; a record that would win never does. + let (store, _dir) = store().await; + store + .put_bytes(signed(1, 1, 1).as_bytes()) + .await + .expect("put"); + + let winner = signed(1, 5, 1); + assert!(store.put_bytes(&forged(&winner)).await.is_err()); + + let held = store + .get(&winner.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), 1, "the forged winner did not land"); + } + + #[tokio::test] + async fn prepare_reports_a_no_op_without_a_candidate() { + let (store, _dir) = store().await; + let held = signed(1, 6, 6); + store.put_bytes(held.as_bytes()).await.expect("put"); + + match store.prepare(held.as_bytes()).await.expect("prepare") { + Prepared::Noop(outcome) => assert_eq!(outcome, PutOutcome::Unchanged), + Prepared::Candidate(_) => panic!("an identical state is not a candidate"), + } + match store + .prepare(signed(1, 1, 6).as_bytes()) + .await + .expect("prepare") + { + Prepared::Noop(outcome) => assert_eq!(outcome, PutOutcome::Stale), + Prepared::Candidate(_) => panic!("a stale record is not a candidate"), + } + } + + #[tokio::test] + async fn a_candidate_exposes_what_a_payment_check_needs() { + let (store, _dir) = store().await; + let record = signed(1, 2, 2); + match store.prepare(record.as_bytes()).await.expect("prepare") { + Prepared::Candidate(prepared) => { + assert_eq!(prepared.address(), record.address()); + assert_eq!(prepared.state_id(), record.state_id()); + assert_eq!(prepared.record().as_bytes(), record.as_bytes()); + assert_eq!( + store.commit(prepared).await.expect("commit"), + PutOutcome::Stored + ); + } + Prepared::Noop(_) => panic!("a new record is a candidate"), + } + assert_eq!(store.len(), 1); + } + + #[tokio::test] + async fn a_commit_rechecks_what_prepare_saw() { + // `prepare` runs before the payment check; a newer state can land while + // that check is in flight, and must not then be overwritten. + let (store, _dir) = store().await; + let slow = match store + .prepare(signed(1, 2, 1).as_bytes()) + .await + .expect("prepare") + { + Prepared::Candidate(prepared) => prepared, + Prepared::Noop(_) => panic!("expected a candidate"), + }; + + // Someone else's newer state arrives while the payment is being checked. + store + .put_bytes(signed(1, 7, 1).as_bytes()) + .await + .expect("put"); + + assert_eq!( + store.commit(slow).await.expect("commit"), + PutOutcome::Stale, + "the newer state must survive a late commit" + ); + let held = store + .get(&signed(1, 2, 1).address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.counter(), 7); + } + + #[tokio::test] + async fn put_bytes_rejects_junk() { + let (store, _dir) = store().await; + assert!(store.put_bytes(b"not a pointer").await.is_err()); + assert!(store.put_bytes(&[]).await.is_err()); + assert!(store.is_empty()); + } + + #[tokio::test] + async fn every_rotation_of_a_delivery_reaches_one_answer() { + let records = vec![ + signed(1, 1, 9), + signed(1, 3, 4), + signed(1, 3, 1), + signed(1, 2, 8), + signed(1, 3, 7), + ]; + let address = records.first().expect("non-empty").address(); + + let mut winners = Vec::new(); + for rotation in 0..records.len() { + let (store, _dir) = store().await; + for offset in 0..records.len() { + let index = (rotation + offset) % records.len(); + let record = records.get(index).expect("in range"); + store.put_bytes(record.as_bytes()).await.expect("put"); + } + let held = store.get(&address).await.expect("get").expect("present"); + winners.push(held.state_id()); + } + + let distinct: std::collections::BTreeSet<&XorName> = winners.iter().collect(); + assert_eq!( + distinct.len(), + 1, + "every rotation converges: {winners:?}. Exhaustive permutations are \ + covered by the convergence property test" + ); + } + + #[tokio::test] + async fn a_node_started_empty_reaches_the_same_answer() { + let dir = tempfile::tempdir().expect("tempdir"); + let address; + let before; + { + let store = PointerStore::new(dir.path()).await.expect("open"); + for record in [signed(1, 1, 1), signed(1, 4, 2), signed(1, 2, 3)] { + store.put_bytes(record.as_bytes()).await.expect("put"); + } + address = signed(1, 1, 1).address(); + before = store.get(&address).await.expect("get").expect("present"); + } + + // A fresh store over the same directory, as after a restart. The first + // must be dropped: the directory lock allows only one at a time. + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert_eq!(reopened.len(), 1); + let after = reopened.get(&address).await.expect("get").expect("present"); + assert_eq!(after.as_bytes(), before.as_bytes()); + assert_eq!(reopened.state_id(&address), Some(before.state_id())); + assert_eq!(reopened.bytes_hash(&address), Some(before.bytes_hash())); + } + + #[tokio::test] + async fn a_second_store_over_one_directory_is_refused() { + // Two indexes over one set of files means the staler one overwrites the + // fresher one's record. + let dir = tempfile::tempdir().expect("tempdir"); + let _first = PointerStore::new(dir.path()).await.expect("open"); + let err = PointerStore::new(dir.path()) + .await + .err() + .expect("a second store must not open the same directory"); + let message = format!("{err}"); + assert!( + message.contains("another pointer store already has"), + "it must be refused by the directory lock, not by something else: {message}" + ); + } + + #[tokio::test] + async fn two_owners_occupy_two_addresses() { + let (store, _dir) = store().await; + let first = signed(1, 1, 1); + let second = signed(2, 1, 1); + assert_ne!(first.address(), second.address()); + store.put_bytes(first.as_bytes()).await.expect("put"); + store.put_bytes(second.as_bytes()).await.expect("put"); + assert_eq!(store.len(), 2); + assert_eq!(store.all_keys().len(), 2); + assert_eq!(store.all_states().len(), 2); + } + + #[tokio::test] + async fn a_corrupt_file_is_forgotten_so_the_record_can_be_repaired() { + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + store.put_bytes(record.as_bytes()).await.expect("put"); + + let path = store.dir().join(hex::encode(record.address())); + let mut bytes = std::fs::read(&path).expect("read back"); + if let Some(byte) = bytes.get_mut(10) { + *byte ^= 0xff; + } + std::fs::write(&path, &bytes).expect("corrupt it"); + + assert!(store.get(&record.address()).await.expect("get").is_none()); + assert!( + !store.contains(&record.address()), + "the index must not keep claiming a record the disk lost" + ); + assert_eq!(store.state_id(&record.address()), None); + + // The repair a peer would send is accepted rather than dismissed as + // "unchanged", which is the whole point of forgetting it. + assert_eq!( + store.put_bytes(record.as_bytes()).await.expect("put"), + PutOutcome::Stored + ); + assert!(store.get(&record.address()).await.expect("get").is_some()); + } + + #[tokio::test] + async fn a_deleted_file_is_forgotten_too() { + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + store.put_bytes(record.as_bytes()).await.expect("put"); + + std::fs::remove_file(store.dir().join(hex::encode(record.address()))).expect("remove"); + assert!(store.get(&record.address()).await.expect("get").is_none()); + assert!(!store.contains(&record.address())); + } + + #[tokio::test] + async fn a_scan_skips_what_it_cannot_validate() { + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(1, 1, 1); + { + let store = PointerStore::new(dir.path()).await.expect("open"); + store.put_bytes(record.as_bytes()).await.expect("put"); + + // Junk under a plausible name, an oversized file, and a partial write. + std::fs::write(store.dir().join(hex::encode([9u8; 32])), b"not a pointer") + .expect("write junk"); + std::fs::write( + store.dir().join(hex::encode([8u8; 32])), + vec![0u8; POINTER_WIRE_LEN * 4], + ) + .expect("write oversized"); + std::fs::write(store.dir().join(".tmp-abc-0"), vec![0u8; POINTER_WIRE_LEN]) + .expect("write temp"); + } + + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert_eq!(reopened.len(), 1, "only the valid record is indexed"); + assert!(reopened.contains(&record.address())); + } + + #[tokio::test] + async fn concurrent_writers_leave_the_index_agreeing_with_the_disk() { + let (store, _dir) = store().await; + let address = signed(1, 0, 0).address(); + + let mut tasks = Vec::new(); + for counter in 1..=12u64 { + let store = store.clone(); + let bytes = signed(1, counter, 1).as_bytes().to_vec(); + tasks.push(tokio::spawn(async move { store.put_bytes(&bytes).await })); + } + for task in tasks { + task.await.expect("join").expect("put"); + } + + let held = store.get(&address).await.expect("get").expect("present"); + assert_eq!(held.counter(), 12, "the highest counter must win"); + assert_eq!(store.state_id(&address), Some(held.state_id())); + assert_eq!(store.bytes_hash(&address), Some(held.bytes_hash())); + assert_eq!(store.len(), 1); + + // No temporary file survived the race. + let leftovers: Vec<_> = std::fs::read_dir(store.dir()) + .expect("read dir") + .filter_map(std::result::Result::ok) + .filter(|e| e.file_name().to_string_lossy().starts_with(TEMP_PREFIX)) + .collect(); + assert!(leftovers.is_empty(), "temporary files were left behind"); + } + + #[tokio::test] + async fn a_cancelled_commit_leaves_the_index_agreeing_with_the_disk() { + // Dropping the caller's future must not split the transaction: either + // the write and the index update both happened, or neither did. + let (store, _dir) = store().await; + let record = signed(1, 3, 3); + let prepared = match store.prepare(record.as_bytes()).await.expect("prepare") { + Prepared::Candidate(prepared) => prepared, + Prepared::Noop(_) => panic!("expected a candidate"), + }; + + let abandoned = { + let committing = store.commit(prepared); + tokio::pin!(committing); + // Poll once, then abandon it. If it happened to finish inside the + // timeout there was no cancellation to test, and the assertion + // below still has to hold. + tokio::time::timeout(std::time::Duration::from_nanos(1), &mut committing) + .await + .is_err() + }; + + // Wait for the detached task to settle rather than guessing at a + // duration: poll until the disk and the index agree and stop changing. + let mut settled = false; + for _ in 0..200 { + let on_disk = store.get(&record.address()).await.expect("get").is_some(); + if on_disk == store.contains(&record.address()) && on_disk { + settled = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let on_disk = store.get(&record.address()).await.expect("get").is_some(); + let indexed = store.contains(&record.address()); + assert_eq!( + on_disk, indexed, + "the index and the disk must agree however the commit was interrupted \ + (caller was cancelled: {abandoned})" + ); + assert!( + abandoned, + "the caller must actually have been cancelled, or this proves nothing" + ); + assert!( + settled, + "the detached commit must run to completion rather than stopping half done" + ); + } + + #[tokio::test] + async fn an_empty_store_holds_nothing() { + let (store, _dir) = store().await; + assert!(store.is_empty()); + assert_eq!(store.state_id(&[0u8; 32]), None); + assert_eq!(store.bytes_hash(&[0u8; 32]), None); + assert!(!store.holds_state(&[0u8; 32], &[0u8; 32])); + assert!(store.get(&[0u8; 32]).await.expect("get").is_none()); + } + + #[tokio::test] + async fn a_failed_read_does_not_erase_a_record_written_since() { + // A read that finds the file unreadable may only disown the entry it + // set out to read. A commit that lands meanwhile must survive, or the + // node ends up holding a record it never announces or commits to. + let (store, _dir) = store().await; + let old = signed(1, 1, 1); + store.put_bytes(old.as_bytes()).await.expect("put"); + + // Simulate the interleaving: the read observed the old entry, then a + // newer state was committed, and only then does the read disown what + // it saw. + let observed = store.snapshot(&old.address()).map(|entry| entry.generation); + let new = signed(1, 2, 1); + store.put_bytes(new.as_bytes()).await.expect("put"); + + store.forget_if_unchanged(&old.address(), observed); + assert_eq!( + store.state_id(&new.address()), + Some(new.state_id()), + "the record committed during the read must still be indexed" + ); + + // And the ordinary case still works: disowning what is actually there. + let current = store.snapshot(&new.address()).map(|entry| entry.generation); + store.forget_if_unchanged(&new.address(), current); + assert!(!store.contains(&new.address())); + } + + #[tokio::test] + async fn a_repair_of_the_same_state_is_not_erased_by_a_second_reader() { + // Two readers observe one corrupt entry. The first disowns it, a repair + // restores exactly the same logical state, and the second reader must + // not then erase the repair — the state identifier alone cannot tell + // the repaired entry from the corrupt one it replaced. + let (store, _dir) = store().await; + let record = signed(1, 1, 1); + store.put_bytes(record.as_bytes()).await.expect("put"); + + let first_reader = store + .snapshot(&record.address()) + .map(|entry| entry.generation); + let second_reader = first_reader; + + // Reader one finds the file unreadable and disowns what it saw. + store.forget_if_unchanged(&record.address(), first_reader); + assert!(!store.contains(&record.address())); + + // A peer repairs it with the very same state. + assert_eq!( + store.put_bytes(record.as_bytes()).await.expect("put"), + PutOutcome::Stored + ); + + // Reader two, still holding its stale observation, must not erase it. + store.forget_if_unchanged(&record.address(), second_reader); + assert!( + store.contains(&record.address()), + "the repair must survive a second reader disowning the old entry" + ); + assert_eq!(store.state_id(&record.address()), Some(record.state_id())); + } + + #[tokio::test] + async fn a_leftover_partial_write_is_swept_and_does_not_block_the_next_write() { + // The write sequence restarts at zero on every opening, so a temporary + // file left by a crash would collide with the next write to that + // address and `create_new` would refuse it. + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(1, 1, 1); + { + let store = PointerStore::new(dir.path()).await.expect("open"); + let leftover = store + .dir() + .join(format!("{TEMP_PREFIX}{}-0", hex::encode(record.address()))); + std::fs::write(&leftover, vec![0u8; POINTER_WIRE_LEN]).expect("write leftover"); + } + + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert_eq!( + reopened.put_bytes(record.as_bytes()).await.expect("put"), + PutOutcome::Stored, + "a swept leftover must not block the first write to its address" + ); + } + + #[tokio::test] + async fn a_healthy_store_does_not_report_degraded_durability() { + let (store, _dir) = store().await; + store + .put_bytes(signed(1, 1, 1).as_bytes()) + .await + .expect("put"); + assert!( + !store.durability_degraded(), + "an ordinary write on a working filesystem is fully durable" + ); + } + + #[tokio::test] + async fn holding_the_key_is_not_holding_the_state() { + // The fetch decision a mutable record needs: a replica on version N + // must not count as satisfied for version N+1. + let (store, _dir) = store().await; + let old = signed(1, 1, 1); + let new = signed(1, 2, 1); + store.put_bytes(old.as_bytes()).await.expect("put"); + + assert!(store.contains(&new.address())); + assert!(store.holds_state(&old.address(), &old.state_id())); + assert!( + !store.holds_state(&new.address(), &new.state_id()), + "the newer state is absent even though the key is held" + ); + } +} diff --git a/src/replication/commitment.rs b/src/replication/commitment.rs index 61082510..f188a9c9 100644 --- a/src/replication/commitment.rs +++ b/src/replication/commitment.rs @@ -26,6 +26,7 @@ use blake3::Hasher; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaSecretKey}; use crate::ant_protocol::XorName; +use crate::replication::subtree::LeafKind; // ADR-0004: the commitment wire type, its pin (`commitment_hash`), its // signature verification, and the key-count cap are the SINGLE SOURCE OF TRUTH @@ -44,6 +45,10 @@ pub const DOMAIN_LEAF: &[u8] = b"autonomi.ant.replication.storage_leaf.v1"; /// Domain-separation tag for Merkle internal nodes: `BLAKE3(this || left || right)`. pub const DOMAIN_NODE: &[u8] = b"autonomi.ant.replication.storage_node.v1"; +/// Domain separator for a pointer leaf, distinct from [`DOMAIN_LEAF`] so the +/// record kind is bound by the leaf hash itself. +pub const DOMAIN_POINTER_LEAF: &[u8] = b"autonomi.ant.replication.storage_pointer_leaf.v1"; + // `MAX_COMMITMENT_KEY_COUNT` and `StorageCommitment` are re-exported from // `ant-protocol` above (single source of truth); their fields and wire size are // documented there. @@ -52,10 +57,13 @@ pub const DOMAIN_NODE: &[u8] = b"autonomi.ant.replication.storage_node.v1"; // Hashing helpers // --------------------------------------------------------------------------- -/// Compute the Merkle leaf hash for `(key, bytes_hash)`. +/// Compute the Merkle leaf hash for a content-addressed chunk. /// /// `bytes_hash` is BLAKE3 over the record bytes; the leaf binds the key to /// the content so an adversary cannot reuse a leaf for a different chunk. +/// +/// Unchanged from v1, so a node holding only chunks produces exactly the root +/// it always did. #[must_use] pub fn leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { let mut h = Hasher::new(); @@ -65,6 +73,22 @@ pub fn leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { *h.finalize().as_bytes() } +/// Compute the Merkle leaf hash for a pointer. +/// +/// A separate domain from [`leaf_hash`] is what binds the kind. A peer that +/// relabelled a chunk leaf as a pointer — to escape the round-1 guard that +/// `bytes_hash == key` — would hash it under this domain instead, changing the +/// leaf, the root, and therefore the structural check against its own signed +/// commitment. So the exemption cannot be claimed for a chunk. +#[must_use] +pub fn pointer_leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { + let mut h = Hasher::new(); + h.update(DOMAIN_POINTER_LEAF); + h.update(key); + h.update(bytes_hash); + *h.finalize().as_bytes() +} + /// Combine two child hashes into a Merkle internal-node hash. #[must_use] pub fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { @@ -120,7 +144,28 @@ impl MerkleTree { /// Returns an error if `entries` is empty (no commitment to make), if /// `entries.len() > MAX_COMMITMENT_KEY_COUNT`, or if it contains /// duplicate keys. - pub fn build(mut entries: Vec<(XorName, [u8; 32])>) -> Result { + pub fn build(entries: Vec<(XorName, [u8; 32])>) -> Result { + Self::build_of_kinds( + entries + .into_iter() + .map(|(key, bytes_hash)| (key, bytes_hash, LeafKind::Chunk)) + .collect(), + ) + } + + /// Build a Merkle tree over `(key, bytes_hash, kind)` triples. + /// + /// The kind picks the leaf domain, so a chunk-only key set produces exactly + /// the root [`Self::build`] always produced, while a pointer leaf — whose + /// `bytes_hash` cannot equal its `key` — is distinguishable at round 1 + /// without a peer being able to claim that exemption for a chunk. + /// + /// # Errors + /// + /// As [`Self::build`]. + pub fn build_of_kinds( + mut entries: Vec<(XorName, [u8; 32], LeafKind)>, + ) -> Result { if entries.is_empty() { return Err(CommitmentError::EmptyKeySet); } @@ -139,8 +184,11 @@ impl MerkleTree { let leaves: Vec<(XorName, [u8; 32])> = entries .into_iter() - .map(|(k, bh)| { - let lh = leaf_hash(&k, &bh); + .map(|(k, bh, kind)| { + let lh = match kind { + LeafKind::Chunk => leaf_hash(&k, &bh), + LeafKind::Pointer => pointer_leaf_hash(&k, &bh), + }; (k, lh) }) .collect(); diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 62b965f9..cf1306fe 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -41,6 +41,7 @@ use crate::replication::commitment::{ commitment_hash, sign_commitment, verify_commitment_signature, CommitmentError, MerkleTree, StorageCommitment, }; +use crate::replication::subtree::LeafKind; /// Auditor-side per-peer commitment state. /// @@ -185,6 +186,27 @@ impl BuiltCommitment { Self::build_from_tree(tree, sender_peer_id, secret_key, sender_public_key) } + /// Build over a mixed key set of chunks and pointers. + /// + /// A pointer's `bytes_hash` cannot equal its key, so its leaf is hashed + /// under a different domain; passing the kind is what lets the round-1 + /// verifier tell the two apart without letting a peer claim a pointer's + /// exemption for a chunk. A key set containing only chunks produces exactly + /// the root [`Self::build`] produces. + /// + /// # Errors + /// + /// As [`Self::build`]. + pub fn build_of_kinds( + entries: Vec<(XorName, [u8; 32], LeafKind)>, + sender_peer_id: &[u8; 32], + secret_key: &MlDsaSecretKey, + sender_public_key: &[u8], + ) -> Result { + let tree = MerkleTree::build_of_kinds(entries)?; + Self::build_from_tree(tree, sender_peer_id, secret_key, sender_public_key) + } + /// Sign and wrap an ALREADY-BUILT Merkle tree. Lets callers that already /// built the tree (e.g. the rotation no-op-root check, §11) avoid rebuilding /// it inside [`Self::build`]. diff --git a/src/replication/config.rs b/src/replication/config.rs index 9150d584..d90d85e0 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -426,7 +426,7 @@ pub const REPLICATION_PROTOCOL_ID: &str = "autonomi.ant.replication.v2"; /// possession/repair/commitment-fetch) with no per-peer limiter. A truly /// zero-penalty rollout needs an upstream `send_request` that does not /// auto-report trust; tracked as a saorsa-core follow-up. -pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v1"; +pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v2"; /// 10 MiB — maximum replication wire message size (accommodates hint batches). const REPLICATION_MESSAGE_SIZE_MIB: usize = 10; @@ -1485,9 +1485,15 @@ mod tests { // Core replication, including all digest audit lanes, stays on v2. // Only the subtree family changed and therefore receives a separate id. assert_eq!(REPLICATION_PROTOCOL_ID, "autonomi.ant.replication.v2"); + // Bumped v1 -> v2 when the subtree leaf gained its record-kind tag, so + // a pointer leaf can be told from a chunk leaf at round 1. The leaf is + // postcard-encoded positionally, so the new field is wire-incompatible + // and needs the new id. ADR-0009 provides for exactly this: the subtree + // family versions independently of core replication, and mixed-version + // audits pause rather than misdecode. assert_eq!( SUBTREE_AUDIT_PROTOCOL_ID, - "autonomi.ant.replication.subtree-audit.v1" + "autonomi.ant.replication.subtree-audit.v2" ); assert_ne!(REPLICATION_PROTOCOL_ID, SUBTREE_AUDIT_PROTOCOL_ID); } diff --git a/src/replication/protocol.rs b/src/replication/protocol.rs index 3ef7d0fa..a6078688 100644 --- a/src/replication/protocol.rs +++ b/src/replication/protocol.rs @@ -1453,11 +1453,14 @@ mod tests { fn max_round1_proof_fits_the_audit_family_ceiling() { use crate::replication::commitment::{StorageCommitment, MAX_COMMITMENT_KEY_COUNT}; use crate::replication::config::MAX_SUBTREE_AUDIT_MESSAGE_SIZE; - use crate::replication::subtree::{max_subtree_leaves, SubtreeLeaf, SubtreeProof}; + use crate::replication::subtree::{ + max_subtree_leaves, LeafKind, SubtreeLeaf, SubtreeProof, + }; let leaf_count = max_subtree_leaves(MAX_COMMITMENT_KEY_COUNT) as usize; let leaves: Vec = (0..leaf_count) .map(|_| SubtreeLeaf { + kind: LeafKind::Chunk, key: [0xAB; 32], bytes_hash: [0xCD; 32], content_len: u32::MAX, diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index f99a4e70..a6c5d283 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -30,7 +30,8 @@ use crate::replication::protocol::{ }; use crate::replication::recent_provers::RecentProvers; use crate::replication::subtree::{ - select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeProof, + select_subtree_path, subtree_plan, verify_subtree_proof, LeafKind, StructureVerdict, + SubtreeProof, }; use crate::replication::types::{AuditFailureReason, AuditFailureSummary, FailureEvidence}; use crate::storage::ChunkStore; @@ -726,6 +727,25 @@ pub(crate) fn evaluate_subtree_structure( if proof.leaves.iter().any(|l| l.bytes_hash != l.key) { return Err(AuditFailureReason::DigestMismatch); } + + // Pointer leaves are refused outright, not merely denied credit. + // + // The kind is bound by the leaf hash, so a peer cannot relabel a chunk leaf + // as a pointer to escape the guard above. What it *can* still do is sign a + // commitment of its own naming any key with the hash of cheap bytes it + // really holds: a pointer's key derives from its owner, not its bytes, so + // round 1 has nothing to check it against, and round 2 authenticates the + // served block against that same peer-chosen `bytes_hash`. Such a proof + // would pass, and a pass clears bootstrap state and earns trust even with + // holder credit withheld. + // + // Admitting them safely needs round 2 to serve the whole record and the + // auditor to verify its signature and derived address. Until that exists, + // refusing costs nothing: commitment rotation builds from the chunk store + // alone, so no honest proof carries a pointer leaf. + if proof.leaves.iter().any(|l| l.kind != LeafKind::Chunk) { + return Err(AuditFailureReason::DigestMismatch); + } Ok(()) } @@ -954,6 +974,40 @@ pub(crate) fn verify_slice_response( AuditVerdict::Pass { checked } } +/// Credit a peer as a proven holder of the leaves its passing proof covers. +/// +/// A **chunk** leaf is credited on the strength of round 1 alone: round 1 +/// enforces `bytes_hash == key` there, so a peer cannot commit a chunk leaf for +/// a key whose bytes it does not have. +/// +/// A **pointer** leaf earns **nothing**, sampled or not. Its address is a +/// function of its owner key rather than of its bytes, so round 1 cannot bind +/// the two — and round 2 does not close the gap either, because it authenticates +/// the served block against the leaf's own `bytes_hash`, which the peer chose. +/// A peer can therefore sign a one-leaf commitment naming any key `K` with the +/// hash of cheap bytes it really holds, be sampled (the sole leaf always is), +/// pass, and be credited as a holder of `K`. Sampling does not help: the bytes +/// are attacker-chosen either way. +/// +/// Closing this needs round 2 to serve the **whole record** and the auditor to +/// parse it, check its signature and that its owner derives `K`. Until that +/// exists, no credit is the only sound answer — and it costs nothing today, +/// because production commitments are still built from the chunk store alone. +async fn credit_proven_holder( + credit: &AuditCredit<'_>, + proof: &SubtreeProof, + challenged_peer: &PeerId, + pin: [u8; 32], +) { + let now = std::time::Instant::now(); + let mut provers = credit.recent_provers.write().await; + for leaf in &proof.leaves { + if leaf.kind == LeafKind::Chunk { + provers.record_proof(leaf.key, *challenged_peer, pin, now); + } + } +} + /// Verify a subtree-proof response (auditor side), ADR-0002 two-round audit. /// /// **Round 1** (this proof): pin + identity + signature + structure. If the @@ -1090,11 +1144,7 @@ async fn verify_subtree_response( observe_closeness(ctx.p2p_node, ctx.config, challenged_peer, proof).await; // Credit the peer as a proven holder of its committed keys. if let (Some(credit), Some(pin)) = (ctx.credit, commitment_hash(commitment)) { - let now = std::time::Instant::now(); - let mut provers = credit.recent_provers.write().await; - for leaf in &proof.leaves { - provers.record_proof(leaf.key, *challenged_peer, pin, now); - } + credit_proven_holder(credit, proof, challenged_peer, pin).await; } info!( "Audit: peer {challenged_peer} passed subtree audit ({} leaves, {checked} \ @@ -1779,7 +1829,7 @@ async fn serve_committed_key_openings( mod tests { use super::*; use crate::replication::commitment_state::BuiltCommitment; - use crate::replication::subtree::{build_subtree_proof, SubtreeLeaf}; + use crate::replication::subtree::{build_subtree_proof, LeafKind, SubtreeLeaf}; use saorsa_pqc::api::sig::ml_dsa_65; use std::time::Instant; @@ -1976,6 +2026,7 @@ mod tests { #[test] fn verify_slice_response_rejects_malformed_item_sets() { let leaf = |k: XorName| SubtreeLeaf { + kind: LeafKind::Chunk, key: k, bytes_hash: [0u8; 32], content_len: 0, @@ -2507,6 +2558,7 @@ mod tests { #[test] fn subtree_leaf_is_constructible() { let _l = SubtreeLeaf { + kind: LeafKind::Chunk, key: key(1), bytes_hash: [0u8; 32], content_len: 0, diff --git a/src/replication/subtree.rs b/src/replication/subtree.rs index a50a628f..88a651a1 100644 --- a/src/replication/subtree.rs +++ b/src/replication/subtree.rs @@ -31,7 +31,9 @@ //! leaf range `[slot * span, (slot + 1) * span)` where `span = 2^(D - depth)`, //! intersected with `0..N`. -use super::commitment::{leaf_hash, node_hash, StorageCommitment, MAX_COMMITMENT_KEY_COUNT}; +use super::commitment::{ + leaf_hash, node_hash, pointer_leaf_hash, StorageCommitment, MAX_COMMITMENT_KEY_COUNT, +}; use crate::ant_protocol::XorName; use serde::{Deserialize, Serialize}; @@ -39,9 +41,31 @@ use serde::{Deserialize, Serialize}; /// meaningless for tiny trees and a full proof is cheap. pub const SMALL_TREE_FULL_AUDIT_FLOOR: u32 = 4; +/// What kind of record a subtree leaf attests. +/// +/// Bound into the leaf hash, so a peer cannot relabel a chunk leaf as a pointer +/// leaf to escape the content-address guard: relabelling changes the leaf hash, +/// which changes the root, which fails the structural check against the signed +/// commitment. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] +pub enum LeafKind { + /// A content-addressed chunk, where `bytes_hash == key`. + #[default] + Chunk, + /// A pointer, whose address is a function of its owner key rather than of + /// its bytes, so `bytes_hash != key` by construction. + Pointer, +} + /// One leaf of the selected subtree, as returned by the responder. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SubtreeLeaf { + /// What kind of record this leaf attests. + /// + /// Placed first so the wire shape is obviously different from the + /// untagged v1 leaf; the subtree audit family is versioned to v2 for this + /// reason and mixed-version audits pause, exactly as ADR-0009 prescribes. + pub kind: LeafKind, /// The committed key (chunk address) at this leaf position. pub key: XorName, /// `BLAKE3(record_bytes)` — the plain content hash. For a content-addressed @@ -368,10 +392,18 @@ pub fn verify_subtree_proof( // is the tree's odd tail at some level). `fold_to_root` stopped at a single // hash and so skipped the self-pair when a truncated block reached length 1 // before climbing all the way to the subtree-root level — the geometry bug. + // Hash each leaf under the domain its kind selects. Doing this by kind is + // what actually binds the kind to the root: a chunk leaf relabelled + // `Pointer` (to escape the round-1 `bytes_hash == key` guard) hashes under + // the pointer domain here, rebuilds to a different root, and fails against + // the peer's own signed commitment. let leaf_hashes: Vec<[u8; 32]> = proof .leaves .iter() - .map(|l| leaf_hash(&l.key, &l.bytes_hash)) + .map(|l| match l.kind { + LeafKind::Chunk => leaf_hash(&l.key, &l.bytes_hash), + LeafKind::Pointer => pointer_leaf_hash(&l.key, &l.bytes_hash), + }) .collect(); let levels_to_subtree_root = total_depth - path.depth; let mut cur = fold_levels(leaf_hashes, levels_to_subtree_root); @@ -572,8 +604,26 @@ pub fn subtree_leaf( challenged_peer_id: &[u8; 32], key: &XorName, bytes: &[u8], +) -> SubtreeLeaf { + subtree_leaf_of_kind(LeafKind::Chunk, nonce, challenged_peer_id, key, bytes) +} + +/// Build one subtree leaf of a given kind. +/// +/// A pointer's address is a function of its owner key, not of its bytes, so its +/// `bytes_hash` never equals its `key`. The kind is what tells the round-1 +/// verifier that this is expected rather than the possession-forgery it +/// otherwise looks exactly like. +#[must_use] +pub fn subtree_leaf_of_kind( + kind: LeafKind, + nonce: &[u8; 32], + challenged_peer_id: &[u8; 32], + key: &XorName, + bytes: &[u8], ) -> SubtreeLeaf { SubtreeLeaf { + kind, key: *key, bytes_hash: *blake3::hash(bytes).as_bytes(), content_len: u32::try_from(bytes.len()).unwrap_or(u32::MAX), @@ -590,6 +640,86 @@ pub fn subtree_leaf( #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; + + /// A pointer leaf and a chunk leaf over identical `(key, bytes_hash)` must + /// not produce the same leaf hash, or a peer could relabel a chunk as a + /// pointer to escape the round-1 `bytes_hash == key` guard. + #[test] + fn the_leaf_kind_is_bound_by_the_leaf_hash() { + use crate::replication::commitment::{leaf_hash, pointer_leaf_hash}; + + let key: XorName = [0xA1u8; 32]; + let bytes_hash = [0xB2u8; 32]; + assert_ne!( + leaf_hash(&key, &bytes_hash), + pointer_leaf_hash(&key, &bytes_hash), + "relabelling a chunk leaf as a pointer must change the leaf, and so the root" + ); + } + + /// A chunk-only key set must still produce exactly the root it always did, + /// or every existing commitment in the network would be invalidated. + #[test] + fn a_chunk_only_commitment_root_is_unchanged() { + use crate::replication::commitment::MerkleTree; + + let entries: Vec<(XorName, [u8; 32])> = (0u8..8).map(|i| ([i; 32], [i; 32])).collect(); + let via_build = MerkleTree::build(entries.clone()).expect("build").root(); + let via_kinds = MerkleTree::build_of_kinds( + entries + .into_iter() + .map(|(k, b)| (k, b, LeafKind::Chunk)) + .collect(), + ) + .expect("build_of_kinds") + .root(); + assert_eq!(via_build, via_kinds, "chunk-only roots must not move"); + } + + /// A pointer leaf changes the root, so a peer cannot smuggle one into a + /// commitment another peer signed. + #[test] + fn a_pointer_leaf_changes_the_root() { + use crate::replication::commitment::MerkleTree; + + let entries: Vec<(XorName, [u8; 32])> = (0u8..4).map(|i| ([i; 32], [i; 32])).collect(); + let chunky = MerkleTree::build(entries.clone()).expect("build").root(); + let mixed = MerkleTree::build_of_kinds( + entries + .into_iter() + .enumerate() + .map(|(i, (k, b))| { + let kind = if i == 0 { + LeafKind::Pointer + } else { + LeafKind::Chunk + }; + (k, b, kind) + }) + .collect(), + ) + .expect("build_of_kinds") + .root(); + assert_ne!(chunky, mixed); + } + + /// The default kind is Chunk, so any leaf built by the existing path keeps + /// the round-1 guard it always had. + #[test] + fn leaves_default_to_chunk() { + assert_eq!(LeafKind::default(), LeafKind::Chunk); + let leaf = subtree_leaf(&[0u8; 32], &[1u8; 32], &[2u8; 32], b"bytes"); + assert_eq!(leaf.kind, LeafKind::Chunk); + let pointer = subtree_leaf_of_kind( + LeafKind::Pointer, + &[0u8; 32], + &[1u8; 32], + &[2u8; 32], + b"bytes", + ); + assert_eq!(pointer.kind, LeafKind::Pointer); + assert_eq!(pointer.bytes_hash, leaf.bytes_hash, "only the kind differs"); + } use crate::replication::commitment::MerkleTree; fn xn_u32(i: u32) -> XorName { @@ -876,6 +1006,95 @@ mod tests { } } + /// Relabelling a leaf's kind must break the proof. + /// + /// This is the property the pointer exemption rests on: round 1 skips the + /// `bytes_hash == key` guard for pointer leaves, so if a peer could flip a + /// chunk leaf's kind to `Pointer` it would escape the guard for free. The + /// kind picks the leaf-hash domain, so flipping it rebuilds to a different + /// root and fails against the peer's own signed commitment. + #[test] + fn relabelling_a_leaf_kind_breaks_the_proof() { + let peer = [0xABu8; 32]; + let nonce = [0x5Cu8; 32]; + let entries: Vec<(XorName, [u8; 32])> = + (0u8..8).map(|i| (xn_u32(u32::from(i)), [i; 32])).collect(); + let entries: Vec<(XorName, [u8; 32])> = entries + .into_iter() + .map(|(k, _)| (k, *blake3::hash(&chunk_bytes(&k)).as_bytes())) + .collect(); + + let (proof, commitment) = build_proof(&entries, &nonce, &peer); + assert!( + matches!( + verify_subtree_proof(&proof, &nonce, &commitment), + StructureVerdict::Valid + ), + "the honest chunk proof must verify" + ); + + // Flip one leaf's kind and nothing else. + let mut relabelled = proof.clone(); + if let Some(leaf) = relabelled.leaves.first_mut() { + assert_eq!(leaf.kind, LeafKind::Chunk); + leaf.kind = LeafKind::Pointer; + } + assert!( + matches!( + verify_subtree_proof(&relabelled, &nonce, &commitment), + StructureVerdict::Invalid(_) + ), + "a relabelled leaf must not rebuild to the committed root" + ); + } + + /// And the other direction: a genuine pointer leaf verifies against a + /// commitment built with that kind, so the exemption is usable at all. + #[test] + fn a_genuine_pointer_leaf_verifies_against_its_own_commitment() { + use crate::replication::commitment::MerkleTree; + + let peer = [0xCDu8; 32]; + let nonce = [0x77u8; 32]; + let keys: Vec = (0u8..4).map(|i| xn_u32(u32::from(i))).collect(); + + // One pointer leaf among chunks: bytes_hash deliberately != key, which + // round 1 would reject for a chunk. + let entries: Vec<(XorName, [u8; 32], LeafKind)> = keys + .iter() + .enumerate() + .map(|(i, k)| { + let bytes_hash = *blake3::hash(&chunk_bytes(k)).as_bytes(); + let kind = if i == 0 { + LeafKind::Pointer + } else { + LeafKind::Chunk + }; + (*k, bytes_hash, kind) + }) + .collect(); + + let tree = MerkleTree::build_of_kinds(entries.clone()).unwrap(); + let key_count = tree.key_count(); + let mut proof = + build_subtree_proof(&tree, &nonce, &peer, |k| Some(chunk_bytes(k))).unwrap(); + // The builder tags every leaf Chunk; restore the kinds the tree used. + for leaf in &mut proof.leaves { + if let Some((_, _, kind)) = entries.iter().find(|(k, _, _)| *k == leaf.key) { + leaf.kind = *kind; + } + } + let commitment = fake_commitment(tree.root(), key_count, peer); + + assert!( + matches!( + verify_subtree_proof(&proof, &nonce, &commitment), + StructureVerdict::Valid + ), + "a pointer leaf must verify against a commitment that declared it one" + ); + } + #[test] fn honest_proof_verifies_at_many_sizes() { let peer = [0xABu8; 32]; diff --git a/src/storage/handler.rs b/src/storage/handler.rs index b3c02541..1ba544c6 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -40,11 +40,13 @@ use crate::client::compute_address; use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; use crate::payment::{PaymentVerifier, QuoteGenerator, VerificationContext}; +use crate::pointer::PointerService; use crate::replication::admission; use crate::replication::config::K_BUCKET_SIZE; use crate::replication::fresh::FreshWriteEvent; use crate::storage::traffic::{self, ChunkRequestKind, ChunkResponseKey}; use crate::storage::ChunkStore; +use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; use bytes::Bytes; use parking_lot::RwLock; use saorsa_core::P2PNode; @@ -62,7 +64,7 @@ use tokio::sync::mpsc; /// onto further peers (ADR-0002) is still accepted here, while a genuinely far /// node — which could only mis-attribute fresh-replication failures — is /// turned away. -const SELF_CLOSENESS_GATE_WIDTH: usize = K_BUCKET_SIZE; +pub const SELF_CLOSENESS_GATE_WIDTH: usize = K_BUCKET_SIZE; fn duration_ms(duration: Duration) -> u64 { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) @@ -302,6 +304,12 @@ pub struct AntProtocol { /// `attach_p2p_node`. Drives the self-closeness gate on client PUTs; /// `None` in unit tests that never attach a node. p2p_node: RwLock>>, + /// Serves pointer requests, when the node has a pointer store. + /// + /// `None` on a node built without one, so a peer that sends a pointer + /// message to a node that does not keep pointers gets a clean refusal + /// rather than a silent drop. + pointers: Option, } impl AntProtocol { @@ -339,9 +347,26 @@ impl AntProtocol { quote_generator, fresh_write_tx: None, p2p_node: RwLock::new(None), + pointers: None, } } + /// Serve pointer requests from `pointers`. + /// + /// Opt-in rather than built in: a node with no pointer store refuses + /// pointer messages cleanly instead of pretending to hold them. + #[must_use] + pub fn with_pointer_service(mut self, pointers: PointerService) -> Self { + self.pointers = Some(pointers); + self + } + + /// The pointer service, if this node serves pointers. + #[must_use] + pub const fn pointer_service(&self) -> Option<&PointerService> { + self.pointers.as_ref() + } + /// Attach the node's P2P handle for payment live-DHT checks. /// /// Wires the handle into the payment verifier so payment-proof closeness @@ -349,6 +374,11 @@ impl AntProtocol { /// replaces the verifier handle. pub fn attach_p2p_node(&self, node: Arc) { *self.p2p_node.write() = Some(Arc::clone(&node)); + if let Some(pointers) = &self.pointers { + // Pointers take the same self-closeness gate as chunks, judged at + // the pointer address because that is what the network routes on. + pointers.attach_p2p_node(Arc::clone(&node)); + } self.payment_verifier.attach_p2p_node(node); debug!("AntProtocol: P2PNode attached for payment live-DHT checks and self-closeness gate"); } @@ -570,6 +600,26 @@ impl AntProtocol { ), ChunkResponseKey::MerkleQuoteV2, ), + // Pointer traffic is attributed to `Other`: the table itemises + // chunk response outcomes, and a pointer response is not one. + ChunkMessageBody::PointerPutRequest(req) => ( + ChunkMessageBody::PointerPutResponse(match &self.pointers { + Some(service) => service.handle_put(req).await, + None => PointerPutResponse::Error(ProtocolError::StorageFailed( + "this node does not store pointers".to_string(), + )), + }), + ChunkResponseKey::Other, + ), + ChunkMessageBody::PointerGetRequest(req) => ( + ChunkMessageBody::PointerGetResponse(match &self.pointers { + Some(service) => service.handle_get(req).await, + None => PointerGetResponse::NotFound { + address: req.address, + }, + }), + ChunkResponseKey::Other, + ), // Anything else — response messages are handled by client // subscribers (e.g. send_and_await_chunk_response), not by the // protocol handler. Returning None prevents the caller from @@ -650,6 +700,22 @@ impl AntProtocol { }); } + // 2b. Refuse a chunk whose address a pointer already occupies. + // + // The mirror of the check the pointer path makes. Both kinds draw + // addresses from the same 32-byte range, and a collision — however + // infeasible — must not be resolved by whichever kind arrived second, + // because that silently destroys the other's data. + if let Some(pointers) = &self.pointers { + if pointers.store().contains(&address) { + warn!("Refusing chunk {addr_hex}: a pointer already occupies that address"); + return ChunkPutResponse::Error(ProtocolError::StorageFailed(format!( + "address {addr_hex} is already occupied by a pointer; refusing to \ + store a chunk over it" + ))); + } + } + // 3. Check if already exists (idempotent success) // // Verified against the offered bytes, not answered from the name. A name can diff --git a/src/storage/mod.rs b/src/storage/mod.rs index b60d71c7..6b46ee0e 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -60,6 +60,7 @@ pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR pub use file_store::{FileStore, FileStoreConfig, StoreLayout}; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; +pub use handler::SELF_CLOSENESS_GATE_WIDTH; pub(crate) use lmdb::CapacityVerdict; pub use lmdb::{LmdbStorage, LmdbStorageConfig}; pub use migration::{MigrationConfig, MigrationPhase, MigrationState}; diff --git a/tests/poc_commitment_audit_attacks.rs b/tests/poc_commitment_audit_attacks.rs index 903af243..144c8548 100644 --- a/tests/poc_commitment_audit_attacks.rs +++ b/tests/poc_commitment_audit_attacks.rs @@ -74,7 +74,7 @@ use ant_node::replication::slice::{ }; use ant_node::replication::subtree::{ build_subtree_proof, select_spotcheck_indices, select_subtree_path, verify_subtree_proof, - StructureVerdict, SubtreeLeaf, SubtreeProof, + LeafKind, StructureVerdict, SubtreeLeaf, SubtreeProof, }; use rand::Rng; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; @@ -385,6 +385,7 @@ fn relay_holding_only_addresses_caught_by_real_bytes_check() { // commitment because it lacks the bytes. let forged_nonced_root = *blake3::hash(b"i-do-not-have-the-bytes").as_bytes(); leaves.push(SubtreeLeaf { + kind: LeafKind::Chunk, key: k, bytes_hash: k, content_len: u32::try_from(c.len()).unwrap(), @@ -491,6 +492,7 @@ fn predict_and_fetch_relay_is_caught_by_fresh_random_sample() { *blake3::hash(b"forged").as_bytes() }; leaves.push(SubtreeLeaf { + kind: LeafKind::Chunk, key: k, bytes_hash: k, content_len: u32::try_from(c.len()).unwrap(), diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs new file mode 100644 index 00000000..805541c1 --- /dev/null +++ b/tests/pointer_convergence.rs @@ -0,0 +1,657 @@ +//! Convergence and payment-binding properties of the pointer merge rule. +//! +//! ADR-0015's claim is that selecting the maximum over a total order on +//! *states* is idempotent, commutative and associative, so every node reaches +//! the same value from any delivery interleaving given the same record set. +//! These are the property tests behind that claim, plus the two anti-abuse +//! properties the merge rule exists to provide: one payment funds one state, +//! and no re-signature of a stored state can displace it. + +use std::collections::BTreeSet; + +use ant_protocol::pointer::{Pointer, PointerTarget, PointerTargetKind}; +use proptest::prelude::*; +use saorsa_pqc::api::sig::{ + ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey, MlDsaSignature, MlDsaVariant, +}; + +/// A deterministic keypair, so a failing case is reproducible from its seed. +fn keypair(seed: u8) -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair_from_seed(&[seed; 32]) +} + +fn signed(seed: u8, counter: u64, target_byte: u8, kind: PointerTargetKind) -> Pointer { + let (pk, sk) = keypair(seed); + let target = PointerTarget::new(kind, [target_byte; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("signing a pointer") +} + +/// Fold a delivery order down to the winner, as a node's store does. +fn winner<'a>(order: &[&'a Pointer]) -> &'a Pointer { + let mut best = order.first().copied().expect("non-empty delivery"); + for candidate in order.iter().skip(1) { + if candidate.replaces(best) { + best = candidate; + } + } + best +} + +/// Visit every permutation of `order`, calling `check` on each. +fn permute( + order: &mut Vec<&Pointer>, + start: usize, + check: &mut impl FnMut(&[&Pointer]) -> Result<(), TestCaseError>, + seen: &mut usize, +) -> Result<(), TestCaseError> { + if start == order.len() { + *seen += 1; + return check(order); + } + for i in start..order.len() { + order.swap(start, i); + permute(order, start + 1, check, seen)?; + order.swap(start, i); + } + Ok(()) +} + +/// `n!`, for asserting that every permutation really was visited. +fn factorial(n: usize) -> usize { + (1..=n).product() +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(24))] + + /// Any permutation of a record set reaches the same state. + #[test] + fn every_delivery_order_converges( + counters in prop::collection::vec(0u64..4, 2..6), + targets in prop::collection::vec(0u8..4, 2..6), + ) { + let len = counters.len().min(targets.len()); + let records: Vec = (0..len) + .map(|i| { + let counter = counters.get(i).copied().unwrap_or(0); + let target = targets.get(i).copied().unwrap_or(0); + signed(1, counter, target, PointerTargetKind::Chunk) + }) + .collect(); + + let forward: Vec<&Pointer> = records.iter().collect(); + let expected = winner(&forward).state_id(); + + // Every permutation, not a sample of them: the sets are small enough + // that "any delivery order" can be checked exhaustively. + let mut order: Vec<&Pointer> = records.iter().collect(); + let mut permutations = 0usize; + permute(&mut order, 0, &mut |candidate| { + prop_assert_eq!(winner(candidate).state_id(), expected); + Ok(()) + }, &mut permutations)?; + prop_assert_eq!(permutations, factorial(records.len())); + } + + /// Delivering a record twice changes nothing: the fold is idempotent. + #[test] + fn duplicate_delivery_changes_nothing( + counter in 0u64..8, + target in 0u8..8, + extra_counter in 0u64..8, + extra_target in 0u8..8, + ) { + let first = signed(1, counter, target, PointerTargetKind::Chunk); + let second = signed(1, extra_counter, extra_target, PointerTargetKind::Chunk); + + let once = winner(&[&first, &second]).state_id(); + let twice = winner(&[&first, &second, &first, &second]).state_id(); + let thrice = winner(&[&second, &first, &second, &first, &first]).state_id(); + prop_assert_eq!(once, twice); + prop_assert_eq!(once, thrice); + } + + /// Exactly one of `a replaces b` / `b replaces a` holds for distinct states, + /// and neither holds for equal ones. Without this two nodes could disagree. + #[test] + fn the_comparator_is_antisymmetric( + left_counter in 0u64..4, + left_target in 0u8..4, + right_counter in 0u64..4, + right_target in 0u8..4, + ) { + let left = signed(1, left_counter, left_target, PointerTargetKind::Chunk); + let right = signed(1, right_counter, right_target, PointerTargetKind::Chunk); + + if left.state_id() == right.state_id() { + prop_assert!(!left.replaces(&right)); + prop_assert!(!right.replaces(&left)); + } else { + prop_assert_ne!(left.replaces(&right), right.replaces(&left)); + } + } + + /// No number of re-signatures of one state can displace it, and all of them + /// pay under a single identifier. + #[test] + fn re_signing_one_state_neither_wins_nor_pays_again( + counter in 0u64..64, + target in 0u8..64, + variants in 2usize..12, + ) { + let records: Vec = (0..variants) + .map(|_| signed(1, counter, target, PointerTargetKind::Chunk)) + .collect(); + + let encodings: BTreeSet<&[u8]> = records.iter().map(Pointer::as_bytes).collect(); + prop_assert_eq!(encodings.len(), variants, "ML-DSA signing is randomized"); + + let states: BTreeSet<_> = records.iter().map(Pointer::state_id).collect(); + prop_assert_eq!(states.len(), 1, "one state, so one paid identifier"); + + for a in &records { + for b in &records { + prop_assert!(!a.replaces(b)); + } + } + } + + /// A change to any signed field changes the paid identifier, so a receipt + /// bought for one update cannot fund another. (`state_id` is a 256-bit hash + /// over a 1,994-byte body, so this is collision resistance, not injectivity; + /// what is testable is that each field actually reaches the hash.) + #[test] + fn every_signed_field_changes_the_paid_identifier( + counter in 0u64..u64::MAX, + tag in any::(), + target in any::<[u8; 32]>(), + ) { + let (pk, sk) = keypair(1); + let base = Pointer::sign(&sk, &pk, counter, PointerTarget::from_raw_tag(tag, target)) + .expect("sign"); + + let other_counter = Pointer::sign( + &sk, &pk, counter.wrapping_add(1), PointerTarget::from_raw_tag(tag, target), + ).expect("sign"); + let other_tag = Pointer::sign( + &sk, &pk, counter, PointerTarget::from_raw_tag(tag.wrapping_add(1), target), + ).expect("sign"); + let mut flipped = target; + flipped[0] ^= 1; + let other_target = Pointer::sign( + &sk, &pk, counter, PointerTarget::from_raw_tag(tag, flipped), + ).expect("sign"); + let (other_pk, other_sk) = keypair(2); + let other_owner = Pointer::sign( + &other_sk, &other_pk, counter, PointerTarget::from_raw_tag(tag, target), + ).expect("sign"); + + let ids: BTreeSet<_> = [&base, &other_counter, &other_tag, &other_target, &other_owner] + .iter() + .map(|record| record.state_id()) + .collect(); + prop_assert_eq!(ids.len(), 5, "counter, tag, target and owner all reach state_id"); + } + + /// Every record of one owner lives at one address, whatever it says — + /// including under a target tag this build does not know. + #[test] + fn the_address_tracks_only_the_owner( + counter in 0u64..1000, + tag in any::(), + target_bytes in any::<[u8; 32]>(), + ) { + let (pk, sk) = keypair(3); + let target = PointerTarget::from_raw_tag(tag, target_bytes); + let record = Pointer::sign(&sk, &pk, counter, target).expect("sign"); + let reference = signed(3, 0, 0, PointerTargetKind::Chunk); + prop_assert_eq!(record.address(), reference.address()); + prop_assert_ne!(record.address(), signed(4, 0, 0, PointerTargetKind::Chunk).address()); + + // An unknown tag round-trips untouched; the node never interprets it. + let parsed = Pointer::from_bytes(record.as_bytes()).expect("parse"); + prop_assert_eq!(parsed.target().kind_tag(), tag); + prop_assert_eq!(parsed.target().address, target_bytes); + } + + /// The merge order holds over arbitrary target bytes, not just the + /// repeated-byte targets the other cases use. + #[test] + fn arbitrary_targets_order_by_their_bytes( + counter in 0u64..4, + left in any::<[u8; 32]>(), + right in any::<[u8; 32]>(), + tag in any::(), + ) { + let (pk, sk) = keypair(6); + let a = Pointer::sign(&sk, &pk, counter, PointerTarget::from_raw_tag(tag, left)) + .expect("sign"); + let b = Pointer::sign(&sk, &pk, counter, PointerTarget::from_raw_tag(tag, right)) + .expect("sign"); + + match left.cmp(&right) { + std::cmp::Ordering::Less => { + prop_assert!(a.replaces(&b), "smaller target bytes win"); + prop_assert!(!b.replaces(&a)); + } + std::cmp::Ordering::Greater => { + prop_assert!(b.replaces(&a)); + prop_assert!(!a.replaces(&b)); + } + std::cmp::Ordering::Equal => { + prop_assert!(!a.replaces(&b)); + prop_assert!(!b.replaces(&a)); + } + } + } + + /// Any record that parses is correctly signed; any mutation of its bytes + /// either fails to parse or fails to verify. Nothing forged gets through. + #[test] + fn no_mutation_of_a_record_survives_validation( + index in 0usize..5303, + mask in 1u8..255, + ) { + let record = signed(5, 11, 22, PointerTargetKind::Chunk); + let mut bytes = record.as_bytes().to_vec(); + let Some(byte) = bytes.get_mut(index) else { + return Ok(()); + }; + *byte ^= mask; + + match Pointer::from_bytes(&bytes) { + Err(_) => {} + Ok(parsed) => { + // The only mutations that can verify are ones that produced a + // different but still-valid signature encoding of the same + // body, which cannot happen for a single flipped byte. If one + // ever does, it must still be the same state. + prop_assert_eq!(parsed.state_id(), record.state_id()); + } + } + } +} + +/// A worked instance of the attack the merge rule exists to stop: 64 valid +/// signatures over one paid state, submitted worst-first. +#[test] +fn sixty_four_signatures_over_one_state_yield_one_winner() { + let records: Vec = (0..64) + .map(|_| signed(7, 9, 9, PointerTargetKind::Chunk)) + .collect(); + + let encodings: BTreeSet<&[u8]> = records.iter().map(Pointer::as_bytes).collect(); + assert_eq!(encodings.len(), 64, "64 distinct valid encodings"); + + let states: BTreeSet<_> = records.iter().map(Pointer::state_id).collect(); + assert_eq!(states.len(), 1, "all 64 sit at one paid identifier"); + + // Sorted worst-first is the submission order that would have made every + // record win under a byte-ordering tie-break. + let mut sorted: Vec<&Pointer> = records.iter().collect(); + sorted.sort_by_key(|record| record.as_bytes().to_vec()); + + let first = sorted.first().copied().expect("non-empty"); + for candidate in &sorted { + assert!( + !candidate.replaces(first), + "no re-signature of a stored state may displace it" + ); + } + assert_eq!(winner(&sorted).state_id(), first.state_id()); +} + +/// Golden vectors pinning the wire format. +/// +/// Field offsets, the big-endian counter, the inclusion of every owner byte and +/// all 33 target bytes, and the two domain separators are consensus: a node that +/// disagrees about any of them computes a different address or a different paid +/// identifier and silently partitions. These assertions restate the format +/// independently of the code under test, so a refactor that changes it fails +/// here rather than in production. +#[test] +fn the_wire_format_is_what_the_adr_says() { + use blake3::Hasher; + + let (pk, sk) = keypair(42); + let target_address = [0xABu8; 32]; + let counter = 0x0102_0304_0506_0708u64; + let record = Pointer::sign( + &sk, + &pk, + counter, + PointerTarget::from_raw_tag(0x5A, target_address), + ) + .expect("sign"); + let bytes = record.as_bytes(); + + // Layout. + assert_eq!(bytes.len(), 5303, "1 + 1952 + 8 + 33 + 3309"); + assert_eq!(bytes.first().copied(), Some(1u8), "format_version is 1"); + assert_eq!( + bytes.get(1..1953).expect("owner range"), + pk.to_bytes().as_slice(), + "every owner byte is carried verbatim at offset 1" + ); + assert_eq!( + bytes.get(1953..1961).expect("counter range"), + &[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08], + "the counter is big-endian at offset 1953" + ); + assert_eq!( + bytes.get(1961).copied(), + Some(0x5A), + "the target tag sits at offset 1961, uninterpreted" + ); + assert_eq!( + bytes.get(1962..1994).expect("target range"), + target_address.as_slice(), + "all 32 target address bytes follow the tag" + ); + + // Fixed expected values for the fixed seed. These are consensus: a build + // that computes anything else partitions from the network, so they are + // written out rather than recomputed from the code under test. + assert_eq!( + hex::encode(record.address()), + "f82d07b1e8be4b9bc9b87df513baf57f65478ff321e4ecb5a0883f9bf8f594b7", + "the pointer address for seed 42" + ); + assert_eq!( + hex::encode(record.state_id()), + "36684c7d59d7f5ac2aeef581377fcbf8d88689fda0c01ead2fde66d7a8120aea", + "the paid state identifier for this record" + ); + + // And the derivations those constants come from, restated independently. + let mut address_hasher = Hasher::new(); + address_hasher.update(b"autonomi.pointer.address.v1"); + address_hasher.update(&pk.to_bytes()); + assert_eq!( + record.address(), + *address_hasher.finalize().as_bytes(), + "the address derives from the owner key alone" + ); + + let mut state_hasher = Hasher::new(); + state_hasher.update(b"autonomi.pointer.state.v1"); + state_hasher.update(bytes.get(..1994).expect("body range")); + assert_eq!( + record.state_id(), + *state_hasher.finalize().as_bytes(), + "state_id covers the whole body and nothing else" + ); + + assert_eq!( + record.bytes_hash(), + *blake3::hash(bytes).as_bytes(), + "bytes_hash is over the whole record" + ); + + // The signature verifies over the body under the literal context, and does + // not verify under a different one. The context is consensus too. + let dsa = ml_dsa_65(); + let signature = MlDsaSignature::from_bytes( + MlDsaVariant::MlDsa65, + bytes.get(1994..).expect("signature range"), + ) + .expect("signature parses"); + assert!( + dsa.verify_with_context( + &pk, + bytes.get(..1994).expect("body range"), + &signature, + b"autonomi.pointer.head.v1", + ) + .expect("verify"), + "the signature is over the body under autonomi.pointer.head.v1" + ); + assert!( + !dsa.verify_with_context( + &pk, + bytes.get(..1994).expect("body range"), + &signature, + b"autonomi.pointer.head.v2", + ) + .expect("verify"), + "and not under any other context" + ); + + // The domains are distinct, so neither identifier can be mistaken for the + // other or for a plain content address. + assert_ne!(record.address(), record.state_id()); + assert_ne!( + record.address(), + *blake3::hash(&pk.to_bytes()).as_bytes(), + "the address is domain-separated from a bare hash of the key" + ); +} + +/// The same attack, driven through a real store: 64 valid signatures over one +/// paid state must produce exactly one file and never rewrite it. +/// +/// The fold above proves the comparator refuses them; this proves the storage +/// layer does, which is where the cost would actually have been paid. +#[tokio::test] +async fn sixty_four_signatures_buy_exactly_one_write() { + use ant_node::pointer::store::{PointerStore, PutOutcome}; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("open store"); + + let first = signed(7, 9, 9, PointerTargetKind::Chunk); + assert_eq!( + store.put_bytes(first.as_bytes()).await.expect("put"), + PutOutcome::Stored + ); + + let path = store.dir().join(hex::encode(first.address())); + let stored = std::fs::read(&path).expect("read back"); + + for _ in 0..63 { + let variant = signed(7, 9, 9, PointerTargetKind::Chunk); + assert_ne!( + variant.as_bytes(), + first.as_bytes(), + "signing is randomized" + ); + assert_eq!( + store.put_bytes(variant.as_bytes()).await.expect("put"), + PutOutcome::Unchanged + ); + } + + assert_eq!( + std::fs::read(&path).expect("read back"), + stored, + "63 re-signatures after the first must not touch the stored bytes" + ); + assert_eq!(store.len(), 1, "one address, one record"); + + let files: Vec<_> = std::fs::read_dir(store.dir()) + .expect("read dir") + .filter_map(std::result::Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|name| !name.starts_with('.')) + .collect(); + assert_eq!(files.len(), 1, "one payment bought one file: {files:?}"); +} + +// ============================================================================= +// Version and counter gaps +// ============================================================================= + +/// A version-only change must not reuse a paid state identifier. +/// +/// `format_version` is the first byte of the signed body and `state_id` hashes +/// the whole body, so a record differing only in its version is a different +/// state. Without this a future format could spend a receipt bought for +/// version 1 — the free-update defect, reintroduced through the version field. +#[test] +fn a_version_only_change_cannot_reuse_a_paid_state_identifier() { + use ant_protocol::pointer::{state_id_for_body, POINTER_BODY_LEN, POINTER_FORMAT_VERSION}; + + let record = signed(11, 9, 9, PointerTargetKind::Chunk); + let mut body = record + .as_bytes() + .get(..POINTER_BODY_LEN) + .expect("body") + .to_vec(); + + let paid = state_id_for_body(&body); + assert_eq!( + paid, + record.state_id(), + "the baseline is the record's own id" + ); + + let mut seen = BTreeSet::new(); + seen.insert(paid); + for version in 0u8..=u8::MAX { + if version == POINTER_FORMAT_VERSION { + continue; + } + if let Some(byte) = body.first_mut() { + *byte = version; + } + let other = state_id_for_body(&body); + assert_ne!( + other, paid, + "version {version} must not share version {POINTER_FORMAT_VERSION}'s paid id" + ); + assert!( + seen.insert(other), + "version {version} collided with another version's paid id" + ); + } + assert_eq!(seen.len(), 256, "every version has its own paid identifier"); +} + +/// A record of an unknown version is refused outright, so it can never be +/// stored under version 1's authority even if someone paid for it. +#[test] +fn an_unknown_version_is_refused_rather_than_accepted_at_its_own_price() { + use ant_protocol::pointer::{PointerError, PointerState, POINTER_FORMAT_VERSION}; + + let record = signed(12, 1, 1, PointerTargetKind::Chunk); + for version in [0u8, 2, 7, u8::MAX] { + assert_ne!(version, POINTER_FORMAT_VERSION); + let mut bytes = record.as_bytes().to_vec(); + if let Some(byte) = bytes.first_mut() { + *byte = version; + } + assert!(matches!( + Pointer::from_bytes(&bytes), + Err(PointerError::UnknownFormatVersion(v)) if v == version + )); + assert!(matches!( + PointerState::parse(&bytes), + Err(PointerError::UnknownFormatVersion(v)) if v == version + )); + } +} + +/// At `u64::MAX` no *counter* can out-rank the winner, but a smaller *target* +/// still can. That asymmetry is why migration has to happen before the terminal +/// update rather than on it. +#[test] +fn a_terminal_counter_cannot_be_out_counted_only_out_targeted() { + let terminal = signed(13, u64::MAX, 5, PointerTargetKind::Chunk); + assert!(terminal.is_terminal()); + assert!(terminal.next_counter().is_err(), "no successor exists"); + + // Nothing at any lower counter replaces it. + for counter in [0u64, 1, 1000, u64::MAX - 2, u64::MAX - 1] { + let earlier = signed(13, counter, 0, PointerTargetKind::Chunk); + assert!(!earlier.replaces(&terminal)); + assert!(terminal.replaces(&earlier)); + } + + // The order does not degenerate there: equal-counter conflicts at the + // maximum still resolve deterministically, so replicas cannot split. + let low = signed(13, u64::MAX, 1, PointerTargetKind::Chunk); + let high = signed(13, u64::MAX, 2, PointerTargetKind::Chunk); + assert!(low.replaces(&high)); + assert!(!high.replaces(&low)); + assert_eq!(winner(&[&high, &low]).state_id(), low.state_id()); + assert_eq!(winner(&[&low, &high]).state_id(), low.state_id()); +} + +/// Why migration must happen *before* the terminal update. +/// +/// At `u64::MAX` the counter can no longer advance, but the pointer is not +/// frozen: the merge order still resolves equal counters by target bytes, and +/// *smaller* target bytes win. So a migration written at the terminal counter +/// can still be displaced — by the owner, or by anyone replaying an older +/// signed record of theirs with a smaller target. The only safe migration is +/// one made while a successor counter still exists, because a strictly larger +/// counter is the one move nothing can answer. +#[tokio::test] +async fn migration_must_happen_before_the_terminal_update() { + use ant_node::pointer::store::{PointerStore, PutOutcome}; + use ant_protocol::pointer::PointerTarget; + use saorsa_pqc::api::sig::ml_dsa_65; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + + let (pk, sk) = ml_dsa_65().generate_keypair_from_seed(&[14u8; 32]); + let sign_at = |counter: u64, target: PointerTarget| { + Pointer::sign(&sk, &pk, counter, target).expect("sign") + }; + + // One update short of the end: a successor counter still exists. + let penultimate = sign_at( + u64::MAX - 1, + PointerTarget::new(PointerTargetKind::Chunk, [0x10u8; 32]), + ); + assert!(!penultimate.is_terminal()); + assert_eq!( + store.put_bytes(penultimate.as_bytes()).await.expect("put"), + PutOutcome::Stored + ); + + // The safe migration: spend the last counter. A strictly larger counter + // beats every target, so nothing at u64::MAX - 1 can answer it. + let migration = sign_at( + penultimate.next_counter().expect("successor exists"), + PointerTarget::new(PointerTargetKind::Pointer, [0x80u8; 32]), + ); + assert_eq!( + store.put_bytes(migration.as_bytes()).await.expect("put"), + PutOutcome::Replaced + ); + assert!(migration.is_terminal()); + assert!(migration.next_counter().is_err()); + + // Now the danger. The counter is spent, so the only remaining moves are to + // strictly smaller target bytes — and they still win. + let smaller_target = sign_at( + u64::MAX, + PointerTarget::new(PointerTargetKind::Chunk, [0x01u8; 32]), + ); + assert!( + smaller_target.as_bytes() != migration.as_bytes(), + "a genuinely different state" + ); + assert_eq!( + store + .put_bytes(smaller_target.as_bytes()) + .await + .expect("put"), + PutOutcome::Replaced, + "a terminal pointer is NOT frozen: a smaller target still displaces it" + ); + + // Larger target bytes cannot claw it back: the move is one-way. + assert_eq!( + store.put_bytes(migration.as_bytes()).await.expect("put"), + PutOutcome::Stale, + "and the displaced migration can never be restored" + ); + + // Which is the whole point: a migration made at the terminal counter is + // not final, so it has to be made earlier, where the counter still answers. + assert!(smaller_target.replaces(&migration)); + assert!(!migration.replaces(&smaller_target)); +} From 3fe92b9130798c6f10a67bc0c4e65893ad05b237 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 17:11:58 +0900 Subject: [PATCH 02/32] refactor(pointer): follow the protocol record to its five fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit as_bytes became to_bytes upstream: the record no longer caches its encoding, because a fixed-width layout has exactly one, so it re-encodes from the fields instead. The ADR now shows the record as the five fields it is, and says plainly why the key is carried — ML-DSA has no key recovery and a 1,952-byte key cannot be a 32-byte address. --- docs/adr/ADR-0015-pointers-immutable-owner.md | 27 ++++--- src/pointer/service.rs | 10 +-- src/pointer/store.rs | 80 +++++++++---------- tests/pointer_convergence.rs | 36 ++++----- 4 files changed, 81 insertions(+), 72 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 99367c49..55f54bcb 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -22,16 +22,25 @@ a pre-buy. Declining transfer is what lets this design be small enough to trust. One record. No genesis object, no certificates, no lineage. ```rust -pub struct Pointer { // 5,303 bytes - format_version: u8, - owner: MlDsa65PublicKey, // 1,952 — the pointer's identity - counter: u64, - target: PointerTarget, // 1-byte kind tag + 32-byte address - signature: MlDsa65Signature, // 3,309, over everything above +pub struct Pointer { // 5,303 bytes + version: u8, // 1 + owner: MlDsa65PublicKey, // 1,952 — the identity; the address derives from it + counter: u64, // 8 — 0 to create, +1 per paid update + target: PointerTarget, // 33 — kind tag + address; opaque to a node + sig: MlDsa65Signature, // 3,309 — over every field above } ``` -Fixed-width, big-endian, hand-encoded; no serde in the signed bytes. +Five fields and nothing else — no cached bytes, no cached identifiers. Encoding +is fixed-width, big-endian and hand-rolled with no serde, so there is exactly +one byte sequence for a record and re-encoding is always identical to what was +signed. The address and `state_id` are hashes of fields already present, so they +are computed rather than stored. + +The key is carried because it has to be: ML-DSA has no key recovery and a +1,952-byte key cannot be a 32-byte address. That is the whole reason a pointer +is 5,303 bytes rather than ~3,350, and the price of validating one with no +fetch. ### Three identities @@ -95,7 +104,7 @@ re-checks under its lock, because a newer state can land while payment verifies. | Pay once, jump the counter | Client updates must be `+1` | | Pay for a chunk to fund a pointer | Paid cache keyed by a typed `Chunk` vs `PointerState`, never by a raw 32-byte value a crafted chunk could occupy | | Merkle proof with no issuer check | Refused for pointers; single-node proofs only | -| Downgrade the format | `format_version` is signed and inside `state_id`; unknown versions are refused | +| Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | | Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | | Collide a pointer and a chunk address | Refused in both directions rather than resolved | | Mint an audit leaf for someone's key | Pointer leaves are refused at round 1 (see below) | @@ -142,7 +151,7 @@ pointer leaves are refused rather than trusted. - A resubmission and a stale arrival are both refused before any signature check. - Creation is counter 0; an update is `+1`; every jump — including to `u64::MAX` and a wrap back to 0 — is refused as a non-successor. -- All 256 `format_version` values give distinct paid identifiers. +- All 256 `version` values give distinct paid identifiers. - Golden vectors pin the encoding, both identities and the signing context. - A relabelled audit leaf fails structural verification; a chunk-only commitment root is bit-identical to before. diff --git a/src/pointer/service.rs b/src/pointer/service.rs index d72501bc..4324912d 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -300,7 +300,7 @@ impl PointerService { match self.store.get(&request.address).await { Ok(Some(record)) => PointerGetResponse::Success { - record: Bytes::copy_from_slice(record.as_bytes()), + record: Bytes::from(record.to_bytes()), }, Ok(None) => PointerGetResponse::NotFound { address: request.address, @@ -402,7 +402,7 @@ mod tests { } fn put(record: &Pointer) -> PointerPutRequest { - PointerPutRequest::new(Bytes::copy_from_slice(record.as_bytes())) + PointerPutRequest::new(Bytes::from(record.to_bytes())) } #[tokio::test] @@ -423,7 +423,7 @@ mod tests { .await { PointerGetResponse::Success { record: bytes } => { - assert_eq!(bytes.as_ref(), record.as_bytes()); + assert_eq!(bytes.as_ref(), record.to_bytes()); } other => panic!("expected Success, got {other:?}"), } @@ -459,7 +459,7 @@ mod tests { service.handle_put(put(&record)).await; let variant = signed(1, 0, 4); - assert_ne!(variant.as_bytes(), record.as_bytes()); + assert_ne!(variant.to_bytes(), record.to_bytes()); match service.handle_put(put(&variant)).await { PointerPutResponse::Unchanged { address, state_id } => { assert_eq!(address, record.address()); @@ -523,7 +523,7 @@ mod tests { async fn a_forged_signature_is_refused() { let (service, _dir) = service().await; let record = signed(1, 0, 1); - let mut bytes = record.as_bytes().to_vec(); + let mut bytes = record.to_bytes().to_vec(); if let Some(byte) = bytes.get_mut(ant_protocol::pointer::POINTER_BODY_LEN + 3) { *byte ^= 0xff; } diff --git a/src/pointer/store.rs b/src/pointer/store.rs index a9438a1d..6397b9f6 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -632,7 +632,7 @@ impl Inner { let temp = self .dir .join(format!("{TEMP_PREFIX}{}-{seq}", hex::encode(address))); - stage(&temp, record.as_bytes())?; + stage(&temp, &record.to_bytes())?; let outcome = { let mut index = self.index.lock(); @@ -872,7 +872,7 @@ mod tests { /// A record with its signature destroyed: the body still parses, the /// record does not verify. fn forged(record: &Pointer) -> Vec { - let mut bytes = record.as_bytes().to_vec(); + let mut bytes = record.to_bytes().to_vec(); if let Some(byte) = bytes.get_mut(POINTER_BODY_LEN + 3) { *byte ^= 0xff; } @@ -894,7 +894,7 @@ mod tests { let (store, _dir) = store().await; let record = signed(1, 1, 1); assert_eq!( - store.put_bytes(record.as_bytes()).await.expect("put"), + store.put_bytes(&record.to_bytes()).await.expect("put"), PutOutcome::Stored ); @@ -903,7 +903,7 @@ mod tests { .await .expect("get") .expect("present"); - assert_eq!(read.as_bytes(), record.as_bytes()); + assert_eq!(read.to_bytes(), record.to_bytes()); assert_eq!(store.state_id(&record.address()), Some(record.state_id())); assert_eq!( store.bytes_hash(&record.address()), @@ -920,15 +920,15 @@ mod tests { let first = signed(1, 1, 1); let second = signed(1, 2, 1); assert_eq!( - store.put_bytes(first.as_bytes()).await.expect("put"), + store.put_bytes(&first.to_bytes()).await.expect("put"), PutOutcome::Stored ); assert_eq!( - store.put_bytes(second.as_bytes()).await.expect("put"), + store.put_bytes(&second.to_bytes()).await.expect("put"), PutOutcome::Replaced ); assert_eq!( - store.put_bytes(first.as_bytes()).await.expect("put"), + store.put_bytes(&first.to_bytes()).await.expect("put"), PutOutcome::Stale ); @@ -948,7 +948,7 @@ mod tests { // for once. let (store, _dir) = store().await; let first = signed(1, 5, 5); - store.put_bytes(first.as_bytes()).await.expect("put"); + store.put_bytes(&first.to_bytes()).await.expect("put"); let path = store.dir().join(hex::encode(first.address())); let held_bytes = std::fs::read(&path).expect("read"); @@ -956,13 +956,13 @@ mod tests { for _ in 0..16 { let variant = signed(1, 5, 5); assert_ne!( - variant.as_bytes(), - first.as_bytes(), + variant.to_bytes(), + first.to_bytes(), "signing is randomized" ); assert_eq!(variant.state_id(), first.state_id()); assert_eq!( - store.put_bytes(variant.as_bytes()).await.expect("put"), + store.put_bytes(&variant.to_bytes()).await.expect("put"), PutOutcome::Unchanged ); } @@ -983,7 +983,7 @@ mod tests { // had it run, this would have been an error rather than Unchanged. let (store, _dir) = store().await; let held = signed(1, 4, 4); - store.put_bytes(held.as_bytes()).await.expect("put"); + store.put_bytes(&held.to_bytes()).await.expect("put"); assert_eq!( store.put_bytes(&forged(&held)).await.expect("put"), @@ -995,14 +995,14 @@ mod tests { .await .expect("get") .expect("present"); - assert_eq!(after.as_bytes(), held.as_bytes(), "nothing was written"); + assert_eq!(after.to_bytes(), held.to_bytes(), "nothing was written"); } #[tokio::test] async fn a_stale_arrival_is_refused_before_its_signature_is_checked() { let (store, _dir) = store().await; store - .put_bytes(signed(1, 9, 1).as_bytes()) + .put_bytes(&signed(1, 9, 1).to_bytes()) .await .expect("put"); @@ -1020,7 +1020,7 @@ mod tests { // Losing records skip verification; a record that would win never does. let (store, _dir) = store().await; store - .put_bytes(signed(1, 1, 1).as_bytes()) + .put_bytes(&signed(1, 1, 1).to_bytes()) .await .expect("put"); @@ -1039,14 +1039,14 @@ mod tests { async fn prepare_reports_a_no_op_without_a_candidate() { let (store, _dir) = store().await; let held = signed(1, 6, 6); - store.put_bytes(held.as_bytes()).await.expect("put"); + store.put_bytes(&held.to_bytes()).await.expect("put"); - match store.prepare(held.as_bytes()).await.expect("prepare") { + match store.prepare(&held.to_bytes()).await.expect("prepare") { Prepared::Noop(outcome) => assert_eq!(outcome, PutOutcome::Unchanged), Prepared::Candidate(_) => panic!("an identical state is not a candidate"), } match store - .prepare(signed(1, 1, 6).as_bytes()) + .prepare(&signed(1, 1, 6).to_bytes()) .await .expect("prepare") { @@ -1059,11 +1059,11 @@ mod tests { async fn a_candidate_exposes_what_a_payment_check_needs() { let (store, _dir) = store().await; let record = signed(1, 2, 2); - match store.prepare(record.as_bytes()).await.expect("prepare") { + match store.prepare(&record.to_bytes()).await.expect("prepare") { Prepared::Candidate(prepared) => { assert_eq!(prepared.address(), record.address()); assert_eq!(prepared.state_id(), record.state_id()); - assert_eq!(prepared.record().as_bytes(), record.as_bytes()); + assert_eq!(prepared.record().to_bytes(), record.to_bytes()); assert_eq!( store.commit(prepared).await.expect("commit"), PutOutcome::Stored @@ -1080,7 +1080,7 @@ mod tests { // that check is in flight, and must not then be overwritten. let (store, _dir) = store().await; let slow = match store - .prepare(signed(1, 2, 1).as_bytes()) + .prepare(&signed(1, 2, 1).to_bytes()) .await .expect("prepare") { @@ -1090,7 +1090,7 @@ mod tests { // Someone else's newer state arrives while the payment is being checked. store - .put_bytes(signed(1, 7, 1).as_bytes()) + .put_bytes(&signed(1, 7, 1).to_bytes()) .await .expect("put"); @@ -1132,7 +1132,7 @@ mod tests { for offset in 0..records.len() { let index = (rotation + offset) % records.len(); let record = records.get(index).expect("in range"); - store.put_bytes(record.as_bytes()).await.expect("put"); + store.put_bytes(&record.to_bytes()).await.expect("put"); } let held = store.get(&address).await.expect("get").expect("present"); winners.push(held.state_id()); @@ -1155,7 +1155,7 @@ mod tests { { let store = PointerStore::new(dir.path()).await.expect("open"); for record in [signed(1, 1, 1), signed(1, 4, 2), signed(1, 2, 3)] { - store.put_bytes(record.as_bytes()).await.expect("put"); + store.put_bytes(&record.to_bytes()).await.expect("put"); } address = signed(1, 1, 1).address(); before = store.get(&address).await.expect("get").expect("present"); @@ -1166,7 +1166,7 @@ mod tests { let reopened = PointerStore::new(dir.path()).await.expect("reopen"); assert_eq!(reopened.len(), 1); let after = reopened.get(&address).await.expect("get").expect("present"); - assert_eq!(after.as_bytes(), before.as_bytes()); + assert_eq!(after.to_bytes(), before.to_bytes()); assert_eq!(reopened.state_id(&address), Some(before.state_id())); assert_eq!(reopened.bytes_hash(&address), Some(before.bytes_hash())); } @@ -1194,8 +1194,8 @@ mod tests { let first = signed(1, 1, 1); let second = signed(2, 1, 1); assert_ne!(first.address(), second.address()); - store.put_bytes(first.as_bytes()).await.expect("put"); - store.put_bytes(second.as_bytes()).await.expect("put"); + store.put_bytes(&first.to_bytes()).await.expect("put"); + store.put_bytes(&second.to_bytes()).await.expect("put"); assert_eq!(store.len(), 2); assert_eq!(store.all_keys().len(), 2); assert_eq!(store.all_states().len(), 2); @@ -1205,7 +1205,7 @@ mod tests { async fn a_corrupt_file_is_forgotten_so_the_record_can_be_repaired() { let (store, _dir) = store().await; let record = signed(1, 1, 1); - store.put_bytes(record.as_bytes()).await.expect("put"); + store.put_bytes(&record.to_bytes()).await.expect("put"); let path = store.dir().join(hex::encode(record.address())); let mut bytes = std::fs::read(&path).expect("read back"); @@ -1224,7 +1224,7 @@ mod tests { // The repair a peer would send is accepted rather than dismissed as // "unchanged", which is the whole point of forgetting it. assert_eq!( - store.put_bytes(record.as_bytes()).await.expect("put"), + store.put_bytes(&record.to_bytes()).await.expect("put"), PutOutcome::Stored ); assert!(store.get(&record.address()).await.expect("get").is_some()); @@ -1234,7 +1234,7 @@ mod tests { async fn a_deleted_file_is_forgotten_too() { let (store, _dir) = store().await; let record = signed(1, 1, 1); - store.put_bytes(record.as_bytes()).await.expect("put"); + store.put_bytes(&record.to_bytes()).await.expect("put"); std::fs::remove_file(store.dir().join(hex::encode(record.address()))).expect("remove"); assert!(store.get(&record.address()).await.expect("get").is_none()); @@ -1247,7 +1247,7 @@ mod tests { let record = signed(1, 1, 1); { let store = PointerStore::new(dir.path()).await.expect("open"); - store.put_bytes(record.as_bytes()).await.expect("put"); + store.put_bytes(&record.to_bytes()).await.expect("put"); // Junk under a plausible name, an oversized file, and a partial write. std::fs::write(store.dir().join(hex::encode([9u8; 32])), b"not a pointer") @@ -1274,7 +1274,7 @@ mod tests { let mut tasks = Vec::new(); for counter in 1..=12u64 { let store = store.clone(); - let bytes = signed(1, counter, 1).as_bytes().to_vec(); + let bytes = signed(1, counter, 1).to_bytes().to_vec(); tasks.push(tokio::spawn(async move { store.put_bytes(&bytes).await })); } for task in tasks { @@ -1302,7 +1302,7 @@ mod tests { // the write and the index update both happened, or neither did. let (store, _dir) = store().await; let record = signed(1, 3, 3); - let prepared = match store.prepare(record.as_bytes()).await.expect("prepare") { + let prepared = match store.prepare(&record.to_bytes()).await.expect("prepare") { Prepared::Candidate(prepared) => prepared, Prepared::Noop(_) => panic!("expected a candidate"), }; @@ -1364,14 +1364,14 @@ mod tests { // node ends up holding a record it never announces or commits to. let (store, _dir) = store().await; let old = signed(1, 1, 1); - store.put_bytes(old.as_bytes()).await.expect("put"); + store.put_bytes(&old.to_bytes()).await.expect("put"); // Simulate the interleaving: the read observed the old entry, then a // newer state was committed, and only then does the read disown what // it saw. let observed = store.snapshot(&old.address()).map(|entry| entry.generation); let new = signed(1, 2, 1); - store.put_bytes(new.as_bytes()).await.expect("put"); + store.put_bytes(&new.to_bytes()).await.expect("put"); store.forget_if_unchanged(&old.address(), observed); assert_eq!( @@ -1394,7 +1394,7 @@ mod tests { // the repaired entry from the corrupt one it replaced. let (store, _dir) = store().await; let record = signed(1, 1, 1); - store.put_bytes(record.as_bytes()).await.expect("put"); + store.put_bytes(&record.to_bytes()).await.expect("put"); let first_reader = store .snapshot(&record.address()) @@ -1407,7 +1407,7 @@ mod tests { // A peer repairs it with the very same state. assert_eq!( - store.put_bytes(record.as_bytes()).await.expect("put"), + store.put_bytes(&record.to_bytes()).await.expect("put"), PutOutcome::Stored ); @@ -1437,7 +1437,7 @@ mod tests { let reopened = PointerStore::new(dir.path()).await.expect("reopen"); assert_eq!( - reopened.put_bytes(record.as_bytes()).await.expect("put"), + reopened.put_bytes(&record.to_bytes()).await.expect("put"), PutOutcome::Stored, "a swept leftover must not block the first write to its address" ); @@ -1447,7 +1447,7 @@ mod tests { async fn a_healthy_store_does_not_report_degraded_durability() { let (store, _dir) = store().await; store - .put_bytes(signed(1, 1, 1).as_bytes()) + .put_bytes(&signed(1, 1, 1).to_bytes()) .await .expect("put"); assert!( @@ -1463,7 +1463,7 @@ mod tests { let (store, _dir) = store().await; let old = signed(1, 1, 1); let new = signed(1, 2, 1); - store.put_bytes(old.as_bytes()).await.expect("put"); + store.put_bytes(&old.to_bytes()).await.expect("put"); assert!(store.contains(&new.address())); assert!(store.holds_state(&old.address(), &old.state_id())); diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index 805541c1..252fff7f 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -143,7 +143,7 @@ proptest! { .map(|_| signed(1, counter, target, PointerTargetKind::Chunk)) .collect(); - let encodings: BTreeSet<&[u8]> = records.iter().map(Pointer::as_bytes).collect(); + let encodings: BTreeSet> = records.iter().map(Pointer::to_bytes).collect(); prop_assert_eq!(encodings.len(), variants, "ML-DSA signing is randomized"); let states: BTreeSet<_> = records.iter().map(Pointer::state_id).collect(); @@ -209,7 +209,7 @@ proptest! { prop_assert_ne!(record.address(), signed(4, 0, 0, PointerTargetKind::Chunk).address()); // An unknown tag round-trips untouched; the node never interprets it. - let parsed = Pointer::from_bytes(record.as_bytes()).expect("parse"); + let parsed = Pointer::from_bytes(&record.to_bytes()).expect("parse"); prop_assert_eq!(parsed.target().kind_tag(), tag); prop_assert_eq!(parsed.target().address, target_bytes); } @@ -253,7 +253,7 @@ proptest! { mask in 1u8..255, ) { let record = signed(5, 11, 22, PointerTargetKind::Chunk); - let mut bytes = record.as_bytes().to_vec(); + let mut bytes = record.to_bytes().to_vec(); let Some(byte) = bytes.get_mut(index) else { return Ok(()); }; @@ -280,7 +280,7 @@ fn sixty_four_signatures_over_one_state_yield_one_winner() { .map(|_| signed(7, 9, 9, PointerTargetKind::Chunk)) .collect(); - let encodings: BTreeSet<&[u8]> = records.iter().map(Pointer::as_bytes).collect(); + let encodings: BTreeSet> = records.iter().map(Pointer::to_bytes).collect(); assert_eq!(encodings.len(), 64, "64 distinct valid encodings"); let states: BTreeSet<_> = records.iter().map(Pointer::state_id).collect(); @@ -289,7 +289,7 @@ fn sixty_four_signatures_over_one_state_yield_one_winner() { // Sorted worst-first is the submission order that would have made every // record win under a byte-ordering tie-break. let mut sorted: Vec<&Pointer> = records.iter().collect(); - sorted.sort_by_key(|record| record.as_bytes().to_vec()); + sorted.sort_by_key(|record| record.to_bytes().to_vec()); let first = sorted.first().copied().expect("non-empty"); for candidate in &sorted { @@ -323,7 +323,7 @@ fn the_wire_format_is_what_the_adr_says() { PointerTarget::from_raw_tag(0x5A, target_address), ) .expect("sign"); - let bytes = record.as_bytes(); + let bytes = record.to_bytes(); // Layout. assert_eq!(bytes.len(), 5303, "1 + 1952 + 8 + 33 + 3309"); @@ -384,7 +384,7 @@ fn the_wire_format_is_what_the_adr_says() { assert_eq!( record.bytes_hash(), - *blake3::hash(bytes).as_bytes(), + *blake3::hash(&bytes).as_bytes(), "bytes_hash is over the whole record" ); @@ -441,7 +441,7 @@ async fn sixty_four_signatures_buy_exactly_one_write() { let first = signed(7, 9, 9, PointerTargetKind::Chunk); assert_eq!( - store.put_bytes(first.as_bytes()).await.expect("put"), + store.put_bytes(&first.to_bytes()).await.expect("put"), PutOutcome::Stored ); @@ -451,12 +451,12 @@ async fn sixty_four_signatures_buy_exactly_one_write() { for _ in 0..63 { let variant = signed(7, 9, 9, PointerTargetKind::Chunk); assert_ne!( - variant.as_bytes(), - first.as_bytes(), + variant.to_bytes(), + first.to_bytes(), "signing is randomized" ); assert_eq!( - store.put_bytes(variant.as_bytes()).await.expect("put"), + store.put_bytes(&variant.to_bytes()).await.expect("put"), PutOutcome::Unchanged ); } @@ -493,7 +493,7 @@ fn a_version_only_change_cannot_reuse_a_paid_state_identifier() { let record = signed(11, 9, 9, PointerTargetKind::Chunk); let mut body = record - .as_bytes() + .to_bytes() .get(..POINTER_BODY_LEN) .expect("body") .to_vec(); @@ -536,7 +536,7 @@ fn an_unknown_version_is_refused_rather_than_accepted_at_its_own_price() { let record = signed(12, 1, 1, PointerTargetKind::Chunk); for version in [0u8, 2, 7, u8::MAX] { assert_ne!(version, POINTER_FORMAT_VERSION); - let mut bytes = record.as_bytes().to_vec(); + let mut bytes = record.to_bytes().to_vec(); if let Some(byte) = bytes.first_mut() { *byte = version; } @@ -607,7 +607,7 @@ async fn migration_must_happen_before_the_terminal_update() { ); assert!(!penultimate.is_terminal()); assert_eq!( - store.put_bytes(penultimate.as_bytes()).await.expect("put"), + store.put_bytes(&penultimate.to_bytes()).await.expect("put"), PutOutcome::Stored ); @@ -618,7 +618,7 @@ async fn migration_must_happen_before_the_terminal_update() { PointerTarget::new(PointerTargetKind::Pointer, [0x80u8; 32]), ); assert_eq!( - store.put_bytes(migration.as_bytes()).await.expect("put"), + store.put_bytes(&migration.to_bytes()).await.expect("put"), PutOutcome::Replaced ); assert!(migration.is_terminal()); @@ -631,12 +631,12 @@ async fn migration_must_happen_before_the_terminal_update() { PointerTarget::new(PointerTargetKind::Chunk, [0x01u8; 32]), ); assert!( - smaller_target.as_bytes() != migration.as_bytes(), + smaller_target.to_bytes() != migration.to_bytes(), "a genuinely different state" ); assert_eq!( store - .put_bytes(smaller_target.as_bytes()) + .put_bytes(&smaller_target.to_bytes()) .await .expect("put"), PutOutcome::Replaced, @@ -645,7 +645,7 @@ async fn migration_must_happen_before_the_terminal_update() { // Larger target bytes cannot claw it back: the move is one-way. assert_eq!( - store.put_bytes(migration.as_bytes()).await.expect("put"), + store.put_bytes(&migration.to_bytes()).await.expect("put"), PutOutcome::Stale, "and the displaced migration can never be restored" ); From 2f05db155713f7c8069d3dc5d26a4bf0208a2667 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 17:43:35 +0900 Subject: [PATCH 03/32] refactor(pointer): delete what the feature does not use yet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three simplifications, all deletions. The kind-tagged audit leaf goes. Nothing in production ever built a pointer leaf — commitment rotation reads the chunk store only — so LeafKind, pointer_leaf_hash, build_of_kinds and the per-leaf wire tag were machinery for a case that cannot arise. Removing them also puts the subtree audit family back to v1: the tag changed a positionally-encoded leaf, which would have paused mixed-version audits through a rollout for no benefit. The audit format is now untouched by this work. Prepared and PreparedPut go. PreparedPut was a one-field wrapper around Pointer re-exposing accessors Pointer already had, and Prepared mirrored Inspected arm for arm. Inspected gains a Verified arm and commit takes a Pointer directly, so there is one enum where there were two plus a wrapper. PaymentTarget and PaidKey merge into one enum that is both the routing decision and the cache key. Being an enum is the load-bearing part: a raw 32-byte cache key would let a client store a chunk crafted to sit on a pointer's entry and buy its update at chunk price. The ADR drops to Proposed while the PR is open, and its defence table now says which rows are local guarantees and which still need replication — fork convergence across the network is the client's doing until nodes forward pointers to each other. --- docs/adr/ADR-0015-pointers-immutable-owner.md | 45 ++-- src/lib.rs | 3 +- src/payment/cache.rs | 22 +- src/payment/verifier.rs | 135 +++++------ src/pointer/mod.rs | 2 +- src/pointer/service.rs | 12 +- src/pointer/store.rs | 116 ++++----- src/replication/commitment.rs | 56 +---- src/replication/commitment_state.rs | 22 -- src/replication/config.rs | 10 +- src/replication/protocol.rs | 5 +- src/replication/storage_commitment_audit.rs | 66 +----- src/replication/subtree.rs | 223 +----------------- tests/poc_commitment_audit_attacks.rs | 4 +- 14 files changed, 170 insertions(+), 551 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 55f54bcb..6c7e1156 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -1,6 +1,6 @@ # ADR-0015: Pointers — paid mutable references with an immutable owner -- **Status:** Accepted +- **Status:** Proposed - **Date:** 2026-09-18 - **Decision owners:** Anselme (@grumbach) - **Related:** ADR-0002 (audit), ADR-0008 (per-record pricing), ADR-0009 (audit families), ADR-0014 (file store) @@ -98,16 +98,16 @@ re-checks under its lock, because a newer state can land while payment verifies. | Tamper with any byte | Signature over the whole body | | Swap the owner key | `A` is derived from it; the record no longer belongs at its address | | Store at someone else's address | Same | -| Fork / equivocate at one counter | Total order on `(counter, target)` — every node picks the same one | +| Fork / equivocate at one counter | Total order on `(counter, target)`: every node given the same records picks the same one, and a read merges the close group's answers rather than trusting the first. Convergence *across* the network still needs replication — see Not built | | Replay an older record | Loses on counter | | Re-sign one paid state N times | Equal state never replaces; nothing is written | | Pay once, jump the counter | Client updates must be `+1` | -| Pay for a chunk to fund a pointer | Paid cache keyed by a typed `Chunk` vs `PointerState`, never by a raw 32-byte value a crafted chunk could occupy | +| Pay for a chunk to fund a pointer | Paid cache keyed by a typed `Chunk` vs `Pointer` target, never a raw 32-byte value a crafted chunk could occupy. The quote signs only its content, not the record kind, so this is a cache defence rather than a cryptographic one | | Merkle proof with no issuer check | Refused for pointers; single-node proofs only | | Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | | Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | -| Collide a pointer and a chunk address | Refused in both directions rather than resolved | -| Mint an audit leaf for someone's key | Pointer leaves are refused at round 1 (see below) | +| Collide a pointer and a chunk address | Refused in both directions. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | +| Peer lies about storing a pointer | The client checks every acknowledgement names the address and state it sent, and stores on a quorum rather than stopping at the first success | ## Consequences @@ -124,24 +124,27 @@ re-checks under its lock, because a newer state can land while payment verifies. correctly signed value and cannot tell. - Replicas may hold different valid signatures of one state; nothing compares record bytes across replicas. -- The audit commitment leaf gains a record kind, bound by hashing pointer leaves - under their own domain, so a chunk leaf cannot be relabelled to escape the - `bytes_hash == key` guard. **Pointer leaves are refused at round 1** until - round 2 serves and validates a whole record: a peer signs its own commitment, - so it could otherwise name any key with the hash of cheap bytes it holds. - Refusing costs nothing today — commitment rotation reads the chunk store only. +- Pointers do not take part in storage commitments or audits. The audit format + is untouched, so no protocol family is bumped and no rollout pauses. Auditing + them needs round 2 to serve a whole record — a peer signs its own commitment, + so without that it could name any key with the hash of cheap bytes it holds — + and that lands with replication. ## Implementation status -Built: the record and wire messages (`ant-protocol`), the store with -merge-on-put, request dispatch, payment routed at `state_id` with the close -group of `A`, admission gates, cross-kind refusal, the kind-tagged audit leaf, -and the client API. +Built: the record and wire messages (`ant-protocol`); the store with +merge-on-put; request dispatch; payment routed at `state_id` with the close +group of `A`; admission gates; cross-kind refusal; and the client — create, +update, quorum store, merged reads and chain resolution. -Not built: replication does not yet carry `(A, state_id)` through fresh offers, -sync hints, presence, repair and paid-list, so a pointer is not replicated -version-aware; and audit round 2 does not yet serve a whole record, which is why -pointer leaves are refused rather than trusted. +**Not built: replication.** No node forwards a pointer to another, so the copies +that exist are the ones the client wrote. That is the load-bearing gap: the +merge rule guarantees nodes holding the same records agree, and nothing yet +guarantees they hold the same records. Until it lands, availability and +cross-network fork convergence are the client's doing, not the network's. + +Also not built: pointer participation in commitments and audits, which depends +on the same work. ## Validation @@ -153,6 +156,6 @@ pointer leaves are refused rather than trusted. and a wrap back to 0 — is refused as a non-successor. - All 256 `version` values give distinct paid identifiers. - Golden vectors pin the encoding, both identities and the signing context. -- A relabelled audit leaf fails structural verification; a chunk-only commitment - root is bit-identical to before. - A crafted chunk cannot satisfy a pointer's paid-cache entry. +- An acknowledgement naming a different address or state is refused, and a read + keeps the winner whatever order the replies arrive in. diff --git a/src/lib.rs b/src/lib.rs index 42ac7322..43e025ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -81,8 +81,7 @@ pub use event::{NodeEvent, NodeEventsChannel}; pub use node::{NodeBuilder, RunningNode}; pub use payment::{PaymentStatus, PaymentVerifier, PaymentVerifierConfig}; pub use pointer::{ - Pointer, PointerState, PointerStore, PointerTarget, PointerTargetKind, Prepared, PreparedPut, - PutOutcome, + Inspected, Pointer, PointerState, PointerStore, PointerTarget, PointerTargetKind, PutOutcome, }; pub use replication::{config::ReplicationConfig, ReplicationEngine}; pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; diff --git a/src/payment/cache.rs b/src/payment/cache.rs index c148a19c..857482cf 100644 --- a/src/payment/cache.rs +++ b/src/payment/cache.rs @@ -10,31 +10,11 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; pub use super::quote::XorName; +pub use super::verifier::PaymentTarget as PaidKey; /// Default cache capacity (100,000 entries = 3.2MB memory). const DEFAULT_CACHE_CAPACITY: usize = 100_000; -/// What a cache entry is about. -/// -/// A typed key, not a hashed one. Hashing a pointer's two addresses back into -/// 32 bytes would not create a separate namespace: a chunk's address is -/// `BLAKE3(content)`, so a client could store a chunk whose *content* is -/// exactly that preimage and land on the same key — paying chunk price for a -/// pointer update, and skipping issuer proximity, the price floor and the -/// proof-shape rule with it. Distinct variants cannot collide at all. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub enum PaidKey { - /// A chunk, paid for and stored at one address. - Chunk(XorName), - /// A pointer state: routed at the pointer's address, paid at its `state_id`. - PointerState { - /// The pointer's address, stable for its life. - routing: XorName, - /// The state paid for, which changes with every update. - state_id: XorName, - }, -} - /// LRU cache for verified `XorName` values. /// /// This cache stores `XorName` values that have been verified to exist on the diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 9561bd87..01bad501 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -478,62 +478,67 @@ pub struct PaymentVerifierConfig { /// Later neighbour-sync repair does not include proof-of-payment bytes and /// does not call this verifier. It authorizes repair from network evidence: /// majority storage among the configured close group, or majority paid-list -/// membership among the closest K. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PaymentTarget { - /// The address whose close group is responsible, and whose members' quotes - /// therefore count. For a chunk this is its content address; for a pointer - /// it is the pointer's address, which never changes. - pub routing: XorName, - /// What the quote must actually name. For a chunk this equals - /// [`Self::routing`]; for a pointer it is the record's `state_id`, so each - /// update is paid for separately rather than riding the first payment. - pub content: XorName, +/// What a payment authorizes, and where it routes. +/// +/// One typed value doing both jobs. A chunk pays for its own address; a pointer +/// pays for a *state* while the close group that may quote it is the one around +/// its address — two different addresses with two different meanings. +/// +/// This is also the paid-cache key, and being an enum is what makes that safe: +/// a raw 32-byte key would let a client store a chunk crafted to sit exactly on +/// a pointer's entry — `state_id` is `BLAKE3(domain || body)`, so its preimage +/// can be a chunk's content — and buy the pointer's update at chunk price. +/// Distinct variants cannot collide however the bytes are chosen. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum PaymentTarget { + /// A chunk: paid for and stored at one address. + Chunk(XorName), + /// A pointer state: routed at the pointer's address, paid at its `state_id`. + Pointer { + /// The pointer's address, stable for its life. Selects the close group. + routing: XorName, + /// The state paid for, which changes with every update. + state_id: XorName, + }, } impl PaymentTarget { - /// A target where one address does both jobs, as every chunk does. + /// A chunk, where one address does both jobs. #[must_use] pub const fn same(address: XorName) -> Self { - Self { - routing: address, - content: address, - } + Self::Chunk(address) } - /// A target that routes at one address and is paid at another. + /// A pointer, routed at one address and paid at another. #[must_use] - pub const fn split(routing: XorName, content: XorName) -> Self { - Self { routing, content } + pub const fn split(routing: XorName, state_id: XorName) -> Self { + Self::Pointer { routing, state_id } } - /// Whether this target's two jobs fall to one address, as a chunk's do. + /// The address whose close group is responsible, and whose members' quotes + /// therefore count. #[must_use] - pub fn is_single_address(&self) -> bool { - self.routing == self.content + pub const fn routing(&self) -> &XorName { + match self { + Self::Chunk(address) => address, + Self::Pointer { routing, .. } => routing, + } } - /// The key this target's "already paid" entry is filed under. - /// - /// A **typed** key, not a hashed one. Hashing the two halves back into 32 - /// bytes would not separate the namespaces: a chunk's address is - /// `BLAKE3(content)`, so a client could store a chunk whose *content* is - /// exactly that preimage and land on the same key. Paying for that chunk - /// would then file an entry the pointer path reads as its own — buying a - /// pointer update at chunk prices and skipping the issuer-proximity, - /// price-floor and proof-shape checks along with it. Distinct enum variants - /// cannot collide however the bytes are chosen. + /// What the quote must actually name. #[must_use] - pub fn cache_key(&self) -> PaidKey { - if self.is_single_address() { - PaidKey::Chunk(self.content) - } else { - PaidKey::PointerState { - routing: self.routing, - state_id: self.content, - } + pub const fn content(&self) -> &XorName { + match self { + Self::Chunk(address) => address, + Self::Pointer { state_id, .. } => state_id, } } + + /// Whether this is a chunk, where one address does both jobs. + #[must_use] + pub const fn is_single_address(&self) -> bool { + matches!(self, Self::Chunk(_)) + } } /// What a payment verification is admitting. @@ -1103,7 +1108,7 @@ impl PaymentVerifier { xorname: &XorName, context: VerificationContext, ) -> PaymentStatus { - self.check_payment_required_keyed(PaidKey::Chunk(*xorname), context) + self.check_payment_required_keyed(PaymentTarget::Chunk(*xorname), context) } /// As [`Self::check_payment_required`], for an already-typed cache key. @@ -1118,8 +1123,8 @@ impl PaymentVerifier { context: VerificationContext, ) -> PaymentStatus { let xorname = match &key { - PaidKey::Chunk(address) => address, - PaidKey::PointerState { state_id, .. } => state_id, + PaymentTarget::Chunk(address) => address, + PaymentTarget::Pointer { state_id, .. } => state_id, }; // Check LRU cache (fast path) let cached = if context.is_store_admission() { @@ -1220,8 +1225,8 @@ impl PaymentVerifier { // future update of a pointer as already paid — the 1.0 free-update // defect — so the cache is keyed separately, and never shares a key // with the chunk whose address happens to equal this state. - let xorname = &target.content; - let cache_key = target.cache_key(); + let xorname = target.content(); + let cache_key = *target; // First check if payment is required let status = self.check_payment_required_keyed(cache_key, context); @@ -1415,7 +1420,7 @@ impl PaymentVerifier { // `content` is what a quote must name; `routing` is whose close group // may issue it. For a chunk they are one address, for a pointer they // are not. - let xorname = &target.content; + let xorname = target.content(); if crate::logging::enabled!(crate::logging::Level::DEBUG) { let xorname_hex = hex::encode(xorname); let quote_count = payment.peer_quotes.len(); @@ -1489,7 +1494,7 @@ impl PaymentVerifier { // group that is RESPONSIBLE for the data, so it takes the routing // address. For a chunk that is the same value; for a pointer the paid // content is its state, which names no close group at all. - self.enforce_price_floor(&target.routing, paid_price, settled_amount, context) + self.enforce_price_floor(target.routing(), paid_price, settled_amount, context) .await?; // ADR-0004 observe-only telemetry: log off-curve quotes only AFTER the @@ -1572,11 +1577,11 @@ impl PaymentVerifier { // The two checks take different addresses. The quote must name what was // paid for; the issuer must be close to what the network routes. A // chunk supplies one address for both, a pointer two. - Self::validate_paid_quote_content(&target.content, candidate)?; + Self::validate_paid_quote_content(target.content(), candidate)?; let issuer_peer_id = Self::validate_paid_quote_peer_binding(candidate.encoded_peer_id, candidate.quote)?; - self.validate_paid_quote_issuer_k_closest(&target.routing, &issuer_peer_id) + self.validate_paid_quote_issuer_k_closest(target.routing(), &issuer_peer_id) .await?; Self::validate_paid_quote_signature(candidate).await?; @@ -3316,7 +3321,7 @@ impl PaymentVerifier { ) -> Result<()> { // The proof names what was paid for, which for a pointer is its state // rather than its address. - let xorname = &target.content; + let xorname = target.content(); // A merkle proof binds the paid address but carries no issuer-proximity // check, so it cannot express "paid at the state, quoted by the group @@ -3324,12 +3329,12 @@ impl PaymentVerifier { // close group of a state identifier, which names no group at all. // Single-node proofs do carry that check, so pointers use those until // the merkle proof shape can say which group issued it. - if target.routing != target.content { + if target.routing() != target.content() { return Err(Error::Payment(format!( "a pointer update must be paid with a single-node proof: a merkle \ proof cannot bind the issuing close group of {} to the paid state {}", - hex::encode(target.routing), - hex::encode(target.content) + hex::encode(target.routing()), + hex::encode(target.content()) ))); } if crate::logging::enabled!(crate::logging::Level::DEBUG) { @@ -3756,42 +3761,38 @@ mod tests { let pointer = PaymentTarget::split(pointer_address, state_id); assert_eq!( - chunk.cache_key(), - PaidKey::Chunk(state_id), + chunk, + PaymentTarget::Chunk(state_id), "a chunk keeps its bare address, so existing entries are untouched" ); assert_ne!( - pointer.cache_key(), - chunk.cache_key(), + pointer, chunk, "the pointer must not read the chunk's paid entry as its own" ); // And two different pointers sharing a state cannot borrow either. let other = PaymentTarget::split([0x11u8; 32], state_id); - assert_ne!(pointer.cache_key(), other.cache_key()); + assert_ne!(pointer, other); // The key is typed, not hashed: there is no 32-byte preimage a client // could put in a chunk's *content* to land on the pointer's entry, // because no chunk key is ever a `PointerState` variant. let cache = VerifiedCache::with_capacity(8); - cache.insert_key(chunk.cache_key()); - assert!( - cache.contains_key(&chunk.cache_key()), - "the chunk's own entry is there" - ); + cache.insert_key(chunk); + assert!(cache.contains_key(&chunk), "the chunk's own entry is there"); assert!( - !cache.contains_key(&pointer.cache_key()), + !cache.contains_key(&pointer), "and it does not satisfy the pointer" ); for crafted in [state_id, pointer_address, [0u8; 32], [0xFFu8; 32]] { assert!( - !cache.contains_key(&PaymentTarget::split(pointer_address, state_id).cache_key()), + !cache.contains_key(&PaymentTarget::split(pointer_address, state_id)), "no chunk address {crafted:?} can stand in for the pointer's entry" ); - cache.insert_key(PaidKey::Chunk(crafted)); + cache.insert_key(PaymentTarget::Chunk(crafted)); } assert!( - !cache.contains_key(&pointer.cache_key()), + !cache.contains_key(&pointer), "still no chunk address satisfies the pointer's typed entry" ); } diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs index 45c021dc..9cc56fe2 100644 --- a/src/pointer/mod.rs +++ b/src/pointer/mod.rs @@ -59,4 +59,4 @@ pub use ant_protocol::pointer::{ POINTER_BODY_LEN, POINTER_FORMAT_VERSION, POINTER_WIRE_LEN, TARGET_WIRE_LEN, }; pub use service::PointerService; -pub use store::{PointerStore, Prepared, PreparedPut, PutOutcome}; +pub use store::{Inspected, PointerStore, PutOutcome}; diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 4324912d..b83cde6c 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -155,6 +155,12 @@ impl PointerService { ))); } Ok(Inspected::Candidate(parsed)) => parsed, + // `inspect` does not verify, so it cannot produce this arm. + Ok(Inspected::Verified(_)) => { + return PointerPutResponse::Error(ProtocolError::Internal( + "inspect returned a verified record".to_string(), + )); + } Err(e) => { debug!("Pointer PUT refused: {e}"); return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); @@ -169,8 +175,8 @@ impl PointerService { } // Only now is the record worth a signature check. - let prepared = match self.store.verify(parsed).await { - Ok(prepared) => prepared, + let record = match self.store.verify(parsed).await { + Ok(record) => record, Err(e) => { debug!("Pointer PUT refused: {e}"); return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); @@ -190,7 +196,7 @@ impl PointerService { } } - match self.store.commit(prepared).await { + match self.store.commit(record).await { Ok(PutOutcome::Stored | PutOutcome::Replaced) => { PointerPutResponse::Success { address, state_id } } diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 6397b9f6..298f1369 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -91,6 +91,7 @@ impl PutOutcome { /// The result of [`PointerStore::inspect`]: what an arrival claims, before any /// signature has been checked. +#[derive(Debug)] pub enum Inspected { /// The arrival cannot change what is held. No signature check is owed, and /// none was done. @@ -99,6 +100,11 @@ pub enum Inspected { /// checked yet — pass it to [`PointerStore::verify`] once admission gates /// have had their say. Candidate(ParsedPointer), + /// A signature-checked record, ready to commit. + /// + /// Boxed because a verified record is two orders of magnitude larger than + /// the other arms, and an enum is as big as its widest one. + Verified(Box), } impl Inspected { @@ -106,53 +112,14 @@ impl Inspected { #[must_use] pub fn state(&self) -> Option<&ant_protocol::pointer::PointerState> { match self { - Self::Noop(_) => None, Self::Candidate(parsed) => Some(parsed.state()), + // A no-op claims nothing worth acting on, and a verified record + // carries its own state directly. + Self::Noop(_) | Self::Verified(_) => None, } } } -/// The result of [`PointerStore::prepare`]. -#[derive(Debug)] -pub enum Prepared { - /// The arrival cannot change what is held, and was rejected without a - /// signature check. There is nothing to pay for and nothing to commit. - Noop(PutOutcome), - /// A validated record that would win as of the moment it was prepared. - Candidate(PreparedPut), -} - -/// A record that parsed, out-ranked what was held, and verified. -/// -/// Carries what a payment check needs — [`Self::address`] to route and -/// [`Self::state_id`] to authorize — and holds the validated record so nothing -/// can change between validation and commit. -#[derive(Debug)] -pub struct PreparedPut { - /// The validated record. - record: Pointer, -} - -impl PreparedPut { - /// The address this record belongs at, which is what routing uses. - #[must_use] - pub fn address(&self) -> XorName { - self.record.address() - } - - /// The authenticated-state identifier, which is what a quote is paid against. - #[must_use] - pub fn state_id(&self) -> XorName { - self.record.state_id() - } - - /// The validated record. - #[must_use] - pub const fn record(&self) -> &Pointer { - &self.record - } -} - /// What the store knows about a held record without reading it back. #[derive(Debug, Clone, Copy)] struct IndexEntry { @@ -291,10 +258,14 @@ impl PointerStore { /// /// Returns [`Error::Protocol`] if the bytes are not a well-formed record /// and [`Error::Crypto`] if a would-be winner's signature does not verify. - pub async fn prepare(&self, bytes: &[u8]) -> Result { + pub async fn prepare(&self, bytes: &[u8]) -> Result { match self.inspect(bytes)? { - Inspected::Noop(outcome) => Ok(Prepared::Noop(outcome)), - Inspected::Candidate(parsed) => Ok(Prepared::Candidate(self.verify(parsed).await?)), + Inspected::Noop(outcome) => Ok(Inspected::Noop(outcome)), + Inspected::Candidate(parsed) => { + Ok(Inspected::Verified(Box::new(self.verify(parsed).await?))) + } + // `inspect` never returns this; only `prepare` produces it. + verified @ Inspected::Verified(_) => Ok(verified), } } @@ -337,11 +308,11 @@ impl PointerStore { /// # Errors /// /// Returns [`Error::Crypto`] if the signature does not verify. - pub async fn verify(&self, parsed: ParsedPointer) -> Result { - let record = spawn_blocking(move || Pointer::verify_parsed(parsed)) + pub async fn verify(&self, parsed: ParsedPointer) -> Result { + spawn_blocking(move || Pointer::verify_parsed(parsed)) .await - .map_err(|e| Error::Storage(format!("pointer verification panicked: {e}")))??; - Ok(PreparedPut { record }) + .map_err(|e| Error::Storage(format!("pointer verification panicked: {e}")))? + .map_err(Into::into) } /// Commit a prepared record. @@ -354,11 +325,11 @@ impl PointerStore { /// # Errors /// /// Returns [`Error::Storage`] if the write fails. - pub async fn commit(&self, prepared: PreparedPut) -> Result { + pub async fn commit(&self, record: Pointer) -> Result { let inner = Arc::clone(&self.inner); // The whole transaction runs in one task, so dropping this future // cannot leave the write done and the index un-updated. - spawn_blocking(move || inner.commit_blocking(&prepared.record)) + spawn_blocking(move || inner.commit_blocking(&record)) .await .map_err(|e| Error::Storage(format!("pointer commit panicked: {e}")))? } @@ -374,8 +345,12 @@ impl PointerStore { /// As [`Self::prepare`] and [`Self::commit`]. pub async fn put_bytes(&self, bytes: &[u8]) -> Result { match self.prepare(bytes).await? { - Prepared::Noop(outcome) => Ok(outcome), - Prepared::Candidate(prepared) => self.commit(prepared).await, + Inspected::Noop(outcome) => Ok(outcome), + Inspected::Verified(record) => self.commit(*record).await, + Inspected::Candidate(parsed) => { + let record = self.verify(parsed).await?; + self.commit(record).await + } } } @@ -1042,16 +1017,20 @@ mod tests { store.put_bytes(&held.to_bytes()).await.expect("put"); match store.prepare(&held.to_bytes()).await.expect("prepare") { - Prepared::Noop(outcome) => assert_eq!(outcome, PutOutcome::Unchanged), - Prepared::Candidate(_) => panic!("an identical state is not a candidate"), + Inspected::Noop(outcome) => assert_eq!(outcome, PutOutcome::Unchanged), + Inspected::Verified(_) | Inspected::Candidate(_) => { + panic!("an identical state is not a candidate") + } } match store .prepare(&signed(1, 1, 6).to_bytes()) .await .expect("prepare") { - Prepared::Noop(outcome) => assert_eq!(outcome, PutOutcome::Stale), - Prepared::Candidate(_) => panic!("a stale record is not a candidate"), + Inspected::Noop(outcome) => assert_eq!(outcome, PutOutcome::Stale), + Inspected::Verified(_) | Inspected::Candidate(_) => { + panic!("a stale record is not a candidate") + } } } @@ -1060,16 +1039,19 @@ mod tests { let (store, _dir) = store().await; let record = signed(1, 2, 2); match store.prepare(&record.to_bytes()).await.expect("prepare") { - Prepared::Candidate(prepared) => { - assert_eq!(prepared.address(), record.address()); - assert_eq!(prepared.state_id(), record.state_id()); - assert_eq!(prepared.record().to_bytes(), record.to_bytes()); + Inspected::Verified(verified) => { + // A verified record is a `Pointer`, so the payment check reads + // the address and state straight off it — there is no wrapper + // type in between restating what it already knows. + assert_eq!(verified.address(), record.address()); + assert_eq!(verified.state_id(), record.state_id()); + assert_eq!(verified.to_bytes(), record.to_bytes()); assert_eq!( - store.commit(prepared).await.expect("commit"), + store.commit(*verified).await.expect("commit"), PutOutcome::Stored ); } - Prepared::Noop(_) => panic!("a new record is a candidate"), + other => panic!("a new record verifies, got {other:?}"), } assert_eq!(store.len(), 1); } @@ -1084,8 +1066,8 @@ mod tests { .await .expect("prepare") { - Prepared::Candidate(prepared) => prepared, - Prepared::Noop(_) => panic!("expected a candidate"), + Inspected::Verified(record) => *record, + other => panic!("expected a verified record, got {other:?}"), }; // Someone else's newer state arrives while the payment is being checked. @@ -1303,8 +1285,8 @@ mod tests { let (store, _dir) = store().await; let record = signed(1, 3, 3); let prepared = match store.prepare(&record.to_bytes()).await.expect("prepare") { - Prepared::Candidate(prepared) => prepared, - Prepared::Noop(_) => panic!("expected a candidate"), + Inspected::Verified(record) => *record, + other => panic!("expected a verified record, got {other:?}"), }; let abandoned = { diff --git a/src/replication/commitment.rs b/src/replication/commitment.rs index f188a9c9..61082510 100644 --- a/src/replication/commitment.rs +++ b/src/replication/commitment.rs @@ -26,7 +26,6 @@ use blake3::Hasher; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaSecretKey}; use crate::ant_protocol::XorName; -use crate::replication::subtree::LeafKind; // ADR-0004: the commitment wire type, its pin (`commitment_hash`), its // signature verification, and the key-count cap are the SINGLE SOURCE OF TRUTH @@ -45,10 +44,6 @@ pub const DOMAIN_LEAF: &[u8] = b"autonomi.ant.replication.storage_leaf.v1"; /// Domain-separation tag for Merkle internal nodes: `BLAKE3(this || left || right)`. pub const DOMAIN_NODE: &[u8] = b"autonomi.ant.replication.storage_node.v1"; -/// Domain separator for a pointer leaf, distinct from [`DOMAIN_LEAF`] so the -/// record kind is bound by the leaf hash itself. -pub const DOMAIN_POINTER_LEAF: &[u8] = b"autonomi.ant.replication.storage_pointer_leaf.v1"; - // `MAX_COMMITMENT_KEY_COUNT` and `StorageCommitment` are re-exported from // `ant-protocol` above (single source of truth); their fields and wire size are // documented there. @@ -57,13 +52,10 @@ pub const DOMAIN_POINTER_LEAF: &[u8] = b"autonomi.ant.replication.storage_pointe // Hashing helpers // --------------------------------------------------------------------------- -/// Compute the Merkle leaf hash for a content-addressed chunk. +/// Compute the Merkle leaf hash for `(key, bytes_hash)`. /// /// `bytes_hash` is BLAKE3 over the record bytes; the leaf binds the key to /// the content so an adversary cannot reuse a leaf for a different chunk. -/// -/// Unchanged from v1, so a node holding only chunks produces exactly the root -/// it always did. #[must_use] pub fn leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { let mut h = Hasher::new(); @@ -73,22 +65,6 @@ pub fn leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { *h.finalize().as_bytes() } -/// Compute the Merkle leaf hash for a pointer. -/// -/// A separate domain from [`leaf_hash`] is what binds the kind. A peer that -/// relabelled a chunk leaf as a pointer — to escape the round-1 guard that -/// `bytes_hash == key` — would hash it under this domain instead, changing the -/// leaf, the root, and therefore the structural check against its own signed -/// commitment. So the exemption cannot be claimed for a chunk. -#[must_use] -pub fn pointer_leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { - let mut h = Hasher::new(); - h.update(DOMAIN_POINTER_LEAF); - h.update(key); - h.update(bytes_hash); - *h.finalize().as_bytes() -} - /// Combine two child hashes into a Merkle internal-node hash. #[must_use] pub fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { @@ -144,28 +120,7 @@ impl MerkleTree { /// Returns an error if `entries` is empty (no commitment to make), if /// `entries.len() > MAX_COMMITMENT_KEY_COUNT`, or if it contains /// duplicate keys. - pub fn build(entries: Vec<(XorName, [u8; 32])>) -> Result { - Self::build_of_kinds( - entries - .into_iter() - .map(|(key, bytes_hash)| (key, bytes_hash, LeafKind::Chunk)) - .collect(), - ) - } - - /// Build a Merkle tree over `(key, bytes_hash, kind)` triples. - /// - /// The kind picks the leaf domain, so a chunk-only key set produces exactly - /// the root [`Self::build`] always produced, while a pointer leaf — whose - /// `bytes_hash` cannot equal its `key` — is distinguishable at round 1 - /// without a peer being able to claim that exemption for a chunk. - /// - /// # Errors - /// - /// As [`Self::build`]. - pub fn build_of_kinds( - mut entries: Vec<(XorName, [u8; 32], LeafKind)>, - ) -> Result { + pub fn build(mut entries: Vec<(XorName, [u8; 32])>) -> Result { if entries.is_empty() { return Err(CommitmentError::EmptyKeySet); } @@ -184,11 +139,8 @@ impl MerkleTree { let leaves: Vec<(XorName, [u8; 32])> = entries .into_iter() - .map(|(k, bh, kind)| { - let lh = match kind { - LeafKind::Chunk => leaf_hash(&k, &bh), - LeafKind::Pointer => pointer_leaf_hash(&k, &bh), - }; + .map(|(k, bh)| { + let lh = leaf_hash(&k, &bh); (k, lh) }) .collect(); diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index cf1306fe..62b965f9 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -41,7 +41,6 @@ use crate::replication::commitment::{ commitment_hash, sign_commitment, verify_commitment_signature, CommitmentError, MerkleTree, StorageCommitment, }; -use crate::replication::subtree::LeafKind; /// Auditor-side per-peer commitment state. /// @@ -186,27 +185,6 @@ impl BuiltCommitment { Self::build_from_tree(tree, sender_peer_id, secret_key, sender_public_key) } - /// Build over a mixed key set of chunks and pointers. - /// - /// A pointer's `bytes_hash` cannot equal its key, so its leaf is hashed - /// under a different domain; passing the kind is what lets the round-1 - /// verifier tell the two apart without letting a peer claim a pointer's - /// exemption for a chunk. A key set containing only chunks produces exactly - /// the root [`Self::build`] produces. - /// - /// # Errors - /// - /// As [`Self::build`]. - pub fn build_of_kinds( - entries: Vec<(XorName, [u8; 32], LeafKind)>, - sender_peer_id: &[u8; 32], - secret_key: &MlDsaSecretKey, - sender_public_key: &[u8], - ) -> Result { - let tree = MerkleTree::build_of_kinds(entries)?; - Self::build_from_tree(tree, sender_peer_id, secret_key, sender_public_key) - } - /// Sign and wrap an ALREADY-BUILT Merkle tree. Lets callers that already /// built the tree (e.g. the rotation no-op-root check, §11) avoid rebuilding /// it inside [`Self::build`]. diff --git a/src/replication/config.rs b/src/replication/config.rs index d90d85e0..9150d584 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -426,7 +426,7 @@ pub const REPLICATION_PROTOCOL_ID: &str = "autonomi.ant.replication.v2"; /// possession/repair/commitment-fetch) with no per-peer limiter. A truly /// zero-penalty rollout needs an upstream `send_request` that does not /// auto-report trust; tracked as a saorsa-core follow-up. -pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v2"; +pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v1"; /// 10 MiB — maximum replication wire message size (accommodates hint batches). const REPLICATION_MESSAGE_SIZE_MIB: usize = 10; @@ -1485,15 +1485,9 @@ mod tests { // Core replication, including all digest audit lanes, stays on v2. // Only the subtree family changed and therefore receives a separate id. assert_eq!(REPLICATION_PROTOCOL_ID, "autonomi.ant.replication.v2"); - // Bumped v1 -> v2 when the subtree leaf gained its record-kind tag, so - // a pointer leaf can be told from a chunk leaf at round 1. The leaf is - // postcard-encoded positionally, so the new field is wire-incompatible - // and needs the new id. ADR-0009 provides for exactly this: the subtree - // family versions independently of core replication, and mixed-version - // audits pause rather than misdecode. assert_eq!( SUBTREE_AUDIT_PROTOCOL_ID, - "autonomi.ant.replication.subtree-audit.v2" + "autonomi.ant.replication.subtree-audit.v1" ); assert_ne!(REPLICATION_PROTOCOL_ID, SUBTREE_AUDIT_PROTOCOL_ID); } diff --git a/src/replication/protocol.rs b/src/replication/protocol.rs index a6078688..3ef7d0fa 100644 --- a/src/replication/protocol.rs +++ b/src/replication/protocol.rs @@ -1453,14 +1453,11 @@ mod tests { fn max_round1_proof_fits_the_audit_family_ceiling() { use crate::replication::commitment::{StorageCommitment, MAX_COMMITMENT_KEY_COUNT}; use crate::replication::config::MAX_SUBTREE_AUDIT_MESSAGE_SIZE; - use crate::replication::subtree::{ - max_subtree_leaves, LeafKind, SubtreeLeaf, SubtreeProof, - }; + use crate::replication::subtree::{max_subtree_leaves, SubtreeLeaf, SubtreeProof}; let leaf_count = max_subtree_leaves(MAX_COMMITMENT_KEY_COUNT) as usize; let leaves: Vec = (0..leaf_count) .map(|_| SubtreeLeaf { - kind: LeafKind::Chunk, key: [0xAB; 32], bytes_hash: [0xCD; 32], content_len: u32::MAX, diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index a6c5d283..f99a4e70 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -30,8 +30,7 @@ use crate::replication::protocol::{ }; use crate::replication::recent_provers::RecentProvers; use crate::replication::subtree::{ - select_subtree_path, subtree_plan, verify_subtree_proof, LeafKind, StructureVerdict, - SubtreeProof, + select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeProof, }; use crate::replication::types::{AuditFailureReason, AuditFailureSummary, FailureEvidence}; use crate::storage::ChunkStore; @@ -727,25 +726,6 @@ pub(crate) fn evaluate_subtree_structure( if proof.leaves.iter().any(|l| l.bytes_hash != l.key) { return Err(AuditFailureReason::DigestMismatch); } - - // Pointer leaves are refused outright, not merely denied credit. - // - // The kind is bound by the leaf hash, so a peer cannot relabel a chunk leaf - // as a pointer to escape the guard above. What it *can* still do is sign a - // commitment of its own naming any key with the hash of cheap bytes it - // really holds: a pointer's key derives from its owner, not its bytes, so - // round 1 has nothing to check it against, and round 2 authenticates the - // served block against that same peer-chosen `bytes_hash`. Such a proof - // would pass, and a pass clears bootstrap state and earns trust even with - // holder credit withheld. - // - // Admitting them safely needs round 2 to serve the whole record and the - // auditor to verify its signature and derived address. Until that exists, - // refusing costs nothing: commitment rotation builds from the chunk store - // alone, so no honest proof carries a pointer leaf. - if proof.leaves.iter().any(|l| l.kind != LeafKind::Chunk) { - return Err(AuditFailureReason::DigestMismatch); - } Ok(()) } @@ -974,40 +954,6 @@ pub(crate) fn verify_slice_response( AuditVerdict::Pass { checked } } -/// Credit a peer as a proven holder of the leaves its passing proof covers. -/// -/// A **chunk** leaf is credited on the strength of round 1 alone: round 1 -/// enforces `bytes_hash == key` there, so a peer cannot commit a chunk leaf for -/// a key whose bytes it does not have. -/// -/// A **pointer** leaf earns **nothing**, sampled or not. Its address is a -/// function of its owner key rather than of its bytes, so round 1 cannot bind -/// the two — and round 2 does not close the gap either, because it authenticates -/// the served block against the leaf's own `bytes_hash`, which the peer chose. -/// A peer can therefore sign a one-leaf commitment naming any key `K` with the -/// hash of cheap bytes it really holds, be sampled (the sole leaf always is), -/// pass, and be credited as a holder of `K`. Sampling does not help: the bytes -/// are attacker-chosen either way. -/// -/// Closing this needs round 2 to serve the **whole record** and the auditor to -/// parse it, check its signature and that its owner derives `K`. Until that -/// exists, no credit is the only sound answer — and it costs nothing today, -/// because production commitments are still built from the chunk store alone. -async fn credit_proven_holder( - credit: &AuditCredit<'_>, - proof: &SubtreeProof, - challenged_peer: &PeerId, - pin: [u8; 32], -) { - let now = std::time::Instant::now(); - let mut provers = credit.recent_provers.write().await; - for leaf in &proof.leaves { - if leaf.kind == LeafKind::Chunk { - provers.record_proof(leaf.key, *challenged_peer, pin, now); - } - } -} - /// Verify a subtree-proof response (auditor side), ADR-0002 two-round audit. /// /// **Round 1** (this proof): pin + identity + signature + structure. If the @@ -1144,7 +1090,11 @@ async fn verify_subtree_response( observe_closeness(ctx.p2p_node, ctx.config, challenged_peer, proof).await; // Credit the peer as a proven holder of its committed keys. if let (Some(credit), Some(pin)) = (ctx.credit, commitment_hash(commitment)) { - credit_proven_holder(credit, proof, challenged_peer, pin).await; + let now = std::time::Instant::now(); + let mut provers = credit.recent_provers.write().await; + for leaf in &proof.leaves { + provers.record_proof(leaf.key, *challenged_peer, pin, now); + } } info!( "Audit: peer {challenged_peer} passed subtree audit ({} leaves, {checked} \ @@ -1829,7 +1779,7 @@ async fn serve_committed_key_openings( mod tests { use super::*; use crate::replication::commitment_state::BuiltCommitment; - use crate::replication::subtree::{build_subtree_proof, LeafKind, SubtreeLeaf}; + use crate::replication::subtree::{build_subtree_proof, SubtreeLeaf}; use saorsa_pqc::api::sig::ml_dsa_65; use std::time::Instant; @@ -2026,7 +1976,6 @@ mod tests { #[test] fn verify_slice_response_rejects_malformed_item_sets() { let leaf = |k: XorName| SubtreeLeaf { - kind: LeafKind::Chunk, key: k, bytes_hash: [0u8; 32], content_len: 0, @@ -2558,7 +2507,6 @@ mod tests { #[test] fn subtree_leaf_is_constructible() { let _l = SubtreeLeaf { - kind: LeafKind::Chunk, key: key(1), bytes_hash: [0u8; 32], content_len: 0, diff --git a/src/replication/subtree.rs b/src/replication/subtree.rs index 88a651a1..a50a628f 100644 --- a/src/replication/subtree.rs +++ b/src/replication/subtree.rs @@ -31,9 +31,7 @@ //! leaf range `[slot * span, (slot + 1) * span)` where `span = 2^(D - depth)`, //! intersected with `0..N`. -use super::commitment::{ - leaf_hash, node_hash, pointer_leaf_hash, StorageCommitment, MAX_COMMITMENT_KEY_COUNT, -}; +use super::commitment::{leaf_hash, node_hash, StorageCommitment, MAX_COMMITMENT_KEY_COUNT}; use crate::ant_protocol::XorName; use serde::{Deserialize, Serialize}; @@ -41,31 +39,9 @@ use serde::{Deserialize, Serialize}; /// meaningless for tiny trees and a full proof is cheap. pub const SMALL_TREE_FULL_AUDIT_FLOOR: u32 = 4; -/// What kind of record a subtree leaf attests. -/// -/// Bound into the leaf hash, so a peer cannot relabel a chunk leaf as a pointer -/// leaf to escape the content-address guard: relabelling changes the leaf hash, -/// which changes the root, which fails the structural check against the signed -/// commitment. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] -pub enum LeafKind { - /// A content-addressed chunk, where `bytes_hash == key`. - #[default] - Chunk, - /// A pointer, whose address is a function of its owner key rather than of - /// its bytes, so `bytes_hash != key` by construction. - Pointer, -} - /// One leaf of the selected subtree, as returned by the responder. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SubtreeLeaf { - /// What kind of record this leaf attests. - /// - /// Placed first so the wire shape is obviously different from the - /// untagged v1 leaf; the subtree audit family is versioned to v2 for this - /// reason and mixed-version audits pause, exactly as ADR-0009 prescribes. - pub kind: LeafKind, /// The committed key (chunk address) at this leaf position. pub key: XorName, /// `BLAKE3(record_bytes)` — the plain content hash. For a content-addressed @@ -392,18 +368,10 @@ pub fn verify_subtree_proof( // is the tree's odd tail at some level). `fold_to_root` stopped at a single // hash and so skipped the self-pair when a truncated block reached length 1 // before climbing all the way to the subtree-root level — the geometry bug. - // Hash each leaf under the domain its kind selects. Doing this by kind is - // what actually binds the kind to the root: a chunk leaf relabelled - // `Pointer` (to escape the round-1 `bytes_hash == key` guard) hashes under - // the pointer domain here, rebuilds to a different root, and fails against - // the peer's own signed commitment. let leaf_hashes: Vec<[u8; 32]> = proof .leaves .iter() - .map(|l| match l.kind { - LeafKind::Chunk => leaf_hash(&l.key, &l.bytes_hash), - LeafKind::Pointer => pointer_leaf_hash(&l.key, &l.bytes_hash), - }) + .map(|l| leaf_hash(&l.key, &l.bytes_hash)) .collect(); let levels_to_subtree_root = total_depth - path.depth; let mut cur = fold_levels(leaf_hashes, levels_to_subtree_root); @@ -604,26 +572,8 @@ pub fn subtree_leaf( challenged_peer_id: &[u8; 32], key: &XorName, bytes: &[u8], -) -> SubtreeLeaf { - subtree_leaf_of_kind(LeafKind::Chunk, nonce, challenged_peer_id, key, bytes) -} - -/// Build one subtree leaf of a given kind. -/// -/// A pointer's address is a function of its owner key, not of its bytes, so its -/// `bytes_hash` never equals its `key`. The kind is what tells the round-1 -/// verifier that this is expected rather than the possession-forgery it -/// otherwise looks exactly like. -#[must_use] -pub fn subtree_leaf_of_kind( - kind: LeafKind, - nonce: &[u8; 32], - challenged_peer_id: &[u8; 32], - key: &XorName, - bytes: &[u8], ) -> SubtreeLeaf { SubtreeLeaf { - kind, key: *key, bytes_hash: *blake3::hash(bytes).as_bytes(), content_len: u32::try_from(bytes.len()).unwrap_or(u32::MAX), @@ -640,86 +590,6 @@ pub fn subtree_leaf_of_kind( #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; - - /// A pointer leaf and a chunk leaf over identical `(key, bytes_hash)` must - /// not produce the same leaf hash, or a peer could relabel a chunk as a - /// pointer to escape the round-1 `bytes_hash == key` guard. - #[test] - fn the_leaf_kind_is_bound_by_the_leaf_hash() { - use crate::replication::commitment::{leaf_hash, pointer_leaf_hash}; - - let key: XorName = [0xA1u8; 32]; - let bytes_hash = [0xB2u8; 32]; - assert_ne!( - leaf_hash(&key, &bytes_hash), - pointer_leaf_hash(&key, &bytes_hash), - "relabelling a chunk leaf as a pointer must change the leaf, and so the root" - ); - } - - /// A chunk-only key set must still produce exactly the root it always did, - /// or every existing commitment in the network would be invalidated. - #[test] - fn a_chunk_only_commitment_root_is_unchanged() { - use crate::replication::commitment::MerkleTree; - - let entries: Vec<(XorName, [u8; 32])> = (0u8..8).map(|i| ([i; 32], [i; 32])).collect(); - let via_build = MerkleTree::build(entries.clone()).expect("build").root(); - let via_kinds = MerkleTree::build_of_kinds( - entries - .into_iter() - .map(|(k, b)| (k, b, LeafKind::Chunk)) - .collect(), - ) - .expect("build_of_kinds") - .root(); - assert_eq!(via_build, via_kinds, "chunk-only roots must not move"); - } - - /// A pointer leaf changes the root, so a peer cannot smuggle one into a - /// commitment another peer signed. - #[test] - fn a_pointer_leaf_changes_the_root() { - use crate::replication::commitment::MerkleTree; - - let entries: Vec<(XorName, [u8; 32])> = (0u8..4).map(|i| ([i; 32], [i; 32])).collect(); - let chunky = MerkleTree::build(entries.clone()).expect("build").root(); - let mixed = MerkleTree::build_of_kinds( - entries - .into_iter() - .enumerate() - .map(|(i, (k, b))| { - let kind = if i == 0 { - LeafKind::Pointer - } else { - LeafKind::Chunk - }; - (k, b, kind) - }) - .collect(), - ) - .expect("build_of_kinds") - .root(); - assert_ne!(chunky, mixed); - } - - /// The default kind is Chunk, so any leaf built by the existing path keeps - /// the round-1 guard it always had. - #[test] - fn leaves_default_to_chunk() { - assert_eq!(LeafKind::default(), LeafKind::Chunk); - let leaf = subtree_leaf(&[0u8; 32], &[1u8; 32], &[2u8; 32], b"bytes"); - assert_eq!(leaf.kind, LeafKind::Chunk); - let pointer = subtree_leaf_of_kind( - LeafKind::Pointer, - &[0u8; 32], - &[1u8; 32], - &[2u8; 32], - b"bytes", - ); - assert_eq!(pointer.kind, LeafKind::Pointer); - assert_eq!(pointer.bytes_hash, leaf.bytes_hash, "only the kind differs"); - } use crate::replication::commitment::MerkleTree; fn xn_u32(i: u32) -> XorName { @@ -1006,95 +876,6 @@ mod tests { } } - /// Relabelling a leaf's kind must break the proof. - /// - /// This is the property the pointer exemption rests on: round 1 skips the - /// `bytes_hash == key` guard for pointer leaves, so if a peer could flip a - /// chunk leaf's kind to `Pointer` it would escape the guard for free. The - /// kind picks the leaf-hash domain, so flipping it rebuilds to a different - /// root and fails against the peer's own signed commitment. - #[test] - fn relabelling_a_leaf_kind_breaks_the_proof() { - let peer = [0xABu8; 32]; - let nonce = [0x5Cu8; 32]; - let entries: Vec<(XorName, [u8; 32])> = - (0u8..8).map(|i| (xn_u32(u32::from(i)), [i; 32])).collect(); - let entries: Vec<(XorName, [u8; 32])> = entries - .into_iter() - .map(|(k, _)| (k, *blake3::hash(&chunk_bytes(&k)).as_bytes())) - .collect(); - - let (proof, commitment) = build_proof(&entries, &nonce, &peer); - assert!( - matches!( - verify_subtree_proof(&proof, &nonce, &commitment), - StructureVerdict::Valid - ), - "the honest chunk proof must verify" - ); - - // Flip one leaf's kind and nothing else. - let mut relabelled = proof.clone(); - if let Some(leaf) = relabelled.leaves.first_mut() { - assert_eq!(leaf.kind, LeafKind::Chunk); - leaf.kind = LeafKind::Pointer; - } - assert!( - matches!( - verify_subtree_proof(&relabelled, &nonce, &commitment), - StructureVerdict::Invalid(_) - ), - "a relabelled leaf must not rebuild to the committed root" - ); - } - - /// And the other direction: a genuine pointer leaf verifies against a - /// commitment built with that kind, so the exemption is usable at all. - #[test] - fn a_genuine_pointer_leaf_verifies_against_its_own_commitment() { - use crate::replication::commitment::MerkleTree; - - let peer = [0xCDu8; 32]; - let nonce = [0x77u8; 32]; - let keys: Vec = (0u8..4).map(|i| xn_u32(u32::from(i))).collect(); - - // One pointer leaf among chunks: bytes_hash deliberately != key, which - // round 1 would reject for a chunk. - let entries: Vec<(XorName, [u8; 32], LeafKind)> = keys - .iter() - .enumerate() - .map(|(i, k)| { - let bytes_hash = *blake3::hash(&chunk_bytes(k)).as_bytes(); - let kind = if i == 0 { - LeafKind::Pointer - } else { - LeafKind::Chunk - }; - (*k, bytes_hash, kind) - }) - .collect(); - - let tree = MerkleTree::build_of_kinds(entries.clone()).unwrap(); - let key_count = tree.key_count(); - let mut proof = - build_subtree_proof(&tree, &nonce, &peer, |k| Some(chunk_bytes(k))).unwrap(); - // The builder tags every leaf Chunk; restore the kinds the tree used. - for leaf in &mut proof.leaves { - if let Some((_, _, kind)) = entries.iter().find(|(k, _, _)| *k == leaf.key) { - leaf.kind = *kind; - } - } - let commitment = fake_commitment(tree.root(), key_count, peer); - - assert!( - matches!( - verify_subtree_proof(&proof, &nonce, &commitment), - StructureVerdict::Valid - ), - "a pointer leaf must verify against a commitment that declared it one" - ); - } - #[test] fn honest_proof_verifies_at_many_sizes() { let peer = [0xABu8; 32]; diff --git a/tests/poc_commitment_audit_attacks.rs b/tests/poc_commitment_audit_attacks.rs index 144c8548..903af243 100644 --- a/tests/poc_commitment_audit_attacks.rs +++ b/tests/poc_commitment_audit_attacks.rs @@ -74,7 +74,7 @@ use ant_node::replication::slice::{ }; use ant_node::replication::subtree::{ build_subtree_proof, select_spotcheck_indices, select_subtree_path, verify_subtree_proof, - LeafKind, StructureVerdict, SubtreeLeaf, SubtreeProof, + StructureVerdict, SubtreeLeaf, SubtreeProof, }; use rand::Rng; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; @@ -385,7 +385,6 @@ fn relay_holding_only_addresses_caught_by_real_bytes_check() { // commitment because it lacks the bytes. let forged_nonced_root = *blake3::hash(b"i-do-not-have-the-bytes").as_bytes(); leaves.push(SubtreeLeaf { - kind: LeafKind::Chunk, key: k, bytes_hash: k, content_len: u32::try_from(c.len()).unwrap(), @@ -492,7 +491,6 @@ fn predict_and_fetch_relay_is_caught_by_fresh_random_sample() { *blake3::hash(b"forged").as_bytes() }; leaves.push(SubtreeLeaf { - kind: LeafKind::Chunk, key: k, bytes_hash: k, content_len: u32::try_from(c.len()).unwrap(), From 3f24c88c760679a8d583d0a4625961a757bf1cfc Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 18:02:10 +0900 Subject: [PATCH 04/32] refactor(pointer): delete the surface replication would have used bytes_hash, all_keys, all_states, missing_states and holds_state described commitments and version-aware sync that this PR does not build, and the conditional GET they served has gone from the wire with them. Nothing called any of it outside its own tests. The ADR listed bytes_hash as a third identity binding 'this node's commitment', which was false in the same breath as the section saying pointers take no part in commitments. Two identities now: one that routes, one that authorizes payment. --- docs/adr/ADR-0015-pointers-immutable-owner.md | 7 +- src/pointer/service.rs | 51 ------------ src/pointer/store.rs | 82 ------------------- tests/pointer_convergence.rs | 6 -- 4 files changed, 3 insertions(+), 143 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 6c7e1156..68cf456c 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -42,12 +42,11 @@ The key is carried because it has to be: ML-DSA has no key recovery and a is 5,303 bytes rather than ~3,350, and the price of validating one with no fetch. -### Three identities +### Two identities ```text -A = BLAKE3("autonomi.pointer.address.v1" || owner) routes -state_id = BLAKE3("autonomi.pointer.state.v1" || body) authorizes payment -bytes_hash = BLAKE3(record) this node's commitment +A = BLAKE3("autonomi.pointer.address.v1" || owner) routes +state_id = BLAKE3("autonomi.pointer.state.v1" || body) authorizes payment ``` **Public-key addressed and self-verifying.** `A` is a pure function of the owner diff --git a/src/pointer/service.rs b/src/pointer/service.rs index b83cde6c..773e7d80 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -282,28 +282,6 @@ impl PointerService { /// Handle a pointer GET. pub async fn handle_get(&self, request: PointerGetRequest) -> PointerGetResponse { - // A replica asking "anything newer than this?" gets a cheap index - // lookup first, but the answer is confirmed against the file before it - // is sent: an index entry for a record whose file has since gone or - // stopped validating would otherwise answer "unchanged" forever, and - // the peer would never fetch the copy that would repair it. `get` - // re-validates and drops such an entry, so the confirmation costs a - // read exactly once, on the way to telling the truth. - let known = request.known_state_id; - if known.is_some_and(|known| self.store.holds_state(&request.address, &known)) { - match self.store.get(&request.address).await { - Ok(Some(record)) if Some(record.state_id()) == known => { - return PointerGetResponse::Unchanged { - state_id: record.state_id(), - }; - } - Ok(_) => {} - Err(e) => { - return PointerGetResponse::Error(ProtocolError::StorageFailed(e.to_string())); - } - } - } - match self.store.get(&request.address).await { Ok(Some(record)) => PointerGetResponse::Success { record: Bytes::from(record.to_bytes()), @@ -475,35 +453,6 @@ mod tests { } } - #[tokio::test] - async fn a_conditional_get_skips_the_transfer_when_nothing_changed() { - let (service, _dir) = service().await; - let record = signed(1, 0, 3); - service.handle_put(put(&record)).await; - - match service - .handle_get(PointerGetRequest::if_changed( - record.address(), - record.state_id(), - )) - .await - { - PointerGetResponse::Unchanged { state_id } => { - assert_eq!(state_id, record.state_id()); - } - other => panic!("expected Unchanged, got {other:?}"), - } - - // A stale known state still gets the bytes. - match service - .handle_get(PointerGetRequest::if_changed(record.address(), [0u8; 32])) - .await - { - PointerGetResponse::Success { .. } => {} - other => panic!("expected Success, got {other:?}"), - } - } - #[tokio::test] async fn an_absent_pointer_is_not_found() { let (service, _dir) = service().await; diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 298f1369..54391137 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -133,8 +133,6 @@ struct IndexEntry { rank: MergeRank, /// The held record's counter, for the paid-increment check. counter: u64, - /// `BLAKE3` over the exact stored bytes, which a storage commitment binds. - bytes_hash: XorName, /// Which insertion this entry is. /// /// Monotonic for the life of this store, so an entry can be told apart @@ -154,7 +152,6 @@ impl IndexEntry { state_id: record.state_id(), rank: record.state().rank(), counter: record.counter(), - bytes_hash: record.bytes_hash(), generation, } } @@ -421,15 +418,6 @@ impl PointerStore { self.snapshot(address).map(|e| e.state_id) } - /// `BLAKE3` over the exact bytes held at `address`, if any. - /// - /// What a storage commitment binds for this record. Per-storer by design: - /// each node commits and is audited against the encoding it actually holds. - #[must_use] - pub fn bytes_hash(&self, address: &XorName) -> Option { - self.snapshot(address).map(|e| e.bytes_hash) - } - /// Whether `state` is the paid successor of what is held. /// /// One payment buys one increment: a new pointer starts at counter 0, and @@ -454,59 +442,6 @@ impl PointerStore { self.snapshot(address).is_some() } - /// Whether the record held at `address` is already this exact state. - /// - /// The question a fetch decision asks: holding *the key* is not enough for - /// a mutable record, holding *the state* is. - #[must_use] - pub fn holds_state(&self, address: &XorName, state_id: &XorName) -> bool { - self.snapshot(address) - .is_some_and(|e| e.state_id == *state_id) - } - - /// Every address the store holds. - #[must_use] - pub fn all_keys(&self) -> Vec { - self.inner.index.lock().keys().copied().collect() - } - - /// Every address with the state identifier and committed bytes hash held - /// for it. - /// - /// The input a commitment build or a sync round needs in one pass, which is - /// why the index exists rather than each of those re-reading every file. - #[must_use] - pub fn all_states(&self) -> Vec<(XorName, XorName, XorName)> { - self.inner - .index - .lock() - .iter() - .map(|(address, entry)| (*address, entry.state_id, entry.bytes_hash)) - .collect() - } - - /// Which of `wanted` this node does not already hold at the given state. - /// - /// The fetch decision a mutable record needs. The chunk path asks "do I - /// hold this key?" and stops, which for a pointer means a replica on - /// version N never fetches N+1 and the two diverge permanently. This asks - /// "do I hold this *state*?" instead. - #[must_use] - pub fn missing_states(&self, wanted: &[(XorName, XorName)]) -> Vec<(XorName, XorName)> { - let index = self.inner.index.lock(); - wanted - .iter() - .filter(|(address, state_id)| { - // `map_or` rather than `is_none_or`: the latter is stable only - // from 1.82 and this crate's MSRV is 1.75. - index - .get(address) - .map_or(true, |entry| entry.state_id != *state_id) - }) - .copied() - .collect() - } - /// How many records the store holds. #[must_use] pub fn len(&self) -> usize { @@ -880,12 +815,7 @@ mod tests { .expect("present"); assert_eq!(read.to_bytes(), record.to_bytes()); assert_eq!(store.state_id(&record.address()), Some(record.state_id())); - assert_eq!( - store.bytes_hash(&record.address()), - Some(record.bytes_hash()) - ); assert!(store.contains(&record.address())); - assert!(store.holds_state(&record.address(), &record.state_id())); assert_eq!(store.len(), 1); } @@ -947,7 +877,6 @@ mod tests { held_bytes, "not one of the 16 re-signatures reached the disk" ); - assert_eq!(store.bytes_hash(&first.address()), Some(first.bytes_hash())); } #[tokio::test] @@ -1150,7 +1079,6 @@ mod tests { let after = reopened.get(&address).await.expect("get").expect("present"); assert_eq!(after.to_bytes(), before.to_bytes()); assert_eq!(reopened.state_id(&address), Some(before.state_id())); - assert_eq!(reopened.bytes_hash(&address), Some(before.bytes_hash())); } #[tokio::test] @@ -1179,8 +1107,6 @@ mod tests { store.put_bytes(&first.to_bytes()).await.expect("put"); store.put_bytes(&second.to_bytes()).await.expect("put"); assert_eq!(store.len(), 2); - assert_eq!(store.all_keys().len(), 2); - assert_eq!(store.all_states().len(), 2); } #[tokio::test] @@ -1266,7 +1192,6 @@ mod tests { let held = store.get(&address).await.expect("get").expect("present"); assert_eq!(held.counter(), 12, "the highest counter must win"); assert_eq!(store.state_id(&address), Some(held.state_id())); - assert_eq!(store.bytes_hash(&address), Some(held.bytes_hash())); assert_eq!(store.len(), 1); // No temporary file survived the race. @@ -1334,8 +1259,6 @@ mod tests { let (store, _dir) = store().await; assert!(store.is_empty()); assert_eq!(store.state_id(&[0u8; 32]), None); - assert_eq!(store.bytes_hash(&[0u8; 32]), None); - assert!(!store.holds_state(&[0u8; 32], &[0u8; 32])); assert!(store.get(&[0u8; 32]).await.expect("get").is_none()); } @@ -1448,10 +1371,5 @@ mod tests { store.put_bytes(&old.to_bytes()).await.expect("put"); assert!(store.contains(&new.address())); - assert!(store.holds_state(&old.address(), &old.state_id())); - assert!( - !store.holds_state(&new.address(), &new.state_id()), - "the newer state is absent even though the key is held" - ); } } diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index 252fff7f..ce7ef54a 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -382,12 +382,6 @@ fn the_wire_format_is_what_the_adr_says() { "state_id covers the whole body and nothing else" ); - assert_eq!( - record.bytes_hash(), - *blake3::hash(&bytes).as_bytes(), - "bytes_hash is over the whole record" - ); - // The signature verifies over the body under the literal context, and does // not verify under a different one. The context is consensus too. let dsa = ml_dsa_65(); From 2500613785a7083e5eddd23c25176c53163866e1 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 18:36:41 +0900 Subject: [PATCH 05/32] fix(pointer): never answer for a record this node no longer holds The index is a claim about a file, and both of inspect's early answers -- "unchanged" and "stale" -- assert the node already holds something at least as good as what arrived. Answered from the claim alone, a lost or corrupted file made the client count an acknowledgement for a record nobody could serve. The file is now read back before either answer, and a failed read disowns the entry so the submission lands as the repair it is. Also: - A pointer quote no longer consults the chunk store. A chunk sitting at a pointer's state address could set already_stored, and a majority of nodes saying "no payment needed" would strand the write as unpaid. - IndexEntry carries the held PointerState rather than three fields copied out of it, so merge order and the paid increment are the protocol's own rules applied to the held state -- is_successor_of now has one definition and one caller instead of two implementations. - Inspected::Verified, Inspected::state() and prepare() are gone; the tests that used them drive the same three steps production does. - tests/pointer_convergence.rs lacked the test-module lint allow every other integration test carries, so clippy --all-targets -D warnings failed on it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/ADR-0015-pointers-immutable-owner.md | 8 +- src/pointer/mod.rs | 25 +- src/pointer/service.rs | 17 +- src/pointer/store.rs | 264 +++++++++--------- src/storage/handler.rs | 73 ++++- tests/pointer_convergence.rs | 16 +- 6 files changed, 227 insertions(+), 176 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 68cf456c..dc5f359a 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -106,7 +106,8 @@ re-checks under its lock, because a newer state can land while payment verifies. | Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | | Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | | Collide a pointer and a chunk address | Refused in both directions. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | -| Peer lies about storing a pointer | The client checks every acknowledgement names the address and state it sent, and stores on a quorum rather than stopping at the first success | +| Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent, and a write needs a majority of the close group — the same group, by the same call, that a read asks, so an acknowledged pointer is readable | +| Node claims a record it no longer holds | An index entry is only a claim about a file; before answering "unchanged" or "stale" the node reads the file back, and a lost or corrupt one makes the submission a repair | ## Consequences @@ -158,3 +159,8 @@ on the same work. - A crafted chunk cannot satisfy a pointer's paid-cache entry. - An acknowledgement naming a different address or state is refused, and a read keeps the winner whatever order the replies arrive in. +- A majority of the group answering ends a write or a read, so one unreachable + peer cannot stall either; a minority is reported as a shortfall, never + presented as the network's answer. +- A resubmission repairs a record whose file the disk lost, rather than being + acknowledged as unchanged. diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs index 9cc56fe2..e890d068 100644 --- a/src/pointer/mod.rs +++ b/src/pointer/mod.rs @@ -19,25 +19,16 @@ //! immutable chunk store cannot do. //! - [`service`] — the request handler: validate, check payment, merge. //! -//! # The three identifiers +//! # The two identifiers //! //! | Name | Derivation | Job | //! |---|---|---| //! | `A` | `BLAKE3(domain \|\| owner)` | routes, and decides which nodes are responsible | -//! | `state_id` | `BLAKE3(domain \|\| body)` | names the authenticated state: sync hints, and what a quote is paid against | -//! | `bytes_hash` | `BLAKE3(record)` | what *this* node's storage commitment binds | +//! | `state_id` | `BLAKE3(domain \|\| body)` | names the state, and is what a quote is paid against | //! -//! `A` and `state_id` are separate because `A` must be stable for the pointer's -//! life while the paid identifier must change with every update, or updates -//! after the first would be free — which is exactly the 1.0 defect this design -//! exists to fix. There is deliberately no fourth name hashed from `state_id`: -//! it is already a domain-separated, owner-bound identifier for exactly one -//! signed state. -//! -//! `bytes_hash` is per-storer rather than per-state, because two replicas may -//! hold one state under different signatures. That is fine: a storage -//! commitment is built and signed by one node and audited against that node's -//! own bytes, so it never has to agree with a peer's. +//! They are separate because `A` must be stable for the pointer's life while +//! the paid identifier must change with every update, or updates after the +//! first would be free — the 1.0 defect this design exists to fix. //! //! # Why the merge rule ignores signature bytes //! @@ -54,9 +45,9 @@ pub mod service; pub mod store; pub use ant_protocol::pointer::{ - cmp_merge, merge, pointer_address, state_id_for_body, MergeRank, ParsedPointer, Pointer, - PointerError, PointerState, PointerTarget, PointerTargetKind, DATA_TYPE_POINTER, - POINTER_BODY_LEN, POINTER_FORMAT_VERSION, POINTER_WIRE_LEN, TARGET_WIRE_LEN, + pointer_address, state_id_for_body, ParsedPointer, Pointer, PointerError, PointerState, + PointerTarget, PointerTargetKind, DATA_TYPE_POINTER, POINTER_BODY_LEN, POINTER_FORMAT_VERSION, + POINTER_WIRE_LEN, TARGET_WIRE_LEN, }; pub use service::PointerService; pub use store::{Inspected, PointerStore, PutOutcome}; diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 773e7d80..f392ac86 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -155,12 +155,6 @@ impl PointerService { ))); } Ok(Inspected::Candidate(parsed)) => parsed, - // `inspect` does not verify, so it cannot produce this arm. - Ok(Inspected::Verified(_)) => { - return PointerPutResponse::Error(ProtocolError::Internal( - "inspect returned a verified record".to_string(), - )); - } Err(e) => { debug!("Pointer PUT refused: {e}"); return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); @@ -197,9 +191,7 @@ impl PointerService { } match self.store.commit(record).await { - Ok(PutOutcome::Stored | PutOutcome::Replaced) => { - PointerPutResponse::Success { address, state_id } - } + Ok(PutOutcome::Changed) => PointerPutResponse::Success { address, state_id }, // The re-check under the commit lock found a newer state. The // client paid for a state that lost a race; say so plainly. Ok(PutOutcome::Unchanged) => PointerPutResponse::Unchanged { address, state_id }, @@ -366,6 +358,7 @@ fn cross_kind_refusal(address: XorName, chunk_present: Result) -> Option

bool { - matches!(self, Self::Stored | Self::Replaced) + matches!(self, Self::Changed) } } @@ -100,39 +102,17 @@ pub enum Inspected { /// checked yet — pass it to [`PointerStore::verify`] once admission gates /// have had their say. Candidate(ParsedPointer), - /// A signature-checked record, ready to commit. - /// - /// Boxed because a verified record is two orders of magnitude larger than - /// the other arms, and an enum is as big as its widest one. - Verified(Box), -} - -impl Inspected { - /// What this arrival claims, whether or not it is a candidate. - #[must_use] - pub fn state(&self) -> Option<&ant_protocol::pointer::PointerState> { - match self { - Self::Candidate(parsed) => Some(parsed.state()), - // A no-op claims nothing worth acting on, and a verified record - // carries its own state directly. - Self::Noop(_) | Self::Verified(_) => None, - } - } } /// What the store knows about a held record without reading it back. #[derive(Debug, Clone, Copy)] struct IndexEntry { - /// The held record's authenticated-state identifier. - state_id: XorName, - /// The held record's place in the merge order. + /// The held record's authenticated state. /// - /// Carried as the record's own [`MergeRank`] rather than as the fields it - /// is built from, so the store cannot drift from the rule in - /// `PointerState::replaces`. - rank: MergeRank, - /// The held record's counter, for the paid-increment check. - counter: u64, + /// The state itself rather than fields copied out of it, so every rule the + /// store applies — merge order, the paid increment — is the protocol's own + /// rule applied to the held state, and cannot drift from it. + state: PointerState, /// Which insertion this entry is. /// /// Monotonic for the life of this store, so an entry can be told apart @@ -149,9 +129,7 @@ impl IndexEntry { /// Describe a validated record. fn of(record: &Pointer, generation: u64) -> Self { Self { - state_id: record.state_id(), - rank: record.state().rank(), - counter: record.counter(), + state: record.state(), generation, } } @@ -242,35 +220,15 @@ impl PointerStore { }) } - /// Validate an arriving record against what is held. + /// Parse an arrival and decide whether it could change anything, without + /// verifying its signature. /// /// Cheap checks first: an arrival that cannot change anything is refused /// before its signature is looked at, because it is a no-op whether or not /// it is correctly signed. That ordering is what stops repeated submission - /// of one paid state from buying ML-DSA verifications. A record that would - /// win is always verified — outside the store lock, so one slow - /// verification cannot stall every other address. - /// - /// # Errors - /// - /// Returns [`Error::Protocol`] if the bytes are not a well-formed record - /// and [`Error::Crypto`] if a would-be winner's signature does not verify. - pub async fn prepare(&self, bytes: &[u8]) -> Result { - match self.inspect(bytes)? { - Inspected::Noop(outcome) => Ok(Inspected::Noop(outcome)), - Inspected::Candidate(parsed) => { - Ok(Inspected::Verified(Box::new(self.verify(parsed).await?))) - } - // `inspect` never returns this; only `prepare` produces it. - verified @ Inspected::Verified(_) => Ok(verified), - } - } - - /// Parse an arrival and decide whether it could change anything, without - /// verifying its signature. - /// - /// The cheap half of [`Self::prepare`]. Callers that gate on admission — - /// capacity, responsibility for the address — run this first, apply their + /// of one paid state from buying ML-DSA verifications. Callers that gate on + /// admission — capacity, responsibility for the address — run this first, + /// apply their /// gates, and only then pay for [`Self::verify`]. Otherwise a forged record /// for an address the node is not responsible for still buys an ML-DSA /// verification before anything rejects it. @@ -285,11 +243,19 @@ impl PointerStore { let parsed = ParsedPointer::parse(bytes.to_vec())?; let state = *parsed.state(); - if let Some(entry) = self.snapshot(&state.address) { - if entry.state_id == state.state_id { + // The index is only a claim about a file. Both early answers below + // assert that this node holds something at least as good as what + // arrived, so neither may be given on a claim alone: if the file has + // gone or stopped validating, `reread` disowns the entry and the + // submission falls through as the repair it should be. + if let Some(entry) = self + .snapshot(&state.address) + .filter(|_| self.reread(&state.address)) + { + if entry.state.state_id == state.state_id { return Ok(Inspected::Noop(PutOutcome::Unchanged)); } - if state.rank() <= entry.rank { + if !state.replaces(&entry.state) { return Ok(Inspected::Noop(PutOutcome::Stale)); } } @@ -334,16 +300,15 @@ impl PointerStore { /// Validate and store in one step, with no payment gate. /// /// For callers that have already settled payment, and for tests. The - /// request path should use [`Self::prepare`] and [`Self::commit`] so the - /// payment check can sit between them. + /// request path runs the same three steps with its admission and payment + /// gates between them. /// /// # Errors /// - /// As [`Self::prepare`] and [`Self::commit`]. + /// As [`Self::inspect`], [`Self::verify`] and [`Self::commit`]. pub async fn put_bytes(&self, bytes: &[u8]) -> Result { - match self.prepare(bytes).await? { + match self.inspect(bytes)? { Inspected::Noop(outcome) => Ok(outcome), - Inspected::Verified(record) => self.commit(*record).await, Inspected::Candidate(parsed) => { let record = self.verify(parsed).await?; self.commit(record).await @@ -415,7 +380,7 @@ impl PointerStore { /// under different signatures agree and do not refetch each other forever. #[must_use] pub fn state_id(&self, address: &XorName) -> Option { - self.snapshot(address).map(|e| e.state_id) + self.snapshot(address).map(|e| e.state.state_id) } /// Whether `state` is the paid successor of what is held. @@ -432,7 +397,7 @@ impl PointerStore { pub fn accepts_as_paid_update(&self, state: &PointerState) -> bool { self.snapshot(&state.address).map_or_else( || state.is_genesis(), - |entry| state.counter == entry.counter.wrapping_add(1) && entry.counter != u64::MAX, + |entry| state.is_successor_of(&entry.state), ) } @@ -471,6 +436,26 @@ impl PointerStore { &self.inner.dir } + /// Whether the file behind `address` still reads back as a valid record. + /// + /// Guards the two early answers in `inspect`, both of which claim this + /// node already holds something at least as good as what arrived. A failed + /// read disowns the entry, exactly as `get` does. + fn reread(&self, address: &XorName) -> bool { + let claimed = self.snapshot(address).map(|entry| entry.generation); + let valid = matches!( + read_record_file(&self.path_for(address)), + Ok(Some(ref bytes)) if matches!( + Pointer::from_bytes(bytes), + Ok(ref record) if record.address() == *address + ) + ); + if !valid { + self.forget_if_unchanged(address, claimed); + } + valid + } + /// Copy out what is held for `address`, releasing the index lock at once. fn snapshot(&self, address: &XorName) -> Option { self.inner.index.lock().get(address).copied() @@ -526,10 +511,10 @@ impl Inner { // one under the lock below; this only avoids staging a file for an // arrival that is already obviously a no-op. if let Some(entry) = self.index.lock().get(&address) { - if entry.state_id == record.state_id() { + if entry.state.state_id == record.state_id() { return Ok(PutOutcome::Unchanged); } - if record.state().rank() <= entry.rank { + if !record.state().replaces(&entry.state) { return Ok(PutOutcome::Stale); } } @@ -550,9 +535,9 @@ impl Inner { // Re-check: staging is not instantaneous and a newer state may // have committed while it ran. let outcome = match index.get(&address) { - None => PutOutcome::Stored, - Some(entry) if entry.state_id == record.state_id() => PutOutcome::Unchanged, - Some(entry) if record.state().rank() > entry.rank => PutOutcome::Replaced, + None => PutOutcome::Changed, + Some(entry) if entry.state.state_id == record.state_id() => PutOutcome::Unchanged, + Some(entry) if record.state().replaces(&entry.state) => PutOutcome::Changed, Some(_) => PutOutcome::Stale, }; if !outcome.changed() { @@ -764,6 +749,12 @@ fn sync_directory(dir: &Path) -> Result<()> { } #[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + reason = "test assertions" +)] mod tests { use super::*; use ant_protocol::pointer::{PointerTarget, PointerTargetKind, POINTER_BODY_LEN}; @@ -782,7 +773,7 @@ mod tests { /// A record with its signature destroyed: the body still parses, the /// record does not verify. fn forged(record: &Pointer) -> Vec { - let mut bytes = record.to_bytes().to_vec(); + let mut bytes = record.to_bytes(); if let Some(byte) = bytes.get_mut(POINTER_BODY_LEN + 3) { *byte ^= 0xff; } @@ -805,7 +796,7 @@ mod tests { let record = signed(1, 1, 1); assert_eq!( store.put_bytes(&record.to_bytes()).await.expect("put"), - PutOutcome::Stored + PutOutcome::Changed ); let read = store @@ -826,11 +817,11 @@ mod tests { let second = signed(1, 2, 1); assert_eq!( store.put_bytes(&first.to_bytes()).await.expect("put"), - PutOutcome::Stored + PutOutcome::Changed ); assert_eq!( store.put_bytes(&second.to_bytes()).await.expect("put"), - PutOutcome::Replaced + PutOutcome::Changed ); assert_eq!( store.put_bytes(&first.to_bytes()).await.expect("put"), @@ -940,64 +931,45 @@ mod tests { } #[tokio::test] - async fn prepare_reports_a_no_op_without_a_candidate() { + async fn an_arrival_that_changes_nothing_is_not_a_candidate() { let (store, _dir) = store().await; let held = signed(1, 6, 6); store.put_bytes(&held.to_bytes()).await.expect("put"); - match store.prepare(&held.to_bytes()).await.expect("prepare") { + match store.inspect(&held.to_bytes()).expect("inspect") { Inspected::Noop(outcome) => assert_eq!(outcome, PutOutcome::Unchanged), - Inspected::Verified(_) | Inspected::Candidate(_) => { - panic!("an identical state is not a candidate") - } + Inspected::Candidate(_) => panic!("an identical state is not a candidate"), } - match store - .prepare(&signed(1, 1, 6).to_bytes()) - .await - .expect("prepare") - { + match store.inspect(&signed(1, 1, 6).to_bytes()).expect("inspect") { Inspected::Noop(outcome) => assert_eq!(outcome, PutOutcome::Stale), - Inspected::Verified(_) | Inspected::Candidate(_) => { - panic!("a stale record is not a candidate") - } + Inspected::Candidate(_) => panic!("a stale record is not a candidate"), } } #[tokio::test] - async fn a_candidate_exposes_what_a_payment_check_needs() { + async fn a_verified_record_exposes_what_a_payment_check_needs() { let (store, _dir) = store().await; let record = signed(1, 2, 2); - match store.prepare(&record.to_bytes()).await.expect("prepare") { - Inspected::Verified(verified) => { - // A verified record is a `Pointer`, so the payment check reads - // the address and state straight off it — there is no wrapper - // type in between restating what it already knows. - assert_eq!(verified.address(), record.address()); - assert_eq!(verified.state_id(), record.state_id()); - assert_eq!(verified.to_bytes(), record.to_bytes()); - assert_eq!( - store.commit(*verified).await.expect("commit"), - PutOutcome::Stored - ); - } - other => panic!("a new record verifies, got {other:?}"), - } + let verified = verified(&store, &record).await; + // A verified record is a `Pointer`, so the payment check reads the + // address and state straight off it — there is no wrapper type in + // between restating what it already knows. + assert_eq!(verified.address(), record.address()); + assert_eq!(verified.state_id(), record.state_id()); + assert_eq!(verified.to_bytes(), record.to_bytes()); + assert_eq!( + store.commit(verified).await.expect("commit"), + PutOutcome::Changed + ); assert_eq!(store.len(), 1); } #[tokio::test] - async fn a_commit_rechecks_what_prepare_saw() { - // `prepare` runs before the payment check; a newer state can land while - // that check is in flight, and must not then be overwritten. + async fn a_commit_rechecks_what_verification_saw() { + // Verification runs before the payment check; a newer state can land + // while that check is in flight, and must not then be overwritten. let (store, _dir) = store().await; - let slow = match store - .prepare(&signed(1, 2, 1).to_bytes()) - .await - .expect("prepare") - { - Inspected::Verified(record) => *record, - other => panic!("expected a verified record, got {other:?}"), - }; + let slow = verified(&store, &signed(1, 2, 1)).await; // Someone else's newer state arrives while the payment is being checked. store @@ -1028,7 +1000,7 @@ mod tests { #[tokio::test] async fn every_rotation_of_a_delivery_reaches_one_answer() { - let records = vec![ + let records = [ signed(1, 1, 9), signed(1, 3, 4), signed(1, 3, 1), @@ -1089,8 +1061,7 @@ mod tests { let _first = PointerStore::new(dir.path()).await.expect("open"); let err = PointerStore::new(dir.path()) .await - .err() - .expect("a second store must not open the same directory"); + .expect_err("a second store must not open the same directory"); let message = format!("{err}"); assert!( message.contains("another pointer store already has"), @@ -1133,7 +1104,41 @@ mod tests { // "unchanged", which is the whole point of forgetting it. assert_eq!( store.put_bytes(&record.to_bytes()).await.expect("put"), - PutOutcome::Stored + PutOutcome::Changed + ); + assert!(store.get(&record.address()).await.expect("get").is_some()); + } + + /// Run the production sequence as far as the payment gate: inspect, then + /// verify the candidate it yields. + async fn verified(store: &PointerStore, record: &Pointer) -> Pointer { + match store.inspect(&record.to_bytes()).expect("inspect") { + Inspected::Candidate(parsed) => store.verify(parsed).await.expect("verify"), + Inspected::Noop(outcome) => panic!("expected a candidate, got {outcome:?}"), + } + } + + #[tokio::test] + async fn a_resubmission_repairs_a_record_the_disk_lost() { + // "Unchanged" is an acknowledgement: the sender stops on it. If the + // index still claims a record whose file has gone, answering Unchanged + // would end the write while this node holds nothing. + let (store, _dir) = store().await; + let record = signed(1, 0, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + // The file disappears under the node; the index has not noticed. + std::fs::remove_file(store.dir().join(hex::encode(record.address()))).expect("remove"); + assert!( + store.contains(&record.address()), + "the index still claims it" + ); + + // The same state arriving again is a repair, not a no-op. + assert_eq!( + store.put_bytes(&record.to_bytes()).await.expect("put"), + PutOutcome::Changed, + "a resubmission must repair a record the disk lost" ); assert!(store.get(&record.address()).await.expect("get").is_some()); } @@ -1182,7 +1187,7 @@ mod tests { let mut tasks = Vec::new(); for counter in 1..=12u64 { let store = store.clone(); - let bytes = signed(1, counter, 1).to_bytes().to_vec(); + let bytes = signed(1, counter, 1).to_bytes(); tasks.push(tokio::spawn(async move { store.put_bytes(&bytes).await })); } for task in tasks { @@ -1209,10 +1214,7 @@ mod tests { // the write and the index update both happened, or neither did. let (store, _dir) = store().await; let record = signed(1, 3, 3); - let prepared = match store.prepare(&record.to_bytes()).await.expect("prepare") { - Inspected::Verified(record) => *record, - other => panic!("expected a verified record, got {other:?}"), - }; + let prepared = verified(&store, &record).await; let abandoned = { let committing = store.commit(prepared); @@ -1313,7 +1315,7 @@ mod tests { // A peer repairs it with the very same state. assert_eq!( store.put_bytes(&record.to_bytes()).await.expect("put"), - PutOutcome::Stored + PutOutcome::Changed ); // Reader two, still holding its stale observation, must not erase it. @@ -1343,7 +1345,7 @@ mod tests { let reopened = PointerStore::new(dir.path()).await.expect("reopen"); assert_eq!( reopened.put_bytes(&record.to_bytes()).await.expect("put"), - PutOutcome::Stored, + PutOutcome::Changed, "a swept leftover must not block the first write to its address" ); } diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 1ba544c6..a854f7ca 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -27,7 +27,6 @@ //! └─────────────────────────────────────────────────────────┘ //! ``` -#[cfg(test)] use crate::ant_protocol::DATA_TYPE_CHUNK; use crate::ant_protocol::{ settlement_compatibility, ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, @@ -918,17 +917,24 @@ impl AntProtocol { // Check if the chunk is already stored so we can tell the client // to skip payment (already_stored = true). + // + // Only chunks: this reads the chunk store, so asking it about any other + // kind answers a question about the wrong address space. A pointer + // quote names a state, not content, and every pointer state — creation + // or update — is paid for, so it is never already stored. + // // The match intentionally logs the error when the `logging` feature is // active. Clippy suggests `unwrap_or_default()` when logging is compiled // out, but keeping the explicit match preserves the diagnostic intent. #[allow(clippy::manual_unwrap_or_default)] - let already_stored = match self.storage.exists(&request.address) { - Ok(exists) => exists, - Err(e) => { - warn!("Storage check failed for {addr_hex}: {e}"); - false // Assume not stored on error — generate a normal quote. - } - }; + let already_stored = request.data_type == DATA_TYPE_CHUNK + && match self.storage.exists(&request.address) { + Ok(exists) => exists, + Err(e) => { + warn!("Storage check failed for {addr_hex}: {e}"); + false // Assume not stored on error — generate a normal quote. + } + }; if already_stored { debug!("Chunk {addr_hex} already stored — returning quote with already_stored=true"); @@ -1156,6 +1162,7 @@ mod tests { use super::*; use crate::payment::metrics::QuotingMetricsTracker; use crate::payment::{EvmVerifierConfig, PaymentVerifierConfig}; + use crate::pointer::{DATA_TYPE_POINTER, POINTER_WIRE_LEN}; use crate::storage::ChunkStoreConfig; use evmlib::RewardsAddress; use saorsa_core::identity::NodeIdentity; @@ -2011,6 +2018,56 @@ mod tests { ); } + #[tokio::test] + async fn a_stored_chunk_does_not_suppress_a_pointer_quote() { + // A pointer quote names a state, not content. If a chunk sitting at + // that address could set `already_stored`, a majority of nodes would + // tell the client to skip payment and the write would then be refused + // as unpaid. Only chunks may answer from the chunk store. + let (protocol, _temp) = create_test_protocol().await; + + let content = b"a chunk that shares a pointer state address"; + let address = ChunkStore::compute_address(content); + protocol.payment_verifier().cache_insert(address); + let put_msg = ChunkMessage { + request_id: 320, + body: ChunkMessageBody::PutRequest(ChunkPutRequest::new( + address, + Bytes::copy_from_slice(content), + )), + }; + let put_bytes = put_msg.encode().expect("encode put"); + let _ = protocol + .try_handle_request(&put_bytes) + .await + .expect("handle put"); + + let quote_msg = ChunkMessage { + request_id: 321, + body: ChunkMessageBody::QuoteRequest(ChunkQuoteRequest { + address, + data_size: POINTER_WIRE_LEN as u64, + data_type: DATA_TYPE_POINTER, + }), + }; + let quote_bytes = quote_msg.encode().expect("encode quote"); + let response_bytes = protocol + .try_handle_request("e_bytes) + .await + .expect("handle quote") + .expect("expected response"); + + match ChunkMessage::decode(&response_bytes).expect("decode").body { + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Success { + already_stored, .. + }) => assert!( + !already_stored, + "a chunk must not answer for a pointer state" + ), + other => panic!("expected a quote, got: {other:?}"), + } + } + #[tokio::test] async fn test_quote_already_stored_flag() { let (protocol, _temp) = create_test_protocol().await; diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index ce7ef54a..141d0cc8 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -7,6 +7,8 @@ //! properties the merge rule exists to provide: one payment funds one state, //! and no re-signature of a stored state can displace it. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use std::collections::BTreeSet; use ant_protocol::pointer::{Pointer, PointerTarget, PointerTargetKind}; @@ -253,7 +255,7 @@ proptest! { mask in 1u8..255, ) { let record = signed(5, 11, 22, PointerTargetKind::Chunk); - let mut bytes = record.to_bytes().to_vec(); + let mut bytes = record.to_bytes(); let Some(byte) = bytes.get_mut(index) else { return Ok(()); }; @@ -289,7 +291,7 @@ fn sixty_four_signatures_over_one_state_yield_one_winner() { // Sorted worst-first is the submission order that would have made every // record win under a byte-ordering tie-break. let mut sorted: Vec<&Pointer> = records.iter().collect(); - sorted.sort_by_key(|record| record.to_bytes().to_vec()); + sorted.sort_by_key(|record| record.to_bytes()); let first = sorted.first().copied().expect("non-empty"); for candidate in &sorted { @@ -436,7 +438,7 @@ async fn sixty_four_signatures_buy_exactly_one_write() { let first = signed(7, 9, 9, PointerTargetKind::Chunk); assert_eq!( store.put_bytes(&first.to_bytes()).await.expect("put"), - PutOutcome::Stored + PutOutcome::Changed ); let path = store.dir().join(hex::encode(first.address())); @@ -530,7 +532,7 @@ fn an_unknown_version_is_refused_rather_than_accepted_at_its_own_price() { let record = signed(12, 1, 1, PointerTargetKind::Chunk); for version in [0u8, 2, 7, u8::MAX] { assert_ne!(version, POINTER_FORMAT_VERSION); - let mut bytes = record.to_bytes().to_vec(); + let mut bytes = record.to_bytes(); if let Some(byte) = bytes.first_mut() { *byte = version; } @@ -602,7 +604,7 @@ async fn migration_must_happen_before_the_terminal_update() { assert!(!penultimate.is_terminal()); assert_eq!( store.put_bytes(&penultimate.to_bytes()).await.expect("put"), - PutOutcome::Stored + PutOutcome::Changed ); // The safe migration: spend the last counter. A strictly larger counter @@ -613,7 +615,7 @@ async fn migration_must_happen_before_the_terminal_update() { ); assert_eq!( store.put_bytes(&migration.to_bytes()).await.expect("put"), - PutOutcome::Replaced + PutOutcome::Changed ); assert!(migration.is_terminal()); assert!(migration.next_counter().is_err()); @@ -633,7 +635,7 @@ async fn migration_must_happen_before_the_terminal_update() { .put_bytes(&smaller_target.to_bytes()) .await .expect("put"), - PutOutcome::Replaced, + PutOutcome::Changed, "a terminal pointer is NOT frozen: a smaller target still displaces it" ); From e04cd7855390b3a4b4a1a859f94c16406150fabb Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 18:51:26 +0900 Subject: [PATCH 06/32] fix(pointer): make the cancelled-commit test deterministic It cancelled the commit with a one-nanosecond timeout and then asserted the cancellation had happened. On a fast runner the commit finishes inside any timeout, so the assertion fired and the CI unit-test job went red while the same test passed locally. It now polls the future exactly once -- that poll hands the write to a blocking task and returns Pending, always -- and drops it. Also: - Inspected loses its nested outcome: Unchanged(state) / Stale(state) / Candidate. The handler reads the address and state off the variant instead of re-parsing the record, which deletes state_of, both "parsed then failed to re-parse" internal errors, and the arm for a no-op outcome that inspect could never return. - The store module doc described a two-step put through prepare(), which no longer exists -- rustdoc's broken-link check would have failed CI on it. Co-Authored-By: Claude Opus 5 (1M context) --- src/pointer/service.rs | 38 +++++---------------- src/pointer/store.rs | 76 +++++++++++++++++++++++------------------- 2 files changed, 50 insertions(+), 64 deletions(-) diff --git a/src/pointer/service.rs b/src/pointer/service.rs index f392ac86..930e200f 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -127,33 +127,20 @@ impl PointerService { // serve is rejected by the gates below without buying an ML-DSA // verification. let parsed = match self.store.inspect(&request.record) { - Ok(Inspected::Noop(PutOutcome::Unchanged)) => { - return match Self::state_of(&request.record) { - Some((address, state_id)) => { - PointerPutResponse::Unchanged { address, state_id } - } - None => PointerPutResponse::Error(ProtocolError::Internal( - "pointer parsed then failed to re-parse".to_string(), - )), + Ok(Inspected::Unchanged(state)) => { + return PointerPutResponse::Unchanged { + address: state.address, + state_id: state.state_id, }; } - Ok(Inspected::Noop(PutOutcome::Stale)) => { - let Some((address, _)) = Self::state_of(&request.record) else { - return PointerPutResponse::Error(ProtocolError::Internal( - "pointer parsed then failed to re-parse".to_string(), - )); - }; - let held = self.store.state_id(&address).unwrap_or_default(); + Ok(Inspected::Stale(state)) => { + // The state named is the one this node keeps, not the one that + // arrived: it is what the sender needs in order to catch up. return PointerPutResponse::Stale { - address, - state_id: held, + address: state.address, + state_id: self.store.state_id(&state.address).unwrap_or_default(), }; } - Ok(Inspected::Noop(other)) => { - return PointerPutResponse::Error(ProtocolError::Internal(format!( - "unexpected no-op outcome {other:?}" - ))); - } Ok(Inspected::Candidate(parsed)) => parsed, Err(e) => { debug!("Pointer PUT refused: {e}"); @@ -307,13 +294,6 @@ impl PointerService { .verify_pointer_payment(&routing_address, &paid_content, proof) .await } - - /// Re-read the address and state a record claims, for a response. - fn state_of(record: &[u8]) -> Option<(XorName, XorName)> { - ant_protocol::pointer::PointerState::parse(record) - .ok() - .map(|state| (state.address, state.state_id)) - } } /// Decide whether a chunk already at `address` blocks this pointer. diff --git a/src/pointer/store.rs b/src/pointer/store.rs index afc12f1d..de54ebe5 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -8,18 +8,20 @@ //! //! # The shape of a write //! -//! A put is two steps, because the payment check sits between them: +//! A put is three steps, because the caller's gates sit between them: //! -//! 1. [`PointerStore::prepare`] parses, compares against what is held and, only -//! if the arrival could win, verifies its signature. It returns either a -//! no-op outcome or a [`PreparedPut`]. -//! 2. The caller verifies payment against the candidate's address and state -//! identifier, then calls [`PointerStore::commit`]. +//! 1. [`PointerStore::inspect`] parses and compares against what is held: the +//! arrival is either unchanged, stale, or a candidate that would win. +//! 2. [`PointerStore::verify`] checks the candidate's signature — after the +//! caller's admission gates, and before it verifies payment. +//! 3. [`PointerStore::commit`] writes it. //! -//! Signature verification happens in step 1 **outside** any store lock, so a -//! flood of unpaid candidates cannot block every other address behind one -//! ML-DSA check. Step 2 re-checks under the lock, because the world may have -//! moved while payment was being verified. +//! Nothing cheap happens after something expensive: a resubmission of what is +//! held is refused before any signature check, so repeatedly submitting one +//! paid state buys no ML-DSA verifications. Step 2 runs **outside** any store +//! lock, so a flood of unpaid candidates cannot block every other address +//! behind one verification, and step 3 re-checks under the lock, because the +//! world may have moved while payment was being verified. //! //! # Atomicity //! @@ -95,9 +97,12 @@ impl PutOutcome { /// signature has been checked. #[derive(Debug)] pub enum Inspected { - /// The arrival cannot change what is held. No signature check is owed, and - /// none was done. - Noop(PutOutcome), + /// The node already holds exactly this state. No signature check is owed: + /// a resubmission of what was paid for and a forgery of it are the same + /// no-op. + Unchanged(PointerState), + /// The arrival loses to what is held, so it changes nothing either. + Stale(PointerState), /// The arrival would win as it stands. Its signature has **not** been /// checked yet — pass it to [`PointerStore::verify`] once admission gates /// have had their say. @@ -253,10 +258,10 @@ impl PointerStore { .filter(|_| self.reread(&state.address)) { if entry.state.state_id == state.state_id { - return Ok(Inspected::Noop(PutOutcome::Unchanged)); + return Ok(Inspected::Unchanged(state)); } if !state.replaces(&entry.state) { - return Ok(Inspected::Noop(PutOutcome::Stale)); + return Ok(Inspected::Stale(state)); } } Ok(Inspected::Candidate(parsed)) @@ -308,7 +313,8 @@ impl PointerStore { /// As [`Self::inspect`], [`Self::verify`] and [`Self::commit`]. pub async fn put_bytes(&self, bytes: &[u8]) -> Result { match self.inspect(bytes)? { - Inspected::Noop(outcome) => Ok(outcome), + Inspected::Unchanged(_) => Ok(PutOutcome::Unchanged), + Inspected::Stale(_) => Ok(PutOutcome::Stale), Inspected::Candidate(parsed) => { let record = self.verify(parsed).await?; self.commit(record).await @@ -757,6 +763,7 @@ fn sync_directory(dir: &Path) -> Result<()> { )] mod tests { use super::*; + use std::future::Future; use ant_protocol::pointer::{PointerTarget, PointerTargetKind, POINTER_BODY_LEN}; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; @@ -937,12 +944,12 @@ mod tests { store.put_bytes(&held.to_bytes()).await.expect("put"); match store.inspect(&held.to_bytes()).expect("inspect") { - Inspected::Noop(outcome) => assert_eq!(outcome, PutOutcome::Unchanged), - Inspected::Candidate(_) => panic!("an identical state is not a candidate"), + Inspected::Unchanged(_) => (), + other => panic!("an identical state is not a candidate, got {other:?}"), } match store.inspect(&signed(1, 1, 6).to_bytes()).expect("inspect") { - Inspected::Noop(outcome) => assert_eq!(outcome, PutOutcome::Stale), - Inspected::Candidate(_) => panic!("a stale record is not a candidate"), + Inspected::Stale(_) => (), + other => panic!("a stale record is not a candidate, got {other:?}"), } } @@ -1114,7 +1121,7 @@ mod tests { async fn verified(store: &PointerStore, record: &Pointer) -> Pointer { match store.inspect(&record.to_bytes()).expect("inspect") { Inspected::Candidate(parsed) => store.verify(parsed).await.expect("verify"), - Inspected::Noop(outcome) => panic!("expected a candidate, got {outcome:?}"), + other => panic!("expected a candidate, got {other:?}"), } } @@ -1216,16 +1223,20 @@ mod tests { let record = signed(1, 3, 3); let prepared = verified(&store, &record).await; - let abandoned = { + { let committing = store.commit(prepared); tokio::pin!(committing); - // Poll once, then abandon it. If it happened to finish inside the - // timeout there was no cancellation to test, and the assertion - // below still has to hold. - tokio::time::timeout(std::time::Duration::from_nanos(1), &mut committing) - .await - .is_err() - }; + // Poll exactly once, then drop. That first poll hands the write to + // a blocking thread and returns `Pending`, so the caller is always + // abandoned mid-commit. A timeout would not be: on a fast machine + // the commit finishes inside any timeout and the test then proves + // nothing, which is how this failed on CI and passed locally. + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + assert!( + committing.as_mut().poll(&mut cx).is_pending(), + "the first poll must hand the write to a blocking task, not finish it" + ); + } // Wait for the detached task to settle rather than guessing at a // duration: poll until the disk and the index agree and stop changing. @@ -1243,12 +1254,7 @@ mod tests { let indexed = store.contains(&record.address()); assert_eq!( on_disk, indexed, - "the index and the disk must agree however the commit was interrupted \ - (caller was cancelled: {abandoned})" - ); - assert!( - abandoned, - "the caller must actually have been cancelled, or this proves nothing" + "the index and the disk must agree however the commit was interrupted" ); assert!( settled, From 597e85e7fd78a33a7599b534201e3e858d377a2e Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 19:12:02 +0900 Subject: [PATCH 07/32] fix(pointer): keep what a node lost, so the loss is not permanent Disowning the index entry for a record whose file had gone left the address looking untouched -- and an address nothing is known about admits only a counter 0 record. So the repair added in the previous commit worked at creation and nowhere else: a pointer that had ever been updated could never be restored. An entry now records that it is no longer on disk rather than disappearing. The node stops serving it and stops counting it, but it still knows what it had, and admits anything at least as good as the lost state -- the merge rule replication would use to catch a replica up. A repair skips no payment: the state it carries was paid for and the rest of the group already serves it. Also: - The read-back compares the whole state, not just the address. The index and the disk are written under one lock, so a valid record for the same address that is not the one the index names means the answer would describe something this node cannot serve. - inspect runs on a blocking thread. It reads a record back off the disk, and a flood of arrivals must not put a blocking read on a runtime worker each. - The ADR no longer claims write and read use one group "by the same call" -- they are the same definition, but each does its own lookup -- and now says plainly that a majority of dishonest peers is not defended against. - PaymentTarget::is_single_address, which only its own test called, is gone. Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/ADR-0015-pointers-immutable-owner.md | 28 +- src/payment/verifier.rs | 11 +- src/pointer/service.rs | 2 +- src/pointer/store.rs | 284 ++++++++++++------ tests/pointer_convergence.rs | 30 +- 5 files changed, 228 insertions(+), 127 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index dc5f359a..5b7718aa 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -45,10 +45,17 @@ fetch. ### Two identities ```text -A = BLAKE3("autonomi.pointer.address.v1" || owner) routes -state_id = BLAKE3("autonomi.pointer.state.v1" || body) authorizes payment +A = BLAKE3::derive_key("autonomi.pointer.address.v1", owner) routes +state_id = BLAKE3::derive_key("autonomi.pointer.state.v1", body) authorizes payment ``` +Derive-key, not a hash of a prefix. A chunk's address is `BLAKE3(content)`, so a +prefix separates nothing: a chunk holding the prefix and an owner key would land +on exactly that owner's address, letting anyone squat an address before its +owner used it, and — since `state_id` is what a pointer's storage is paid +against — letting one settled quote buy both a pointer and a chunk. Derive-key +is a different function, so no content hashes into either space. + **Public-key addressed and self-verifying.** `A` is a pure function of the owner key, and the key is in the record, so a node validates a pointer from its own bytes: one hash, one signature check, no fetch. @@ -101,13 +108,13 @@ re-checks under its lock, because a newer state can land while payment verifies. | Replay an older record | Loses on counter | | Re-sign one paid state N times | Equal state never replaces; nothing is written | | Pay once, jump the counter | Client updates must be `+1` | -| Pay for a chunk to fund a pointer | Paid cache keyed by a typed `Chunk` vs `Pointer` target, never a raw 32-byte value a crafted chunk could occupy. The quote signs only its content, not the record kind, so this is a cache defence rather than a cryptographic one | +| Pay for a chunk to fund a pointer | A quote signs its content, not the record kind, so the defence is that the content cannot be shared: `state_id` is a derive-key output and no chunk can sit at one. The paid cache is keyed by a typed `Chunk` vs `Pointer` target as well, so the two never alias even in memory | | Merkle proof with no issuer check | Refused for pointers; single-node proofs only | | Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | | Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | -| Collide a pointer and a chunk address | Refused in both directions. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | -| Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent, and a write needs a majority of the close group — the same group, by the same call, that a read asks, so an acknowledged pointer is readable | -| Node claims a record it no longer holds | An index entry is only a claim about a file; before answering "unchanged" or "stale" the node reads the file back, and a lost or corrupt one makes the submission a repair | +| Collide a pointer and a chunk address | Only a genuine BLAKE3 collision can produce one, and it is refused in both directions anyway. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | +| Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent, and a write needs a majority of the close group. A read asks the same group by the same definition, so the two quorums intersect — though each does its own lookup, so churn between them is not covered. A majority of dishonest peers is not defended against at all: there is no storage receipt beyond quorum | +| Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, because an address nothing is known about admits only a counter 0 record, and a loss above that would otherwise be permanent | ## Consequences @@ -162,5 +169,10 @@ on the same work. - A majority of the group answering ends a write or a read, so one unreachable peer cannot stall either; a minority is reported as a shortfall, never presented as the network's answer. -- A resubmission repairs a record whose file the disk lost, rather than being - acknowledged as unchanged. +- A resubmission repairs a record whose file the disk lost — at any counter, + not just at creation — rather than being acknowledged as unchanged. A file + swapped for a different valid record is not answered for either. +- No chunk content produces a pointer address or a paid identifier. +- End to end against a live testnet with real settlement: create, update, read + back, resolve a chain to its chunk, repeat a stored state, read an address + nobody wrote, and refuse a signed record that skips the counter. diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 01bad501..0832f72c 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -533,12 +533,6 @@ impl PaymentTarget { Self::Pointer { state_id, .. } => state_id, } } - - /// Whether this is a chunk, where one address does both jobs. - #[must_use] - pub const fn is_single_address(&self) -> bool { - matches!(self, Self::Chunk(_)) - } } /// What a payment verification is admitting. @@ -3798,10 +3792,7 @@ mod tests { } #[test] - fn a_single_address_target_is_reported_as_one() { - assert!(PaymentTarget::same([1u8; 32]).is_single_address()); - assert!(!PaymentTarget::split([1u8; 32], [2u8; 32]).is_single_address()); - } + fn a_single_address_target_is_reported_as_one() {} fn create_test_verifier() -> PaymentVerifier { let config = PaymentVerifierConfig { diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 930e200f..332497f1 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -126,7 +126,7 @@ impl PointerService { // checked yet, so a forged record for an address this node does not // serve is rejected by the gates below without buying an ML-DSA // verification. - let parsed = match self.store.inspect(&request.record) { + let parsed = match self.store.inspect(&request.record).await { Ok(Inspected::Unchanged(state)) => { return PointerPutResponse::Unchanged { address: state.address, diff --git a/src/pointer/store.rs b/src/pointer/store.rs index de54ebe5..69d402eb 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -42,7 +42,7 @@ use std::collections::HashMap; use std::fs::{File, OpenOptions}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use fs2::FileExt; @@ -69,8 +69,7 @@ pub enum PutOutcome { /// The record was written: either nothing was held, or it won the merge. /// /// One outcome rather than two, because nothing downstream treats "stored" - /// differently from "replaced" — both mean the node now holds this state - /// and both are what `changed()` reports. + /// differently from "replaced": both mean the node now holds this state. Changed, /// The held record is the same authenticated state. Nothing was written. /// @@ -82,17 +81,6 @@ pub enum PutOutcome { Stale, } -impl PutOutcome { - /// Whether this outcome changed what the node holds. - /// - /// Only a change is worth announcing to replication or counting towards a - /// commitment rebuild. - #[must_use] - pub const fn changed(self) -> bool { - matches!(self, Self::Changed) - } -} - /// The result of [`PointerStore::inspect`]: what an arrival claims, before any /// signature has been checked. #[derive(Debug)] @@ -118,6 +106,15 @@ struct IndexEntry { /// store applies — merge order, the paid increment — is the protocol's own /// rule applied to the held state, and cannot drift from it. state: PointerState, + /// Whether the file behind `state` is still there and still that record. + /// + /// A read that finds it gone or different clears this rather than dropping + /// the entry. The node stops serving the record, because it does not have + /// it — but it still knows what it had, and that is what lets the state be + /// restored. Dropping the entry would leave the address looking untouched, + /// where only a counter 0 record is admissible, so a pointer that had ever + /// been updated could never be repaired. + on_disk: bool, /// Which insertion this entry is. /// /// Monotonic for the life of this store, so an entry can be told apart @@ -135,6 +132,7 @@ impl IndexEntry { fn of(record: &Pointer, generation: u64) -> Self { Self { state: record.state(), + on_disk: true, generation, } } @@ -163,12 +161,6 @@ struct Inner { write_seq: AtomicU64, /// Source of index generations, monotonic for this store's lifetime. generation: AtomicU64, - /// Set if a directory sync ever failed after a commit. - /// - /// Those writes are stored and visible; what is uncertain is whether they - /// survive a power loss. Reporting them as failures would be wrong, and - /// saying nothing would overstate the guarantee, so the store records it. - durability_degraded: AtomicBool, /// Held for the store's lifetime; releasing it releases the directory. _lock_file: File, } @@ -219,7 +211,6 @@ impl PointerStore { index: Mutex::new(index), write_seq: AtomicU64::new(0), generation: AtomicU64::new(next_generation), - durability_degraded: AtomicBool::new(false), _lock_file: lock_file, }), }) @@ -241,26 +232,33 @@ impl PointerStore { /// # Errors /// /// Returns [`Error::Protocol`] if the bytes are not a well-formed record. - pub fn inspect(&self, bytes: &[u8]) -> Result { + pub async fn inspect(&self, bytes: &[u8]) -> Result { + // Off the executor: deciding this reads the held record back off the + // disk, and a flood of arrivals must not put a blocking read on a + // runtime worker for each one. + let store = self.clone(); + let bytes = bytes.to_vec(); + spawn_blocking(move || store.inspect_blocking(&bytes)) + .await + .map_err(|e| Error::Storage(format!("pointer inspection panicked: {e}")))? + } + + /// The body of [`Self::inspect`], on a blocking thread. + fn inspect_blocking(&self, bytes: &[u8]) -> Result { // Parsing decodes the owner key once; `verify` reuses that parse rather // than decoding again. The bytes travel with it, so the two cannot be // mismatched. let parsed = ParsedPointer::parse(bytes.to_vec())?; let state = *parsed.state(); - // The index is only a claim about a file. Both early answers below - // assert that this node holds something at least as good as what - // arrived, so neither may be given on a claim alone: if the file has - // gone or stopped validating, `reread` disowns the entry and the - // submission falls through as the repair it should be. - if let Some(entry) = self - .snapshot(&state.address) - .filter(|_| self.reread(&state.address)) - { - if entry.state.state_id == state.state_id { + // Both early answers below assert that this node holds something at + // least as good as what arrived, so neither may be given on the index's + // word alone. + if let Some(held) = self.held_state(&state.address) { + if held.state_id == state.state_id { return Ok(Inspected::Unchanged(state)); } - if !state.replaces(&entry.state) { + if !state.replaces(&held) { return Ok(Inspected::Stale(state)); } } @@ -312,7 +310,7 @@ impl PointerStore { /// /// As [`Self::inspect`], [`Self::verify`] and [`Self::commit`]. pub async fn put_bytes(&self, bytes: &[u8]) -> Result { - match self.inspect(bytes)? { + match self.inspect(bytes).await? { Inspected::Unchanged(_) => Ok(PutOutcome::Unchanged), Inspected::Stale(_) => Ok(PutOutcome::Stale), Inspected::Candidate(parsed) => { @@ -351,7 +349,7 @@ impl PointerStore { let validated = match read { Ok(Some(validated)) => validated, Ok(None) => { - self.forget_if_unchanged(address, claimed); + self.disown_if_unchanged(address, claimed); return Ok(None); } Err(e) => return Err(e), @@ -365,7 +363,7 @@ impl PointerStore { from the index", hex::encode(address) ); - self.forget_if_unchanged(address, claimed); + self.disown_if_unchanged(address, claimed); Ok(None) } Err(e) => { @@ -373,7 +371,7 @@ impl PointerStore { "Pointer file at {} does not validate ({e}); dropping it from the index", hex::encode(address) ); - self.forget_if_unchanged(address, claimed); + self.disown_if_unchanged(address, claimed); Ok(None) } } @@ -386,7 +384,9 @@ impl PointerStore { /// under different signatures agree and do not refetch each other forever. #[must_use] pub fn state_id(&self, address: &XorName) -> Option { - self.snapshot(address).map(|e| e.state.state_id) + self.snapshot(address) + .filter(|entry| entry.on_disk) + .map(|entry| entry.state.state_id) } /// Whether `state` is the paid successor of what is held. @@ -399,24 +399,43 @@ impl PointerStore { /// Only the client path asks this. Replication uses the merge rule instead, /// so a replica that missed an update can still catch up rather than being /// stuck behind a gap it can never fill. + /// + /// A record this node knew and lost takes that same merge rule: anything at + /// least as good as the lost state restores it. The increment rule exists to + /// stop an owner buying one state and skipping to it, and a repair skips + /// nothing — the state it carries was paid for and the rest of the group + /// already serves it. Holding a lost address to the increment rule would + /// make every loss above counter 0 permanent, because only a counter 0 + /// record is admissible at an address nothing is known about. #[must_use] pub fn accepts_as_paid_update(&self, state: &PointerState) -> bool { self.snapshot(&state.address).map_or_else( || state.is_genesis(), - |entry| state.is_successor_of(&entry.state), + |entry| { + if entry.on_disk { + state.is_successor_of(&entry.state) + } else { + state.state_id == entry.state.state_id || state.replaces(&entry.state) + } + }, ) } /// Whether a record is held at `address`. #[must_use] pub fn contains(&self, address: &XorName) -> bool { - self.snapshot(address).is_some() + self.snapshot(address).is_some_and(|entry| entry.on_disk) } /// How many records the store holds. #[must_use] pub fn len(&self) -> usize { - self.inner.index.lock().len() + self.inner + .index + .lock() + .values() + .filter(|entry| entry.on_disk) + .count() } /// Whether the store holds nothing. @@ -425,41 +444,39 @@ impl PointerStore { self.len() == 0 } - /// Whether any committed write could not have its directory entry flushed. - /// - /// Those records are stored and readable now; what is uncertain is whether - /// they survive a power loss. A put still reports success, because the - /// write did happen — this is how an operator learns the filesystem is not - /// giving the store what it asks for. - #[must_use] - pub fn durability_degraded(&self) -> bool { - self.inner.durability_degraded.load(Ordering::Relaxed) - } - /// Directory holding the records. #[must_use] pub fn dir(&self) -> &Path { &self.inner.dir } - /// Whether the file behind `address` still reads back as a valid record. + /// The state this node can actually serve at `address`, having read it + /// back. /// - /// Guards the two early answers in `inspect`, both of which claim this - /// node already holds something at least as good as what arrived. A failed - /// read disowns the entry, exactly as `get` does. - fn reread(&self, address: &XorName) -> bool { - let claimed = self.snapshot(address).map(|entry| entry.generation); - let valid = matches!( + /// The index is a claim about a file; this is that claim checked. The file + /// must still be there and must still be the record the index names — + /// comparing the state, not merely the address, because the disk and the + /// index are updated under one lock and an answer taken between the two + /// would otherwise describe a record that is no longer the one held. + /// + /// Only the structure is parsed: these bytes verified when they were + /// committed, and the question here is what is held, not whether it is + /// authentic. A failed check disowns the entry, so the arrival that found + /// it becomes a repair. + fn held_state(&self, address: &XorName) -> Option { + let entry = self.snapshot(address).filter(|entry| entry.on_disk)?; + let serves = matches!( read_record_file(&self.path_for(address)), Ok(Some(ref bytes)) if matches!( - Pointer::from_bytes(bytes), - Ok(ref record) if record.address() == *address + PointerState::parse(bytes), + Ok(ref state) if *state == entry.state ) ); - if !valid { - self.forget_if_unchanged(address, claimed); + if !serves { + self.disown_if_unchanged(address, Some(entry.generation)); + return None; } - valid + Some(entry.state) } /// Copy out what is held for `address`, releasing the index lock at once. @@ -467,8 +484,11 @@ impl PointerStore { self.inner.index.lock().get(address).copied() } - /// Drop the index entry for `address`, but only if it is still the exact - /// entry the caller found unreadable. + /// Stop serving `address`, but only if the entry is still the exact one + /// the caller found unreadable. + /// + /// The entry stays, marked as no longer on disk: what was lost is what says + /// which states can restore it. /// /// A read is not atomic with a write. Removing unconditionally would let a /// slow read of a corrupt file erase the entry for a record committed while @@ -477,15 +497,17 @@ impl PointerStore { /// rather than the state identifier also covers the case where the record /// written meanwhile is a *repair of the same state*, which a state /// comparison could not tell apart from the entry being disowned. - fn forget_if_unchanged(&self, address: &XorName, claimed: Option) { + fn disown_if_unchanged(&self, address: &XorName, claimed: Option) { let Some(claimed) = claimed else { // Nothing was claimed when the read began, so there is nothing this - // read is entitled to remove. + // read is entitled to disown. return; }; let mut index = self.inner.index.lock(); - if index.get(address).map(|entry| entry.generation) == Some(claimed) { - index.remove(address); + if let Some(entry) = index.get_mut(address) { + if entry.generation == claimed { + entry.on_disk = false; + } } } @@ -516,7 +538,7 @@ impl Inner { // A cheap look before doing any work. The authoritative check is the // one under the lock below; this only avoids staging a file for an // arrival that is already obviously a no-op. - if let Some(entry) = self.index.lock().get(&address) { + if let Some(entry) = self.index.lock().get(&address).filter(|e| e.on_disk) { if entry.state.state_id == record.state_id() { return Ok(PutOutcome::Unchanged); } @@ -541,12 +563,14 @@ impl Inner { // Re-check: staging is not instantaneous and a newer state may // have committed while it ran. let outcome = match index.get(&address) { - None => PutOutcome::Changed, + // Nothing held, or a record this node lost: either way the + // write must happen, whatever state it carries. + None | Some(IndexEntry { on_disk: false, .. }) => PutOutcome::Changed, Some(entry) if entry.state.state_id == record.state_id() => PutOutcome::Unchanged, Some(entry) if record.state().replaces(&entry.state) => PutOutcome::Changed, Some(_) => PutOutcome::Stale, }; - if !outcome.changed() { + if outcome != PutOutcome::Changed { let _ = std::fs::remove_file(&temp); return Ok(outcome); } @@ -568,10 +592,10 @@ impl Inner { // Durability of the directory entry, after the commit and outside the // lock. The record is already stored and indexed, so a failure here - // cannot be reported as "nothing happened"; it is recorded instead, and - // `durability_degraded` reports it. + // cannot be reported as "nothing happened"; it is logged instead, which + // is how an operator learns the filesystem is not giving the store what + // it asks for. if let Err(e) = sync_directory(&self.dir) { - self.durability_degraded.store(true, Ordering::Relaxed); warn!( "{} is stored and indexed, but {} could not be synced: {e}. It is \ visible now; its survival across a power loss depends on the \ @@ -763,9 +787,9 @@ fn sync_directory(dir: &Path) -> Result<()> { )] mod tests { use super::*; - use std::future::Future; use ant_protocol::pointer::{PointerTarget, PointerTargetKind, POINTER_BODY_LEN}; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; + use std::future::Future; fn keypair(seed: u8) -> (MlDsaPublicKey, MlDsaSecretKey) { ml_dsa_65().generate_keypair_from_seed(&[seed; 32]) @@ -943,11 +967,15 @@ mod tests { let held = signed(1, 6, 6); store.put_bytes(&held.to_bytes()).await.expect("put"); - match store.inspect(&held.to_bytes()).expect("inspect") { + match store.inspect(&held.to_bytes()).await.expect("inspect") { Inspected::Unchanged(_) => (), other => panic!("an identical state is not a candidate, got {other:?}"), } - match store.inspect(&signed(1, 1, 6).to_bytes()).expect("inspect") { + match store + .inspect(&signed(1, 1, 6).to_bytes()) + .await + .expect("inspect") + { Inspected::Stale(_) => (), other => panic!("a stale record is not a candidate, got {other:?}"), } @@ -1119,7 +1147,7 @@ mod tests { /// Run the production sequence as far as the payment gate: inspect, then /// verify the candidate it yields. async fn verified(store: &PointerStore, record: &Pointer) -> Pointer { - match store.inspect(&record.to_bytes()).expect("inspect") { + match store.inspect(&record.to_bytes()).await.expect("inspect") { Inspected::Candidate(parsed) => store.verify(parsed).await.expect("verify"), other => panic!("expected a candidate, got {other:?}"), } @@ -1150,6 +1178,81 @@ mod tests { assert!(store.get(&record.address()).await.expect("get").is_some()); } + #[tokio::test] + async fn a_lost_record_above_counter_zero_is_still_repairable() { + // The admission rule alone would make this impossible: an address the + // node knows nothing about admits only a counter 0 record, so an entry + // that was *forgotten* on a failed read could never be restored above + // genesis, and every loss would be permanent. + let (store, _dir) = store().await; + for counter in 0..=3u64 { + store + .put_bytes(&signed(1, counter, 1).to_bytes()) + .await + .expect("put"); + } + let held = signed(1, 3, 1); + + std::fs::remove_file(store.dir().join(hex::encode(held.address()))).expect("remove"); + // A read notices the loss and stops the node answering for it. + assert!(store.get(&held.address()).await.expect("get").is_none()); + assert!(!store.contains(&held.address()), "it is not served"); + assert_eq!(store.state_id(&held.address()), None); + + // What it lost is what it will take back, and so is anything newer. + assert!( + store.accepts_as_paid_update(&held.state()), + "the state this node lost must be admissible again" + ); + assert!( + store.accepts_as_paid_update(&signed(1, 9, 1).state()), + "so must a newer state the rest of the group has moved on to" + ); + assert!( + !store.accepts_as_paid_update(&signed(1, 2, 1).state()), + "but not one that loses to what was lost" + ); + + assert_eq!( + store.put_bytes(&held.to_bytes()).await.expect("put"), + PutOutcome::Changed, + "the repair must write, not be answered as unchanged" + ); + let back = store + .get(&held.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(back.counter(), 3); + assert_eq!(back.state_id(), held.state_id()); + + // And the increment rule is back in force now that it holds one. + assert!(store.accepts_as_paid_update(&signed(1, 4, 1).state())); + assert!(!store.accepts_as_paid_update(&signed(1, 6, 1).state())); + } + + #[tokio::test] + async fn a_file_swapped_for_another_valid_record_is_not_answered_for() { + // The index names one state; the disk holds a different, perfectly + // valid one. Answering from the index would acknowledge a state this + // node cannot serve. + let (store, _dir) = store().await; + let indexed = signed(1, 4, 1); + store.put_bytes(&indexed.to_bytes()).await.expect("put"); + + let other = signed(1, 9, 1); + std::fs::write( + store.dir().join(hex::encode(indexed.address())), + other.to_bytes(), + ) + .expect("swap the file"); + + match store.inspect(&indexed.to_bytes()).await.expect("inspect") { + Inspected::Candidate(_) => (), + other => panic!("a state the node cannot serve must not be answered for: {other:?}"), + } + } + #[tokio::test] async fn a_deleted_file_is_forgotten_too() { let (store, _dir) = store().await; @@ -1286,7 +1389,7 @@ mod tests { let new = signed(1, 2, 1); store.put_bytes(&new.to_bytes()).await.expect("put"); - store.forget_if_unchanged(&old.address(), observed); + store.disown_if_unchanged(&old.address(), observed); assert_eq!( store.state_id(&new.address()), Some(new.state_id()), @@ -1295,7 +1398,7 @@ mod tests { // And the ordinary case still works: disowning what is actually there. let current = store.snapshot(&new.address()).map(|entry| entry.generation); - store.forget_if_unchanged(&new.address(), current); + store.disown_if_unchanged(&new.address(), current); assert!(!store.contains(&new.address())); } @@ -1315,7 +1418,7 @@ mod tests { let second_reader = first_reader; // Reader one finds the file unreadable and disowns what it saw. - store.forget_if_unchanged(&record.address(), first_reader); + store.disown_if_unchanged(&record.address(), first_reader); assert!(!store.contains(&record.address())); // A peer repairs it with the very same state. @@ -1325,7 +1428,7 @@ mod tests { ); // Reader two, still holding its stale observation, must not erase it. - store.forget_if_unchanged(&record.address(), second_reader); + store.disown_if_unchanged(&record.address(), second_reader); assert!( store.contains(&record.address()), "the repair must survive a second reader disowning the old entry" @@ -1356,19 +1459,6 @@ mod tests { ); } - #[tokio::test] - async fn a_healthy_store_does_not_report_degraded_durability() { - let (store, _dir) = store().await; - store - .put_bytes(&signed(1, 1, 1).to_bytes()) - .await - .expect("put"); - assert!( - !store.durability_degraded(), - "an ordinary write on a working filesystem is fully durable" - ); - } - #[tokio::test] async fn holding_the_key_is_not_holding_the_state() { // The fetch decision a mutable record needs: a replica on version N diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index 141d0cc8..d8c01774 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -356,34 +356,42 @@ fn the_wire_format_is_what_the_adr_says() { // written out rather than recomputed from the code under test. assert_eq!( hex::encode(record.address()), - "f82d07b1e8be4b9bc9b87df513baf57f65478ff321e4ecb5a0883f9bf8f594b7", + "83671eca6ee18b38987207922fdc23a1c378818c8a076b69dcbd40509f8786a8", "the pointer address for seed 42" ); assert_eq!( hex::encode(record.state_id()), - "36684c7d59d7f5ac2aeef581377fcbf8d88689fda0c01ead2fde66d7a8120aea", + "2d611afab3f72642b0196d99faebb74fb9b4f659f0f0b3b4f73dc85847efab42", "the paid state identifier for this record" ); // And the derivations those constants come from, restated independently. - let mut address_hasher = Hasher::new(); - address_hasher.update(b"autonomi.pointer.address.v1"); - address_hasher.update(&pk.to_bytes()); assert_eq!( record.address(), - *address_hasher.finalize().as_bytes(), + blake3::derive_key("autonomi.pointer.address.v1", &pk.to_bytes()), "the address derives from the owner key alone" ); - - let mut state_hasher = Hasher::new(); - state_hasher.update(b"autonomi.pointer.state.v1"); - state_hasher.update(bytes.get(..1994).expect("body range")); assert_eq!( record.state_id(), - *state_hasher.finalize().as_bytes(), + blake3::derive_key( + "autonomi.pointer.state.v1", + bytes.get(..1994).expect("body range") + ), "state_id covers the whole body and nothing else" ); + // Derive-key, not a hash of a prefix: a chunk is addressed by BLAKE3 over + // its content, so a prefix construction would put both identities inside + // the chunk address space for anyone who could write the preimage. + let mut prefixed = Hasher::new(); + prefixed.update(b"autonomi.pointer.address.v1"); + prefixed.update(&pk.to_bytes()); + assert_ne!( + record.address(), + *prefixed.finalize().as_bytes(), + "the address must not be reachable as a plain hash" + ); + // The signature verifies over the body under the literal context, and does // not verify under a different one. The context is consensus too. let dsa = ml_dsa_65(); From 4b89425f93339e66a4b21ff3975e3c33e211dd7e Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 19:16:07 +0900 Subject: [PATCH 08/32] fix(pointer): do not ask Windows to fsync a directory A directory cannot be opened as a file on Windows without backup semantics, so the post-rename directory sync failed on every write there. It was the Windows unit-test job's only failure on this branch, and the warning it logged was not a durability warning -- it was the wrong question asked of the wrong filesystem. The sync is now Unix-only, where a directory entry is a thing you can flush; NTFS orders the rename's own metadata. Co-Authored-By: Claude Opus 5 (1M context) --- src/pointer/store.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 69d402eb..6189f315 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -763,9 +763,8 @@ fn stage(temp: &Path, bytes: &[u8]) -> Result<()> { /// Flush the directory entry a rename created. /// /// Without it a crash can leave the entry unflushed and the record invisible on -/// restart. Opening a directory is not portable, so a directory that cannot be -/// opened is reported as success with a note: there is nothing to sync and -/// nothing went wrong with the write. +/// restart. +#[cfg(unix)] fn sync_directory(dir: &Path) -> Result<()> { match File::open(dir) { Ok(handle) => handle @@ -778,6 +777,17 @@ fn sync_directory(dir: &Path) -> Result<()> { } } +/// As above, where there is no such thing to ask for. +/// +/// A directory cannot be opened as a file on Windows without backup semantics, +/// so the Unix form fails on every write there — which is not a durability +/// warning, it is the wrong question. NTFS orders the rename's own metadata. +#[cfg(not(unix))] +#[allow(clippy::unnecessary_wraps, reason = "one signature for both platforms")] +fn sync_directory(_dir: &Path) -> Result<()> { + Ok(()) +} + #[cfg(test)] #[allow( clippy::unwrap_used, From 14bd165c76b1b5c64f648883d4d19403ebc4ff1e Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 19:46:51 +0900 Subject: [PATCH 09/32] docs(pointer): say what the identities are, and what is not defended The comments still described BLAKE3 over a domain prefix and the preimage that construction allowed; the identities are derive-key outputs now, and reaching one from a chunk takes a collision across two BLAKE3 modes rather than content anyone can write down. The typed paid-cache key does not rest on that either way, which is the point of it being typed. The ADR now also states two things it was quiet about: a read returns a state only when two answering peers name it, because one peer serving an owner-signed state nobody paid for would otherwise be believed by every reader; and a node that loses a record can repair it while running but not across a restart, where a missing file leaves nothing to remember. That, and a node joining a close group after a pointer exists, are the same missing mechanism -- replication. Also removes the empty test left behind when is_single_address was deleted. Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/ADR-0015-pointers-immutable-owner.md | 40 ++++++++++++++----- src/lib.rs | 2 +- src/payment/verifier.rs | 25 ++++++------ src/pointer/mod.rs | 11 ++++- src/pointer/service.rs | 8 ++-- 5 files changed, 57 insertions(+), 29 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 5b7718aa..15df9bb5 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -34,8 +34,8 @@ pub struct Pointer { // 5,303 bytes Five fields and nothing else — no cached bytes, no cached identifiers. Encoding is fixed-width, big-endian and hand-rolled with no serde, so there is exactly one byte sequence for a record and re-encoding is always identical to what was -signed. The address and `state_id` are hashes of fields already present, so they -are computed rather than stored. +signed. The address and `state_id` are derived from fields +already present, so they are computed rather than stored. The key is carried because it has to be: ML-DSA has no key recovery and a 1,952-byte key cannot be a 32-byte address. That is the whole reason a pointer @@ -54,7 +54,10 @@ prefix separates nothing: a chunk holding the prefix and an owner key would land on exactly that owner's address, letting anyone squat an address before its owner used it, and — since `state_id` is what a pointer's storage is paid against — letting one settled quote buy both a pointer and a chunk. Derive-key -is a different function, so no content hashes into either space. +is a different BLAKE3 mode, so neither identity is reachable from any content a +chunk could hold. The two spaces are both 32 bytes and nothing proves them +disjoint; what changed is that landing on one now takes a collision rather than +a preimage anyone can write down. **Public-key addressed and self-verifying.** `A` is a pure function of the owner key, and the key is in the record, so a node validates a pointer from its own @@ -94,8 +97,12 @@ length → version → structure → compare with held → admission → signatu Cheap first. A resubmission of what is held is refused before any signature check. Admission (capacity, responsibility for `A`) precedes the signature, so a -forged record for someone else's address buys no cryptography. The commit -re-checks under its lock, because a newer state can land while payment verifies. +forged record for someone else's address buys no cryptography. "Compare with +held" reads the held record back rather than trusting the index, so no answer +describes a record the node cannot serve. The commit re-checks under its lock, +because a newer state can land while payment verifies. Every step off the async +executor: the read, the signature check and the write each run on a blocking +thread, so a flood of arrivals cannot occupy the runtime's workers. ## What this defends against @@ -113,7 +120,8 @@ re-checks under its lock, because a newer state can land while payment verifies. | Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | | Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | | Collide a pointer and a chunk address | Only a genuine BLAKE3 collision can produce one, and it is refused in both directions anyway. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | -| Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent, and a write needs a majority of the close group. A read asks the same group by the same definition, so the two quorums intersect — though each does its own lookup, so churn between them is not covered. A majority of dishonest peers is not defended against at all: there is no storage receipt beyond quorum | +| Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent. A read asks the same group by the same definition, so the quorums intersect — though each does its own lookup, so churn between them is not covered | +| One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. A dishonest *majority* is not defended against: there is no storage receipt beyond quorum | | Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, because an address nothing is known about admits only a counter 0 record, and a loss above that would otherwise be permanent | ## Consequences @@ -121,6 +129,10 @@ re-checks under its lock, because a newer state can land while payment verifies. - Validating a pointer needs nothing but the pointer — no quorum, no lineage, no Sybil exposure in ownership. - Creation is one record and one payment, with no retention dependency. +- A pointer write must reach one more peer than a chunk write does. A chunk is + self-proving, so one copy settles it; a pointer read has to decide which of + several signed states is current, and that answer has to come from more than + one peer. - **Ownership cannot change.** Handover is indirection: point at a new pointer the recipient owns. The old owner keeps write access forever, so it is a revocable forwarding state, not a sale. @@ -150,6 +162,14 @@ merge rule guarantees nodes holding the same records agree, and nothing yet guarantees they hold the same records. Until it lands, availability and cross-network fork convergence are the client's doing, not the network's. +Two consequences follow from it and land with it. A node that joins a close +group after a pointer was created can never obtain it: the increment rule admits +only a counter 0 record at an address nothing is known about. And a node that +loses a record can repair it while it is running — it keeps what it lost, and +takes back that state or any that replaces it — but not across a restart, where +a missing file leaves nothing to remember. Both are the same missing mechanism: +a node cannot ask another node for a record. + Also not built: pointer participation in commitments and audits, which depends on the same work. @@ -166,9 +186,11 @@ on the same work. - A crafted chunk cannot satisfy a pointer's paid-cache entry. - An acknowledgement naming a different address or state is refused, and a read keeps the winner whatever order the replies arrive in. -- A majority of the group answering ends a write or a read, so one unreachable - peer cannot stall either; a minority is reported as a shortfall, never - presented as the network's answer. +- A quorum answering ends a write or a read, so one unreachable peer cannot + stall either; a minority is reported as a shortfall, never presented as the + network's answer, and neither is a state only one peer named. +- The write and read thresholds overlap in at least the two peers a read + demands, at every group width from 1 to 64. - A resubmission repairs a record whose file the disk lost — at any counter, not just at creation — rather than being acknowledged as unchanged. A file swapped for a different valid record is not answered for either. diff --git a/src/lib.rs b/src/lib.rs index 43e025ab..cfb41cdb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,7 @@ //! Two data types: //! - **Chunk**: Immutable content-addressed data (hash(value) == key) //! - **Pointer**: A paid mutable reference signed by an immutable owner, stored -//! at `BLAKE3(domain || owner_key)` (see [`mod@pointer`] and ADR-0015) +//! at an address derived from the owner key (see [`mod@pointer`] and ADR-0015) //! //! ## Example //! diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 0832f72c..ba3bb8ed 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -485,10 +485,11 @@ pub struct PaymentVerifierConfig { /// its address — two different addresses with two different meanings. /// /// This is also the paid-cache key, and being an enum is what makes that safe: -/// a raw 32-byte key would let a client store a chunk crafted to sit exactly on -/// a pointer's entry — `state_id` is `BLAKE3(domain || body)`, so its preimage -/// can be a chunk's content — and buy the pointer's update at chunk price. -/// Distinct variants cannot collide however the bytes are chosen. +/// a raw 32-byte key would file both kinds under one value, so anything that +/// put a chunk on a pointer's identifier would buy the pointer's update at +/// chunk price. `state_id` is a `derive_key` output, which no chunk address can +/// reach, so that now takes a BLAKE3 collision — and distinct variants cannot +/// collide however the bytes are chosen, so the cache does not depend on it. #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub enum PaymentTarget { /// A chunk: paid for and stored at one address. @@ -3740,12 +3741,13 @@ mod tests { /// A chunk whose address equals a pointer's `state_id` must not be able to /// pay for that pointer's update. /// - /// `state_id` is `BLAKE3(domain || body)`, so a client can store a chunk - /// whose *content* is exactly `domain || body`; that chunk's address is the - /// pointer's state identifier. If both filed their "already paid" entry - /// under that one value, paying chunk price for the chunk would buy the - /// pointer update — and skip issuer proximity, the price floor and the - /// proof-shape rule with it. + /// Reaching a pointer's `state_id` with a chunk now takes a BLAKE3 + /// collision across two modes — it is a `derive_key` output, and a chunk + /// address is a plain hash. This is the second line: were the two ever to + /// meet at one value, filing both "already paid" entries under it would let + /// chunk price buy a pointer update, skipping issuer proximity, the price + /// floor and the proof-shape rule with it. The typed key does not depend on + /// the addresses being unreachable from each other. #[test] fn a_chunk_cannot_pay_for_a_pointer_that_shares_its_address() { let state_id: XorName = [0x5Au8; 32]; @@ -3791,9 +3793,6 @@ mod tests { ); } - #[test] - fn a_single_address_target_is_reported_as_one() {} - fn create_test_verifier() -> PaymentVerifier { let config = PaymentVerifierConfig { evm: EvmVerifierConfig::default(), diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs index e890d068..8c142a93 100644 --- a/src/pointer/mod.rs +++ b/src/pointer/mod.rs @@ -23,8 +23,15 @@ //! //! | Name | Derivation | Job | //! |---|---|---| -//! | `A` | `BLAKE3(domain \|\| owner)` | routes, and decides which nodes are responsible | -//! | `state_id` | `BLAKE3(domain \|\| body)` | names the state, and is what a quote is paid against | +//! | `A` | `derive_key("autonomi.pointer.address.v1", owner)` | routes, and decides which nodes are responsible | +//! | `state_id` | `derive_key("autonomi.pointer.state.v1", body)` | names the state, and is what a quote is paid against | +//! +//! BLAKE3's derive-key mode, not a hash of a prefix and the input. A chunk is +//! addressed by a plain hash of its content, so a prefix would put both of +//! these inside the chunk address space for anyone who could write the +//! preimage — squatting an address before its owner used it, or buying a +//! pointer and a chunk with one payment. Nothing rules out a collision between +//! the two modes, but nothing produces one either. //! //! They are separate because `A` must be stable for the pointer's life while //! the paid identifier must change with every update, or updates after the diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 332497f1..77b825d4 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -299,10 +299,10 @@ impl PointerService { /// Decide whether a chunk already at `address` blocks this pointer. /// /// Separated from the handler because the branch cannot be reached in a test -/// any other way: a pointer address is `BLAKE3(domain || owner)` and a chunk -/// address is `BLAKE3(content)`, so occupying both with real data would take an -/// actual hash collision. The decision is what matters, so the decision is what -/// is tested. +/// any other way: a pointer address comes out of BLAKE3's derive-key mode and a +/// chunk address out of a plain hash, so occupying both with real data would +/// take a collision across the two. The decision is what matters, so the +/// decision is what is tested. /// /// `Some(response)` means refuse. Refusing is the only safe answer: whichever /// kind were chosen, the other's data would be destroyed, and the node cannot From d2c598ed7b8cee1b33c528b12b7aa48eee98b4ab Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 20:07:21 +0900 Subject: [PATCH 10/32] docs(pointer): state the Byzantine boundary and drop the impossibility claims The ADR contradicted itself about the hash spaces on consecutive lines: it said neither identity was reachable from any chunk content, then said nothing proved the two ranges disjoint. Only the second is true. What derive-key buys is cost, not impossibility. It also understated where the defence ends. A read needs two peers to name a state, so one peer cannot decide what a pointer says -- but two colluding ones can, and since only the owner can sign, what that buys is the owner's own updates unpaid. Raising the bar only raises the number of nodes to grind: the pointer's address follows a key the owner chooses and node ids are choosable too, so an owner determined to sit beside their own pointer reaches any fixed threshold. Replication and audits are what actually answer it, and neither is built. The table says so now. And a read no longer necessarily ends at the answer quorum -- it keeps asking while the states it has seen are uncorroborated -- so the consequence that said it did is corrected. Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/ADR-0015-pointers-immutable-owner.md | 26 ++++++++++++------- src/payment/verifier.rs | 11 ++++---- src/pointer/mod.rs | 4 +-- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 15df9bb5..88572c0a 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -54,10 +54,11 @@ prefix separates nothing: a chunk holding the prefix and an owner key would land on exactly that owner's address, letting anyone squat an address before its owner used it, and — since `state_id` is what a pointer's storage is paid against — letting one settled quote buy both a pointer and a chunk. Derive-key -is a different BLAKE3 mode, so neither identity is reachable from any content a -chunk could hold. The two spaces are both 32 bytes and nothing proves them -disjoint; what changed is that landing on one now takes a collision rather than -a preimage anyone can write down. +is a different BLAKE3 mode. Both still produce 32 bytes and the ranges are not +disjoint; what changed is the cost. Landing a chunk on a pointer identity now +means finding a preimage under one mode for an output of the other, which is +the security assumption BLAKE3 is built on, rather than a string anyone can +write down. **Public-key addressed and self-verifying.** `A` is a pure function of the owner key, and the key is in the record, so a node validates a pointer from its own @@ -115,13 +116,14 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. | Replay an older record | Loses on counter | | Re-sign one paid state N times | Equal state never replaces; nothing is written | | Pay once, jump the counter | Client updates must be `+1` | -| Pay for a chunk to fund a pointer | A quote signs its content, not the record kind, so the defence is that the content cannot be shared: `state_id` is a derive-key output and no chunk can sit at one. The paid cache is keyed by a typed `Chunk` vs `Pointer` target as well, so the two never alias even in memory | +| Pay for a chunk to fund a pointer | A quote signs its content, not the record kind, so the defence is that the content cannot be shared: putting a chunk on a `state_id` means breaking BLAKE3 across its two modes. The paid cache is keyed by a typed `Chunk` vs `Pointer` target as well, so the two never alias even in memory, whatever the bytes | | Merkle proof with no issuer check | Refused for pointers; single-node proofs only | | Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | | Unknown target kind | Carried, never interpreted — a node stores 33 opaque bytes | -| Collide a pointer and a chunk address | Only a genuine BLAKE3 collision can produce one, and it is refused in both directions anyway. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | +| Collide a pointer and a chunk address | Takes a cross-mode BLAKE3 break, and is refused in both directions anyway. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | | Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent. A read asks the same group by the same definition, so the quorums intersect — though each does its own lookup, so churn between them is not covered | -| One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. A dishonest *majority* is not defended against: there is no storage receipt beyond quorum | +| One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | +| Two peers decide it | **Not defended against.** Two colluding close-group peers clear the bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Raising the bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold. What actually answers it is replication and audits, neither of which is built | | Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, because an address nothing is known about admits only a counter 0 record, and a loss above that would otherwise be permanent | ## Consequences @@ -186,9 +188,13 @@ on the same work. - A crafted chunk cannot satisfy a pointer's paid-cache entry. - An acknowledgement naming a different address or state is refused, and a read keeps the winner whatever order the replies arrive in. -- A quorum answering ends a write or a read, so one unreachable peer cannot - stall either; a minority is reported as a shortfall, never presented as the - network's answer, and neither is a state only one peer named. +- A write ends as soon as its quorum acknowledges, so one unreachable peer + cannot stall it. A read ends when a quorum has answered *and* one of the + states they named has the backing a read demands; short of that it keeps + asking, and if the group is exhausted without either, it reports a shortfall + rather than presenting one peer's word as the network's answer. +- A state only one peer named does not suppress the state the others agree on, + in any arrival order. - The write and read thresholds overlap in at least the two peers a read demands, at every group width from 1 to 64. - A resubmission repairs a record whose file the disk lost — at any counter, diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index ba3bb8ed..74848ac4 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -487,9 +487,10 @@ pub struct PaymentVerifierConfig { /// This is also the paid-cache key, and being an enum is what makes that safe: /// a raw 32-byte key would file both kinds under one value, so anything that /// put a chunk on a pointer's identifier would buy the pointer's update at -/// chunk price. `state_id` is a `derive_key` output, which no chunk address can -/// reach, so that now takes a BLAKE3 collision — and distinct variants cannot -/// collide however the bytes are chosen, so the cache does not depend on it. +/// chunk price. `state_id` is a `derive_key` output, so putting a chunk on one +/// is a preimage problem rather than a string anyone can write down — and +/// distinct variants cannot collide however the bytes are chosen, so the cache +/// does not rest on that assumption either. #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub enum PaymentTarget { /// A chunk: paid for and stored at one address. @@ -3746,8 +3747,8 @@ mod tests { /// address is a plain hash. This is the second line: were the two ever to /// meet at one value, filing both "already paid" entries under it would let /// chunk price buy a pointer update, skipping issuer proximity, the price - /// floor and the proof-shape rule with it. The typed key does not depend on - /// the addresses being unreachable from each other. + /// floor and the proof-shape rule with it. The typed key holds whatever the + /// two hash modes do. #[test] fn a_chunk_cannot_pay_for_a_pointer_that_shares_its_address() { let state_id: XorName = [0x5Au8; 32]; diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs index 8c142a93..683640aa 100644 --- a/src/pointer/mod.rs +++ b/src/pointer/mod.rs @@ -30,8 +30,8 @@ //! addressed by a plain hash of its content, so a prefix would put both of //! these inside the chunk address space for anyone who could write the //! preimage — squatting an address before its owner used it, or buying a -//! pointer and a chunk with one payment. Nothing rules out a collision between -//! the two modes, but nothing produces one either. +//! pointer and a chunk with one payment. The two modes still share a 32-byte +//! range; what changed is that crossing it is a preimage problem. //! //! They are separate because `A` must be stable for the pointer's life while //! the paid identifier must change with every update, or updates after the From a249fe38827767c0f25fd08b1d3535e9796091b6 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 20:11:59 +0900 Subject: [PATCH 11/32] docs(pointer): the validation list should not claim more than a test can show Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/ADR-0015-pointers-immutable-owner.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 88572c0a..70efbd85 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -200,7 +200,9 @@ on the same work. - A resubmission repairs a record whose file the disk lost — at any counter, not just at creation — rather than being acknowledged as unchanged. A file swapped for a different valid record is not answered for either. -- No chunk content produces a pointer address or a paid identifier. +- The chunk preimages the old prefix construction handed out no longer land on + either identity. (No test can say more: that no content does is the preimage + assumption, not a property one can check.) - End to end against a live testnet with real settlement: create, update, read back, resolve a chain to its chunk, repeat a stored state, read an address nobody wrote, and refuse a signed record that skips the counter. From 89fd13b5905206a2dd9ca97cdf263a49f1acf693 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 20:34:22 +0900 Subject: [PATCH 12/32] fix(pointer): take the tie-break winner, and test convergence through the handler The paid gate required exactly counter + 1, so a node holding one state refused another at the same counter even though both were separately paid and the merge rule says which wins. Give two nodes those states in opposite orders and each keeps a different record for ever -- the fork the merge rule exists to prevent, put back by the gate standing in front of it. The convergence tests never saw it: they drive the store directly, below the gate. The store now asks is_paid_update_of, which is "wins the merge and does not skip", and two new service tests drive the real request handler: two nodes given the same two states in opposite orders agree, and a counter jump is still refused after a tie-break has been taken. Also corrects two docs that claimed more than the code checks. get serves the file rather than the index, so a file replaced by a different validly signed record for the same address is served -- the reader verifies it and the read quorum decides between replicas that disagree. And the held-state check parses the body, so corruption inside the signature passes it and is caught on the next read; verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be. Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/ADR-0015-pointers-immutable-owner.md | 5 +- src/payment/verifier.rs | 4 +- src/pointer/service.rs | 64 +++++++++++++++++++ src/pointer/store.rs | 36 ++++++++--- tests/pointer_convergence.rs | 2 +- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index 70efbd85..cbe5eb18 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -124,7 +124,7 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. | Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent. A read asks the same group by the same definition, so the quorums intersect — though each does its own lookup, so churn between them is not covered | | One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | | Two peers decide it | **Not defended against.** Two colluding close-group peers clear the bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Raising the bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold. What actually answers it is replication and audits, neither of which is built | -| Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, because an address nothing is known about admits only a counter 0 record, and a loss above that would otherwise be permanent | +| Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, because an address nothing is known about admits only a counter 0 record, and a loss above that would otherwise be permanent. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | ## Consequences @@ -200,6 +200,9 @@ on the same work. - A resubmission repairs a record whose file the disk lost — at any counter, not just at creation — rather than being acknowledged as unchanged. A file swapped for a different valid record is not answered for either. +- Two nodes given the same two paid states in opposite orders keep the same + record, through the request handler and its admission gate rather than the + store alone. - The chunk preimages the old prefix construction handed out no longer land on either identity. (No test can say more: that no content does is the preimage assumption, not a property one can check.) diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 74848ac4..6720ee20 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -3743,8 +3743,8 @@ mod tests { /// pay for that pointer's update. /// /// Reaching a pointer's `state_id` with a chunk now takes a BLAKE3 - /// collision across two modes — it is a `derive_key` output, and a chunk - /// address is a plain hash. This is the second line: were the two ever to + /// preimage under one BLAKE3 mode for an output of the other — it is a + /// `derive_key` output, and a chunk address is a plain hash. This is the second line: were the two ever to /// meet at one value, filing both "already paid" entries under it would let /// chunk price buy a pointer update, skipping issuer proximity, the price /// floor and the proof-shape rule with it. The typed key holds whatever the diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 77b825d4..f28a4f08 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -407,6 +407,70 @@ mod tests { } } + #[tokio::test] + async fn two_nodes_given_the_same_states_in_opposite_orders_agree() { + // The convergence property, driven through the request handler rather + // than the store. The gate in front of the merge rule is part of the + // production path, and a gate that admits records by arrival order + // would leave these two nodes holding different records for ever -- + // which is the fork the merge rule exists to prevent. + let (first_node, _a) = service().await; + let (second_node, _b) = service().await; + + // Two separately paid states at one counter. The merge rule says the + // smaller target wins, whichever arrives first. + let loser = signed(1, 0, 9); + let winner = signed(1, 0, 1); + assert!(winner.replaces(&loser)); + + for record in [&loser, &winner] { + first_node.handle_put(put(record)).await; + } + for record in [&winner, &loser] { + second_node.handle_put(put(record)).await; + } + + for (name, node) in [("first", &first_node), ("second", &second_node)] { + match node + .handle_get(PointerGetRequest::new(winner.address())) + .await + { + PointerGetResponse::Success { record: bytes } => assert_eq!( + Pointer::from_bytes(&bytes).expect("parse").state_id(), + winner.state_id(), + "the {name} node kept the wrong record" + ), + other => panic!("expected the winner, got {other:?}"), + } + } + + // And the node that already had the winner refused to go back. + assert!(matches!( + second_node.handle_put(put(&loser)).await, + PointerPutResponse::Stale { .. } + )); + } + + #[tokio::test] + async fn a_counter_jump_is_still_refused_after_a_tie_break() { + // Taking a tie-break winner must not loosen the increment rule: the + // counter has not moved, so the next state is still exactly one on. + let (service, _dir) = service().await; + service.handle_put(put(&signed(1, 0, 9))).await; + assert!(matches!( + service.handle_put(put(&signed(1, 0, 1))).await, + PointerPutResponse::Success { .. } + )); + assert!(matches!( + service.handle_put(put(&signed(1, 7, 1))).await, + PointerPutResponse::PaymentRequired { .. } + )); + assert!(matches!( + service.handle_put(put(&signed(1, 1, 1))).await, + PointerPutResponse::Success { .. } + )); + } + #[tokio::test] async fn re_submitting_a_held_state_is_unchanged_not_success() { // A client that retries must be able to tell "your update landed" from diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 6189f315..4f932cc8 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -322,10 +322,19 @@ impl PointerStore { /// Read the record held at `address`, if any. /// - /// Re-validates on the way out, so a file corrupted under the node is - /// reported as missing rather than served as authentic — and the index - /// entry for it is dropped, so the node will accept a fresh copy of that - /// state instead of answering "unchanged" to its own repair. + /// Re-validates on the way out: the signature is checked and the record + /// must belong at the address asked for, so a file corrupted under the node + /// is reported as missing rather than served as authentic, and the entry is + /// disowned so the node will take a fresh copy instead of answering + /// "unchanged" to its own repair. + /// + /// The file is what is served, not the index. A file replaced under the + /// node by a *different* record that is genuinely signed for this address + /// is served as held — it is a real record, the reader verifies it, and the + /// read quorum is what decides between replicas that disagree. What the + /// index is not allowed to do is claim a state the file does not have — + /// that check guards [`Self::inspect`]'s two early answers, the ones that + /// assert this node already holds something. /// /// # Errors /// @@ -389,12 +398,15 @@ impl PointerStore { .map(|entry| entry.state.state_id) } - /// Whether `state` is the paid successor of what is held. + /// Whether `state` is a paid update of what is held. /// /// One payment buys one increment: a new pointer starts at counter 0, and - /// an update must be exactly one past what this node holds. Without it an - /// owner pays once, jumps the counter, skips every intermediate payment and - /// strands the pointer at a counter nothing can advance. + /// an update either advances the counter by one or wins the target + /// tie-break at the counter already held. Without the bound an owner pays + /// once, jumps the counter, skips every intermediate payment and strands + /// the pointer where nothing can advance it; without the tie-break, two + /// separately paid states at one counter would leave every node holding + /// whichever reached it first. /// /// Only the client path asks this. Replication uses the merge rule instead, /// so a replica that missed an update can still catch up rather than being @@ -413,7 +425,7 @@ impl PointerStore { || state.is_genesis(), |entry| { if entry.on_disk { - state.is_successor_of(&entry.state) + state.is_paid_update_of(&entry.state) } else { state.state_id == entry.state.state_id || state.replaces(&entry.state) } @@ -461,7 +473,11 @@ impl PointerStore { /// /// Only the structure is parsed: these bytes verified when they were /// committed, and the question here is what is held, not whether it is - /// authentic. A failed check disowns the entry, so the arrival that found + /// authentic. Corruption inside the signature therefore passes here — the + /// state is read from the body — and is caught by [`Self::get`], which + /// verifies and disowns. Checking it here instead would put an ML-DSA + /// verification in front of the payment gate, which is the one place it + /// must not be. A failed check disowns the entry, so the arrival that found /// it becomes a repair. fn held_state(&self, address: &XorName) -> Option { let entry = self.snapshot(address).filter(|entry| entry.on_disk)?; diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index d8c01774..2a48dd16 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -389,7 +389,7 @@ fn the_wire_format_is_what_the_adr_says() { assert_ne!( record.address(), *prefixed.finalize().as_bytes(), - "the address must not be reachable as a plain hash" + "the prefix construction must not still produce the address" ); // The signature verifies over the body under the literal context, and does From 97edcc33616f430880ca5198452d31f07825a8d0 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 18 Sep 2026 20:56:01 +0900 Subject: [PATCH 13/32] docs(pointer): say at most one increment, and put back the doc I split Two corrections, both mine. The ADR and the admission comment still said every update is exactly counter + 1 after the rule changed to admit a separately paid tie-break winner at the counter already held. What holds now is that one payment buys one state and at most one increment: a tie-break moves the pointer without advancing the counter, but it must strictly descend in target bytes and each step is bought on its own state_id, so it buys nothing an ordinary update would not. And PaymentTarget had been inserted between VerificationContext's doc comment and its enum, so the long explanation of admission paths documented the wrong type and VerificationContext was left with a one-line stand-in. PaymentTarget now sits above that block with its own doc. Co-Authored-By: Claude Opus 5 (1M context) --- docs/adr/ADR-0015-pointers-immutable-owner.md | 9 ++- src/payment/verifier.rs | 62 +++++++++---------- src/pointer/service.rs | 13 ++-- 3 files changed, 46 insertions(+), 38 deletions(-) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0015-pointers-immutable-owner.md index cbe5eb18..a0b8585a 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0015-pointers-immutable-owner.md @@ -70,8 +70,13 @@ would make every update after the first free. ### Pay to create, pay to update -Creation is `counter = 0`. Each update is `counter + 1`. Both are paid against -their own `state_id`, so **one payment buys exactly one increment**. +Creation is `counter = 0`. An update is `counter + 1`, or the same counter with +a smaller target — the merge rule's tie-break, which two concurrent updates must +both be able to land on or they leave the group split. Every one of them is paid +against its own `state_id`, so **one payment buys one state and at most one +increment**. A tie-break moves the pointer without advancing the counter, but it +must strictly descend in target bytes and each step is bought separately, so it +buys nothing an ordinary update would not. The client path enforces `+1`. Replication accepts any strictly greater counter, because a replica that missed an update must be able to catch up; refusing the diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 6720ee20..839acb37 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -448,36 +448,6 @@ pub struct PaymentVerifierConfig { pub price_floor: PriceFloorConfig, } -/// The fresh admission path a payment proof is being verified for. -/// -/// - **`ClientPut`** — the node is admitting a chunk store from a direct -/// client PUT. The verifier applies store-strength cache semantics and live -/// payment checks. -/// - **`FreshReplication`** — the node is admitting a chunk store via the -/// immediate fresh-write fan-out. The receiver is about to store the newly -/// written chunk as if the client PUT it there directly, so this context is -/// verified EXACTLY like `ClientPut` (store-strength cache semantics, same -/// live checks, same price-floor policy). It exists as a separate variant so -/// price-floor telemetry can distinguish direct ingress from fan-out — the -/// two paths can legitimately diverge during commitment rotation, and the -/// floor policy for fan-out must be tunable from observed data without -/// touching direct-PUT behaviour. -/// - **`PaidListAdmission`** — the node is admitting fresh paid-list metadata. -/// It runs the same live payment checks, but writes a weaker cache entry -/// that does not authorize future chunk stores. The price floor never -/// applies here: paid-list records reprice no fresh economic decision. -/// -/// The caller must check local receiver/admission membership before invoking -/// the verifier for replication admission: fresh chunk replication requires -/// local close-group responsibility, and fresh paid-list replication requires -/// local paid-list close-group membership. Direct client PUT deliberately does -/// not perform a receiver-responsibility gate. The verifier itself only checks -/// payment proof validity and that the paid quote's issuer is in the K closest -/// peers for the quoted chunk address. -/// -/// Later neighbour-sync repair does not include proof-of-payment bytes and -/// does not call this verifier. It authorizes repair from network evidence: -/// majority storage among the configured close group, or majority paid-list /// What a payment authorizes, and where it routes. /// /// One typed value doing both jobs. A chunk pays for its own address; a pointer @@ -537,7 +507,37 @@ impl PaymentTarget { } } -/// What a payment verification is admitting. +/// The fresh admission path a payment proof is being verified for. +/// +/// - **`ClientPut`** — the node is admitting a chunk store from a direct +/// client PUT. The verifier applies store-strength cache semantics and live +/// payment checks. +/// - **`FreshReplication`** — the node is admitting a chunk store via the +/// immediate fresh-write fan-out. The receiver is about to store the newly +/// written chunk as if the client PUT it there directly, so this context is +/// verified EXACTLY like `ClientPut` (store-strength cache semantics, same +/// live checks, same price-floor policy). It exists as a separate variant so +/// price-floor telemetry can distinguish direct ingress from fan-out — the +/// two paths can legitimately diverge during commitment rotation, and the +/// floor policy for fan-out must be tunable from observed data without +/// touching direct-PUT behaviour. +/// - **`PaidListAdmission`** — the node is admitting fresh paid-list metadata. +/// It runs the same live payment checks, but writes a weaker cache entry +/// that does not authorize future chunk stores. The price floor never +/// applies here: paid-list records reprice no fresh economic decision. +/// +/// The caller must check local receiver/admission membership before invoking +/// the verifier for replication admission: fresh chunk replication requires +/// local close-group responsibility, and fresh paid-list replication requires +/// local paid-list close-group membership. Direct client PUT deliberately does +/// not perform a receiver-responsibility gate. The verifier itself only checks +/// payment proof validity and that the paid quote's issuer is in the K closest +/// peers for the quoted chunk address. +/// +/// Later neighbour-sync repair does not include proof-of-payment bytes and +/// does not call this verifier. It authorizes repair from network evidence: +/// majority storage among the configured close group, or majority paid-list +/// membership among the closest K. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VerificationContext { /// The node is admitting a chunk store from a direct client PUT, with diff --git a/src/pointer/service.rs b/src/pointer/service.rs index f28a4f08..ffed9053 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -204,11 +204,14 @@ impl PointerService { ) -> Option { let address = state.address; - // One payment buys one increment. A create is counter 0 and an update - // is exactly one past what this node holds; anything else would let an - // owner pay once and jump the counter, skipping every intermediate - // payment. Replication does not come through here — it merges on the - // counter order, so a replica behind a gap can still catch up. + // One payment buys one state and at most one increment. A create is + // counter 0; an update is one past what this node holds, or the + // tie-break winner at that same counter, which two concurrent updates + // must both be able to land on or the group stays split. What is + // refused is a jump, which would let an owner pay once and skip every + // intermediate payment. Replication does not come through here — it + // merges on the counter order, so a replica behind a gap can still + // catch up. if !self.store.accepts_as_paid_update(state) { debug!( "Rejecting pointer PUT for {}: counter {} is not the paid successor", From 30c08faa3903ebe8e42ef9cdd82c39de692db965 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 11:43:44 +0900 Subject: [PATCH 14/32] docs(pointer): renumber the pointer ADR to 0016 ADR-0015 was taken by direct browser clients over WebRTC-direct while this branch was open, and the governance check refuses two ADRs wearing one number. Renames the file, its title, and every reference to it across the node. --- ...-immutable-owner.md => ADR-0016-pointers-immutable-owner.md} | 2 +- src/lib.rs | 2 +- src/pointer/mod.rs | 2 +- tests/pointer_convergence.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename docs/adr/{ADR-0015-pointers-immutable-owner.md => ADR-0016-pointers-immutable-owner.md} (99%) diff --git a/docs/adr/ADR-0015-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md similarity index 99% rename from docs/adr/ADR-0015-pointers-immutable-owner.md rename to docs/adr/ADR-0016-pointers-immutable-owner.md index a0b8585a..f54904ff 100644 --- a/docs/adr/ADR-0015-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -1,4 +1,4 @@ -# ADR-0015: Pointers — paid mutable references with an immutable owner +# ADR-0016: Pointers — paid mutable references with an immutable owner - **Status:** Proposed - **Date:** 2026-09-18 diff --git a/src/lib.rs b/src/lib.rs index cfb41cdb..068f5147 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,7 @@ //! Two data types: //! - **Chunk**: Immutable content-addressed data (hash(value) == key) //! - **Pointer**: A paid mutable reference signed by an immutable owner, stored -//! at an address derived from the owner key (see [`mod@pointer`] and ADR-0015) +//! at an address derived from the owner key (see [`mod@pointer`] and ADR-0016) //! //! ## Example //! diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs index 683640aa..766113b0 100644 --- a/src/pointer/mod.rs +++ b/src/pointer/mod.rs @@ -1,6 +1,6 @@ //! Pointers — paid mutable references with an immutable owner. //! -//! Implements `docs/adr/ADR-0015-pointers-immutable-owner.md`. +//! Implements `docs/adr/ADR-0016-pointers-immutable-owner.md`. //! //! A pointer is a mutable, owner-signed reference stored at an address derived //! from the owner's public key. Ownership is fixed at creation: there is no diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index 2a48dd16..f5a8ccf5 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -1,6 +1,6 @@ //! Convergence and payment-binding properties of the pointer merge rule. //! -//! ADR-0015's claim is that selecting the maximum over a total order on +//! ADR-0016's claim is that selecting the maximum over a total order on //! *states* is idempotent, commutative and associative, so every node reaches //! the same value from any delivery interleaving given the same record set. //! These are the property tests behind that claim, plus the two anti-abuse From d2eaff1af3d1f34d29be16900cf8fe8bf0a39339 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 12:02:53 +0900 Subject: [PATCH 15/32] chore: take main's lockfile rather than the pre-rebase one The rebase carried this branch's old Cargo.lock forward, which quietly downgraded dozens of transitive dependencies main had moved on from. It is now main's lock with one entry repinned: ant-protocol, for the pointer record and its wire messages. --- Cargo.lock | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0010ecbd..c653951a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3292,7 +3292,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -7119,7 +7119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core", + "windows-core 0.61.2", "windows-future", "windows-link 0.1.3", "windows-numerics", @@ -7131,7 +7131,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core", + "windows-core 0.61.2", ] [[package]] @@ -7143,8 +7143,21 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", - "windows-result", - "windows-strings", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -7153,7 +7166,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core", + "windows-core 0.61.2", "windows-link 0.1.3", "windows-threading", ] @@ -7198,7 +7211,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core", + "windows-core 0.61.2", "windows-link 0.1.3", ] @@ -7211,6 +7224,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -7220,6 +7242,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.48.0" From 0f4db79e7c7e944681ece2f1ae2ee180abcea849 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 12:34:12 +0900 Subject: [PATCH 16/32] chore: keep main's lockfile edges, and index the pointer ADR Repinning ant-protocol with cargo update also re-resolved an unrelated Windows edge, putting winapi-util back on windows-sys 0.48.0 where main had moved it to 0.61.2. The lock is now main's with the one ant-protocol entry edited by hand, so nothing else moves. Also adds ADR-0016 to the index, and records there that browser clients cannot reach a pointer yet and what opening that path would take. --- Cargo.lock | 2 +- docs/adr/ADR-0016-pointers-immutable-owner.md | 9 +++++++++ docs/adr/README.md | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c653951a..16e0f41e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7103,7 +7103,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index f54904ff..aff67879 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -180,6 +180,15 @@ a node cannot ask another node for a record. Also not built: pointer participation in commitments and audits, which depends on the same work. +**Not built: browser clients.** ADR-0015's WebRTC-direct transport admits, +sanitizes and classifies message kinds by an explicit list, and pointer requests +are in none of them. The client's pointer API is therefore native-only rather +than compiled for a transport that would reject it. Reaching a pointer from a +browser needs four things, each a deliberate decision at a security boundary: +admit the two request kinds, let the response sanitizer pass their replies, +classify a pointer GET as a read and a pointer PUT as paid-exclusive, and give +the browser client the same quorum and corroboration rules the native one uses. + ## Validation - Every delivery order of a record set converges to one value, exhaustively over diff --git a/docs/adr/README.md b/docs/adr/README.md index 8fc52bc4..5061984e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -35,3 +35,4 @@ See [`TOOLING.md`](./TOOLING.md) for `adrs`, `adr-kit`, and AI harness setup. - [ADR-0014: One File Per Chunk, and Retiring LMDB Without Losing Data](./ADR-0014-file-based-chunk-store-and-lmdb-retirement.md) - [ADR-0013: Settlement version and pre-payment compatibility](./ADR-0013-settlement-version-and-pre-payment-compatibility.md) - [ADR-0015: Direct browser clients over WebRTC Direct](./ADR-0015-direct-browser-clients-over-webrtc-direct.md) +- [ADR-0016: Pointers — paid mutable references with an immutable owner](./ADR-0016-pointers-immutable-owner.md) From 0c9c8789e59dec7b1d2bdabfea0c0fa861cbeca9 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 12:41:55 +0900 Subject: [PATCH 17/32] feat(pointer): itemise pointer RPC traffic Pointer requests and responses were counted as Other, the bucket for non-request and unknown future variants. That is what every other request kind avoids, and it makes paid pointer traffic indistinguishable from messages the table does not understand. Both request kinds and all eight response outcomes now have their own keys, on a third summary line because group 2 is already near tracing's field cap. Unchanged and Stale are itemised apart from Success, since a re-submission and a lost race are exactly what you want to tell apart when counting paid writes. The classifiers are exhaustive -- the pointer response enums are not non_exhaustive -- so a new variant is a compile error here rather than silently becoming Other. --- src/storage/handler.rs | 24 +++++----- src/storage/traffic.rs | 102 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 20 deletions(-) diff --git a/src/storage/handler.rs b/src/storage/handler.rs index a854f7ca..d380200e 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -599,26 +599,26 @@ impl AntProtocol { ), ChunkResponseKey::MerkleQuoteV2, ), - // Pointer traffic is attributed to `Other`: the table itemises - // chunk response outcomes, and a pointer response is not one. - ChunkMessageBody::PointerPutRequest(req) => ( - ChunkMessageBody::PointerPutResponse(match &self.pointers { + ChunkMessageBody::PointerPutRequest(req) => { + let response = match &self.pointers { Some(service) => service.handle_put(req).await, None => PointerPutResponse::Error(ProtocolError::StorageFailed( "this node does not store pointers".to_string(), )), - }), - ChunkResponseKey::Other, - ), - ChunkMessageBody::PointerGetRequest(req) => ( - ChunkMessageBody::PointerGetResponse(match &self.pointers { + }; + let key = ChunkResponseKey::of_pointer_put(&response); + (ChunkMessageBody::PointerPutResponse(response), key) + } + ChunkMessageBody::PointerGetRequest(req) => { + let response = match &self.pointers { Some(service) => service.handle_get(req).await, None => PointerGetResponse::NotFound { address: req.address, }, - }), - ChunkResponseKey::Other, - ), + }; + let key = ChunkResponseKey::of_pointer_get(&response); + (ChunkMessageBody::PointerGetResponse(response), key) + } // Anything else — response messages are handled by client // subscribers (e.g. send_and_await_chunk_response), not by the // protocol handler. Returning None prevents the caller from diff --git a/src/storage/traffic.rs b/src/storage/traffic.rs index dc316f4e..35688e8f 100644 --- a/src/storage/traffic.rs +++ b/src/storage/traffic.rs @@ -13,6 +13,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use crate::ant_protocol::{ChunkGetResponse, ChunkMessageBody, ChunkPutResponse}; +use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; /// Kind of an inbound chunk message, for the rx table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -23,6 +24,8 @@ pub enum ChunkRequestKind { MerkleQuote, QuoteV2, MerkleQuoteV2, + PointerGet, + PointerPut, /// A non-request variant (responses meant for client subscribers) or an /// unknown future variant. Other, @@ -31,7 +34,7 @@ pub enum ChunkRequestKind { } impl ChunkRequestKind { - const N: usize = 8; + const N: usize = 10; const fn index(self) -> usize { match self { @@ -41,8 +44,10 @@ impl ChunkRequestKind { Self::MerkleQuote => 3, Self::QuoteV2 => 4, Self::MerkleQuoteV2 => 5, - Self::Other => 6, - Self::DecodeError => 7, + Self::PointerGet => 6, + Self::PointerPut => 7, + Self::Other => 8, + Self::DecodeError => 9, } } } @@ -61,13 +66,21 @@ pub enum ChunkResponseKey { MerkleQuote, QuoteV2, MerkleQuoteV2, + PointerGetSuccess, + PointerGetNotFound, + PointerGetError, + PointerPutSuccess, + PointerPutUnchanged, + PointerPutStale, + PointerPutPaymentRequired, + PointerPutError, /// A response variant this table does not itemise (e.g. an `Other` /// outcome on a `#[non_exhaustive]` enum). Other, } impl ChunkResponseKey { - const N: usize = 12; + const N: usize = 20; const fn index(self) -> usize { match self { @@ -82,7 +95,15 @@ impl ChunkResponseKey { Self::MerkleQuote => 8, Self::QuoteV2 => 9, Self::MerkleQuoteV2 => 10, - Self::Other => 11, + Self::PointerGetSuccess => 11, + Self::PointerGetNotFound => 12, + Self::PointerGetError => 13, + Self::PointerPutSuccess => 14, + Self::PointerPutUnchanged => 15, + Self::PointerPutStale => 16, + Self::PointerPutPaymentRequired => 17, + Self::PointerPutError => 18, + Self::Other => 19, } } } @@ -97,6 +118,8 @@ impl ChunkRequestKind { ChunkMessageBody::MerkleCandidateQuoteRequest(_) => Self::MerkleQuote, ChunkMessageBody::QuoteRequestV2(_) => Self::QuoteV2, ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) => Self::MerkleQuoteV2, + ChunkMessageBody::PointerGetRequest(_) => Self::PointerGet, + ChunkMessageBody::PointerPutRequest(_) => Self::PointerPut, _ => Self::Other, } } @@ -113,6 +136,34 @@ impl ChunkResponseKey { } } + /// Classify a pointer GET response by outcome. + /// + /// Exhaustive, unlike the chunk classifiers: the pointer response enums are + /// not `#[non_exhaustive]`, so a new variant is a compile error here rather + /// than silently becoming `Other`. + pub fn of_pointer_get(response: &PointerGetResponse) -> Self { + match response { + PointerGetResponse::Success { .. } => Self::PointerGetSuccess, + PointerGetResponse::NotFound { .. } => Self::PointerGetNotFound, + PointerGetResponse::Error(_) => Self::PointerGetError, + } + } + + /// Classify a pointer PUT response by outcome. + /// + /// `Unchanged` and `Stale` are itemised separately from `Success` because + /// they are what a re-submission and a lost race look like, and telling + /// those apart is the point of counting paid writes at all. + pub fn of_pointer_put(response: &PointerPutResponse) -> Self { + match response { + PointerPutResponse::Success { .. } => Self::PointerPutSuccess, + PointerPutResponse::Unchanged { .. } => Self::PointerPutUnchanged, + PointerPutResponse::Stale { .. } => Self::PointerPutStale, + PointerPutResponse::PaymentRequired { .. } => Self::PointerPutPaymentRequired, + PointerPutResponse::Error(_) => Self::PointerPutError, + } + } + /// Classify a PUT response by outcome. pub fn of_put(response: &ChunkPutResponse) -> Self { match response { @@ -160,10 +211,11 @@ pub fn record_send_failed(bytes: usize) { /// Emit the cumulative chunk-RPC traffic as INFO summary lines, target /// `ant_node::storage::traffic`. /// -/// Flat snake-case keys like the replication summary. Two lines sharing the +/// Flat snake-case keys like the replication summary. Three lines sharing the /// same target and message, distinguished by `group`: rx by request kind -/// (`group = 1`) and tx by kind × outcome (`group = 2`), keeping each under -/// `tracing`'s 32-field cap. +/// (`group = 1`), chunk and quote tx by kind × outcome (`group = 2`), and +/// pointer tx by kind × outcome (`group = 3`). Pointers take a line of their +/// own because `group = 2` is already close to `tracing`'s 32-field cap. pub fn log_chunk_rpc_traffic_summary() { use ChunkRequestKind as Q; use ChunkResponseKey as R; @@ -183,6 +235,8 @@ pub fn log_chunk_rpc_traffic_summary() { quote_v2_rx_bytes = rb(Q::QuoteV2), quote_v2_rx_count = rc(Q::QuoteV2), merkle_quote_v2_rx_bytes = rb(Q::MerkleQuoteV2), merkle_quote_v2_rx_count = rc(Q::MerkleQuoteV2), + pointer_get_rx_bytes = rb(Q::PointerGet), pointer_get_rx_count = rc(Q::PointerGet), + pointer_put_rx_bytes = rb(Q::PointerPut), pointer_put_rx_count = rc(Q::PointerPut), other_rx_bytes = rb(Q::Other), other_rx_count = rc(Q::Other), decode_error_rx_bytes = rb(Q::DecodeError), decode_error_rx_count = rc(Q::DecodeError), "chunk rpc traffic summary (cumulative)" @@ -210,6 +264,28 @@ pub fn log_chunk_rpc_traffic_summary() { send_failed_tx_count = SEND_FAILED_COUNT.load(Ordering::Relaxed), "chunk rpc traffic summary (cumulative)" ); + + crate::logging::info!( + target: "ant_node::storage::traffic", + group = 3, + pointer_get_success_tx_bytes = tb(R::PointerGetSuccess), + pointer_get_success_tx_count = tc(R::PointerGetSuccess), + pointer_get_not_found_tx_bytes = tb(R::PointerGetNotFound), + pointer_get_not_found_tx_count = tc(R::PointerGetNotFound), + pointer_get_error_tx_bytes = tb(R::PointerGetError), + pointer_get_error_tx_count = tc(R::PointerGetError), + pointer_put_success_tx_bytes = tb(R::PointerPutSuccess), + pointer_put_success_tx_count = tc(R::PointerPutSuccess), + pointer_put_unchanged_tx_bytes = tb(R::PointerPutUnchanged), + pointer_put_unchanged_tx_count = tc(R::PointerPutUnchanged), + pointer_put_stale_tx_bytes = tb(R::PointerPutStale), + pointer_put_stale_tx_count = tc(R::PointerPutStale), + pointer_put_payment_required_tx_bytes = tb(R::PointerPutPaymentRequired), + pointer_put_payment_required_tx_count = tc(R::PointerPutPaymentRequired), + pointer_put_error_tx_bytes = tb(R::PointerPutError), + pointer_put_error_tx_count = tc(R::PointerPutError), + "chunk rpc traffic summary (cumulative)" + ); } #[cfg(test)] @@ -225,6 +301,8 @@ mod tests { ChunkRequestKind::MerkleQuote, ChunkRequestKind::QuoteV2, ChunkRequestKind::MerkleQuoteV2, + ChunkRequestKind::PointerGet, + ChunkRequestKind::PointerPut, ChunkRequestKind::Other, ChunkRequestKind::DecodeError, ]; @@ -245,6 +323,14 @@ mod tests { ChunkResponseKey::MerkleQuote, ChunkResponseKey::QuoteV2, ChunkResponseKey::MerkleQuoteV2, + ChunkResponseKey::PointerGetSuccess, + ChunkResponseKey::PointerGetNotFound, + ChunkResponseKey::PointerGetError, + ChunkResponseKey::PointerPutSuccess, + ChunkResponseKey::PointerPutUnchanged, + ChunkResponseKey::PointerPutStale, + ChunkResponseKey::PointerPutPaymentRequired, + ChunkResponseKey::PointerPutError, ChunkResponseKey::Other, ]; let mut seen = std::collections::HashSet::new(); From c755739c4bedce74a5c853b50723bbcdfa7c5d02 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 13:04:06 +0900 Subject: [PATCH 18/32] fix(pointer): charge a pointer write against the disk before making it A pointer PUT asked check_capacity(), which asks whether a write of *zero* bytes would fit and charges nothing, and then wrote 5,303 bytes. The file store names that race itself: dozens of handlers can each pass against one cached measurement before any of them has written a byte, and collectively cross the configured reserve. Chunks avoid it by reserving; pointers did not. Pointer writes now take a reservation for the size a record actually is and hold it until the write lands, settled by net growth -- a create commits the charge, a replacement or a no-op releases it, because the disk only grows in the first case. The early admission check asks about the same size rather than about nothing, so a client still learns a full disk before paying. Three tests: a full disk refuses and stores nothing; eight concurrent creations each take and settle a charge; and sixty-four writes that change nothing give their charges back, which is the drop path -- a charge stranded there would be permanent, and enough of them would make an empty disk look full until the process restarted. The narrow race itself, where the reserve is crossed between check and write, cannot be driven deterministically without a test hook into free-space measurement; these cover the paths, not the interleaving. --- src/pointer/service.rs | 175 ++++++++++++++++++++++++++++++++++++- src/storage/chunk_store.rs | 13 +++ src/storage/file_store.rs | 14 ++- src/storage/mod.rs | 1 + 4 files changed, 198 insertions(+), 5 deletions(-) diff --git a/src/pointer/service.rs b/src/pointer/service.rs index ffed9053..816d5144 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -44,6 +44,7 @@ use crate::payment::PaymentVerifier; use crate::pointer::store::{Inspected, PointerStore, PutOutcome}; use crate::replication::admission; use crate::storage::{ChunkStore, SELF_CLOSENESS_GATE_WIDTH}; +use ant_protocol::pointer::POINTER_WIRE_LEN; /// Handles pointer requests against a [`PointerStore`]. #[derive(Clone)] @@ -177,7 +178,42 @@ impl PointerService { } } - match self.store.commit(record).await { + // Charge the bytes this write will take, and hold the charge until it + // lands. Checking capacity and then writing is the race the file store + // exists to close: concurrent writers all pass one cached measurement + // before any of them has written a byte, and cross the reserve + // together. A record is a fixed `POINTER_WIRE_LEN`, so that is the + // whole charge. + let reservation = match &self.chunks { + Some(chunks) => match chunks.reserve(POINTER_WIRE_LEN as u64) { + Ok(reservation) => Some(reservation), + Err(e) => { + debug!("Rejecting pointer PUT for {}: {e}", hex::encode(address)); + return PointerPutResponse::Error(ProtocolError::StorageFailed(e.to_string())); + } + }, + None => None, + }; + // Whether this write grows the disk or overwrites a record already + // there. Racy by nature — another writer may create it in between — and + // wrong only towards counting bytes that are not there, which the next + // measurement corrects. The opposite error would under-count and let + // the reserve be crossed. + let replacing = self.store.contains(&address); + + let outcome = self.store.commit(record).await; + match (&outcome, replacing) { + // A new file landed: the charge becomes bytes on disk. + (Ok(PutOutcome::Changed), false) => { + if let Some(reservation) = reservation { + reservation.commit(); + } + } + // Replaced in place, or nothing written. Dropping releases it. + _ => drop(reservation), + } + + match outcome { Ok(PutOutcome::Changed) => PointerPutResponse::Success { address, state_id }, // The re-check under the commit lock found a newer state. The // client paid for a state that lost a race; say so plainly. @@ -233,8 +269,11 @@ impl PointerService { if let Some(refusal) = cross_kind_refusal(address, chunks.exists(&address)) { return Some(refusal); } - // Capacity before payment, as the chunk path does. - if let Err(e) = chunks.check_capacity() { + // Capacity before payment, as the chunk path does, and for the + // size a record actually is rather than for nothing. The binding + // charge is taken at the commit; this only avoids paying to find + // out the disk is full. + if let Err(e) = chunks.check_capacity_for(POINTER_WIRE_LEN as u64) { debug!("Rejecting pointer PUT for {}: {e}", hex::encode(address)); return Some(PointerPutResponse::Error(ProtocolError::StorageFailed( e.to_string(), @@ -361,6 +400,136 @@ mod tests { (PointerService::new(store), dir) } + /// A service whose chunk store guards a disk that cannot take another byte. + async fn service_with_full_disk() -> (PointerService, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = PointerStore::new(dir.path()).await.expect("store"); + let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + // Larger than any disk, so every capacity question answers "full". + disk_reserve: u64::MAX, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + ( + PointerService::new(store).with_chunk_store(Arc::new(chunks)), + dir, + ) + } + + #[tokio::test] + async fn a_full_disk_refuses_a_pointer_before_it_is_written() { + // The write must be charged against the disk, not merely checked + // against it: a check that passes and a write that follows are the + // race the file store exists to close. With no room at all, the + // reservation cannot be taken and nothing lands. + let (service, _dir) = service_with_full_disk().await; + let record = signed(1, 0, 1); + + match service.handle_put(put(&record)).await { + PointerPutResponse::Error(ProtocolError::StorageFailed(message)) => { + assert!( + message.contains("disk space") || message.contains("reserve"), + "a full disk should say so, got: {message}" + ); + } + other => panic!("a full disk must refuse the write, got {other:?}"), + } + + assert!( + matches!( + service + .handle_get(PointerGetRequest::new(record.address())) + .await, + PointerGetResponse::NotFound { .. } + ), + "nothing may be stored when the disk had no room for it" + ); + } + + #[tokio::test] + async fn concurrent_creations_are_each_charged_against_the_disk() { + // Every one of these would pass a bare capacity *check* against one + // cached measurement. What stops them collectively crossing the + // reserve is that each holds a charge until its write lands. + let (service, dir) = service().await; + let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + let service = service.with_chunk_store(Arc::new(chunks)); + + let mut writes = Vec::new(); + for seed in 1..=8u8 { + let record = signed(seed, 0, 1); + let service = service.clone(); + writes.push(tokio::spawn(async move { + (record.address(), service.handle_put(put(&record)).await) + })); + } + + for write in writes { + let (address, response) = write.await.expect("join"); + assert!( + matches!(response, PointerPutResponse::Success { .. }), + "a disk with room must take the write, got {response:?}" + ); + assert!(matches!( + service.handle_get(PointerGetRequest::new(address)).await, + PointerGetResponse::Success { .. } + )); + } + assert_eq!(service.store().len(), 8); + } + + #[tokio::test] + async fn writes_that_change_nothing_give_their_charge_back() { + // The charge is released by dropping the reservation, which is the path + // a re-submission and a stale arrival take. A charge that leaked there + // would be permanent — nothing else decrements it — and enough of them + // would make an empty disk look full until the process restarted. + let (service, dir) = service().await; + let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + let service = service.with_chunk_store(Arc::new(chunks)); + + let held = signed(1, 0, 1); + service.handle_put(put(&held)).await; + for _ in 0..32 { + // Same state: nothing is written, so nothing may stay charged. + assert!(matches!( + service.handle_put(put(&held)).await, + PointerPutResponse::Unchanged { .. } + )); + // And a losing state: also nothing written. + assert!(matches!( + service.handle_put(put(&signed(1, 0, 9))).await, + PointerPutResponse::Stale { .. } + )); + } + + // If those 64 no-ops had each stranded a charge, this would be refused. + assert!(matches!( + service.handle_put(put(&signed(2, 0, 1))).await, + PointerPutResponse::Success { .. } + )); + } + fn put(record: &Pointer) -> PointerPutRequest { PointerPutRequest::new(Bytes::from(record.to_bytes())) } diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index 66fe3ece..36731dfb 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -1023,6 +1023,19 @@ impl ChunkStore { self.files.check_capacity_for(bytes) } + /// Charge `bytes` against the disk before writing them. + /// + /// For the pointer store, which keeps its own files on this disk. Checking capacity + /// and then writing is a race the file store names explicitly: concurrent writers all + /// pass one cached measurement and cross the reserve together. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the disk cannot take `bytes` more. + pub(crate) fn reserve(&self, bytes: u64) -> Result { + self.files.reserve_bytes(bytes) + } + /// Wait until every blocking task in either backing has finished. pub async fn wait_idle(&self) { self.files.wait_idle().await; diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs index 8a6140c8..9784e087 100644 --- a/src/storage/file_store.rs +++ b/src/storage/file_store.rs @@ -372,7 +372,7 @@ impl CapacityGuard { /// /// Held by whatever is actually doing the write, so the charge is released even if the /// caller's future is dropped and only the blocking closure survives. -struct Reservation { +pub(crate) struct Reservation { /// The guard this was taken from. capacity: Arc, /// Payload size, before rounding. @@ -383,7 +383,7 @@ struct Reservation { impl Reservation { /// The write landed: move the charge from in-flight to written. - fn commit(mut self) { + pub(crate) fn commit(mut self) { self.capacity.commit_reservation(self.bytes); self.settled = true; } @@ -1514,6 +1514,16 @@ impl FileStore { self.capacity.check(bytes) } + /// Charge `bytes` against the disk before writing them, releasing the charge if the + /// write does not happen. + /// + /// For stores that share this disk but keep their own files — the pointer store does. + /// They must not check and then write: that is the race [`CapacityGuard::reserve`] + /// exists to close. + pub(crate) fn reserve_bytes(&self, bytes: u64) -> Result { + self.capacity.reserve(bytes) + } + /// Force the next capacity question to re-measure the filesystem. /// /// Called after the legacy environment is removed, because that is a step change in diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 6b46ee0e..ae795c6e 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -57,6 +57,7 @@ pub(crate) mod traffic; pub use crate::ant_protocol::XorName; pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR}; +pub(crate) use file_store::Reservation; pub use file_store::{FileStore, FileStoreConfig, StoreLayout}; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; From fef2f14cc1bff66da16b159d14a3040502be996a Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 13:24:22 +0900 Subject: [PATCH 19/32] fix(pointer): let the charge travel with the write Two holes in the reservation, both found by review. The charge was settled in handle_put while commit moved the write into a blocking task. Drop that future after it spawns and the charge is released while the detached closure goes on to publish the file -- exactly the case the Reservation type says it exists for. It now moves into commit and is settled inside the transaction that owns the write. And settling by observed net growth could under-count, not only over-count as I claimed. The check read the index before the rename; a file the index claims can be gone, so what looked like a replacement grows the disk after all, and the charge was released for bytes that landed. Every write that lands now commits its charge. Over-counting is the safe direction and the next measurement corrects it. Staged bytes that cannot be removed keep their charge rather than releasing it, since they are still on the disk. The test that claimed to cover the release path could not reach it -- Unchanged and Stale return from inspect, before any reservation is taken -- so it is replaced by one at the store level that loses a commit race, which is the reachable way to hold a charge and write nothing. inspect, verify and commit are pub(crate) now: they are the request path's internal steps, put_bytes is the public one, and a public method cannot take a crate-private reservation anyway. --- docs/adr/ADR-0016-pointers-immutable-owner.md | 2 +- src/lib.rs | 2 +- src/pointer/mod.rs | 2 +- src/pointer/service.rs | 64 +-------- src/pointer/store.rs | 124 +++++++++++++++--- 5 files changed, 112 insertions(+), 82 deletions(-) diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index aff67879..4d28e72b 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -144,7 +144,7 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. the recipient owns. The old owner keeps write access forever, so it is a revocable forwarding state, not a sale. - **Key compromise is permanent.** No rotation, no recovery. -- The inlined key costs 1,920 bytes on every read, forever — a deliberate trade +- The inlined key costs 1,952 bytes on every read, forever — a deliberate trade for self-contained validation. - Determinism is not freshness: an eclipsed reader can be handed an older, correctly signed value and cannot tell. diff --git a/src/lib.rs b/src/lib.rs index 068f5147..852c837a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -81,7 +81,7 @@ pub use event::{NodeEvent, NodeEventsChannel}; pub use node::{NodeBuilder, RunningNode}; pub use payment::{PaymentStatus, PaymentVerifier, PaymentVerifierConfig}; pub use pointer::{ - Inspected, Pointer, PointerState, PointerStore, PointerTarget, PointerTargetKind, PutOutcome, + Pointer, PointerState, PointerStore, PointerTarget, PointerTargetKind, PutOutcome, }; pub use replication::{config::ReplicationConfig, ReplicationEngine}; pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig, LmdbStorage, LmdbStorageConfig}; diff --git a/src/pointer/mod.rs b/src/pointer/mod.rs index 766113b0..c4787f39 100644 --- a/src/pointer/mod.rs +++ b/src/pointer/mod.rs @@ -57,4 +57,4 @@ pub use ant_protocol::pointer::{ POINTER_WIRE_LEN, TARGET_WIRE_LEN, }; pub use service::PointerService; -pub use store::{Inspected, PointerStore, PutOutcome}; +pub use store::{PointerStore, PutOutcome}; diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 816d5144..86fc25e1 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -194,26 +194,10 @@ impl PointerService { }, None => None, }; - // Whether this write grows the disk or overwrites a record already - // there. Racy by nature — another writer may create it in between — and - // wrong only towards counting bytes that are not there, which the next - // measurement corrects. The opposite error would under-count and let - // the reserve be crossed. - let replacing = self.store.contains(&address); - - let outcome = self.store.commit(record).await; - match (&outcome, replacing) { - // A new file landed: the charge becomes bytes on disk. - (Ok(PutOutcome::Changed), false) => { - if let Some(reservation) = reservation { - reservation.commit(); - } - } - // Replaced in place, or nothing written. Dropping releases it. - _ => drop(reservation), - } - - match outcome { + // The charge travels with the write. Settling it here instead would + // release it the moment this future is dropped, while the blocking + // transaction it started runs on and publishes the file. + match self.store.commit(record, reservation).await { Ok(PutOutcome::Changed) => PointerPutResponse::Success { address, state_id }, // The re-check under the commit lock found a newer state. The // client paid for a state that lost a race; say so plainly. @@ -490,46 +474,6 @@ mod tests { assert_eq!(service.store().len(), 8); } - #[tokio::test] - async fn writes_that_change_nothing_give_their_charge_back() { - // The charge is released by dropping the reservation, which is the path - // a re-submission and a stale arrival take. A charge that leaked there - // would be permanent — nothing else decrements it — and enough of them - // would make an empty disk look full until the process restarted. - let (service, dir) = service().await; - let chunks = ChunkStore::new(crate::storage::ChunkStoreConfig { - root_dir: dir.path().to_path_buf(), - verify_on_read: false, - max_map_size: 0, - disk_reserve: 0, - migration: crate::storage::MigrationConfig::default(), - }) - .await - .expect("chunk store"); - let service = service.with_chunk_store(Arc::new(chunks)); - - let held = signed(1, 0, 1); - service.handle_put(put(&held)).await; - for _ in 0..32 { - // Same state: nothing is written, so nothing may stay charged. - assert!(matches!( - service.handle_put(put(&held)).await, - PointerPutResponse::Unchanged { .. } - )); - // And a losing state: also nothing written. - assert!(matches!( - service.handle_put(put(&signed(1, 0, 9))).await, - PointerPutResponse::Stale { .. } - )); - } - - // If those 64 no-ops had each stranded a charge, this would be refused. - assert!(matches!( - service.handle_put(put(&signed(2, 0, 1))).await, - PointerPutResponse::Success { .. } - )); - } - fn put(record: &Pointer) -> PointerPutRequest { PointerPutRequest::new(Bytes::from(record.to_bytes())) } diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 4f932cc8..a1e91f9c 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -10,11 +10,14 @@ //! //! A put is three steps, because the caller's gates sit between them: //! -//! 1. [`PointerStore::inspect`] parses and compares against what is held: the -//! arrival is either unchanged, stale, or a candidate that would win. -//! 2. [`PointerStore::verify`] checks the candidate's signature — after the -//! caller's admission gates, and before it verifies payment. -//! 3. [`PointerStore::commit`] writes it. +//! 1. `inspect` parses and compares against what is held: the arrival is +//! either unchanged, stale, or a candidate that would win. +//! 2. `verify` checks the candidate's signature — after the caller's +//! admission gates, and before it verifies payment. +//! 3. `commit` writes it, and settles the disk charge taken for it. +//! +//! Those three are internal to the crate; [`PointerStore::put_bytes`] is the +//! same sequence in one call, for callers with nothing to do in between. //! //! Nothing cheap happens after something expensive: a resubmission of what is //! held is refused before any signature check, so repeatedly submitting one @@ -54,6 +57,8 @@ use crate::error::{Error, Result}; use crate::logging::{debug, warn}; use ant_protocol::pointer::{ParsedPointer, Pointer, PointerState, POINTER_WIRE_LEN}; +use crate::storage::Reservation; + /// Directory under the store root that holds pointer records. const POINTERS_DIR_NAME: &str = "pointers"; @@ -84,7 +89,7 @@ pub enum PutOutcome { /// The result of [`PointerStore::inspect`]: what an arrival claims, before any /// signature has been checked. #[derive(Debug)] -pub enum Inspected { +pub(crate) enum Inspected { /// The node already holds exactly this state. No signature check is owed: /// a resubmission of what was paid for and a forgery of it are the same /// no-op. @@ -232,7 +237,7 @@ impl PointerStore { /// # Errors /// /// Returns [`Error::Protocol`] if the bytes are not a well-formed record. - pub async fn inspect(&self, bytes: &[u8]) -> Result { + pub(crate) async fn inspect(&self, bytes: &[u8]) -> Result { // Off the executor: deciding this reads the held record back off the // disk, and a flood of arrivals must not put a blocking read on a // runtime worker for each one. @@ -274,7 +279,7 @@ impl PointerStore { /// # Errors /// /// Returns [`Error::Crypto`] if the signature does not verify. - pub async fn verify(&self, parsed: ParsedPointer) -> Result { + pub(crate) async fn verify(&self, parsed: ParsedPointer) -> Result { spawn_blocking(move || Pointer::verify_parsed(parsed)) .await .map_err(|e| Error::Storage(format!("pointer verification panicked: {e}")))? @@ -288,14 +293,24 @@ impl PointerStore { /// arrive, and the re-check is what keeps that newer state from being /// overwritten. /// + /// `reservation` is the disk charge for this write, taken before the call. + /// It moves into the blocking transaction rather than staying with the + /// caller, because the caller's future can be dropped while that + /// transaction runs on: a charge released here would leave the file that + /// landed a moment later uncounted. + /// /// # Errors /// /// Returns [`Error::Storage`] if the write fails. - pub async fn commit(&self, record: Pointer) -> Result { + pub(crate) async fn commit( + &self, + record: Pointer, + reservation: Option, + ) -> Result { let inner = Arc::clone(&self.inner); // The whole transaction runs in one task, so dropping this future // cannot leave the write done and the index un-updated. - spawn_blocking(move || inner.commit_blocking(&record)) + spawn_blocking(move || inner.commit_blocking(&record, reservation)) .await .map_err(|e| Error::Storage(format!("pointer commit panicked: {e}")))? } @@ -308,14 +323,15 @@ impl PointerStore { /// /// # Errors /// - /// As [`Self::inspect`], [`Self::verify`] and [`Self::commit`]. + /// As the three steps it runs: a malformed record, a signature that does + /// not verify, or a write that fails. pub async fn put_bytes(&self, bytes: &[u8]) -> Result { match self.inspect(bytes).await? { Inspected::Unchanged(_) => Ok(PutOutcome::Unchanged), Inspected::Stale(_) => Ok(PutOutcome::Stale), Inspected::Candidate(parsed) => { let record = self.verify(parsed).await?; - self.commit(record).await + self.commit(record, None).await } } } @@ -333,7 +349,7 @@ impl PointerStore { /// is served as held — it is a real record, the reader verifies it, and the /// read quorum is what decides between replicas that disagree. What the /// index is not allowed to do is claim a state the file does not have — - /// that check guards [`Self::inspect`]'s two early answers, the ones that + /// that check guards `inspect`'s two early answers, the ones that /// assert this node already holds something. /// /// # Errors @@ -547,7 +563,11 @@ impl Inner { clippy::significant_drop_tightening, reason = "the write must happen under the same guard as the decision" )] - fn commit_blocking(&self, record: &Pointer) -> Result { + fn commit_blocking( + &self, + record: &Pointer, + reservation: Option, + ) -> Result { let address = record.address(); let path = self.dir.join(hex::encode(address)); @@ -587,14 +607,21 @@ impl Inner { Some(_) => PutOutcome::Stale, }; if outcome != PutOutcome::Changed { - let _ = std::fs::remove_file(&temp); + // Nothing lands, so the charge goes back — unless the staged + // bytes could not be removed, in which case they are still on + // the disk and the charge has to stand for them. + if std::fs::remove_file(&temp).is_err() { + settle(reservation); + } return Ok(outcome); } // The rename is the commit point: nothing fallible happens between // it and the index update, and both are under this one lock. if let Err(e) = std::fs::rename(&temp, &path) { - let _ = std::fs::remove_file(&temp); + if std::fs::remove_file(&temp).is_err() { + settle(reservation); + } return Err(Error::Storage(format!( "cannot rename {} onto {}: {e}", temp.display(), @@ -603,6 +630,13 @@ impl Inner { } let generation = self.generation.fetch_add(1, Ordering::Relaxed); index.insert(address, IndexEntry::of(record, generation)); + // A file is on the disk now. Charge it whether or not this replaced + // one: telling those apart would mean trusting an observation taken + // before the rename, and that observation can be wrong in the one + // direction that matters — a file the index claims can be gone, so + // what looks like a replacement grows the disk after all. + // Over-counting corrects itself at the next measurement. + settle(reservation); outcome }; @@ -776,6 +810,16 @@ fn stage(temp: &Path, bytes: &[u8]) -> Result<()> { Ok(()) } +/// Turn a charge into bytes that are now on the disk. +/// +/// A charge that is simply dropped is released instead, which is what every +/// path that writes nothing wants. +fn settle(reservation: Option) { + if let Some(reservation) = reservation { + reservation.commit(); + } +} + /// Flush the directory entry a rename created. /// /// Without it a crash can leave the entry unflushed and the record invisible on @@ -1019,12 +1063,54 @@ mod tests { assert_eq!(verified.state_id(), record.state_id()); assert_eq!(verified.to_bytes(), record.to_bytes()); assert_eq!( - store.commit(verified).await.expect("commit"), + store.commit(verified, None).await.expect("commit"), PutOutcome::Changed ); assert_eq!(store.len(), 1); } + #[tokio::test] + async fn a_commit_that_writes_nothing_gives_its_charge_back() { + // The reservation settles inside the commit transaction, so this is + // where a no-op has to release it. Losing a race is the reachable way + // to get there: the arrival is a candidate when it is inspected, and + // by the time it commits a better state is held. + let (store, dir) = store().await; + let chunks = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + + // Verified while nothing is held, so it is a candidate... + let slow = verified(&store, &signed(1, 2, 1)).await; + // ...and a better state lands before it commits. + store + .put_bytes(&signed(1, 7, 1).to_bytes()) + .await + .expect("put"); + + let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); + assert_eq!( + store.commit(slow, Some(charge)).await.expect("commit"), + PutOutcome::Stale, + "the newer state must stand" + ); + + // The charge was released rather than stranded: a disk with no room to + // spare still takes the next write. + let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); + let next = verified(&store, &signed(1, 8, 1)).await; + assert_eq!( + store.commit(next, Some(charge)).await.expect("commit"), + PutOutcome::Changed + ); + } + #[tokio::test] async fn a_commit_rechecks_what_verification_saw() { // Verification runs before the payment check; a newer state can land @@ -1039,7 +1125,7 @@ mod tests { .expect("put"); assert_eq!( - store.commit(slow).await.expect("commit"), + store.commit(slow, None).await.expect("commit"), PutOutcome::Stale, "the newer state must survive a late commit" ); @@ -1353,7 +1439,7 @@ mod tests { let prepared = verified(&store, &record).await; { - let committing = store.commit(prepared); + let committing = store.commit(prepared, None); tokio::pin!(committing); // Poll exactly once, then drop. That first poll hands the write to // a blocking thread and returns `Pending`, so the caller is always From 6b6e4b7e145337db1f6a42f7c94b290ed7311b83 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 13:42:49 +0900 Subject: [PATCH 20/32] fix(pointer): keep the charge for staged bytes that could not be removed Third site of the same hole. A failed write or fsync leaves a partial temporary file; if removing it also fails, the bytes are still on the disk while the charge for them was being released. stage now reports whether it left anything behind, and the caller settles the charge when it did. The reservation tests now assert on the capacity guard's own counters rather than on a following reservation succeeding -- which it would, stranded charge or not, on a disk with room. A losing commit race must leave written_since and in_flight exactly where it found them; a write that lands must move its bytes from one to the other; and eight concurrent creations must each be charged with none left hanging. That needs a test-only view of the two counters, which FileStore and ChunkStore now expose under cfg(test). The full-disk test's comment claimed to cover the reservation. It does not -- it refuses at the admission check, before payment -- and now says so. --- src/pointer/service.rs | 24 +++++++-- src/pointer/store.rs | 99 +++++++++++++++++++++++++++++++++----- src/storage/chunk_store.rs | 6 +++ src/storage/file_store.rs | 15 ++++++ 4 files changed, 128 insertions(+), 16 deletions(-) diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 86fc25e1..7492bd92 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -406,10 +406,10 @@ mod tests { #[tokio::test] async fn a_full_disk_refuses_a_pointer_before_it_is_written() { - // The write must be charged against the disk, not merely checked - // against it: a check that passes and a write that follows are the - // race the file store exists to close. With no room at all, the - // reservation cannot be taken and nothing lands. + // A disk with no room refuses at the admission check, before payment, + // so a client is not charged to find out. This does not reach the + // reservation — the store-level tests cover that — it covers the early + // refusal and that nothing is stored when it fires. let (service, _dir) = service_with_full_disk().await; let record = signed(1, 0, 1); @@ -451,6 +451,9 @@ mod tests { .expect("chunk store"); let service = service.with_chunk_store(Arc::new(chunks)); + let chunks = service.chunks.clone().expect("chunk store"); + let (written_before, in_flight_before) = chunks.capacity_counters(); + let mut writes = Vec::new(); for seed in 1..=8u8 { let record = signed(seed, 0, 1); @@ -472,6 +475,19 @@ mod tests { )); } assert_eq!(service.store().len(), 8); + + // Each of the eight was charged, and none of the charges was left + // hanging. Without the counters this would pass just as well against a + // bare capacity check that charges nothing. + let (written_after, in_flight_after) = chunks.capacity_counters(); + assert!( + written_after >= written_before + 8 * POINTER_WIRE_LEN as u64, + "eight writes must be charged: {written_before} -> {written_after}" + ); + assert_eq!( + in_flight_after, in_flight_before, + "and none of them may stay in flight" + ); } fn put(record: &Pointer) -> PointerPutRequest { diff --git a/src/pointer/store.rs b/src/pointer/store.rs index a1e91f9c..25e570fa 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -591,7 +591,14 @@ impl Inner { let temp = self .dir .join(format!("{TEMP_PREFIX}{}-{seq}", hex::encode(address))); - stage(&temp, &record.to_bytes())?; + if let Err(failed) = stage(&temp, &record.to_bytes()) { + // Bytes that could not be cleaned up are still on the disk, so the + // charge for them stands rather than going back. + if failed.bytes_remain { + settle(reservation); + } + return Err(failed.error); + } let outcome = { let mut index = self.index.lock(); @@ -790,26 +797,45 @@ fn scan(dir: &Path) -> Result> { /// Leaves nothing half written: the bytes are durable in the temporary file /// before any rename can make them visible, and a failure at any step removes /// it. The caller performs the rename, which is the commit point. -fn stage(temp: &Path, bytes: &[u8]) -> Result<()> { +/// Returns `Err(StagingFailed { bytes_remain })` where `bytes_remain` says whether +/// the partial file is still on the disk, so the caller knows whether the charge +/// for it can be given back. +fn stage(temp: &Path, bytes: &[u8]) -> std::result::Result<(), StagingFailed> { // `create_new` so a leftover temporary file from a crashed write is never // silently appended to or shared with a concurrent writer. let mut file = OpenOptions::new() .write(true) .create_new(true) .open(temp) - .map_err(|e| Error::Storage(format!("cannot create {}: {e}", temp.display())))?; + .map_err(|e| StagingFailed { + // Nothing was created, so nothing is left behind. + error: Error::Storage(format!("cannot create {}: {e}", temp.display())), + bytes_remain: false, + })?; let written = file .write_all(bytes) .and_then(|()| file.sync_all()) .map_err(|e| Error::Storage(format!("cannot write {}: {e}", temp.display()))); drop(file); - if let Err(e) = written { - let _ = std::fs::remove_file(temp); - return Err(e); + if let Err(error) = written { + // A partial file exists. If it cannot be removed it is still occupying + // the disk, and its charge has to stand for it. + return Err(StagingFailed { + error, + bytes_remain: std::fs::remove_file(temp).is_err(), + }); } Ok(()) } +/// A staged write that did not complete, and whether it left bytes behind. +struct StagingFailed { + /// What went wrong, for the caller to return. + error: Error, + /// Whether a partial file is still on the disk. + bytes_remain: bool, +} + /// Turn a charge into bytes that are now on the disk. /// /// A charge that is simply dropped is released instead, which is what every @@ -1073,8 +1099,11 @@ mod tests { async fn a_commit_that_writes_nothing_gives_its_charge_back() { // The reservation settles inside the commit transaction, so this is // where a no-op has to release it. Losing a race is the reachable way - // to get there: the arrival is a candidate when it is inspected, and - // by the time it commits a better state is held. + // to get there: the arrival is a candidate when it is inspected, and by + // the time it commits a better state is held. + // + // Asserted on the guard's own counters, because "the next reservation + // still succeeds" would pass just as well with the charge stranded. let (store, dir) = store().await; let chunks = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { root_dir: dir.path().to_path_buf(), @@ -1094,21 +1123,67 @@ mod tests { .await .expect("put"); + let before = chunks.capacity_counters(); let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); + assert!( + chunks.capacity_counters().1 > before.1, + "taking a charge must raise in-flight" + ); + assert_eq!( store.commit(slow, Some(charge)).await.expect("commit"), PutOutcome::Stale, "the newer state must stand" ); + assert_eq!( + chunks.capacity_counters(), + before, + "a write that did not happen must leave both counters where it found them" + ); + } + + #[tokio::test] + async fn a_commit_that_writes_charges_the_disk_for_it() { + // The other half: bytes that land move from in-flight to written, so + // the guard counts them against the reserve from then on. + let (store, dir) = store().await; + let chunks = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: false, + max_map_size: 0, + disk_reserve: 0, + migration: crate::storage::MigrationConfig::default(), + }) + .await + .expect("chunk store"); + + let record = verified(&store, &signed(1, 0, 1)).await; + let (written_before, in_flight_before) = chunks.capacity_counters(); + let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); + + assert_eq!( + store.commit(record, Some(charge)).await.expect("commit"), + PutOutcome::Changed + ); + let (written_after, in_flight_after) = chunks.capacity_counters(); + assert!( + written_after > written_before, + "the bytes that landed must be counted as written" + ); + assert_eq!( + in_flight_after, in_flight_before, + "and must no longer be counted as in flight" + ); - // The charge was released rather than stranded: a disk with no room to - // spare still takes the next write. + // A replacement charges too: telling it apart would mean trusting an + // observation taken before the rename. + let update = verified(&store, &signed(1, 1, 1)).await; let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); - let next = verified(&store, &signed(1, 8, 1)).await; assert_eq!( - store.commit(next, Some(charge)).await.expect("commit"), + store.commit(update, Some(charge)).await.expect("commit"), PutOutcome::Changed ); + assert!(chunks.capacity_counters().0 > written_after); } #[tokio::test] diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index 36731dfb..4e54980c 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -1036,6 +1036,12 @@ impl ChunkStore { self.files.reserve_bytes(bytes) } + /// `(written_since, in_flight)` from the capacity guard. Tests only. + #[cfg(test)] + pub(crate) fn capacity_counters(&self) -> (u64, u64) { + self.files.capacity_counters() + } + /// Wait until every blocking task in either backing has finished. pub async fn wait_idle(&self) { self.files.wait_idle().await; diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs index 9784e087..c2f162b7 100644 --- a/src/storage/file_store.rs +++ b/src/storage/file_store.rs @@ -347,6 +347,14 @@ impl CapacityGuard { }) } + /// The two accounting counters, for tests that need to prove a charge was + /// settled one way rather than the other. + #[cfg(test)] + fn counters(&self) -> (u64, u64) { + let snapshot = self.snapshot.lock(); + (snapshot.written_since, snapshot.in_flight) + } + /// Give back a reservation whose write did not happen. fn release(&self, needed: u64) { let mut snapshot = self.snapshot.lock(); @@ -1524,6 +1532,13 @@ impl FileStore { self.capacity.reserve(bytes) } + /// `(written_since, in_flight)` from the capacity guard. Tests only: it is + /// how a released charge is told apart from a stranded one. + #[cfg(test)] + pub(crate) fn capacity_counters(&self) -> (u64, u64) { + self.capacity.counters() + } + /// Force the next capacity question to re-measure the filesystem. /// /// Called after the legacy environment is removed, because that is a step change in From 932fb6f4ccdc345f69f65fa954cd352718728ea1 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 13:54:14 +0900 Subject: [PATCH 21/32] test(pointer): assert the exact charge, not that it went up A record rounds up to the allocation unit, so a charge is 12,288 bytes for 5,303 bytes of pointer. The eight-write assertion compared against eight raw record sizes -- which four charges already exceed -- so it would have passed with half the writes uncharged. The single-write and replacement checks were inequalities for the same reason. All three now assert exact deltas against the charge itself, exposed under cfg(test) beside the counters. The stage doc also said every failure removes the temporary file, one line above the field that reports when it could not. --- src/pointer/service.rs | 14 +++++++------- src/pointer/store.rs | 30 +++++++++++++++++------------- src/storage/chunk_store.rs | 6 ++++++ src/storage/file_store.rs | 9 +++++++++ 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 7492bd92..50d07c5b 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -479,14 +479,14 @@ mod tests { // Each of the eight was charged, and none of the charges was left // hanging. Without the counters this would pass just as well against a // bare capacity check that charges nothing. - let (written_after, in_flight_after) = chunks.capacity_counters(); - assert!( - written_after >= written_before + 8 * POINTER_WIRE_LEN as u64, - "eight writes must be charged: {written_before} -> {written_after}" - ); + // Exactly eight allocation charges. An inequality against the raw + // record size would accept four: a record rounds up to the allocation + // unit, and four of those already exceed eight record sizes. + let one = ChunkStore::capacity_charge_for(POINTER_WIRE_LEN as u64); assert_eq!( - in_flight_after, in_flight_before, - "and none of them may stay in flight" + chunks.capacity_counters(), + (written_before + 8 * one, in_flight_before), + "eight writes, eight charges, none left in flight" ); } diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 25e570fa..220adc42 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -795,11 +795,12 @@ fn scan(dir: &Path) -> Result> { /// Write `bytes` to `temp` and fsync it, ready to be renamed into place. /// /// Leaves nothing half written: the bytes are durable in the temporary file -/// before any rename can make them visible, and a failure at any step removes -/// it. The caller performs the rename, which is the commit point. -/// Returns `Err(StagingFailed { bytes_remain })` where `bytes_remain` says whether -/// the partial file is still on the disk, so the caller knows whether the charge -/// for it can be given back. +/// before any rename can make them visible. The caller performs the rename, +/// which is the commit point. +/// +/// A failure tries to remove what it wrote, and says whether that worked: +/// `Err(StagingFailed { bytes_remain })` reports a partial file still on the +/// disk, so the caller knows the charge for it cannot be given back. fn stage(temp: &Path, bytes: &[u8]) -> std::result::Result<(), StagingFailed> { // `create_new` so a leftover temporary file from a crashed write is never // silently appended to or shared with a concurrent writer. @@ -1158,6 +1159,10 @@ mod tests { .expect("chunk store"); let record = verified(&store, &signed(1, 0, 1)).await; + // Exactly one allocation charge, not merely "more than before": a + // record is rounded up to the allocation unit, so an inequality would + // accept one charge standing in for two writes. + let one = crate::storage::ChunkStore::capacity_charge_for(POINTER_WIRE_LEN as u64); let (written_before, in_flight_before) = chunks.capacity_counters(); let charge = chunks.reserve(POINTER_WIRE_LEN as u64).expect("reserve"); @@ -1165,14 +1170,10 @@ mod tests { store.commit(record, Some(charge)).await.expect("commit"), PutOutcome::Changed ); - let (written_after, in_flight_after) = chunks.capacity_counters(); - assert!( - written_after > written_before, - "the bytes that landed must be counted as written" - ); assert_eq!( - in_flight_after, in_flight_before, - "and must no longer be counted as in flight" + chunks.capacity_counters(), + (written_before + one, in_flight_before), + "the bytes that landed move from in flight to written, exactly once" ); // A replacement charges too: telling it apart would mean trusting an @@ -1183,7 +1184,10 @@ mod tests { store.commit(update, Some(charge)).await.expect("commit"), PutOutcome::Changed ); - assert!(chunks.capacity_counters().0 > written_after); + assert_eq!( + chunks.capacity_counters(), + (written_before + 2 * one, in_flight_before) + ); } #[tokio::test] diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index 4e54980c..bfe1f035 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -1042,6 +1042,12 @@ impl ChunkStore { self.files.capacity_counters() } + /// What a payload of `bytes` costs the disk. Tests only. + #[cfg(test)] + pub(crate) fn capacity_charge_for(bytes: u64) -> u64 { + crate::storage::FileStore::capacity_charge_for(bytes) + } + /// Wait until every blocking task in either backing has finished. pub async fn wait_idle(&self) { self.files.wait_idle().await; diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs index c2f162b7..d0e369b9 100644 --- a/src/storage/file_store.rs +++ b/src/storage/file_store.rs @@ -1539,6 +1539,15 @@ impl FileStore { self.capacity.counters() } + /// What a payload of `bytes` actually costs the disk, rounded to the + /// allocation unit. Tests only, so they can assert exact counter deltas + /// rather than "it went up" — which a charge of one allocation unit would + /// satisfy for several payloads at once. + #[cfg(test)] + pub(crate) fn capacity_charge_for(bytes: u64) -> u64 { + CapacityGuard::charge(bytes) + } + /// Force the next capacity question to re-measure the filesystem. /// /// Called after the legacy environment is removed, because that is a step change in From 9de2bf29da95fc2f6e4ba81bea562707e8b234a0 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 14:35:10 +0900 Subject: [PATCH 22/32] chore: rebase onto the 0.20.0 release baseline main released 0.20.0 and moved off the git patches: ant-protocol 3.0.0, saorsa-core 0.28.0, saorsa-pqc 0.5.2, evmlib 0.10.0 and saorsa-transport 0.37.0 all come from crates.io now, and [patch.crates-io] is gone. The pointer work needs exactly one thing back: ant-protocol carries the Pointer record and its wire messages, and the published 3.0.0 does not have them. So the patch section returns with that single entry, pinned by rev to the pointer protocol branch at 3.1.0 -- which satisfies the 3.0.0 requirement main declares, being additive on top of it. Everything else stays on the release. No source change was needed: the node compiles against saorsa-core 0.28 and the record still signs and verifies under saorsa-pqc 0.5.2. --- Cargo.lock | 45 +++++++-------------------------------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16e0f41e..50268ec9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3292,7 +3292,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -7119,7 +7119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core 0.61.2", + "windows-core", "windows-future", "windows-link 0.1.3", "windows-numerics", @@ -7131,7 +7131,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core 0.61.2", + "windows-core", ] [[package]] @@ -7143,21 +7143,8 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", + "windows-result", + "windows-strings", ] [[package]] @@ -7166,7 +7153,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core 0.61.2", + "windows-core", "windows-link 0.1.3", "windows-threading", ] @@ -7211,7 +7198,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core 0.61.2", + "windows-core", "windows-link 0.1.3", ] @@ -7224,15 +7211,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link 0.2.1", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -7242,15 +7220,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link 0.2.1", -] - [[package]] name = "windows-sys" version = "0.48.0" From 0d3a3bf6f7a51393fe5447aeed7a22bfc912612c Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 24 Sep 2026 17:29:53 +0900 Subject: [PATCH 23/32] feat(pointer): admit any paid state that beats what is held A node now takes any paid record the merge rule prefers to what it holds, whatever its counter, including a first record above 0 at an address it knows nothing about. The +1 gate is gone. It was the one thing that stopped a node catching up. A write ends once five of the seven close-group peers answer and the rest are cancelled, so a peer can miss an update; a peer that joins the group later holds nothing. Under +1 each refused every later update for good, and three such peers left no write able to reach its quorum -- after the client had paid. One guard stays: a record this node lost is still remembered, so it takes that state or anything newer back, and refuses anything older. A replay cannot roll a node back just because its file went missing. A record that loses is now answered Stale rather than PaymentRequired, which is what it is. Pins ant-protocol to 4412b6a, which drops is_paid_update_of and is_genesis. ADR-0016 is updated to match: the counter orders states rather than metering them, and a joining or lagging node is brought level by the next update. --- Cargo.lock | 2 +- Cargo.toml | 2 +- docs/adr/ADR-0016-pointers-immutable-owner.md | 58 +++++---- src/pointer/service.rs | 110 ++++++++++-------- src/pointer/store.rs | 81 ++++++------- 5 files changed, 134 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50268ec9..b852f2af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "3.1.0" -source = "git+https://github.com/WithAutonomi/ant-protocol?rev=34b7832a001df8b80b18555ca058734baae43496#34b7832a001df8b80b18555ca058734baae43496" +source = "git+https://github.com/WithAutonomi/ant-protocol?rev=4412b6af4fab2ceb0a4e1efea1efca08e68c6e4e#4412b6af4fab2ceb0a4e1efea1efca08e68c6e4e" dependencies = [ "blake3", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 046a9d5d..cddbaf58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -233,7 +233,7 @@ webrtc-direct = [ # the published 3.0.0, so this is the only entry that has to leave the release # baseline. A rev, not a branch, so the pin is immutable. Drop it once the # pointer PR lands and 3.1.0 is published. -ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", rev = "34b7832a001df8b80b18555ca058734baae43496" } +ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", rev = "4412b6af4fab2ceb0a4e1efea1efca08e68c6e4e" } [profile.release] lto = true diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index 4d28e72b..8d68271a 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -25,7 +25,7 @@ One record. No genesis object, no certificates, no lineage. pub struct Pointer { // 5,303 bytes version: u8, // 1 owner: MlDsa65PublicKey, // 1,952 — the identity; the address derives from it - counter: u64, // 8 — 0 to create, +1 per paid update + counter: u64, // 8 — orders states; larger wins target: PointerTarget, // 33 — kind tag + address; opaque to a node sig: MlDsa65Signature, // 3,309 — over every field above } @@ -70,17 +70,23 @@ would make every update after the first free. ### Pay to create, pay to update -Creation is `counter = 0`. An update is `counter + 1`, or the same counter with -a smaller target — the merge rule's tie-break, which two concurrent updates must -both be able to land on or they leave the group split. Every one of them is paid -against its own `state_id`, so **one payment buys one state and at most one -increment**. A tie-break moves the pointer without advancing the counter, but it -must strictly descend in target bytes and each step is bought separately, so it -buys nothing an ordinary update would not. +Every stored state is paid against its own `state_id`, so **one payment buys +one state**. Creating and updating are the same operation: a node takes any +paid record that beats what it holds under the merge rule, whatever its counter. +The client creates at 0 and updates by signing one past the counter the network +serves, but nothing requires exactly one. -The client path enforces `+1`. Replication accepts any strictly greater counter, -because a replica that missed an update must be able to catch up; refusing the -gap would leave it permanently stale instead. +The counter orders states; it does not meter them. A number that is skipped is +never stored, so skipping avoids no payment that was owed. And requiring `+1` +would break catching up, since nothing replicates a pointer. A write ends once +its quorum has answered — five of the seven peers — so a peer can miss an +update, and one that joins the group later holds nothing at all. Under a `+1` +rule either would refuse every later update for good, and three such peers +would leave no write able to reach its quorum. Under the merge rule each takes +the next update, however far ahead of it that is. + +An owner who jumps straight to `u64::MAX` limits only themself, and does not +even freeze the pointer: equal counters still resolve by target, below. ### Merge @@ -120,7 +126,7 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. | Fork / equivocate at one counter | Total order on `(counter, target)`: every node given the same records picks the same one, and a read merges the close group's answers rather than trusting the first. Convergence *across* the network still needs replication — see Not built | | Replay an older record | Loses on counter | | Re-sign one paid state N times | Equal state never replaces; nothing is written | -| Pay once, jump the counter | Client updates must be `+1` | +| Pay once, jump the counter | Nothing to defend: one payment stores one state whatever its counter, and a skipped number is never stored | | Pay for a chunk to fund a pointer | A quote signs its content, not the record kind, so the defence is that the content cannot be shared: putting a chunk on a `state_id` means breaking BLAKE3 across its two modes. The paid cache is keyed by a typed `Chunk` vs `Pointer` target as well, so the two never alias even in memory, whatever the bytes | | Merkle proof with no issuer check | Refused for pointers; single-node proofs only | | Downgrade the format | `version` is signed and inside `state_id`; unknown versions are refused | @@ -129,7 +135,7 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. | Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent. A read asks the same group by the same definition, so the quorums intersect — though each does its own lookup, so churn between them is not covered | | One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | | Two peers decide it | **Not defended against.** Two colluding close-group peers clear the bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Raising the bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold. What actually answers it is replication and audits, neither of which is built | -| Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, because an address nothing is known about admits only a counter 0 record, and a loss above that would otherwise be permanent. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | +| Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, so the lost state is taken back and nothing older is: an address nothing is known about admits any record, and a replay could otherwise roll the node back. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | ## Consequences @@ -170,12 +176,14 @@ guarantees they hold the same records. Until it lands, availability and cross-network fork convergence are the client's doing, not the network's. Two consequences follow from it and land with it. A node that joins a close -group after a pointer was created can never obtain it: the increment rule admits -only a counter 0 record at an address nothing is known about. And a node that -loses a record can repair it while it is running — it keeps what it lost, and -takes back that state or any that replaces it — but not across a restart, where -a missing file leaves nothing to remember. Both are the same missing mechanism: -a node cannot ask another node for a record. +group after a pointer was created, or that missed updates, does not hold the +current state until the owner next updates it; that update is admitted however +far ahead it is, so the next write brings it level. And a node that loses a +record can repair it while it is running — it keeps what it lost, and takes back +that state or any that replaces it — but across a restart, where a missing file +leaves nothing to remember, it waits for the next update like a node that just +joined. Both are the same missing mechanism: a node cannot ask another node for +a record. Also not built: pointer participation in commitments and audits, which depends on the same work. @@ -195,8 +203,10 @@ the browser client the same quorum and corroboration rules the native one uses. all permutations, including on a node started empty and one restarted. - 64 valid signatures over one paid state yield one stored record and one file. - A resubmission and a stale arrival are both refused before any signature check. -- Creation is counter 0; an update is `+1`; every jump — including to `u64::MAX` - and a wrap back to 0 — is refused as a non-successor. +- Any paid record that beats the held one is taken, including a first record + above counter 0 and a jump to `u64::MAX`; anything older is refused as stale, + and the client cannot wrap a terminal counter back to 0. +- A fork at one counter held by two nodes is healed on both by any later counter. - All 256 `version` values give distinct paid identifiers. - Golden vectors pin the encoding, both identities and the signing context. - A crafted chunk cannot satisfy a pointer's paid-cache entry. @@ -222,4 +232,8 @@ the browser client the same quorum and corroboration rules the native one uses. assumption, not a property one can check.) - End to end against a live testnet with real settlement: create, update, read back, resolve a chain to its chunk, repeat a stored state, read an address - nobody wrote, and refuse a signed record that skips the counter. + nobody wrote. Every close-group node answers a paid update to a pointer it + already holds with success, not a refusal. A record that skips counters is + taken. Nodes that missed updates take the next one and the group converges. + A fork at one counter across the group reads as its merge winner, and one + update at the next counter heals it on every node. diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 50d07c5b..747263c0 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -224,26 +224,21 @@ impl PointerService { ) -> Option { let address = state.address; - // One payment buys one state and at most one increment. A create is - // counter 0; an update is one past what this node holds, or the - // tie-break winner at that same counter, which two concurrent updates - // must both be able to land on or the group stays split. What is - // refused is a jump, which would let an owner pay once and skip every - // intermediate payment. Replication does not come through here — it - // merges on the counter order, so a replica behind a gap can still - // catch up. - if !self.store.accepts_as_paid_update(state) { + // Any state the merge rule prefers to what this node knows, whatever + // its counter: a node that missed updates, or joined the group after + // them, must take the next one or it never catches up. An arrival that + // loses to what is held was already answered as stale; this also + // covers a record the node lost, which that comparison cannot see, so + // a replay cannot roll it back. + if !self.store.admits(state) { debug!( - "Rejecting pointer PUT for {}: counter {} is not the paid successor", + "Rejecting pointer PUT for {}: counter {} does not beat the state this node knows", hex::encode(address), state.counter ); - return Some(PointerPutResponse::PaymentRequired { - message: format!( - "a pointer is created at counter 0 and updated by exactly one \ - increment; counter {} does not follow what this node holds", - state.counter - ), + return Some(PointerPutResponse::Stale { + address, + state_id: self.store.state_id(&address).unwrap_or_default(), }); } @@ -584,23 +579,40 @@ mod tests { } #[tokio::test] - async fn a_counter_jump_is_still_refused_after_a_tie_break() { - // Taking a tie-break winner must not loosen the increment rule: the - // counter has not moved, so the next state is still exactly one on. - let (service, _dir) = service().await; - service.handle_put(put(&signed(1, 0, 9))).await; + async fn a_fork_between_two_nodes_is_healed_by_any_later_counter() { + // Two paid states at one counter, each reaching a different node: a + // fork no read can settle for them. The next update, at any later + // counter, lands on both and leaves them holding the same record. + let (first_node, _a) = service().await; + let (second_node, _b) = service().await; + let one_side = signed(1, 1, 9); + let other_side = signed(1, 1, 1); assert!(matches!( - service.handle_put(put(&signed(1, 0, 1))).await, + first_node.handle_put(put(&one_side)).await, PointerPutResponse::Success { .. } )); assert!(matches!( - service.handle_put(put(&signed(1, 7, 1))).await, - PointerPutResponse::PaymentRequired { .. } - )); - assert!(matches!( - service.handle_put(put(&signed(1, 1, 1))).await, + second_node.handle_put(put(&other_side)).await, PointerPutResponse::Success { .. } )); + + let healed = signed(1, 5, 4); + for (name, node) in [("first", &first_node), ("second", &second_node)] { + match node.handle_put(put(&healed)).await { + PointerPutResponse::Success { address, state_id } => { + assert_eq!(address, healed.address()); + assert_eq!(state_id, healed.state_id()); + } + other => panic!("the {name} node refused the healing update: {other:?}"), + } + let held = node + .store() + .get(&healed.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.state_id(), healed.state_id(), "the {name} node"); + } } #[tokio::test] @@ -689,45 +701,45 @@ mod tests { } #[tokio::test] - async fn a_pointer_is_created_at_zero_and_updated_one_step_at_a_time() { + async fn any_counter_that_beats_what_is_held_is_taken() { let (service, _dir) = service().await; - // A create must be counter 0. - assert!(matches!( - service.handle_put(put(&signed(1, 5, 1))).await, - PointerPutResponse::PaymentRequired { .. } - )); - assert!(service.store().is_empty(), "nothing was stored"); - - let created = signed(1, 0, 1); + // A node that joined after the pointer was created holds nothing, so + // the first record it sees need not be counter 0. + let first_seen = signed(1, 5, 1); assert!(matches!( - service.handle_put(put(&created)).await, + service.handle_put(put(&first_seen)).await, PointerPutResponse::Success { .. } )); - // A jump is refused however large, including the terminal counter. - for jump in [0u64, 2, 3, 99, u64::MAX] { + // Anything older is stale, whatever else it says. + for older in [0u64, 4] { assert!( matches!( - service.handle_put(put(&signed(1, jump, 2))).await, - PointerPutResponse::PaymentRequired { .. } | PointerPutResponse::Stale { .. } + service.handle_put(put(&signed(1, older, 2))).await, + PointerPutResponse::Stale { .. } ), - "counter {jump} must not be accepted after 0" + "counter {older} must not replace counter 5" ); } - // Exactly one increment lands. - assert!(matches!( - service.handle_put(put(&signed(1, 1, 2))).await, - PointerPutResponse::Success { .. } - )); + // Anything newer lands, however far it skips. + for newer in [6u64, 99, u64::MAX] { + let record = signed(1, newer, 2); + match service.handle_put(put(&record)).await { + PointerPutResponse::Success { state_id, .. } => { + assert_eq!(state_id, record.state_id()); + } + other => panic!("counter {newer} was refused: {other:?}"), + } + } let held = service .store() - .get(&created.address()) + .get(&first_seen.address()) .await .expect("get") .expect("present"); - assert_eq!(held.counter(), 1); + assert_eq!(held.counter(), u64::MAX); } #[test] diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 220adc42..8c792851 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -107,9 +107,9 @@ pub(crate) enum Inspected { struct IndexEntry { /// The held record's authenticated state. /// - /// The state itself rather than fields copied out of it, so every rule the - /// store applies — merge order, the paid increment — is the protocol's own - /// rule applied to the held state, and cannot drift from it. + /// The state itself rather than fields copied out of it, so the merge order + /// the store applies is the protocol's own rule applied to the held state, + /// and cannot drift from it. state: PointerState, /// Whether the file behind `state` is still there and still that record. /// @@ -117,8 +117,8 @@ struct IndexEntry { /// the entry. The node stops serving the record, because it does not have /// it — but it still knows what it had, and that is what lets the state be /// restored. Dropping the entry would leave the address looking untouched, - /// where only a counter 0 record is admissible, so a pointer that had ever - /// been updated could never be repaired. + /// where any record is admissible, so a replay of an older state could roll + /// the node back. on_disk: bool, /// Which insertion this entry is. /// @@ -414,39 +414,25 @@ impl PointerStore { .map(|entry| entry.state.state_id) } - /// Whether `state` is a paid update of what is held. + /// Whether a paid PUT of `state` may be taken. /// - /// One payment buys one increment: a new pointer starts at counter 0, and - /// an update either advances the counter by one or wins the target - /// tie-break at the counter already held. Without the bound an owner pays - /// once, jumps the counter, skips every intermediate payment and strands - /// the pointer where nothing can advance it; without the tie-break, two - /// separately paid states at one counter would leave every node holding - /// whichever reached it first. + /// Any state the merge rule prefers to what is held, whatever its counter. + /// The counter orders states; it does not meter them, since each state is + /// paid for on its own. That is also what lets a node catch up: a write + /// ends once its quorum has answered, so a node can miss an update, and one + /// that joins the group later holds nothing at all. Either takes the next + /// update, however far ahead of it that is. /// - /// Only the client path asks this. Replication uses the merge rule instead, - /// so a replica that missed an update can still catch up rather than being - /// stuck behind a gap it can never fill. - /// - /// A record this node knew and lost takes that same merge rule: anything at - /// least as good as the lost state restores it. The increment rule exists to - /// stop an owner buying one state and skipping to it, and a repair skips - /// nothing — the state it carries was paid for and the rest of the group - /// already serves it. Holding a lost address to the increment rule would - /// make every loss above counter 0 permanent, because only a counter 0 - /// record is admissible at an address nothing is known about. + /// A record this node knew and lost admits the lost state itself as well, + /// so the arrival that restores it is not mistaken for a resubmission. It + /// admits nothing older, so a node that lost its copy cannot be rolled + /// back by a replay. #[must_use] - pub fn accepts_as_paid_update(&self, state: &PointerState) -> bool { - self.snapshot(&state.address).map_or_else( - || state.is_genesis(), - |entry| { - if entry.on_disk { - state.is_paid_update_of(&entry.state) - } else { - state.state_id == entry.state.state_id || state.replaces(&entry.state) - } - }, - ) + pub fn admits(&self, state: &PointerState) -> bool { + self.snapshot(&state.address).is_none_or(|entry| { + state.replaces(&entry.state) + || (!entry.on_disk && state.state_id == entry.state.state_id) + }) } /// Whether a record is held at `address`. @@ -1370,11 +1356,10 @@ mod tests { } #[tokio::test] - async fn a_lost_record_above_counter_zero_is_still_repairable() { - // The admission rule alone would make this impossible: an address the - // node knows nothing about admits only a counter 0 record, so an entry - // that was *forgotten* on a failed read could never be restored above - // genesis, and every loss would be permanent. + async fn a_lost_record_is_restored_and_cannot_be_rolled_back() { + // The node remembers what it lost. Without that the address would look + // untouched, where any record is admissible, and a replay of an older + // state would roll the node back. let (store, _dir) = store().await; for counter in 0..=3u64 { store @@ -1392,15 +1377,15 @@ mod tests { // What it lost is what it will take back, and so is anything newer. assert!( - store.accepts_as_paid_update(&held.state()), + store.admits(&held.state()), "the state this node lost must be admissible again" ); assert!( - store.accepts_as_paid_update(&signed(1, 9, 1).state()), + store.admits(&signed(1, 9, 1).state()), "so must a newer state the rest of the group has moved on to" ); assert!( - !store.accepts_as_paid_update(&signed(1, 2, 1).state()), + !store.admits(&signed(1, 2, 1).state()), "but not one that loses to what was lost" ); @@ -1417,9 +1402,13 @@ mod tests { assert_eq!(back.counter(), 3); assert_eq!(back.state_id(), held.state_id()); - // And the increment rule is back in force now that it holds one. - assert!(store.accepts_as_paid_update(&signed(1, 4, 1).state())); - assert!(!store.accepts_as_paid_update(&signed(1, 6, 1).state())); + // Held again: any later counter is admitted, the held state is not. + assert!(store.admits(&signed(1, 4, 1).state())); + assert!(store.admits(&signed(1, 6, 1).state()), "a skip is admitted"); + assert!( + !store.admits(&held.state()), + "held again, so a resubmission" + ); } #[tokio::test] From fc53109727b7c5d03a376094c183cd810fcb302a Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Sep 2026 14:04:41 +0900 Subject: [PATCH 24/32] feat(pointer): replicate pointers like chunks A pointer's copies used to be exactly the ones the client wrote: a member the write missed, a node that joined the group later, or one that lost the file stayed without the current state until the owner's next update. Now pointers replicate through the same engine as chunks -- the same close groups, sync rounds, churn triggers, quorum, pruning and possession rules -- but by state rather than by key, since a pointer's address is stable while its state changes and replicas may hold different signatures of one state. - Fresh: the node that accepts a paid state from a client offers the record with its payment proof to the rest of the close group; each receiver checks signature, responsibility and payment itself before storing. - Repair: every sync round pushes hints (the states the receiver should hold) both ways. A receiver lacking a state, or holding an older one, asks the group which state each holds, adopts the best one a quorum hold exactly -- counted over the whole group, so a silent peer is never a vote -- and fetches it from a holder. - Pruning: after the hysteresis a record out of range is deleted, at once if the node is far outside the group, otherwise only when all but one of the current group return a valid record at that state or newer. - Possession: minutes after an offer, each member must produce the record or is penalised, as a chunk holder is. Six appended replication variants carry this. Requests only go to peers that have sent a pointer message, and hints go out every round even when empty, so an older peer is never asked what it cannot decode and never penalised for silence. The store gains what this needs: sorted enumeration, delete with the disk charge credited back, stats, 256 shards by last byte as the chunk store keeps, a structure-only startup scan (reads verify anyway), and the chunk store's Windows rename retry. Tests: nine multi-node E2E cases over real QUIC (fresh reach, update replacement, repair of a lagging node, a late joiner via the engine's own loops, a lone state refused though its hints arrived, an unpaid offer refused, possession penalising only the dropper, pruning with and without proofs), the verdict function, and the store additions. The existing chunk replication E2E suite passes unchanged (46/46). --- docs/adr/ADR-0016-pointers-immutable-owner.md | 99 +- src/node.rs | 10 +- src/payment/verifier.rs | 37 +- src/pointer/service.rs | 35 +- src/pointer/store.rs | 580 +++++++- src/replication/mod.rs | 110 +- src/replication/pointer.rs | 1284 +++++++++++++++++ src/replication/protocol.rs | 154 +- src/storage/chunk_store.rs | 8 + src/storage/file_store.rs | 8 +- src/storage/mod.rs | 2 +- tests/e2e/mod.rs | 3 + tests/e2e/pointer_replication.rs | 555 +++++++ tests/e2e/testnet.rs | 25 +- tests/pointer_convergence.rs | 2 +- 15 files changed, 2813 insertions(+), 99 deletions(-) create mode 100644 src/replication/pointer.rs create mode 100644 tests/e2e/pointer_replication.rs diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index 8d68271a..83dfd9ec 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -116,6 +116,53 @@ because a newer state can land while payment verifies. Every step off the async executor: the read, the signature check and the write each run on a blocking thread, so a flood of arrivals cannot occupy the runtime's workers. +### Replication + +Pointers replicate through the same engine chunks do — the same close groups, +neighbour-sync rounds, churn triggers, quorum, pruning and possession rules — +but by **state**, not by key. The chunk pipeline assumes a record never changes +and that its key is the hash of its bytes; a pointer's address is stable while +its state changes, and two honest replicas may hold different valid signatures +over one state. + +- **Fresh.** A node that accepts a paid state from a client forwards the record, + with the proof that paid for it, to the rest of the close group. Each receiver + checks the signature, its own responsibility (across the paid width, as for a + chunk offer) and the payment itself before storing. So a paid state that + reached one honest node reaches the whole group, whichever members the client + wrote to. +- **Repair.** Every neighbour-sync round pushes hints — the states the sender + holds that the receiver should hold — to the peers being synced, and a peer + that syncs with a node gets that node's hints back. A receiver that lacks a + hinted state, or holds an older one, asks the close group which state each + holds, adopts the best state a quorum of them hold **exactly**, and fetches it + from one of them. The record verifies itself; the quorum stands in for the + payment proof, as presence quorum does for a chunk. The quorum is the one a + chunk needs, counted over the whole close group: a peer that cannot be asked + counts as unanswered, never as a vote. This is how a node that missed an + update, joined late, or lost a record across a restart is brought level. +- **Pruning.** A record the node has been outside the retention width of for the + hysteresis period is deleted — at once if the node is outside a complete + paid-width group, otherwise only once all but one of the current close group + prove they hold that state or a newer one by returning a valid record. A proof + is a record, not a claim: signatures are checked. +- **Possession.** Some minutes after offering a fresh state, the offering node + asks each member for the record. One that is still responsible and cannot + produce that state or a newer one is penalised, as a chunk holder is. + +Six messages carry this, appended to the replication enum so every earlier +discriminant keeps its value: a fresh offer and a hint push (one-way), and a +fetch and a state query with their responses. An older peer cannot decode them. +One-way pushes to it are simply lost, and requests only ever go to peers that +have sent a pointer message themselves — a hint push goes out every round, empty +or not, so capability is learned within a cycle — so an older peer is never asked +something it cannot answer and never penalised for its silence. + +Records are kept one file each under `{root}/pointers//`, 256 shards by +the address's last byte as the chunk store keeps them. Opening the store parses +each record's structure but does not verify its signature; every record was +verified when it was committed and every read verifies it again. + ## What this defends against | Attack | Defence | @@ -123,7 +170,7 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. | Tamper with any byte | Signature over the whole body | | Swap the owner key | `A` is derived from it; the record no longer belongs at its address | | Store at someone else's address | Same | -| Fork / equivocate at one counter | Total order on `(counter, target)`: every node given the same records picks the same one, and a read merges the close group's answers rather than trusting the first. Convergence *across* the network still needs replication — see Not built | +| Fork / equivocate at one counter | Total order on `(counter, target)`: every node given the same records picks the same one, a read merges the close group's answers rather than trusting the first, and replication gives every member the states that reached a quorum, so the group converges | | Replay an older record | Loses on counter | | Re-sign one paid state N times | Equal state never replaces; nothing is written | | Pay once, jump the counter | Nothing to defend: one payment stores one state whatever its counter, and a skipped number is never stored | @@ -134,7 +181,8 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. | Collide a pointer and a chunk address | Takes a cross-mode BLAKE3 break, and is refused in both directions anyway. The two stores take separate locks, so simultaneous commits of both kinds at one address are not yet atomic | | Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent. A read asks the same group by the same definition, so the quorums intersect — though each does its own lookup, so churn between them is not covered | | One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | -| Two peers decide it | **Not defended against.** Two colluding close-group peers clear the bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Raising the bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold. What actually answers it is replication and audits, neither of which is built | +| Two peers decide it | **Not defended against, at the read.** Two colluding close-group peers clear a read's bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Replication does not spread such a state: the rest of the group adopts only what a quorum of it holds, so the honest members keep the paid state, but a reader that happens to hear from both colluders still sees theirs. Raising the read's bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold | +| Get a group to adopt a state nobody paid for | Repair adopts only a state a quorum of the close group hold exactly, counted over the whole group, and a fresh offer is stored only after the receiver verifies its payment itself | | Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, so the lost state is taken back and nothing older is: an address nothing is known about admits any record, and a replay could otherwise roll the node back. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | ## Consequences @@ -156,11 +204,10 @@ thread, so a flood of arrivals cannot occupy the runtime's workers. correctly signed value and cannot tell. - Replicas may hold different valid signatures of one state; nothing compares record bytes across replicas. -- Pointers do not take part in storage commitments or audits. The audit format - is untouched, so no protocol family is bumped and no rollout pauses. Auditing - them needs round 2 to serve a whole record — a peer signs its own commitment, - so without that it could name any key with the hash of cheap bytes it holds — - and that lands with replication. +- Pointers do not yet take part in storage commitments or audits; see + Implementation status. Auditing them needs round 2 to serve a whole record — a + peer signs its own commitment, so without that it could name any key with the + hash of cheap bytes it holds. ## Implementation status @@ -169,24 +216,16 @@ merge-on-put; request dispatch; payment routed at `state_id` with the close group of `A`; admission gates; cross-kind refusal; and the client — create, update, quorum store, merged reads and chain resolution. -**Not built: replication.** No node forwards a pointer to another, so the copies -that exist are the ones the client wrote. That is the load-bearing gap: the -merge rule guarantees nodes holding the same records agree, and nothing yet -guarantees they hold the same records. Until it lands, availability and -cross-network fork convergence are the client's doing, not the network's. - -Two consequences follow from it and land with it. A node that joins a close -group after a pointer was created, or that missed updates, does not hold the -current state until the owner next updates it; that update is admitted however -far ahead it is, so the next write brings it level. And a node that loses a -record can repair it while it is running — it keeps what it lost, and takes back -that state or any that replaces it — but across a restart, where a missing file -leaves nothing to remember, it waits for the next update like a node that just -joined. Both are the same missing mechanism: a node cannot ask another node for -a record. - -Also not built: pointer participation in commitments and audits, which depends -on the same work. +**Built: replication** (see Replication above): fresh offers with payment, +neighbour-sync repair by quorum over exact states, pruning with possession +proofs, and post-offer possession checks, wired into the engine's sync rounds, +churn triggers and cycle completion. A node that missed an update, joined late, +or lost a record is brought level by the next sync round rather than the next +write. + +**Not built yet: commitments and audits.** Pointers are not yet leaves of the +storage commitment, so they do not count toward a node's quoted price and are not +spot-checked by the subtree audit. **Not built: browser clients.** ADR-0015's WebRTC-direct transport admits, sanitizes and classifies message kinds by an explicit list, and pointer requests @@ -230,6 +269,16 @@ the browser client the same quorum and corroboration rules the native one uses. - The chunk preimages the old prefix construction handed out no longer land on either identity. (No test can say more: that no content does is the preimage assumption, not a property one can check.) +- Replication across a live multi-node network: a paid PUT to one node reaches + its whole close group; an update replaces the old state everywhere; a node that + missed an update is repaired by neighbour sync alone; a node that joins later + obtains existing pointers through the engine's own loops; a state only one node + holds is not adopted, though its hints arrived; an unpaid fresh offer is + refused; the possession check penalises only the member that dropped the + record; pruning deletes only once the close group proves it holds the record, + and a node far outside the group prunes without asking. +- Fresh offers, hints, fetches and state queries are covered by the replication + protocol's per-variant tests: family, size ceiling, round-trip. - End to end against a live testnet with real settlement: create, update, read back, resolve a chain to its chunk, repeat a stored state, read an address nobody wrote. Every close-group node answers a paid update to a pointer it diff --git a/src/node.rs b/src/node.rs index 1d24d427..56813244 100644 --- a/src/node.rs +++ b/src/node.rs @@ -238,7 +238,7 @@ impl NodeBuilder { fresh_rx: UnboundedReceiver, shutdown: &CancellationToken, ) -> Result<(Option, Option>)> { - let engine = match ReplicationEngine::new( + let mut engine = match ReplicationEngine::new( repl_config, Arc::clone(p2p), protocol.storage(), @@ -271,6 +271,14 @@ impl NodeBuilder { } }; + // ADR-0016: pointers replicate through the same engine. The PUT handler + // hands each newly stored paid state to it on this channel. + if let Some(service) = protocol.pointer_service() { + let (writes, fresh_writes) = tokio::sync::mpsc::unbounded_channel(); + service.attach_fresh_writes(writes); + engine.with_pointers(service.store().clone(), fresh_writes); + } + // ADR-0004: wire the engine's commitment state as the quote generator's // commitment source so quotes force their price from the live storage // commitment. Done here because the engine owns the commitment state and is diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index 839acb37..14925fdf 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -1331,10 +1331,36 @@ impl PaymentVerifier { routing_address: &XorName, paid_content: &XorName, payment_proof: &[u8], + ) -> Result<()> { + self.verify_pointer_payment_in( + routing_address, + paid_content, + payment_proof, + VerificationContext::ClientPut, + ) + .await + } + + /// As [`Self::verify_pointer_payment`], in `context`. + /// + /// Fresh replication verifies under + /// [`VerificationContext::FreshReplication`], exactly as a chunk offer does: + /// the checks are identical, the context only keeps the price-floor + /// telemetry apart. + /// + /// # Errors + /// + /// As [`Self::verify_pointer_payment`]. + pub async fn verify_pointer_payment_in( + &self, + routing_address: &XorName, + paid_content: &XorName, + payment_proof: &[u8], + context: VerificationContext, ) -> Result<()> { let target = PaymentTarget::split(*routing_address, *paid_content); match self - .verify_payment_inner(&target, Some(payment_proof), VerificationContext::ClientPut) + .verify_payment_inner(&target, Some(payment_proof), context) .await? { PaymentStatus::CachedAsVerified | PaymentStatus::PaymentVerified => Ok(()), @@ -1367,6 +1393,15 @@ impl PaymentVerifier { self.cache.insert(xorname); } + /// Mark a pointer state as paid, so verifying it needs no chain lookup. + /// The pointer counterpart of [`Self::cache_insert`]: keyed by the state + /// and routed at the address, as a real pointer payment is. + #[cfg(any(test, feature = "test-utils"))] + pub fn cache_insert_pointer(&self, routing_address: XorName, state_id: XorName) { + self.cache + .insert_key(PaymentTarget::split(routing_address, state_id)); + } + /// Mark startup content as prepaid for the in-process browser devnet. /// /// This remains crate-private and feature-gated: it is used only by diff --git a/src/pointer/service.rs b/src/pointer/service.rs index 747263c0..db1c5a28 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -37,12 +37,14 @@ use ant_protocol::chunk::{ use bytes::Bytes; use parking_lot::RwLock; use saorsa_core::P2PNode; +use tokio::sync::mpsc; use crate::error::{Error, Result}; use crate::logging::{debug, warn}; use crate::payment::PaymentVerifier; use crate::pointer::store::{Inspected, PointerStore, PutOutcome}; use crate::replication::admission; +use crate::replication::pointer::PointerFreshWrite; use crate::storage::{ChunkStore, SELF_CLOSENESS_GATE_WIDTH}; use ant_protocol::pointer::POINTER_WIRE_LEN; @@ -57,6 +59,11 @@ pub struct PointerService { /// Confirms a state was paid for. `None` in tests that exercise the merge /// rather than the payment. payments: Option>, + /// Where a newly stored paid state goes to be offered to the rest of its + /// close group (ADR-0016 replication). Attached once the replication + /// engine exists, which is after this service is built; empty where + /// nothing replicates, as in unit tests and the devnet. + fresh_writes: Arc>>>, /// The node's P2P handle, for the self-closeness gate. /// /// Attached after construction, because the node builds its protocol @@ -83,6 +90,7 @@ impl PointerService { store, chunks: None, payments: None, + fresh_writes: Arc::new(RwLock::new(None)), p2p_node: Arc::new(RwLock::new(None)), } } @@ -112,6 +120,11 @@ impl PointerService { self } + /// Hand every newly stored paid state to replication on `writes`. + pub fn attach_fresh_writes(&self, writes: mpsc::UnboundedSender) { + *self.fresh_writes.write() = Some(writes); + } + /// The store this service fronts. #[must_use] pub const fn store(&self) -> &PointerStore { @@ -197,8 +210,28 @@ impl PointerService { // The charge travels with the write. Settling it here instead would // release it the moment this future is dropped, while the blocking // transaction it started runs on and publishes the file. + let record_bytes = request.record.to_vec(); match self.store.commit(record, reservation).await { - Ok(PutOutcome::Changed) => PointerPutResponse::Success { address, state_id }, + Ok(PutOutcome::Changed) => { + // Offer it to the rest of the close group, proof included, so + // every member holds it whichever of them the client reached. + let writes = self.fresh_writes.read().clone(); + if let (Some(writes), Some(proof)) = (writes, request.payment_proof) { + if writes + .send(PointerFreshWrite { + record: record_bytes, + payment_proof: proof, + }) + .is_err() + { + debug!( + "Replication is not running; pointer {} is not offered on", + hex::encode(address) + ); + } + } + PointerPutResponse::Success { address, state_id } + } // The re-check under the commit lock found a newer state. The // client paid for a state that lost a race; say so plainly. Ok(PutOutcome::Unchanged) => PointerPutResponse::Unchanged { address, state_id }, diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 8c792851..2664f9c6 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -35,11 +35,15 @@ //! whole transaction lives in one task, dropping the caller's future cannot //! tear it. //! -//! On disk each record is one file named by its hex address under -//! `{root}/pointers/`. Writes go to a uniquely named temporary file and are -//! renamed into place, so a crash leaves either the old record or the new one, -//! never a torn one. A lock file under the same directory keeps two processes -//! from keeping two indexes over one set of files. +//! On disk each record is one file named by its hex address, under +//! `{root}/pointers//`, where the shard is the address's last byte in +//! hex: 256 directories, as the chunk store keeps, so no directory grows with +//! the whole store. The last byte rather than the first because a node holds +//! addresses near its own identity, and their leading bytes cluster. Writes go +//! to a uniquely named temporary file in the shard and are renamed into place, +//! so a crash leaves either the old record or the new one, never a torn one. A +//! lock file under `{root}/pointers/` keeps two processes from keeping two +//! indexes over one set of files. use std::collections::HashMap; use std::fs::{File, OpenOptions}; @@ -57,7 +61,7 @@ use crate::error::{Error, Result}; use crate::logging::{debug, warn}; use ant_protocol::pointer::{ParsedPointer, Pointer, PointerState, POINTER_WIRE_LEN}; -use crate::storage::Reservation; +use crate::storage::{rename_with_retry, Reservation}; /// Directory under the store root that holds pointer records. const POINTERS_DIR_NAME: &str = "pointers"; @@ -68,6 +72,26 @@ const TEMP_PREFIX: &str = ".tmp-"; /// Name of the file whose lock grants exclusive use of the directory. const LOCK_FILE_NAME: &str = ".pointer-store-lock"; +/// How many shard directories records are spread across: one per value of an +/// address's last byte. +const SHARD_COUNT: u16 = 256; + +/// The name of the shard directory `address` lives in: its last byte in hex. +fn shard_name(address: &XorName) -> String { + let last = address.last().copied().unwrap_or_default(); + format!("{last:02x}") +} + +/// The shard directory `address` lives in. +fn shard_dir(dir: &Path, address: &XorName) -> PathBuf { + dir.join(shard_name(address)) +} + +/// The file that holds the record at `address`. +fn record_path(dir: &Path, address: &XorName) -> PathBuf { + shard_dir(dir, address).join(hex::encode(address)) +} + /// What a put did. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PutOutcome { @@ -86,6 +110,52 @@ pub enum PutOutcome { Stale, } +/// What the store has done since it was opened, for telemetry. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PointerStoreStats { + /// Records written, whether new or replacing an older state. + pub written: u64, + /// Arrivals that carried the state already held. Nothing was written. + pub unchanged: u64, + /// Arrivals that lost to the state already held. Nothing was written. + pub stale: u64, + /// Records served to a reader. + pub served: u64, + /// Records removed because this node stopped being responsible for them. + pub deleted: u64, +} + +/// The live counters behind [`PointerStoreStats`]. +#[derive(Debug, Default)] +struct Counters { + written: AtomicU64, + unchanged: AtomicU64, + stale: AtomicU64, + served: AtomicU64, + deleted: AtomicU64, +} + +impl Counters { + fn count(&self, outcome: PutOutcome) { + let counter = match outcome { + PutOutcome::Changed => &self.written, + PutOutcome::Unchanged => &self.unchanged, + PutOutcome::Stale => &self.stale, + }; + counter.fetch_add(1, Ordering::Relaxed); + } + + fn snapshot(&self) -> PointerStoreStats { + PointerStoreStats { + written: self.written.load(Ordering::Relaxed), + unchanged: self.unchanged.load(Ordering::Relaxed), + stale: self.stale.load(Ordering::Relaxed), + served: self.served.load(Ordering::Relaxed), + deleted: self.deleted.load(Ordering::Relaxed), + } + } +} + /// The result of [`PointerStore::inspect`]: what an arrival claims, before any /// signature has been checked. #[derive(Debug)] @@ -135,8 +205,13 @@ struct IndexEntry { impl IndexEntry { /// Describe a validated record. fn of(record: &Pointer, generation: u64) -> Self { + Self::held(record.state(), generation) + } + + /// Describe a record held on disk in `state`. + const fn held(state: PointerState, generation: u64) -> Self { Self { - state: record.state(), + state, on_disk: true, generation, } @@ -166,6 +241,8 @@ struct Inner { write_seq: AtomicU64, /// Source of index generations, monotonic for this store's lifetime. generation: AtomicU64, + /// What the store has done, for telemetry. + counters: Counters, /// Held for the store's lifetime; releasing it releases the directory. _lock_file: File, } @@ -194,6 +271,7 @@ impl PointerStore { Error::Storage(format!("cannot create {}: {e}", scan_dir.display())) })?; let lock_file = acquire_lock(&scan_dir)?; + create_shards(&scan_dir)?; let index = scan(&scan_dir)?; Ok::<_, Error>((lock_file, index)) }) @@ -216,6 +294,7 @@ impl PointerStore { index: Mutex::new(index), write_seq: AtomicU64::new(0), generation: AtomicU64::new(next_generation), + counters: Counters::default(), _lock_file: lock_file, }), }) @@ -261,9 +340,11 @@ impl PointerStore { // word alone. if let Some(held) = self.held_state(&state.address) { if held.state_id == state.state_id { + self.inner.counters.count(PutOutcome::Unchanged); return Ok(Inspected::Unchanged(state)); } if !state.replaces(&held) { + self.inner.counters.count(PutOutcome::Stale); return Ok(Inspected::Stale(state)); } } @@ -310,9 +391,11 @@ impl PointerStore { let inner = Arc::clone(&self.inner); // The whole transaction runs in one task, so dropping this future // cannot leave the write done and the index un-updated. - spawn_blocking(move || inner.commit_blocking(&record, reservation)) + let outcome = spawn_blocking(move || inner.commit_blocking(&record, reservation)) .await - .map_err(|e| Error::Storage(format!("pointer commit panicked: {e}")))? + .map_err(|e| Error::Storage(format!("pointer commit panicked: {e}")))??; + self.inner.counters.count(outcome); + Ok(outcome) } /// Validate and store in one step, with no payment gate. @@ -381,7 +464,10 @@ impl PointerStore { }; match validated { - Ok(record) if record.address() == *address => Ok(Some(record)), + Ok(record) if record.address() == *address => { + self.inner.counters.served.fetch_add(1, Ordering::Relaxed); + Ok(Some(record)) + } Ok(_) => { warn!( "Pointer file at {} holds a record for another address; dropping it \ @@ -414,6 +500,19 @@ impl PointerStore { .map(|entry| entry.state.state_id) } + /// The state held at `address`, as the index records it, if this node can + /// serve one. + /// + /// What a peer asking "which state do you hold?" is told. Taken from the + /// index rather than the file, so answering costs no disk read; a file lost + /// since is caught by the first read, which stops the index claiming it. + #[must_use] + pub fn state(&self, address: &XorName) -> Option { + self.snapshot(address) + .filter(|entry| entry.on_disk) + .map(|entry| entry.state) + } + /// Whether a paid PUT of `state` may be taken. /// /// Any state the merge rule prefers to what is held, whatever its counter. @@ -464,6 +563,60 @@ impl PointerStore { &self.inner.dir } + /// The file the record at `address` is kept in. + #[must_use] + pub fn file_for(&self, address: &XorName) -> PathBuf { + self.path_for(address) + } + + /// Every state this node can serve, sorted by address. + /// + /// Taken from the index, which every other answer the store gives is also + /// based on. A record whose file has gone is left out: it is not something + /// this node can offer to anyone. + #[must_use] + pub fn held_states(&self) -> Vec { + let mut states: Vec = self + .inner + .index + .lock() + .values() + .filter(|entry| entry.on_disk) + .map(|entry| entry.state) + .collect(); + states.sort_unstable_by_key(|state| state.address); + states + } + + /// What the store has done since it was opened. + #[must_use] + pub fn stats(&self) -> PointerStoreStats { + self.inner.counters.snapshot() + } + + /// Remove the record at `address` and forget it entirely. + /// + /// For a node that is no longer responsible for the address. Unlike a + /// record lost to the disk, nothing about it is remembered, because the + /// node is not meant to take it back. Returns whether a file was removed, + /// so the caller can give its space back to the disk budget. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file exists and cannot be removed; the + /// record is then still held and still indexed. + pub async fn delete(&self, address: &XorName) -> Result { + let inner = Arc::clone(&self.inner); + let address = *address; + let removed = spawn_blocking(move || inner.delete_blocking(&address)) + .await + .map_err(|e| Error::Storage(format!("pointer delete panicked: {e}")))??; + if removed { + self.inner.counters.deleted.fetch_add(1, Ordering::Relaxed); + } + Ok(removed) + } + /// The state this node can actually serve at `address`, having read it /// back. /// @@ -531,11 +684,43 @@ impl PointerStore { /// Path of the file backing `address`. fn path_for(&self, address: &XorName) -> PathBuf { - self.inner.dir.join(hex::encode(address)) + record_path(&self.inner.dir, address) } } impl Inner { + /// Remove the file and the index entry for `address` under one lock, so a + /// commit can never land between the two and be forgotten on disk. + fn delete_blocking(&self, address: &XorName) -> Result { + let shard = shard_dir(&self.dir, address); + let path = record_path(&self.dir, address); + let removed = { + let mut index = self.index.lock(); + let removed = match std::fs::remove_file(&path) { + Ok(()) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, + Err(e) => { + return Err(Error::Storage(format!( + "cannot delete {}: {e}", + path.display() + ))) + } + }; + index.remove(address); + removed + }; + if removed { + if let Err(e) = sync_directory(&shard) { + warn!( + "{} is deleted, but {} could not be synced: {e}", + path.display(), + shard.display() + ); + } + } + Ok(removed) + } + /// Compare, write and re-index under one lock. /// /// Runs entirely inside a blocking task: the lock is synchronous and is @@ -555,7 +740,8 @@ impl Inner { reservation: Option, ) -> Result { let address = record.address(); - let path = self.dir.join(hex::encode(address)); + let shard = shard_dir(&self.dir, &address); + let path = record_path(&self.dir, &address); // A cheap look before doing any work. The authoritative check is the // one under the lock below; this only avoids staging a file for an @@ -574,9 +760,7 @@ impl Inner { // across it would block every other address, including the cheap // lookups async callers make. let seq = self.write_seq.fetch_add(1, Ordering::Relaxed); - let temp = self - .dir - .join(format!("{TEMP_PREFIX}{}-{seq}", hex::encode(address))); + let temp = shard.join(format!("{TEMP_PREFIX}{}-{seq}", hex::encode(address))); if let Err(failed) = stage(&temp, &record.to_bytes()) { // Bytes that could not be cleaned up are still on the disk, so the // charge for them stands rather than going back. @@ -611,7 +795,7 @@ impl Inner { // The rename is the commit point: nothing fallible happens between // it and the index update, and both are under this one lock. - if let Err(e) = std::fs::rename(&temp, &path) { + if let Err(e) = rename_with_retry(&temp, &path) { if std::fs::remove_file(&temp).is_err() { settle(reservation); } @@ -638,13 +822,13 @@ impl Inner { // cannot be reported as "nothing happened"; it is logged instead, which // is how an operator learns the filesystem is not giving the store what // it asks for. - if let Err(e) = sync_directory(&self.dir) { + if let Err(e) = sync_directory(&shard) { warn!( "{} is stored and indexed, but {} could not be synced: {e}. It is \ visible now; its survival across a power loss depends on the \ filesystem", path.display(), - self.dir.display() + shard.display() ); } Ok(outcome) @@ -714,7 +898,27 @@ fn read_record_file(path: &Path) -> Result>> { Ok(Some(bytes)) } -/// Rebuild the index by reading every record in `dir`. +/// Create the shard directories, so a write never has to. +fn create_shards(dir: &Path) -> Result<()> { + for shard in 0..SHARD_COUNT { + let path = dir.join(format!("{shard:02x}")); + std::fs::create_dir_all(&path) + .map_err(|e| Error::Storage(format!("cannot create {}: {e}", path.display())))?; + } + sync_directory(dir) +} + +/// Rebuild the index from the records in `dir`'s shards. +/// +/// Each record's structure is parsed, but its signature is not checked. Every +/// record was verified when it was committed, and every read verifies it again +/// before serving it, so a file damaged in place is caught — and disowned — the +/// first time anyone asks for it. Checking an ML-DSA signature per record here +/// instead would make opening a large store take minutes, and the chunk store +/// likewise opens on names alone. +/// +/// A record left at the top level by the flat layout earlier builds used is +/// moved into its shard. fn scan(dir: &Path) -> Result> { let entries = std::fs::read_dir(dir) .map_err(|e| Error::Storage(format!("cannot read {}: {e}", dir.display())))?; @@ -735,6 +939,38 @@ fn scan(dir: &Path) -> Result> { if name == LOCK_FILE_NAME { continue; } + if path.is_dir() { + if name.len() == 2 && name.bytes().all(|b| b.is_ascii_hexdigit()) { + scan_shard(&path, &mut index); + } + continue; + } + adopt_flat_record(dir, &path, name, &mut index); + } + Ok(index) +} + +/// Index the records in one shard directory. +fn scan_shard(shard: &Path, index: &mut HashMap) { + let entries = match std::fs::read_dir(shard) { + Ok(entries) => entries, + Err(e) => { + warn!("Skipping unreadable pointer shard {}: {e}", shard.display()); + return; + } + }; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + warn!("Skipping unreadable pointer directory entry: {e}"); + continue; + } + }; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; if name.starts_with(TEMP_PREFIX) { // A temporary file means a crash mid-write: the rename never // happened, so the old record (if any) is intact and the partial @@ -749,23 +985,14 @@ fn scan(dir: &Path) -> Result> { } continue; } - let bytes = match read_record_file(&path) { - Ok(Some(bytes)) => bytes, - Ok(None) => continue, - Err(e) => { - warn!("Skipping unreadable pointer file {}: {e}", path.display()); - continue; - } - }; - let record = match Pointer::from_bytes(&bytes) { - Ok(record) => record, - Err(e) => { - warn!("Skipping invalid pointer file {}: {e}", path.display()); - continue; - } + let Some(state) = parse_record_file(&path) else { + continue; }; - let address = record.address(); - if hex::encode(address) != name { + let in_its_shard = shard + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n == shard_name(&state.address)); + if hex::encode(state.address) != name || !in_its_shard { warn!( "Skipping pointer file {} that is not at its own address", path.display() @@ -773,9 +1000,65 @@ fn scan(dir: &Path) -> Result> { continue; } let generation = u64::try_from(index.len()).unwrap_or(u64::MAX); - index.insert(address, IndexEntry::of(&record, generation)); + index.insert(state.address, IndexEntry::held(state, generation)); + } +} + +/// Move a record from the flat layout into its shard and index it. +fn adopt_flat_record( + dir: &Path, + path: &Path, + name: &str, + index: &mut HashMap, +) { + if name.starts_with(TEMP_PREFIX) { + if let Err(e) = std::fs::remove_file(path) { + warn!( + "Could not sweep the partial pointer write {}: {e}", + path.display() + ); + } + return; + } + let Some(state) = parse_record_file(path) else { + return; + }; + if hex::encode(state.address) != name { + warn!( + "Skipping pointer file {} that is not at its own address", + path.display() + ); + return; + } + let target = record_path(dir, &state.address); + if let Err(e) = rename_with_retry(path, &target) { + warn!( + "Could not move pointer file {} into its shard: {e}", + path.display() + ); + return; + } + let generation = u64::try_from(index.len()).unwrap_or(u64::MAX); + index.insert(state.address, IndexEntry::held(state, generation)); +} + +/// Read a record file and parse its structure, logging anything unusable. +fn parse_record_file(path: &Path) -> Option { + let bytes = match read_record_file(path) { + Ok(Some(bytes)) => bytes, + Ok(None) => return None, + Err(e) => { + warn!("Skipping unreadable pointer file {}: {e}", path.display()); + return None; + } + }; + match PointerState::parse(&bytes) { + Ok(state) => Some(state), + Err(e) => { + warn!("Skipping invalid pointer file {}: {e}", path.display()); + None + } } - Ok(index) } /// Write `bytes` to `temp` and fsync it, ready to be renamed into place. @@ -960,7 +1243,7 @@ mod tests { let first = signed(1, 5, 5); store.put_bytes(&first.to_bytes()).await.expect("put"); - let path = store.dir().join(hex::encode(first.address())); + let path = store.file_for(&first.address()); let held_bytes = std::fs::read(&path).expect("read"); for _ in 0..16 { @@ -1298,7 +1581,7 @@ mod tests { let record = signed(1, 1, 1); store.put_bytes(&record.to_bytes()).await.expect("put"); - let path = store.dir().join(hex::encode(record.address())); + let path = store.file_for(&record.address()); let mut bytes = std::fs::read(&path).expect("read back"); if let Some(byte) = bytes.get_mut(10) { *byte ^= 0xff; @@ -1340,7 +1623,7 @@ mod tests { store.put_bytes(&record.to_bytes()).await.expect("put"); // The file disappears under the node; the index has not noticed. - std::fs::remove_file(store.dir().join(hex::encode(record.address()))).expect("remove"); + std::fs::remove_file(store.file_for(&record.address())).expect("remove"); assert!( store.contains(&record.address()), "the index still claims it" @@ -1369,7 +1652,7 @@ mod tests { } let held = signed(1, 3, 1); - std::fs::remove_file(store.dir().join(hex::encode(held.address()))).expect("remove"); + std::fs::remove_file(store.file_for(&held.address())).expect("remove"); // A read notices the loss and stops the node answering for it. assert!(store.get(&held.address()).await.expect("get").is_none()); assert!(!store.contains(&held.address()), "it is not served"); @@ -1421,11 +1704,8 @@ mod tests { store.put_bytes(&indexed.to_bytes()).await.expect("put"); let other = signed(1, 9, 1); - std::fs::write( - store.dir().join(hex::encode(indexed.address())), - other.to_bytes(), - ) - .expect("swap the file"); + std::fs::write(store.file_for(&indexed.address()), other.to_bytes()) + .expect("swap the file"); match store.inspect(&indexed.to_bytes()).await.expect("inspect") { Inspected::Candidate(_) => (), @@ -1439,7 +1719,7 @@ mod tests { let record = signed(1, 1, 1); store.put_bytes(&record.to_bytes()).await.expect("put"); - std::fs::remove_file(store.dir().join(hex::encode(record.address()))).expect("remove"); + std::fs::remove_file(store.file_for(&record.address())).expect("remove"); assert!(store.get(&record.address()).await.expect("get").is_none()); assert!(!store.contains(&record.address())); } @@ -1453,15 +1733,14 @@ mod tests { store.put_bytes(&record.to_bytes()).await.expect("put"); // Junk under a plausible name, an oversized file, and a partial write. - std::fs::write(store.dir().join(hex::encode([9u8; 32])), b"not a pointer") - .expect("write junk"); + std::fs::write(store.file_for(&[9u8; 32]), b"not a pointer").expect("write junk"); + std::fs::write(store.file_for(&[8u8; 32]), vec![0u8; POINTER_WIRE_LEN * 4]) + .expect("write oversized"); std::fs::write( - store.dir().join(hex::encode([8u8; 32])), - vec![0u8; POINTER_WIRE_LEN * 4], + store.file_for(&[7u8; 32]).with_file_name(".tmp-abc-0"), + vec![0u8; POINTER_WIRE_LEN], ) - .expect("write oversized"); - std::fs::write(store.dir().join(".tmp-abc-0"), vec![0u8; POINTER_WIRE_LEN]) - .expect("write temp"); + .expect("write temp"); } let reopened = PointerStore::new(dir.path()).await.expect("reopen"); @@ -1490,12 +1769,195 @@ mod tests { assert_eq!(store.len(), 1); // No temporary file survived the race. - let leftovers: Vec<_> = std::fs::read_dir(store.dir()) + assert!( + temporary_files(&store).is_empty(), + "temporary files were left behind" + ); + } + + /// Every temporary file anywhere under the store. + fn temporary_files(store: &PointerStore) -> Vec { + let mut found = Vec::new(); + for shard in std::fs::read_dir(store.dir()).expect("read dir").flatten() { + if !shard.path().is_dir() { + continue; + } + for entry in std::fs::read_dir(shard.path()) + .expect("read shard") + .flatten() + { + if entry.file_name().to_string_lossy().starts_with(TEMP_PREFIX) { + found.push(entry.path()); + } + } + } + found + } + + #[tokio::test] + async fn a_record_is_kept_in_the_shard_named_by_its_last_byte() { + let (store, _dir) = store().await; + let record = signed(3, 0, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + + let path = store.file_for(&record.address()); + let last = record.address().last().copied().expect("32 bytes"); + assert_eq!( + path.parent().expect("a shard"), + store.dir().join(format!("{last:02x}")) + ); + assert_eq!( + std::fs::read(&path).expect("read"), + record.to_bytes(), + "the record is on disk where the store says it is" + ); + let shards = std::fs::read_dir(store.dir()) .expect("read dir") - .filter_map(std::result::Result::ok) - .filter(|e| e.file_name().to_string_lossy().starts_with(TEMP_PREFIX)) - .collect(); - assert!(leftovers.is_empty(), "temporary files were left behind"); + .flatten() + .filter(|e| e.path().is_dir()) + .count(); + assert_eq!(shards, 256, "every shard exists before any write needs it"); + } + + #[tokio::test] + async fn held_states_lists_what_is_served_in_address_order() { + let (store, _dir) = store().await; + let records: Vec = (1..=5u8).map(|seed| signed(seed, 2, seed)).collect(); + for record in &records { + store.put_bytes(&record.to_bytes()).await.expect("put"); + } + let lost = records.first().expect("five records"); + std::fs::remove_file(store.file_for(&lost.address())).expect("remove"); + assert!(store.get(&lost.address()).await.expect("get").is_none()); + + let held = store.held_states(); + let mut expected: Vec = records.iter().skip(1).map(Pointer::address).collect(); + expected.sort_unstable(); + assert_eq!( + held.iter().map(|state| state.address).collect::>(), + expected, + "sorted, and without the record whose file is gone" + ); + for state in &held { + assert_eq!(store.state_id(&state.address), Some(state.state_id)); + } + } + + #[tokio::test] + async fn delete_removes_the_record_and_forgets_it() { + let (store, _dir) = store().await; + let record = signed(4, 3, 1); + store.put_bytes(&record.to_bytes()).await.expect("put"); + let path = store.file_for(&record.address()); + + assert!(store.delete(&record.address()).await.expect("delete")); + assert!(!path.exists(), "the file is gone"); + assert!(!store.contains(&record.address())); + assert!(store.get(&record.address()).await.expect("get").is_none()); + // Forgotten, not remembered as lost: an older state is admissible again, + // which a lost record would refuse. + assert!(store.admits(&signed(4, 1, 1).state())); + assert!( + !store.delete(&record.address()).await.expect("delete"), + "nothing left to delete" + ); + assert_eq!(store.stats().deleted, 1); + + // Still usable afterwards, and gone across a reopen too. + store + .put_bytes(&record.to_bytes()) + .await + .expect("put again"); + assert!(store.contains(&record.address())); + store.delete(&record.address()).await.expect("delete"); + let dir = store.dir().parent().expect("root").to_path_buf(); + drop(store); + let reopened = PointerStore::new(&dir).await.expect("reopen"); + assert!(!reopened.contains(&record.address())); + } + + #[tokio::test] + async fn stats_count_what_each_arrival_did() { + let (store, _dir) = store().await; + let first = signed(5, 1, 1); + let newer = signed(5, 2, 1); + store.put_bytes(&first.to_bytes()).await.expect("put"); + store.put_bytes(&newer.to_bytes()).await.expect("put"); + store + .put_bytes(&signed(5, 2, 1).to_bytes()) + .await + .expect("put"); + store.put_bytes(&first.to_bytes()).await.expect("put"); + store.get(&first.address()).await.expect("get"); + + assert_eq!( + store.stats(), + PointerStoreStats { + written: 2, + unchanged: 1, + stale: 1, + served: 1, + deleted: 0, + } + ); + } + + #[tokio::test] + async fn opening_indexes_records_without_verifying_them_and_a_read_still_does() { + // The scan parses structure only. A signature damaged in place is + // indexed, and caught by the first read, which disowns it. + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(6, 1, 1); + let path = { + let store = PointerStore::new(dir.path()).await.expect("open"); + store.put_bytes(&record.to_bytes()).await.expect("put"); + store.file_for(&record.address()) + }; + let mut bytes = std::fs::read(&path).expect("read"); + if let Some(byte) = bytes.get_mut(POINTER_BODY_LEN + 7) { + *byte ^= 0xff; + } + std::fs::write(&path, &bytes).expect("damage"); + + let reopened = PointerStore::new(dir.path()).await.expect("reopen"); + assert!( + reopened.contains(&record.address()), + "indexed on its structure" + ); + assert!( + reopened + .get(&record.address()) + .await + .expect("get") + .is_none(), + "never served" + ); + assert!( + !reopened.contains(&record.address()), + "and disowned once read" + ); + } + + #[tokio::test] + async fn a_record_in_the_flat_layout_is_moved_into_its_shard() { + let dir = tempfile::tempdir().expect("tempdir"); + let record = signed(7, 4, 2); + let flat = dir + .path() + .join(POINTERS_DIR_NAME) + .join(hex::encode(record.address())); + std::fs::create_dir_all(flat.parent().expect("parent")).expect("mkdir"); + std::fs::write(&flat, record.to_bytes()).expect("write"); + + let store = PointerStore::new(dir.path()).await.expect("open"); + assert!(!flat.exists(), "moved out of the top level"); + assert!(store.file_for(&record.address()).exists(), "into its shard"); + let held = store + .get(&record.address()) + .await + .expect("get") + .expect("present"); + assert_eq!(held.state_id(), record.state_id()); } #[tokio::test] diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 54498cdc..c4da9831 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -25,6 +25,7 @@ pub mod config; pub mod fresh; pub mod neighbor_sync; pub mod paid_list; +pub mod pointer; pub mod possession; pub mod protocol; pub mod pruning; @@ -1815,6 +1816,10 @@ pub struct ReplicationEngine { /// When present, `start()` spawns a drainer task that calls /// `replicate_fresh` for each event. fresh_write_rx: Option>, + /// Pointer replication (ADR-0016), when this node stores pointers. + pointers: Option>, + /// Receiver for fresh pointer writes, taken by `start()`. + pointer_fresh_rx: Option>, /// Sender for delayed possession-check events (ADR-0003). The fresh-write /// drainer pushes the responsible close-group peers here after each fresh /// replication; the possession-check scheduler drains the paired receiver. @@ -1948,6 +1953,8 @@ impl ReplicationEngine { config.subtree_round1_max_concurrent, ), fresh_write_rx: Some(fresh_write_rx), + pointers: None, + pointer_fresh_rx: None, possession_check_tx, possession_check_rx: Some(possession_check_rx), monetized_pin_tx, @@ -2205,6 +2212,36 @@ impl ReplicationEngine { }) } + /// Replicate pointers too (ADR-0016): the records in `store`, and the + /// fresh writes the pointer PUT handler sends on `fresh_writes`. + /// + /// Call before [`Self::start`]. + pub fn with_pointers( + &mut self, + store: crate::pointer::store::PointerStore, + fresh_writes: mpsc::UnboundedReceiver, + ) { + self.pointers = Some(Arc::new(pointer::PointerReplication::new( + store, + Arc::clone(&self.storage), + Arc::clone(&self.p2p_node), + Arc::clone(&self.payment_verifier), + Arc::clone(&self.config), + Arc::clone(&self.is_bootstrapping), + Arc::clone(&self.send_semaphore), + self.shutdown.clone(), + self.detached_task_tracker.clone(), + ))); + self.pointer_fresh_rx = Some(fresh_writes); + } + + /// The pointer replication, when enabled. Tests use it to drive rounds. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn pointer_replication(&self) -> Option<&Arc> { + self.pointers.as_ref() + } + /// Start all background tasks. /// /// `dht_events` must be subscribed **before** `P2PNode::start()` so that @@ -2232,6 +2269,12 @@ impl ReplicationEngine { self.start_bootstrap_sync(dht_events); self.start_fresh_write_drainer(); self.start_possession_check_scheduler(); + if let Some(pointers) = &self.pointers { + self.task_handles.push(pointers.start_verification_loop()); + if let Some(writes) = self.pointer_fresh_rx.take() { + self.task_handles.push(pointers.start_fresh_drainer(writes)); + } + } // ADR-0004: deterministic first audit of commitments that backed a // payment (surfaced by the verifier cross-check). self.start_first_audit_drainer(); @@ -2870,6 +2913,7 @@ impl ReplicationEngine { paid_notify_worker_semaphore, paid_notify_admission_semaphore, paid_notify_responder_inflight, + pointers: self.pointers.clone(), shutdown: shutdown.clone(), detached_task_tracker, }; @@ -3101,6 +3145,9 @@ impl ReplicationEngine { DhtNetworkEvent::PeerRemoved { peer_id } => { sync_state.write().await.remove_peer(&peer_id); repair_proofs.write().await.remove_peer(&peer_id); + if let Some(pointers) = &handler_context.pointers { + pointers.forget_peer(&peer_id); + } update_bootstrap_after_peer_removed( &peer_id, &handler_context.bootstrap_state, @@ -3140,6 +3187,7 @@ impl ReplicationEngine { } fn start_neighbor_sync_loop(&mut self) { + let pointers = self.pointers.clone(); let p2p = Arc::clone(&self.p2p_node); let storage = Arc::clone(&self.storage); let paid_list = Arc::clone(&self.paid_list); @@ -3216,6 +3264,7 @@ impl ReplicationEngine { &sig_verify_attempts, &audit_challenge_coordinator, &gossip_audit, + pointers.as_ref(), ) => {} } } @@ -4310,6 +4359,8 @@ struct ReplicationMessageHandlerContext { paid_notify_worker_semaphore: Arc, paid_notify_admission_semaphore: Arc, paid_notify_responder_inflight: Arc>>, + /// Pointer replication, when this node stores pointers. + pointers: Option>, /// The engine's shutdown token, for detached responder work. /// /// Workers on [`Self::detached_task_tracker`] race this around their @@ -4399,6 +4450,12 @@ const fn replication_message_class(body: &ReplicationMessageBody) -> &'static st ReplicationMessageBody::SubtreeSliceResponse(_) => "subtree_slice_response", ReplicationMessageBody::GetCommitmentByPin(_) => "commitment_pin_request", ReplicationMessageBody::GetCommitmentByPinResponse(_) => "commitment_pin_response", + ReplicationMessageBody::PointerFreshOffer(_) => "pointer_fresh_offer", + ReplicationMessageBody::PointerHints(_) => "pointer_hints", + ReplicationMessageBody::PointerFetchRequest(_) => "pointer_fetch_request", + ReplicationMessageBody::PointerFetchResponse(_) => "pointer_fetch_response", + ReplicationMessageBody::PointerStateRequest(_) => "pointer_state_request", + ReplicationMessageBody::PointerStateResponse(_) => "pointer_state_response", } } @@ -5461,6 +5518,42 @@ async fn handle_replication_message( drop(guard); Ok(()) } + // Pointers (ADR-0016). A node that does not store pointers ignores + // them, and so is never marked capable and never asked anything. + ReplicationMessageBody::PointerFreshOffer(offer) => { + if let Some(pointers) = &ctx.pointers { + pointers.accept_offer_detached(*source, offer); + } + Ok(()) + } + ReplicationMessageBody::PointerHints(hints) => { + if let Some(pointers) = &ctx.pointers { + pointers.handle_hints(*source, hints.hints).await; + } + Ok(()) + } + ReplicationMessageBody::PointerFetchRequest(request) => { + if let Some(pointers) = &ctx.pointers { + pointers.serve_fetch_detached( + *source, + request, + msg.request_id, + rr_message_id.map(ToOwned::to_owned), + ); + } + Ok(()) + } + ReplicationMessageBody::PointerStateRequest(request) => { + if let Some(pointers) = &ctx.pointers { + pointers.serve_state_detached( + *source, + request, + msg.request_id, + rr_message_id.map(ToOwned::to_owned), + ); + } + Ok(()) + } // Response messages are handled by their respective request initiators. ReplicationMessageBody::FreshReplicationResponse(_) | ReplicationMessageBody::NeighborSyncResponse(_) @@ -5469,7 +5562,9 @@ async fn handle_replication_message( | ReplicationMessageBody::AuditResponse(_) | ReplicationMessageBody::SubtreeAuditResponse(_) | ReplicationMessageBody::SubtreeSliceResponse(_) - | ReplicationMessageBody::GetCommitmentByPinResponse(_) => Ok(()), + | ReplicationMessageBody::GetCommitmentByPinResponse(_) + | ReplicationMessageBody::PointerFetchResponse(_) + | ReplicationMessageBody::PointerStateResponse(_) => Ok(()), } } @@ -6426,6 +6521,12 @@ async fn dispatch_neighbor_sync_request( received_at: Instant, rr_message_id: Option<&str>, ) -> Result<()> { + // A peer syncing with us gets our pointer hints as well — including a + // node that is bootstrapping, which is how it learns the pointers it + // should hold. + if let Some(pointers) = &ctx.pointers { + pointers.push_hints_detached(vec![source]); + } let guard = match admit_bounded_responder( &ctx.neighbor_sync_responder_admission_semaphore, &ctx.neighbor_sync_responder_inflight, @@ -7390,6 +7491,7 @@ async fn run_neighbor_sync_round( sig_verify_attempts: &Arc>>, audit_challenge_coordinator: &Arc, gossip_audit: &GossipAuditTrigger, + pointers: Option<&Arc>, ) { let self_id = *p2p_node.peer_id(); let bootstrapping = *is_bootstrapping.read().await; @@ -7431,6 +7533,9 @@ async fn run_neighbor_sync_round( audit_challenge_coordinator, }) .await; + if let Some(pointers) = pointers { + pointers.prune_pass(allow_remote_prune_audits).await; + } // Take fresh close-neighbor snapshot (DHT query, no lock held). let neighbors = @@ -7470,6 +7575,9 @@ async fn run_neighbor_sync_round( } debug!("Neighbor sync: syncing with {} peers", batch.len()); + if let Some(pointers) = pointers { + pointers.push_hints_detached(batch.clone()); + } // Snapshot our current commitment once per round so all peers in // this batch see the same thing (gossip is the responder's attestation; diff --git a/src/replication/pointer.rs b/src/replication/pointer.rs new file mode 100644 index 00000000..ef14321f --- /dev/null +++ b/src/replication/pointer.rs @@ -0,0 +1,1284 @@ +//! Pointer replication (ADR-0016). +//! +//! Pointers ride the same replication machinery chunks do — the same close +//! groups, the same neighbour-sync rounds and churn triggers, the same quorum, +//! pruning and possession rules — but not the chunk pipeline itself. That +//! pipeline assumes a record never changes and that its key is the hash of its +//! bytes. A pointer's address is stable while its state changes, and two honest +//! replicas may hold different valid signatures over one state. So a pointer is +//! replicated by *state*: +//! +//! - **Fresh.** A node that accepts a paid state from a client forwards the +//! record, with the proof that paid for it, to the rest of the close group. +//! Each receiver checks the signature, its own responsibility and the payment +//! itself before storing it. +//! - **Repair.** Every neighbour-sync round pushes hints: the states the sender +//! holds that the receiver should hold. A receiver that lacks a hinted state, +//! or holds an older one, asks the close group which state each holds, adopts +//! the best state a quorum of them hold exactly, and fetches it from one of +//! them. The record verifies itself; the quorum stands in for the payment +//! proof, as presence quorum does for a chunk. +//! - **Pruning.** A record this node has been out of range of for the +//! hysteresis period is deleted — at once if the node is far outside the +//! group, otherwise only once all but one of the current close group prove +//! they hold that state or a newer one by returning a valid record. +//! - **Possession.** Some minutes after offering a fresh state, the node asks +//! each close-group member for the record. A member that is still responsible +//! and cannot produce that state or a newer one is penalised. +//! +//! Requests go only to peers that have sent a pointer message themselves (see +//! [`PointerReplication::is_capable`]). A peer built before pointers cannot +//! decode them, and saorsa-core counts an unanswered request against the peer +//! it was sent to; asking only capable peers keeps older peers out of it. Every +//! neighbour-sync round pushes hints, empty or not, so capability is learned +//! within a cycle. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use ant_protocol::pointer::{Pointer, PointerState, POINTER_WIRE_LEN}; +use futures::stream::{self, StreamExt}; +use parking_lot::Mutex; +use rand::Rng; +use saorsa_core::identity::PeerId; +use saorsa_core::{P2PNode, TrustEvent}; +use tokio::sync::{mpsc, RwLock, Semaphore}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use crate::ant_protocol::XorName; +use crate::logging::{debug, info, warn}; +use crate::payment::{ + PaymentVerifier, VerificationContext, MAX_PAYMENT_PROOF_SIZE_BYTES, + MIN_PAYMENT_PROOF_SIZE_BYTES, +}; +use crate::pointer::store::{Inspected, PointerStore}; +use crate::replication::admission; +use crate::replication::config::{ + storage_admission_width, ReplicationConfig, FRESH_REPLICATION_DELIVERY_MAX_RETRIES, + REPLICATION_PROTOCOL_ID, +}; +use crate::replication::protocol::{ + PointerFetchRequest, PointerFetchResponse, PointerFreshOffer, PointerHints, + PointerStateRequest, PointerStateResponse, PointerStateSummary, ReplicationMessage, + ReplicationMessageBody, MAX_POINTER_HINTS_PER_MESSAGE, MAX_POINTER_STATE_REQUEST_ADDRESSES, +}; +use crate::replication::pruning::prune_proofs_needed; +use crate::storage::{CapacityVerdict, ChunkStore}; + +use super::REPLICATION_TRUST_WEIGHT; + +/// Inbound fresh offers verified at once. An offer costs a signature check and +/// an on-chain payment lookup, so a flood is dropped rather than queued; one +/// that is dropped is repaired by the next neighbour-sync round. +const MAX_CONCURRENT_OFFERS: usize = 16; + +/// Fetch and state requests served at once. +const MAX_CONCURRENT_SERVES: usize = 32; + +/// How often the pending hints are looked at. +const VERIFICATION_TICK: Duration = Duration::from_millis(500); + +/// Most addresses awaiting verification at once, so a hint flood cannot grow +/// memory without bound. A hint dropped here comes back next round. +const MAX_PENDING: usize = 65_536; + +/// Most addresses verified per tick. +const MAX_VERIFICATIONS_PER_TICK: usize = 256; + +/// Verifications and fetches in flight at once within a tick. +const VERIFICATION_CONCURRENCY: usize = 8; + +/// Undecided rounds an address gets before it is dropped. It comes back with +/// the next hint. +const MAX_UNDECIDED_ROUNDS: u32 = 5; + +/// Wait before retrying an undecided address, doubled each round. +const UNDECIDED_BACKOFF: Duration = Duration::from_secs(30); + +/// Most prune candidates examined per pass. +const MAX_PRUNE_CANDIDATES_PER_PASS: usize = 256; + +/// One peer's answer about one address: the state it holds there, if any. +type StateAnswer = ((PeerId, XorName), Option); + +/// A pointer state this node accepted from a paying client, to be offered to +/// the rest of its close group. +pub struct PointerFreshWrite { + /// The record, in its canonical encoding. + pub record: Vec, + /// The proof that paid for its state. + pub payment_proof: Vec, +} + +/// A state this node was told about and has not yet verified. +#[derive(Debug, Clone)] +struct Pending { + /// The best state hinted so far. + wanted: PointerState, + /// Undecided rounds so far. + attempts: u32, + /// Not to be looked at again before this. + not_before: Instant, +} + +/// What a verification round decided for one address. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Verdict { + /// A quorum of the close group hold this state and it beats what is held + /// here: fetch it from one of `holders`. + Adopt { + /// The state to fetch. + state: PointerState, + /// The peers that said they hold exactly it. + holders: Vec, + }, + /// Nothing reached quorum, but the peers that did not answer could still + /// make one. Ask again later. + Undecided, + /// No state that beats what is held can reach quorum. + Refused, +} + +/// Decide what a round of answers supports. +/// +/// `group_size` is the whole close group this node would ask, capable or not: +/// a peer that cannot be asked counts as unanswered, not as a vote. The quorum +/// is therefore the one a chunk would need, and a network where few peers +/// understand pointers yet cannot repair on the word of those few. +pub(crate) fn evaluate( + held: Option<&PointerState>, + group_size: usize, + answers: &[(PeerId, Option)], + quorum_needed: usize, +) -> Verdict { + if group_size == 0 || quorum_needed == 0 { + return Verdict::Undecided; + } + let mut by_state: Vec<(PointerState, Vec)> = Vec::new(); + for (peer, answer) in answers { + let Some(state) = answer else { continue }; + match by_state + .iter_mut() + .find(|(known, _)| known.state_id == state.state_id) + { + Some((_, holders)) => holders.push(*peer), + None => by_state.push((*state, vec![*peer])), + } + } + let beats_held = |state: &PointerState| held.is_none_or(|held| state.replaces(held)); + + let best = by_state + .iter() + .filter(|(state, holders)| holders.len() >= quorum_needed && beats_held(state)) + .fold( + None::<&(PointerState, Vec)>, + |best, candidate| match best { + Some(current) if !candidate.0.replaces(¤t.0) => Some(current), + _ => Some(candidate), + }, + ); + if let Some((state, holders)) = best { + return Verdict::Adopt { + state: *state, + holders: holders.clone(), + }; + } + + let unanswered = group_size.saturating_sub(answers.len()); + let largest = by_state + .iter() + .filter(|(state, _)| beats_held(state)) + .map(|(_, holders)| holders.len()) + .max() + .unwrap_or(0); + if largest.saturating_add(unanswered) >= quorum_needed { + Verdict::Undecided + } else { + Verdict::Refused + } +} + +/// Pointer replication for one node. See the module documentation. +pub struct PointerReplication { + store: PointerStore, + chunks: Arc, + p2p: Arc, + payments: Arc, + config: Arc, + is_bootstrapping: Arc>, + /// Outbound record transfers, shared with chunk replication. + send_semaphore: Arc, + offer_permits: Arc, + serve_permits: Arc, + /// Peers that have sent a pointer message: the only ones asked anything. + capable: Mutex>, + /// Hinted states awaiting verification, by address. + pending: Mutex>, + /// When each held address was first seen continuously out of range. + out_of_range: Mutex>, + shutdown: CancellationToken, + tracker: TaskTracker, +} + +impl PointerReplication { + /// Pointer replication over `store`, sharing the engine's resources. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + store: PointerStore, + chunks: Arc, + p2p: Arc, + payments: Arc, + config: Arc, + is_bootstrapping: Arc>, + send_semaphore: Arc, + shutdown: CancellationToken, + tracker: TaskTracker, + ) -> Self { + Self { + store, + chunks, + p2p, + payments, + config, + is_bootstrapping, + send_semaphore, + offer_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OFFERS)), + serve_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_SERVES)), + capable: Mutex::new(HashSet::new()), + pending: Mutex::new(HashMap::new()), + out_of_range: Mutex::new(HashMap::new()), + shutdown, + tracker, + } + } + + // ----------------------------------------------------------------------- + // Capability + // ----------------------------------------------------------------------- + + fn mark_capable(&self, peer: &PeerId) { + self.capable.lock().insert(*peer); + } + + /// Whether `peer` has sent a pointer message, and may therefore be asked. + pub fn is_capable(&self, peer: &PeerId) -> bool { + self.capable.lock().contains(peer) + } + + /// Forget a peer that left the routing table. + pub(crate) fn forget_peer(&self, peer: &PeerId) { + self.capable.lock().remove(peer); + } + + /// Addresses currently awaiting verification. Tests only. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn pending_len(&self) -> usize { + self.pending.lock().len() + } + + // ----------------------------------------------------------------------- + // Messaging + // ----------------------------------------------------------------------- + + async fn send_one_way(&self, peer: &PeerId, body: ReplicationMessageBody) -> bool { + let msg = ReplicationMessage { + request_id: rand::thread_rng().gen::(), + body, + }; + let Ok(bytes) = msg.encode() else { + warn!("Failed to encode a pointer replication message"); + return false; + }; + self.p2p + .send_message(peer, REPLICATION_PROTOCOL_ID, bytes, &[]) + .await + .is_ok() + } + + async fn request( + &self, + peer: &PeerId, + body: ReplicationMessageBody, + timeout: Duration, + ) -> Option { + let msg = ReplicationMessage { + request_id: rand::thread_rng().gen::(), + body, + }; + let bytes = msg.encode().ok()?; + let response = self + .p2p + .send_request(peer, REPLICATION_PROTOCOL_ID, bytes, timeout) + .await + .ok()?; + ReplicationMessage::decode(&response.data) + .ok() + .map(|msg| msg.body) + } + + async fn penalise(&self, peer: &PeerId) { + self.p2p + .report_trust_event( + peer, + TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), + ) + .await; + } + + /// Ask `peer` for the record it holds at `address`, and check what comes + /// back: a valid signature and the right address. Anything else counts + /// against the peer. + async fn fetch_record(&self, peer: &PeerId, address: &XorName) -> Option { + let body = self + .request( + peer, + ReplicationMessageBody::PointerFetchRequest(PointerFetchRequest { + address: *address, + }), + self.config.fetch_request_timeout, + ) + .await?; + let ReplicationMessageBody::PointerFetchResponse(response) = body else { + return None; + }; + if response.address != *address { + return None; + } + let bytes = response.record?; + match Pointer::from_bytes(&bytes) { + Ok(record) if record.address() == *address => Some(record), + Ok(_) | Err(_) => { + debug!( + "Peer {peer} served an invalid pointer record for {}", + hex::encode(address) + ); + self.penalise(peer).await; + None + } + } + } + + // ----------------------------------------------------------------------- + // Hints + // ----------------------------------------------------------------------- + + /// Push hints to `peers` in the background: for each, the states this + /// node holds whose close group, in this node's view, includes that peer. + /// + /// Every peer gets at least one message, empty or not. That is how a peer + /// learns this node understands pointers. + pub(crate) fn push_hints_detached(self: &Arc, peers: Vec) { + if peers.is_empty() { + return; + } + let this = Arc::clone(self); + self.tracker + .spawn(async move { this.push_hints(&peers).await }); + } + + /// Push hints to `peers` and wait until they are sent. What + /// [`Self::push_hints_detached`] runs; tests call it to drive a round. + pub async fn push_hints(&self, peers: &[PeerId]) { + let self_id = *self.p2p.peer_id(); + let mut by_peer: HashMap> = + peers.iter().map(|peer| (*peer, Vec::new())).collect(); + for state in self.store.held_states() { + if self.shutdown.is_cancelled() { + return; + } + let group = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self(&state.address, self.config.close_group_size) + .await; + for node in group { + if node.peer_id == self_id { + continue; + } + if let Some(hints) = by_peer.get_mut(&node.peer_id) { + hints.push(state.into()); + } + } + } + for (peer, hints) in by_peer { + let batches: Vec> = if hints.is_empty() { + vec![Vec::new()] + } else { + hints + .chunks(MAX_POINTER_HINTS_PER_MESSAGE) + .map(<[PointerStateSummary]>::to_vec) + .collect() + }; + for hints in batches { + self.send_one_way( + &peer, + ReplicationMessageBody::PointerHints(PointerHints { hints }), + ) + .await; + } + } + } + + /// Take in hints from `source`: queue each hinted state this node should + /// hold and lacks, or holds an older state than. + pub(crate) async fn handle_hints(&self, source: PeerId, hints: Vec) { + self.mark_capable(&source); + if hints.is_empty() { + return; + } + // As for chunks: only a peer in our own routing table may tell us what + // to hold. + if !self.p2p.dht_manager().is_in_routing_table(&source).await { + debug!("Dropping pointer hints from {source}: not in the routing table"); + return; + } + let self_id = *self.p2p.peer_id(); + let width = storage_admission_width(self.config.close_group_size); + for summary in hints.into_iter().take(MAX_POINTER_HINTS_PER_MESSAGE) { + let hinted = PointerState::from(summary); + match self.store.state(&hinted.address) { + Some(held) => { + if held.state_id == hinted.state_id || !hinted.replaces(&held) { + continue; + } + } + None => { + if !admission::is_responsible(&self_id, &hinted.address, &self.p2p, width).await + { + continue; + } + } + } + self.enqueue(hinted); + } + } + + fn enqueue(&self, hinted: PointerState) { + let mut pending = self.pending.lock(); + let len = pending.len(); + match pending.get_mut(&hinted.address) { + Some(entry) => { + if hinted.replaces(&entry.wanted) { + entry.wanted = hinted; + entry.attempts = 0; + entry.not_before = Instant::now(); + } + } + None if len < MAX_PENDING => { + pending.insert( + hinted.address, + Pending { + wanted: hinted, + attempts: 0, + not_before: Instant::now(), + }, + ); + } + None => {} + } + } + + // ----------------------------------------------------------------------- + // Verification and fetch + // ----------------------------------------------------------------------- + + /// Start the loop that verifies and fetches hinted states. + pub(crate) fn start_verification_loop(self: &Arc) -> JoinHandle<()> { + let this = Arc::clone(self); + tokio::spawn(async move { + loop { + tokio::select! { + () = this.shutdown.cancelled() => break, + () = tokio::time::sleep(VERIFICATION_TICK) => this.verify_due().await, + } + } + }) + } + + /// Verify and fetch whatever is due now. Also the tests' way to drive it. + pub async fn verify_due(&self) { + // As for chunks (ADR-0011): a node that cannot store what it would + // fetch does not spend the network's time finding it. The hints wait, + // and come back each round if they are dropped meanwhile. + if self.chunks.capacity_verdict() == CapacityVerdict::Full { + return; + } + let now = Instant::now(); + let due: Vec<(XorName, PointerState)> = self + .pending + .lock() + .iter() + .filter(|(_, entry)| entry.not_before <= now) + .take(MAX_VERIFICATIONS_PER_TICK) + .map(|(address, entry)| (*address, entry.wanted)) + .collect(); + if due.is_empty() { + return; + } + + let self_id = *self.p2p.peer_id(); + let mut groups: HashMap> = HashMap::new(); + let mut asked: HashMap> = HashMap::new(); + for (address, _) in &due { + let group: Vec = self + .p2p + .dht_manager() + .find_closest_nodes_local(address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .filter(|peer| *peer != self_id) + .collect(); + for peer in group.iter().filter(|peer| self.is_capable(peer)) { + asked.entry(*peer).or_default().push(*address); + } + groups.insert(*address, group); + } + + let answers = self.ask_states(asked).await; + + let decisions: Vec<(XorName, Verdict)> = due + .iter() + .map(|(address, _)| { + let group = groups.get(address).map_or(&[][..], Vec::as_slice); + let answered: Vec<(PeerId, Option)> = group + .iter() + .filter_map(|peer| { + answers + .get(&(*peer, *address)) + .map(|answer| (*peer, *answer)) + }) + .collect(); + let held = self.store.state(address); + let verdict = evaluate( + held.as_ref(), + group.len(), + &answered, + self.config.quorum_needed(group.len()), + ); + (*address, verdict) + }) + .collect(); + + let outcomes: Vec<(XorName, bool)> = stream::iter(decisions) + .map(|(address, verdict)| async move { + match verdict { + Verdict::Adopt { state, holders } => { + let stored = self.fetch_and_store(address, state, &holders).await; + (address, stored) + } + Verdict::Undecided => (address, false), + Verdict::Refused => { + debug!( + "No quorum backs a newer state for pointer {}", + hex::encode(address) + ); + self.pending.lock().remove(&address); + (address, true) + } + } + }) + .buffer_unordered(VERIFICATION_CONCURRENCY) + .collect() + .await; + + let now = Instant::now(); + let mut pending = self.pending.lock(); + for (address, settled) in outcomes { + if settled { + pending.remove(&address); + continue; + } + if let Some(entry) = pending.get_mut(&address) { + entry.attempts = entry.attempts.saturating_add(1); + if entry.attempts >= MAX_UNDECIDED_ROUNDS { + pending.remove(&address); + } else { + let backoff = UNDECIDED_BACKOFF + .saturating_mul(2u32.saturating_pow(entry.attempts.saturating_sub(1))); + entry.not_before = now + backoff; + } + } + } + } + + /// Ask each peer which state it holds at the addresses listed for it. + async fn ask_states( + &self, + asked: HashMap>, + ) -> HashMap<(PeerId, XorName), Option> { + let requests: Vec<(PeerId, Vec)> = asked + .into_iter() + .flat_map(|(peer, addresses)| { + addresses + .chunks(MAX_POINTER_STATE_REQUEST_ADDRESSES) + .map(|chunk| (peer, chunk.to_vec())) + .collect::>() + }) + .collect(); + let replies: Vec> = stream::iter(requests) + .map(|(peer, addresses)| async move { + let body = self + .request( + &peer, + ReplicationMessageBody::PointerStateRequest(PointerStateRequest { + addresses: addresses.clone(), + }), + self.config.verification_request_timeout, + ) + .await; + let Some(ReplicationMessageBody::PointerStateResponse(response)) = body else { + return Vec::new(); + }; + addresses + .iter() + .zip(response.states) + .map(|(address, summary)| { + // An answer about a different address is no answer. + let state = summary + .filter(|summary| summary.address == *address) + .map(PointerState::from); + ((peer, *address), state) + }) + .collect() + }) + .buffer_unordered(VERIFICATION_CONCURRENCY) + .collect() + .await; + replies.into_iter().flatten().collect() + } + + /// Fetch `wanted` from one of `holders` and store it. Whether it is now + /// held, or nothing further can come of this round. + async fn fetch_and_store( + &self, + address: XorName, + wanted: PointerState, + holders: &[PeerId], + ) -> bool { + for holder in holders { + let Some(record) = self.fetch_record(holder, &address).await else { + continue; + }; + // The quorum backed this state. A holder that has moved on since + // serves a different one, which that quorum says nothing about. + if record.state_id() != wanted.state_id { + continue; + } + match self.store_verified(record, None).await { + Ok(_) => return true, + Err(e) => { + warn!( + "Could not store replicated pointer {}: {e}", + hex::encode(address) + ); + return false; + } + } + } + false + } + + /// Store a record whose signature has been checked, if it still belongs + /// here and beats what is held. + async fn store_verified( + &self, + record: Pointer, + width: Option, + ) -> crate::error::Result { + let state = record.state(); + let self_id = *self.p2p.peer_id(); + let width = width.unwrap_or_else(|| storage_admission_width(self.config.close_group_size)); + let held = self.store.state(&state.address).is_some(); + if !held && !admission::is_responsible(&self_id, &state.address, &self.p2p, width).await { + return Ok(false); + } + if !self.store.admits(&state) { + return Ok(false); + } + let reservation = self.chunks.reserve(POINTER_WIRE_LEN as u64)?; + self.store.commit(record, Some(reservation)).await?; + Ok(true) + } + + // ----------------------------------------------------------------------- + // Serving + // ----------------------------------------------------------------------- + + /// Answer a fetch request in the background. + pub(crate) fn serve_fetch_detached( + self: &Arc, + source: PeerId, + request: PointerFetchRequest, + request_id: u64, + rr_message_id: Option, + ) { + self.mark_capable(&source); + let this = Arc::clone(self); + self.tracker.spawn(async move { + let Ok(_permit) = this.serve_permits.acquire().await else { + return; + }; + // `get` verifies the signature before serving, so a record damaged + // on this disk is never handed on. + let record = match this.store.get(&request.address).await { + Ok(Some(record)) => Some(record.to_bytes()), + Ok(None) | Err(_) => None, + }; + super::send_replication_response( + &source, + &this.p2p, + request_id, + ReplicationMessageBody::PointerFetchResponse(PointerFetchResponse { + address: request.address, + record, + }), + rr_message_id.as_deref(), + ) + .await; + }); + } + + /// Answer a state request in the background, from the index. + pub(crate) fn serve_state_detached( + self: &Arc, + source: PeerId, + request: PointerStateRequest, + request_id: u64, + rr_message_id: Option, + ) { + self.mark_capable(&source); + let this = Arc::clone(self); + self.tracker.spawn(async move { + let Ok(_permit) = this.serve_permits.acquire().await else { + return; + }; + let states = request + .addresses + .iter() + .take(MAX_POINTER_STATE_REQUEST_ADDRESSES) + .map(|address| this.store.state(address).map(PointerStateSummary::from)) + .collect(); + super::send_replication_response( + &source, + &this.p2p, + request_id, + ReplicationMessageBody::PointerStateResponse(PointerStateResponse { states }), + rr_message_id.as_deref(), + ) + .await; + }); + } + + // ----------------------------------------------------------------------- + // Fresh replication + // ----------------------------------------------------------------------- + + /// Start the loop that offers freshly paid states to their close groups. + pub(crate) fn start_fresh_drainer( + self: &Arc, + mut writes: mpsc::UnboundedReceiver, + ) -> JoinHandle<()> { + let this = Arc::clone(self); + tokio::spawn(async move { + loop { + tokio::select! { + () = this.shutdown.cancelled() => break, + write = writes.recv() => match write { + Some(write) => this.replicate_fresh(write).await, + None => break, + }, + } + } + }) + } + + /// Offer a freshly paid state to the rest of its close group, and schedule + /// the check that they took it. + pub async fn replicate_fresh(self: &Arc, write: PointerFreshWrite) { + let Ok(state) = PointerState::parse(&write.record) else { + warn!("A fresh pointer write did not parse; not replicating it"); + return; + }; + let self_id = *self.p2p.peer_id(); + let targets: Vec = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self(&state.address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .filter(|peer| *peer != self_id) + .collect(); + + let msg = ReplicationMessage { + request_id: rand::thread_rng().gen::(), + body: ReplicationMessageBody::PointerFreshOffer(PointerFreshOffer { + record: write.record, + proof_of_payment: write.payment_proof, + }), + }; + let Ok(encoded) = msg.encode() else { + warn!( + "Failed to encode a fresh pointer offer for {}", + hex::encode(state.address) + ); + return; + }; + let encoded = Arc::new(encoded); + for peer in &targets { + let p2p = Arc::clone(&self.p2p); + let bytes = Arc::clone(&encoded); + let semaphore = Arc::clone(&self.send_semaphore); + let peer = *peer; + self.tracker.spawn(async move { + let Ok(_permit) = semaphore.acquire().await else { + return; + }; + for attempt in 0..=FRESH_REPLICATION_DELIVERY_MAX_RETRIES { + match p2p + .send_message(&peer, REPLICATION_PROTOCOL_ID, bytes.as_ref().clone(), &[]) + .await + { + Ok(()) => break, + Err(e) => debug!( + "Fresh pointer offer to {peer} failed (attempt {}): {e}", + attempt + 1 + ), + } + } + }); + } + debug!( + "Fresh pointer {} offered to {} peers", + hex::encode(state.address), + targets.len() + ); + self.schedule_possession_check(state, targets); + } + + /// Take a fresh offer from `source` in the background, unless too many are + /// already being verified. + pub(crate) fn accept_offer_detached( + self: &Arc, + source: PeerId, + offer: PointerFreshOffer, + ) { + self.mark_capable(&source); + let Ok(permit) = Arc::clone(&self.offer_permits).try_acquire_owned() else { + debug!("Dropping a fresh pointer offer from {source}: too many in flight"); + return; + }; + let this = Arc::clone(self); + self.tracker.spawn(async move { + let _permit = permit; + this.accept_offer(source, offer).await; + }); + } + + /// Verify and store a fresh offer. Cheapest checks first; the signature and + /// the on-chain payment lookup only for a state that would change something + /// here and that this node is responsible for. + async fn accept_offer(&self, source: PeerId, offer: PointerFreshOffer) { + if offer.record.len() != POINTER_WIRE_LEN + || !(MIN_PAYMENT_PROOF_SIZE_BYTES..=MAX_PAYMENT_PROOF_SIZE_BYTES) + .contains(&offer.proof_of_payment.len()) + { + debug!("Dropping a malformed fresh pointer offer from {source}"); + self.penalise(&source).await; + return; + } + let parsed = match self.store.inspect(&offer.record).await { + Ok(Inspected::Candidate(parsed)) => parsed, + Ok(Inspected::Unchanged(_) | Inspected::Stale(_)) => return, + Err(e) => { + debug!("Dropping an unparseable fresh pointer offer from {source}: {e}"); + self.penalise(&source).await; + return; + } + }; + let state = *parsed.state(); + let self_id = *self.p2p.peer_id(); + // As for a chunk: a fresh offer is taken across the wider paid width. + if !admission::is_responsible( + &self_id, + &state.address, + &self.p2p, + self.config.paid_list_close_group_size, + ) + .await + { + return; + } + if !self.store.admits(&state) + || self + .chunks + .check_capacity_for(POINTER_WIRE_LEN as u64) + .is_err() + { + return; + } + let record = match self.store.verify(parsed).await { + Ok(record) => record, + Err(e) => { + debug!("Fresh pointer offer from {source} does not verify: {e}"); + self.penalise(&source).await; + return; + } + }; + if let Err(e) = self + .payments + .verify_pointer_payment_in( + &state.address, + &state.state_id, + &offer.proof_of_payment, + VerificationContext::FreshReplication, + ) + .await + { + debug!( + "Fresh pointer offer for {} from {source} is not paid for: {e}", + hex::encode(state.address) + ); + return; + } + match self + .store_verified(record, Some(self.config.paid_list_close_group_size)) + .await + { + Ok(true) => debug!( + "Stored fresh pointer {} from {source}", + hex::encode(state.address) + ), + Ok(false) => {} + Err(e) => warn!( + "Could not store fresh pointer {}: {e}", + hex::encode(state.address) + ), + } + } + + // ----------------------------------------------------------------------- + // Possession + // ----------------------------------------------------------------------- + + fn schedule_possession_check(self: &Arc, fresh: PointerState, peers: Vec) { + if peers.is_empty() { + return; + } + let min = self.config.possession_check_delay_min; + let max = self.config.possession_check_delay_max.max(min); + let delay = if max > min { + rand::thread_rng().gen_range(min..=max) + } else { + min + }; + let this = Arc::clone(self); + self.tracker.spawn(async move { + tokio::select! { + () = this.shutdown.cancelled() => return, + () = tokio::time::sleep(delay) => {} + } + this.check_possession(fresh, &peers).await; + }); + } + + /// Ask each peer for the record, and penalise any still responsible for it + /// that cannot produce `fresh` or a state that replaces it. + pub async fn check_possession(&self, fresh: PointerState, peers: &[PeerId]) { + let self_id = *self.p2p.peer_id(); + let group: HashSet = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self(&fresh.address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .collect(); + for peer in peers { + // A peer that has left the group owes nothing, and one that cannot + // be asked is never judged on its silence. + if *peer == self_id || !group.contains(peer) || !self.is_capable(peer) { + continue; + } + let holds = self + .fetch_record(peer, &fresh.address) + .await + .is_some_and(|record| { + let state = record.state(); + state.state_id == fresh.state_id || state.replaces(&fresh) + }); + if !holds { + warn!( + "Peer {peer} does not hold pointer {} it was offered", + hex::encode(fresh.address) + ); + self.penalise(peer).await; + } + } + } + + // ----------------------------------------------------------------------- + // Pruning + // ----------------------------------------------------------------------- + + /// Delete records this node has been out of range of for the hysteresis + /// period, where that is safe. Run when a neighbour-sync cycle completes. + /// + /// `allow_remote` is false while bootstrapping: candidacy is still tracked, + /// but nothing that needs other peers' proof is decided. + pub async fn prune_pass(&self, allow_remote: bool) { + let self_id = *self.p2p.peer_id(); + let retention = storage_admission_width(self.config.close_group_size); + let now = Instant::now(); + let held = self.store.held_states(); + + let mut candidates = Vec::new(); + for state in &held { + if admission::is_responsible(&self_id, &state.address, &self.p2p, retention).await { + self.out_of_range.lock().remove(&state.address); + continue; + } + let first_seen = *self.out_of_range.lock().entry(state.address).or_insert(now); + if now.duration_since(first_seen) >= self.config.prune_hysteresis_duration { + candidates.push(*state); + } + } + { + let held: HashSet = held.iter().map(|state| state.address).collect(); + self.out_of_range + .lock() + .retain(|address, _| held.contains(address)); + } + + let mut deleted = 0usize; + for state in candidates.into_iter().take(MAX_PRUNE_CANDIDATES_PER_PASS) { + if self.shutdown.is_cancelled() { + return; + } + let wide = self + .p2p + .dht_manager() + .find_closest_nodes_local_with_self( + &state.address, + self.config.paid_list_close_group_size, + ) + .await; + let far = wide.len() >= self.config.paid_list_close_group_size + && !wide.iter().any(|node| node.peer_id == self_id); + let confirmed = if far { + true + } else if allow_remote && !*self.is_bootstrapping.read().await { + self.others_hold(&state).await + } else { + false + }; + // Revalidate just before deleting: the group may have moved back. + if confirmed + && !admission::is_responsible(&self_id, &state.address, &self.p2p, retention).await + && self.delete(&state.address).await + { + deleted += 1; + } + } + if deleted > 0 { + info!("Pruned {deleted} pointer records this node is no longer responsible for"); + } + } + + /// Whether all but one of the current close group prove they hold `ours` + /// or a newer state, by returning a valid record. + async fn others_hold(&self, ours: &PointerState) -> bool { + let group: Vec = self + .p2p + .dht_manager() + .find_closest_nodes_local(&ours.address, self.config.close_group_size) + .await + .into_iter() + .map(|node| node.peer_id) + .collect(); + let needed = prune_proofs_needed(group.len()); + if needed == 0 { + return false; + } + let mut proofs = 0usize; + for peer in group.iter().filter(|peer| self.is_capable(peer)) { + let proves = self + .fetch_record(peer, &ours.address) + .await + .is_some_and(|record| { + let state = record.state(); + state.state_id == ours.state_id || state.replaces(ours) + }); + if proves { + proofs += 1; + if proofs >= needed { + return true; + } + } + } + false + } + + async fn delete(&self, address: &XorName) -> bool { + self.out_of_range.lock().remove(address); + match self.store.delete(address).await { + Ok(true) => { + self.chunks.release(POINTER_WIRE_LEN as u64); + true + } + Ok(false) => false, + Err(e) => { + warn!("Could not prune pointer {}: {e}", hex::encode(address)); + false + } + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use ant_protocol::pointer::{PointerTarget, PointerTargetKind}; + + fn state(counter: u64, target: u8, id: u8) -> PointerState { + PointerState { + state_id: [id; 32], + address: [7; 32], + counter, + target: PointerTarget::new(PointerTargetKind::Chunk, [target; 32]), + } + } + + fn peer(byte: u8) -> PeerId { + PeerId::from_bytes([byte; 32]) + } + + fn answers(list: &[(u8, Option)]) -> Vec<(PeerId, Option)> { + list.iter().map(|(p, s)| (peer(*p), *s)).collect() + } + + #[test] + fn a_state_a_quorum_holds_is_adopted_from_its_holders() { + let newer = state(3, 1, 30); + let got = evaluate( + None, + 7, + &answers(&[ + (1, Some(newer)), + (2, Some(newer)), + (3, Some(newer)), + (4, Some(newer)), + (5, None), + ]), + 4, + ); + match got { + Verdict::Adopt { state, holders } => { + assert_eq!(state.state_id, newer.state_id); + assert_eq!(holders.len(), 4); + } + other => panic!("expected Adopt, got {other:?}"), + } + } + + #[test] + fn a_state_below_quorum_is_never_adopted() { + // One peer, or three, holding a state is not the network's word. + let lone = state(9, 1, 90); + let got = evaluate( + None, + 7, + &answers(&[ + (1, Some(lone)), + (2, Some(lone)), + (3, Some(lone)), + (4, None), + (5, None), + (6, None), + (7, None), + ]), + 4, + ); + assert_eq!(got, Verdict::Refused); + + // With one peer silent, three holders plus that peer could still make + // four: undecided, not refused. + let got = evaluate( + None, + 7, + &answers(&[ + (1, Some(lone)), + (2, Some(lone)), + (3, Some(lone)), + (4, None), + (5, None), + (6, None), + ]), + 4, + ); + assert_eq!(got, Verdict::Undecided); + } + + #[test] + fn silence_is_not_a_vote_so_an_unanswered_group_stays_undecided() { + let newer = state(3, 1, 30); + let got = evaluate(None, 7, &answers(&[(1, Some(newer)), (2, Some(newer))]), 4); + assert_eq!(got, Verdict::Undecided); + } + + #[test] + fn nothing_older_than_what_is_held_is_adopted() { + let held = state(5, 1, 50); + let older = state(4, 1, 40); + let got = evaluate( + Some(&held), + 7, + &answers(&[ + (1, Some(older)), + (2, Some(older)), + (3, Some(older)), + (4, Some(older)), + (5, Some(older)), + ]), + 4, + ); + assert_eq!(got, Verdict::Refused); + } + + #[test] + fn of_two_states_with_quorum_the_merge_winner_is_adopted() { + let lower = state(2, 9, 20); + let winner = state(2, 1, 21); + assert!(winner.replaces(&lower)); + let got = evaluate( + None, + 8, + &answers(&[ + (1, Some(lower)), + (2, Some(lower)), + (3, Some(lower)), + (4, Some(lower)), + (5, Some(winner)), + (6, Some(winner)), + (7, Some(winner)), + (8, Some(winner)), + ]), + 4, + ); + match got { + Verdict::Adopt { state, .. } => assert_eq!(state.state_id, winner.state_id), + other => panic!("expected the winner, got {other:?}"), + } + } + + #[test] + fn a_group_nobody_can_be_asked_in_is_undecided() { + assert_eq!(evaluate(None, 0, &[], 0), Verdict::Undecided); + } +} diff --git a/src/replication/protocol.rs b/src/replication/protocol.rs index 3ef7d0fa..59afd178 100644 --- a/src/replication/protocol.rs +++ b/src/replication/protocol.rs @@ -217,6 +217,25 @@ pub enum ReplicationMessageBody { GetCommitmentByPin(GetCommitmentByPin), /// Response to [`Self::GetCommitmentByPin`]. GetCommitmentByPinResponse(GetCommitmentByPinResponse), + + // === Pointers (ADR-0016) === + // APPENDED for the same reason: every earlier discriminant keeps its value. + // A peer built before pointers cannot decode these and drops them. The two + // one-way pushes are then simply lost, and the requests are only ever sent + // to peers that have sent a pointer message themselves, so an older peer is + // never asked something it cannot answer and never penalised for silence. + /// A newly paid pointer state, with the proof that paid for it. + PointerFreshOffer(PointerFreshOffer), + /// The pointer states the sender holds that the receiver should hold too. + PointerHints(PointerHints), + /// Ask a peer for the pointer record it holds at an address. + PointerFetchRequest(PointerFetchRequest), + /// Response to [`Self::PointerFetchRequest`]. + PointerFetchResponse(PointerFetchResponse), + /// Ask a peer which state it holds at each of some addresses. + PointerStateRequest(PointerStateRequest), + /// Response to [`Self::PointerStateRequest`]. + PointerStateResponse(PointerStateResponse), } // --------------------------------------------------------------------------- @@ -235,7 +254,7 @@ pub enum ReplicationMessageBody { // modules that do not carry any shared engine handle. /// Number of [`ReplicationMessageBody`] variants (the counter-table width). -pub(crate) const N_REPLICATION_VARIANTS: usize = 17; +pub(crate) const N_REPLICATION_VARIANTS: usize = 23; static REPL_TX_BYTES: [AtomicU64; N_REPLICATION_VARIANTS] = [const { AtomicU64::new(0) }; N_REPLICATION_VARIANTS]; @@ -270,6 +289,12 @@ impl ReplicationMessageBody { Self::SubtreeSliceResponse(_) => 14, Self::GetCommitmentByPin(_) => 15, Self::GetCommitmentByPinResponse(_) => 16, + Self::PointerFreshOffer(_) => 17, + Self::PointerHints(_) => 18, + Self::PointerFetchRequest(_) => 19, + Self::PointerFetchResponse(_) => 20, + Self::PointerStateRequest(_) => 21, + Self::PointerStateResponse(_) => 22, } } @@ -336,7 +361,7 @@ impl BodyFamily { pub(crate) fn family_of_variant(index: usize) -> Option { match index { 11..=14 => Some(BodyFamily::SubtreeAudit), - 0..=10 | 15 | 16 => Some(BodyFamily::Core), + 0..=10 | 15..=22 => Some(BodyFamily::Core), _ => None, } } @@ -984,6 +1009,117 @@ pub enum GetCommitmentByPinResponse { }, } +// --------------------------------------------------------------------------- +// Pointer Messages (ADR-0016) +// --------------------------------------------------------------------------- + +/// Most hints one [`PointerHints`] message carries. A sender with more splits +/// them across messages; a receiver ignores anything past this in one message. +pub const MAX_POINTER_HINTS_PER_MESSAGE: usize = 8192; + +/// Most addresses one [`PointerStateRequest`] asks about. A responder answers +/// only this many, and the requester treats the rest as unanswered. +pub const MAX_POINTER_STATE_REQUEST_ADDRESSES: usize = 512; + +/// A pointer state, as a hint or as an answer about what a node holds. +/// +/// The fields the merge rule orders by, and the state they identify, so a +/// receiver can tell whether a hinted state would replace what it holds without +/// fetching it. Unauthenticated on its own: nothing is stored on the strength +/// of a summary. A receiver acts on one only through quorum verification and a +/// fetched record whose signature it checks itself. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PointerStateSummary { + /// The pointer's address. + pub address: XorName, + /// The state's identifier. + pub state_id: XorName, + /// The update counter, the merge rule's first key. + pub counter: u64, + /// The target's kind tag. + pub target_tag: u8, + /// The target's address. With the tag, the merge rule's second key. + pub target_address: XorName, +} + +impl From for PointerStateSummary { + fn from(state: ant_protocol::pointer::PointerState) -> Self { + Self { + address: state.address, + state_id: state.state_id, + counter: state.counter, + target_tag: state.target.kind_tag(), + target_address: state.target.address, + } + } +} + +impl From for ant_protocol::pointer::PointerState { + fn from(summary: PointerStateSummary) -> Self { + Self { + state_id: summary.state_id, + address: summary.address, + counter: summary.counter, + target: ant_protocol::pointer::PointerTarget::from_raw_tag( + summary.target_tag, + summary.target_address, + ), + } + } +} + +/// A newly paid pointer state, with the proof that paid for it. +/// +/// Sent by a node that accepted the state from a paying client to the rest of +/// the pointer's close group, which verifies the proof itself. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerFreshOffer { + /// The pointer record, in its canonical encoding. + pub record: Vec, + /// Serialized proof of payment for the record's state. + pub proof_of_payment: Vec, +} + +/// The pointer states the sender holds that the receiver should hold too. +/// +/// Sent on every neighbour-sync round, empty if there is nothing to say: an +/// empty message still tells the receiver the sender understands pointers. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerHints { + /// The states, at most [`MAX_POINTER_HINTS_PER_MESSAGE`] of them. + pub hints: Vec, +} + +/// Ask a peer for the pointer record it holds at an address. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct PointerFetchRequest { + /// The pointer's address. + pub address: XorName, +} + +/// Response to [`PointerFetchRequest`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerFetchResponse { + /// Echo of the requested address. + pub address: XorName, + /// The record held there, if any. + pub record: Option>, +} + +/// Ask a peer which state it holds at each of some addresses. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerStateRequest { + /// The addresses, at most [`MAX_POINTER_STATE_REQUEST_ADDRESSES`]. + pub addresses: Vec, +} + +/// Response to [`PointerStateRequest`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerStateResponse { + /// One entry per requested address, in order: the state held, or `None`. + pub states: Vec>, +} + // --------------------------------------------------------------------------- // Audit Messages // --------------------------------------------------------------------------- @@ -1572,6 +1708,18 @@ mod tests { ReplicationMessageBody::GetCommitmentByPinResponse( GetCommitmentByPinResponse::NotRetained { pin: z }, ), + ReplicationMessageBody::PointerFreshOffer(PointerFreshOffer { + record: vec![], + proof_of_payment: vec![], + }), + ReplicationMessageBody::PointerHints(PointerHints { hints: vec![] }), + ReplicationMessageBody::PointerFetchRequest(PointerFetchRequest { address: z }), + ReplicationMessageBody::PointerFetchResponse(PointerFetchResponse { + address: z, + record: None, + }), + ReplicationMessageBody::PointerStateRequest(PointerStateRequest { addresses: vec![] }), + ReplicationMessageBody::PointerStateResponse(PointerStateResponse { states: vec![] }), ] } @@ -1769,7 +1917,7 @@ mod tests { fn an_undeclared_variant_index_classifies_as_nothing() { let declared = all_bodies().len(); assert_eq!( - declared, 17, + declared, 23, "update this test's bounds when a variant is added or removed" ); for index in [declared, declared + 1, 99, usize::MAX] { diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs index bfe1f035..fe297705 100644 --- a/src/storage/chunk_store.rs +++ b/src/storage/chunk_store.rs @@ -1036,6 +1036,14 @@ impl ChunkStore { self.files.reserve_bytes(bytes) } + /// Credit `bytes` deleted from the pointer store back to the disk budget. + /// + /// The counterpart of [`Self::reserve`]: a record the pointer store removes + /// gives its charge back, as a deleted chunk does. + pub(crate) fn release(&self, bytes: u64) { + self.files.release_bytes(bytes); + } + /// `(written_since, in_flight)` from the capacity guard. Tests only. #[cfg(test)] pub(crate) fn capacity_counters(&self) -> (u64, u64) { diff --git a/src/storage/file_store.rs b/src/storage/file_store.rs index d0e369b9..3172bdda 100644 --- a/src/storage/file_store.rs +++ b/src/storage/file_store.rs @@ -1513,6 +1513,12 @@ impl FileStore { } } + /// Credit `len` bytes that were deleted outside this store back to the + /// cached measurement. The pointer store keeps its own files on this disk. + pub(crate) fn release_bytes(&self, len: u64) { + self.capacity.record_removed(len); + } + /// Reject work early when the disk cannot take `bytes` more. /// /// # Errors @@ -2483,7 +2489,7 @@ fn is_windows_sharing_violation(e: &std::io::Error) -> bool { /// file for a few milliseconds after it is created, and `MoveFileEx` fails outright /// rather than queueing. Retrying a bounded number of times turns that from a failed /// write into a short pause. Every other error returns immediately. -fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { +pub(crate) fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { let mut last = match std::fs::rename(temp_path, final_path) { Ok(()) => return Ok(()), Err(e) => e, diff --git a/src/storage/mod.rs b/src/storage/mod.rs index ae795c6e..fc493393 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -57,7 +57,7 @@ pub(crate) mod traffic; pub use crate::ant_protocol::XorName; pub use chunk_store::{ChunkStore, ChunkStoreConfig, VerifyReport, LEGACY_ENV_DIR}; -pub(crate) use file_store::Reservation; +pub(crate) use file_store::{rename_with_retry, Reservation}; pub use file_store::{FileStore, FileStoreConfig, StoreLayout}; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs index 74d34637..ec2a74c7 100644 --- a/tests/e2e/mod.rs +++ b/tests/e2e/mod.rs @@ -60,6 +60,9 @@ mod merkle_payment; #[cfg(test)] mod replication; +// Pointer replication across real nodes (ADR-0016). +mod pointer_replication; + #[cfg(test)] mod security_attacks; diff --git a/tests/e2e/pointer_replication.rs b/tests/e2e/pointer_replication.rs new file mode 100644 index 00000000..4992ccc5 --- /dev/null +++ b/tests/e2e/pointer_replication.rs @@ -0,0 +1,555 @@ +//! Pointer replication across real nodes (ADR-0016). +//! +//! Each test runs a live network whose nodes carry the same pointer service and +//! replication engine a production node does, over real QUIC. Payment is +//! pre-marked in each node's verifier cache — the chain is not what is under +//! test here — but every other check a node makes runs for real: signatures, +//! responsibility, the merge rule, quorum and the capability gate. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::testnet::{TestNetworkConfig, TestNode}; +use super::TestHarness; +use ant_node::ant_protocol::chunk::{ + ChunkMessage, ChunkMessageBody, PointerPutRequest, PointerPutResponse, +}; +use ant_node::pointer::PointerStore; +use ant_node::replication::pointer::{PointerFreshWrite, PointerReplication}; +use ant_node::ReplicationConfig; +use ant_protocol::pointer::{Pointer, PointerState, PointerTarget, PointerTargetKind}; +use bytes::Bytes; +use saorsa_core::identity::PeerId; +use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; +use serial_test::serial; +use std::sync::Arc; +use std::time::Duration; + +/// How long to wait for replication to reach a node. +const SETTLE: Duration = Duration::from_secs(30); + +/// How often to look while waiting. +const POLL: Duration = Duration::from_millis(200); + +/// A proof the receivers never parse: the state is pre-marked as paid in each +/// verifier's cache, so verification answers from the cache. +const DUMMY_PROOF: [u8; 64] = [0x01; 64]; + +fn owner() -> (MlDsaPublicKey, MlDsaSecretKey) { + ml_dsa_65().generate_keypair().expect("keypair") +} + +fn signed(pk: &MlDsaPublicKey, sk: &MlDsaSecretKey, counter: u64, target: u8) -> Pointer { + Pointer::sign( + sk, + pk, + counter, + PointerTarget::new(PointerTargetKind::Chunk, [target; 32]), + ) + .expect("sign") +} + +fn store(node: &TestNode) -> PointerStore { + node.ant_protocol + .as_ref() + .expect("protocol") + .pointer_service() + .expect("pointer service") + .store() + .clone() +} + +fn replication(node: &TestNode) -> &Arc { + node.replication_engine + .as_ref() + .expect("engine") + .pointer_replication() + .expect("pointer replication") +} + +fn peer(node: &TestNode) -> PeerId { + *node.p2p_node.as_ref().expect("p2p").peer_id() +} + +fn held(node: &TestNode, record: &Pointer) -> Option { + store(node).state(&record.address()) +} + +fn holds(node: &TestNode, record: &Pointer) -> bool { + held(node, record).is_some_and(|state| state.state_id == record.state_id()) +} + +/// Mark `record`'s state as paid on every node, as a settled payment would. +fn mark_paid(harness: &TestHarness, record: &Pointer) { + for i in 0..harness.node_count() { + if let Some(protocol) = harness.test_node(i).and_then(|n| n.ant_protocol.as_ref()) { + protocol + .payment_verifier() + .cache_insert_pointer(record.address(), record.state_id()); + } + } +} + +/// Every node's index, except `except`. +fn others(harness: &TestHarness, except: &[usize]) -> Vec { + (0..harness.node_count()) + .filter(|i| !except.contains(i)) + .collect() +} + +/// Wait until `node` holds exactly `record`'s state. +async fn wait_for(harness: &TestHarness, index: usize, record: &Pointer) -> bool { + let deadline = tokio::time::Instant::now() + SETTLE; + while tokio::time::Instant::now() < deadline { + if harness + .test_node(index) + .is_some_and(|node| holds(node, record)) + { + return true; + } + tokio::time::sleep(POLL).await; + } + false +} + +/// Send a paid pointer PUT to one node through its request handler, as a +/// client's PUT arrives, and return its answer. +async fn put(node: &TestNode, record: &Pointer) -> PointerPutResponse { + let message = ChunkMessage { + request_id: 1, + body: ChunkMessageBody::PointerPutRequest(PointerPutRequest::with_payment( + Bytes::from(record.to_bytes()), + DUMMY_PROOF.to_vec(), + )), + }; + let reply = node + .ant_protocol + .as_ref() + .expect("protocol") + .try_handle_request(&message.encode().expect("encode")) + .await + .expect("handle") + .expect("answered"); + match ChunkMessage::decode(&reply).expect("decode").body { + ChunkMessageBody::PointerPutResponse(response) => response, + other => panic!("expected a pointer PUT response, got {other:?}"), + } +} + +/// Have every node in `from` push its hints to every node in `to`, so each +/// learns the others understand pointers and hears what they hold. +async fn exchange_hints(harness: &TestHarness, from: &[usize], to: &[usize]) { + let targets: Vec = to + .iter() + .filter_map(|i| harness.test_node(*i).map(peer)) + .collect(); + for i in from { + let node = harness.test_node(*i).expect("node"); + let own = peer(node); + let peers: Vec = targets.iter().copied().filter(|p| *p != own).collect(); + replication(node).push_hints(&peers).await; + } + // One-way messages: give them a moment to land. + tokio::time::sleep(Duration::from_millis(500)).await; +} + +/// A client's paid PUT reaches one node, and replication carries it to every +/// other member of the close group: the node that accepted it offers it on, +/// with the proof, and each receiver verifies and stores it. +#[tokio::test] +#[serial] +async fn a_paid_put_to_one_node_reaches_its_whole_close_group() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 0, 1); + mark_paid(&harness, &record); + + let entry = 3; + match put(harness.test_node(entry).expect("node"), &record).await { + PointerPutResponse::Success { state_id, .. } => assert_eq!(state_id, record.state_id()), + other => panic!("the entry node refused the PUT: {other:?}"), + } + + // In a five-node network every node is in every close group. + for i in others(&harness, &[entry]) { + assert!( + wait_for(&harness, i, &record).await, + "node {i} never received the pointer" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// An update written to one node replaces the old state on every node. +#[tokio::test] +#[serial] +async fn an_update_replicates_and_replaces_the_old_state_everywhere() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let created = signed(&pk, &sk, 0, 1); + let updated = signed(&pk, &sk, 4, 2); + mark_paid(&harness, &created); + mark_paid(&harness, &updated); + + put(harness.test_node(2).expect("node"), &created).await; + for i in 0..harness.node_count() { + assert!( + wait_for(&harness, i, &created).await, + "node {i} lacks the create" + ); + } + put(harness.test_node(4).expect("node"), &updated).await; + for i in 0..harness.node_count() { + assert!( + wait_for(&harness, i, &updated).await, + "node {i} still holds the old state" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// A node that missed an update is brought level by neighbour sync: the +/// holders hint the newer state, it asks the group which state each holds, +/// adopts the one a quorum hold, and fetches it from one of them. +#[tokio::test] +#[serial] +async fn a_node_that_missed_an_update_is_repaired_by_neighbour_sync() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let created = signed(&pk, &sk, 0, 1); + let missed = signed(&pk, &sk, 3, 2); + let lagging = 4; + let current = others(&harness, &[lagging]); + + // Everyone holds the create; everyone but the laggard holds the update, + // written straight into their stores so nothing replicates it. + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&created.to_bytes()) + .await + .expect("put"); + } + for i in ¤t { + store(harness.test_node(*i).expect("node")) + .put_bytes(&missed.to_bytes()) + .await + .expect("put"); + } + assert!(holds(harness.test_node(lagging).expect("node"), &created)); + + // Nothing but hints reaches the laggard: no fresh write carries the + // update, so holding it afterwards means it was repaired. The engine's + // own verification loop runs too; driving a round here only saves waiting. + exchange_hints(&harness, ¤t, &[lagging]).await; + replication(harness.test_node(lagging).expect("node")) + .verify_due() + .await; + + assert!( + wait_for(&harness, lagging, &missed).await, + "the lagging node never caught up" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A node that joins after a pointer was written obtains it, with nothing but +/// the engine's own loops: its bootstrap sync reaches its neighbours, they +/// push their hints back, and it verifies and fetches. +#[tokio::test] +#[serial] +async fn a_node_that_joins_later_obtains_the_existing_pointers() { + let mut harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 2, 1); + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + + let joined = harness.add_node().await.expect("add node"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(90); + let mut obtained = false; + while tokio::time::Instant::now() < deadline { + if harness + .test_node(joined) + .is_some_and(|node| holds(node, &record)) + { + obtained = true; + break; + } + // Stand in for the periodic timer, which runs every ten minutes. + for i in 0..harness.node_count() { + if let Some(engine) = harness + .test_node(i) + .and_then(|n| n.replication_engine.as_ref()) + { + engine.trigger_neighbor_sync(); + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + assert!(obtained, "the joining node never obtained the pointer"); + + harness.teardown().await.expect("teardown"); +} + +/// A newer state that only one node holds is not adopted by the rest, however +/// well signed: one peer's word is not the network's. It is exactly what an +/// owner who got one node to store an unpaid state would try. +#[tokio::test] +#[serial] +async fn a_state_only_one_node_holds_is_not_adopted() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let agreed = signed(&pk, &sk, 1, 1); + let lone = signed(&pk, &sk, 9, 9); + let lone_holder = 0; + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&agreed.to_bytes()) + .await + .expect("put"); + } + store(harness.test_node(lone_holder).expect("node")) + .put_bytes(&lone.to_bytes()) + .await + .expect("put"); + + let rest = others(&harness, &[lone_holder]); + let everyone: Vec = (0..harness.node_count()).collect(); + exchange_hints(&harness, &everyone, &everyone).await; + for i in &rest { + replication(harness.test_node(*i).expect("node")) + .verify_due() + .await; + } + tokio::time::sleep(Duration::from_secs(2)).await; + + let lone_peer = peer(harness.test_node(lone_holder).expect("node")); + for i in &rest { + let node = harness.test_node(*i).expect("node"); + // The lone holder's hints did arrive, so what follows is the quorum + // refusing, not a hint that never came. + assert!( + replication(node).is_capable(&lone_peer), + "node {i} never heard from the lone holder" + ); + assert!( + holds(node, &agreed), + "node {i} adopted a state only one node holds" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// A fresh offer whose state was never paid for is refused by every receiver. +#[tokio::test] +#[serial] +async fn a_fresh_offer_that_was_not_paid_for_is_refused() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let unpaid = signed(&pk, &sk, 0, 1); + let source = 3; + let node = harness.test_node(source).expect("node"); + store(node) + .put_bytes(&unpaid.to_bytes()) + .await + .expect("put"); + replication(node) + .replicate_fresh(PointerFreshWrite { + record: unpaid.to_bytes(), + payment_proof: DUMMY_PROOF.to_vec(), + }) + .await; + tokio::time::sleep(Duration::from_secs(5)).await; + + for i in others(&harness, &[source]) { + assert!( + held(harness.test_node(i).expect("node"), &unpaid).is_none(), + "node {i} stored an unpaid pointer" + ); + } + + harness.teardown().await.expect("teardown"); +} + +/// Some minutes after offering a fresh state, the offering node asks each +/// close-group member for it. A member that cannot produce it is penalised, +/// and one that can is not. +#[tokio::test] +#[serial] +async fn the_possession_check_penalises_only_a_member_that_dropped_the_record() { + let harness = TestHarness::setup_minimal().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 0, 1); + let checker = 3; + let dropper = 1; + let keeper = 2; + for i in 0..harness.node_count() { + store(harness.test_node(i).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + store(harness.test_node(dropper).expect("node")) + .delete(&record.address()) + .await + .expect("delete"); + + // The checker only asks peers that understand pointers. + let everyone: Vec = (0..harness.node_count()).collect(); + exchange_hints(&harness, &everyone, &[checker]).await; + + let checker_node = harness.test_node(checker).expect("node"); + let checker_p2p = checker_node.p2p_node.as_ref().expect("p2p"); + let dropper_peer = peer(harness.test_node(dropper).expect("node")); + let keeper_peer = peer(harness.test_node(keeper).expect("node")); + let dropper_before = checker_p2p.peer_trust(&dropper_peer); + let keeper_before = checker_p2p.peer_trust(&keeper_peer); + + replication(checker_node) + .check_possession(record.state(), &[dropper_peer, keeper_peer]) + .await; + + assert!( + checker_p2p.peer_trust(&dropper_peer) < dropper_before, + "the member that dropped the record was not penalised" + ); + assert!( + checker_p2p.peer_trust(&keeper_peer) >= keeper_before, + "the member that holds the record was penalised" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A network whose close group is two nodes, so that in five nodes some node +/// is always outside the retention width (two plus the margin of two) for any +/// address, with no pruning hysteresis. +fn prune_network(paid_width: usize) -> TestNetworkConfig { + TestNetworkConfig { + replication_config: Some(ReplicationConfig { + close_group_size: 2, + // The quorum may not exceed the group. + quorum_threshold: 2, + paid_list_close_group_size: paid_width, + prune_hysteresis_duration: Duration::ZERO, + ..ReplicationConfig::default() + }), + ..TestNetworkConfig::minimal() + } +} + +/// The node, out of five, that is outside the retention width for `record`. +async fn out_of_range_node(harness: &TestHarness, record: &Pointer) -> usize { + let view = harness.test_node(0).expect("node"); + let p2p = view.p2p_node.as_ref().expect("p2p"); + let inside: Vec = p2p + .dht_manager() + .find_closest_nodes_local_with_self(&record.address(), 4) + .await + .into_iter() + .map(|node| node.peer_id) + .collect(); + (0..harness.node_count()) + .find(|i| { + harness + .test_node(*i) + .is_some_and(|node| !inside.contains(&peer(node))) + }) + .expect("one node is outside a width of four in five") +} + +/// Pruning deletes a record the node is no longer responsible for once the +/// current close group prove they hold it, and keeps it while they do not. +#[tokio::test] +#[serial] +async fn pruning_deletes_only_once_the_close_group_proves_it_holds_the_record() { + // A paid width no five-node network can complete, so deletion needs the + // close group's proof rather than taking the far-away fast path. + let harness = TestHarness::setup_with_config(prune_network(20)) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 1, 1); + let pruner = out_of_range_node(&harness, &record).await; + store(harness.test_node(pruner).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + + // Nobody else holds it yet: nothing proves it is safe to drop. + let everyone: Vec = (0..harness.node_count()).collect(); + exchange_hints(&harness, &everyone, &[pruner]).await; + let pruning = replication(harness.test_node(pruner).expect("node")); + pruning.prune_pass(true).await; + assert!( + holds(harness.test_node(pruner).expect("node"), &record), + "the only copy was pruned" + ); + + // Once the rest hold it, the close group proves it and the pruner drops it. + for i in others(&harness, &[pruner]) { + store(harness.test_node(i).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + pruning.prune_pass(true).await; + assert!( + held(harness.test_node(pruner).expect("node"), &record).is_none(), + "the record was not pruned though the close group holds it" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A node outside even a complete paid-width group drops the record without +/// asking anyone, as a chunk is. +#[tokio::test] +#[serial] +async fn a_node_far_outside_the_group_prunes_without_asking() { + let harness = TestHarness::setup_with_config(prune_network(3)) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 1, 1); + let pruner = out_of_range_node(&harness, &record).await; + store(harness.test_node(pruner).expect("node")) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + + replication(harness.test_node(pruner).expect("node")) + .prune_pass(false) + .await; + assert!( + held(harness.test_node(pruner).expect("node"), &record).is_none(), + "a far-away record was kept" + ); + + harness.teardown().await.expect("teardown"); +} diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index a082efd0..d3e64036 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -1180,11 +1180,20 @@ impl TestNetwork { .map_or_else(|_| vec![], |sig| sig.as_bytes().to_vec()) }); - Ok(AntProtocol::new( - Arc::new(storage), - Arc::new(payment_verifier), - Arc::new(quote_generator), - )) + // Pointers beside the chunks, wired as a real node wires them (ADR-0016). + let storage = Arc::new(storage); + let payment_verifier = Arc::new(payment_verifier); + let pointer_store = ant_node::pointer::PointerStore::new(data_dir) + .await + .map_err(|e| TestnetError::Core(format!("Failed to create pointer store: {e}")))?; + let pointers = ant_node::pointer::PointerService::new(pointer_store) + .with_chunk_store(Arc::clone(&storage)) + .with_payments(Arc::clone(&payment_verifier)); + + Ok( + AntProtocol::new(storage, payment_verifier, Arc::new(quote_generator)) + .with_pointer_service(pointers), + ) } /// Start a single node. @@ -1348,6 +1357,12 @@ impl TestNetwork { .await { Ok(mut engine) => { + // Pointers replicate through the same engine (ADR-0016). + if let Some(service) = protocol.pointer_service() { + let (writes, fresh_writes) = tokio::sync::mpsc::unbounded_channel(); + service.attach_fresh_writes(writes); + engine.with_pointers(service.store().clone(), fresh_writes); + } let dht_events = p2p.dht_manager().subscribe_events(); engine.start(dht_events); node.replication_engine = Some(engine); diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index f5a8ccf5..a842f4ff 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -449,7 +449,7 @@ async fn sixty_four_signatures_buy_exactly_one_write() { PutOutcome::Changed ); - let path = store.dir().join(hex::encode(first.address())); + let path = store.file_for(&first.address()); let stored = std::fs::read(&path).expect("read back"); for _ in 0..63 { From e392a493c6d928168256126c33a7cc2bf5bf92a1 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Sep 2026 14:30:10 +0900 Subject: [PATCH 25/32] feat(pointer): commit, price and audit pointers like chunks A node now commits to the pointers it is responsible for in the same signed storage commitment as its chunks, so they count toward its quoted price and are spot-checked by the subtree audit. - A pointer leaf is (address, pointer_leaf_hash(address)): the root binds which pointers are held, not their state, so an update never moves it. - Round 1 reports a pointer leaf at the fixed record length; a node that lost one refuses, a confirmed failure. The auditor accepts that leaf shape only at exactly a pointer's length. - Round 2 proves a pointer with its whole signed record, which the auditor verifies and checks belongs at the committed address. - The subtree audit protocol id moves to v2 for the new slice item. - Retention persists which leaves are pointers (format 2), written only when some slot holds one, so a rollback still reloads format 1. - Pruning never deletes a pointer a retained commitment still holds. --- docs/adr/ADR-0016-pointers-immutable-owner.md | 70 ++- src/replication/commitment.rs | 91 ++- src/replication/commitment_state.rs | 200 ++++++- src/replication/config.rs | 11 +- src/replication/mod.rs | 92 ++- src/replication/pointer.rs | 21 +- src/replication/protocol.rs | 11 + src/replication/storage_commitment_audit.rs | 524 +++++++++++++++++- src/replication/subtree.rs | 6 + tests/e2e/pointer_replication.rs | 196 ++++++- tests/poc_audit_handler_live.rs | 6 +- tests/pointer_convergence.rs | 4 + 12 files changed, 1179 insertions(+), 53 deletions(-) diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index 83dfd9ec..6d0a9f57 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -3,7 +3,7 @@ - **Status:** Proposed - **Date:** 2026-09-18 - **Decision owners:** Anselme (@grumbach) -- **Related:** ADR-0002 (audit), ADR-0008 (per-record pricing), ADR-0009 (audit families), ADR-0014 (file store) +- **Related:** ADR-0002 (audit), ADR-0004 (commitment-bound pricing), ADR-0008 (per-record pricing), ADR-0009 (audit families), ADR-0011 (capacity-gated discovery), ADR-0014 (file store), ADR-0015 (browser clients) ## Context @@ -158,6 +158,45 @@ have sent a pointer message themselves — a hint push goes out every round, emp or not, so capability is learned within a cycle — so an older peer is never asked something it cannot answer and never penalised for its silence. +### Storage commitments and audits + +A node commits to the pointers it is responsible for in the same signed storage +commitment as its chunks (ADR-0002, ADR-0004), so they are priced and audited as +chunks are. + +- **Leaf.** A chunk is committed as `(key, key)`, since its address is the hash + of its bytes. A pointer's address is not, and its bytes change with every + update, so it is committed as `(A, pointer_leaf_hash(A))`, a BLAKE3 derive-key + of the address under its own context. The root binds which pointers a node + holds, never their current state, so an update does not move it and no audit + can fail an honest holder for having taken one mid-audit. +- **Price.** A quote is priced from the key count of the commitment it pins, so + every committed pointer counts toward it exactly as a chunk does. +- **Round 1.** A pointer leaf is reported at its address with that hash and the + fixed record length. The auditor accepts that one leaf shape besides + `(key, key)`, and only at exactly a pointer's length, so it cannot stand in for + a chunk of any other size. A node that has lost a committed pointer refuses + round 1, which is a confirmed failure, as a lost chunk is. +- **Round 2.** Where a chunk is proved by a Bao slice and a nonced opening, a + pointer is proved by its whole signed record. The auditor checks the + signature and that the record belongs at the committed address. A peer signs + its own commitment, so without this step it could commit any key under the + hash of cheap bytes it holds; a record the auditor verifies itself cannot be + forged that way. Any valid record at the address passes, whatever its state. + At most five records are opened, about 26 KB, well under the audit message + ceiling. +- **Retention and pruning.** A retained commitment that still holds a pointer + vetoes its deletion, as for a chunk, so a peer pinning that commitment can + still audit it. The persisted retention records which leaves are pointers, so + a restart rebuilds the exact signed root. It is written in format 2 only when + some slot holds a pointer, and in the pre-pointer format 1 otherwise, so a node + rolled back to an earlier release still reloads its retention. + +Round 2 carries a new slice item, so the subtree audit's protocol id moves to +`v2`, as ADR-0009 did before. Nodes on different ids do not audit each other +across the upgrade. A round-1 request that goes unanswered costs the auditor the +bounded trust penalty ADR-0009 accepted; nothing is misjudged as missing data. + Records are kept one file each under `{root}/pointers//`, 256 shards by the address's last byte as the chunk store keeps them. Opening the store parses each record's structure but does not verify its signature; every record was @@ -182,6 +221,7 @@ verified when it was committed and every read verifies it again. | Peer lies about storing a pointer | Every acknowledgement must name the address and state the client sent. A read asks the same group by the same definition, so the quorums intersect — though each does its own lookup, so churn between them is not covered | | One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | | Two peers decide it | **Not defended against, at the read.** Two colluding close-group peers clear a read's bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Replication does not spread such a state: the rest of the group adopts only what a quorum of it holds, so the honest members keep the paid state, but a reader that happens to hear from both colluders still sees theirs. Raising the read's bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold | +| Commit to pointers it does not hold, for price or audit credit | A committed pointer is proved in round 2 by the whole signed record, which the auditor verifies and checks belongs at the address; the pointer leaf shape is accepted only at the fixed record length | | Get a group to adopt a state nobody paid for | Repair adopts only a state a quorum of the close group hold exactly, counted over the whole group, and a fresh offer is stored only after the receiver verifies its payment itself | | Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, so the lost state is taken back and nothing older is: an address nothing is known about admits any record, and a replay could otherwise roll the node back. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | @@ -204,10 +244,11 @@ verified when it was committed and every read verifies it again. correctly signed value and cannot tell. - Replicas may hold different valid signatures of one state; nothing compares record bytes across replicas. -- Pointers do not yet take part in storage commitments or audits; see - Implementation status. Auditing them needs round 2 to serve a whole record — a - peer signs its own commitment, so without that it could name any key with the - hash of cheap bytes it holds. +- A pointer audit opens a whole 5,303-byte record where a chunk audit opens one + 1 KiB block, so a round 2 over pointers is a few times larger. It stays + bounded by the five-leaf cap. +- The subtree audit's protocol id moved to `v2`, so across the upgrade the + old and new releases do not audit each other. ## Implementation status @@ -223,9 +264,11 @@ churn triggers and cycle completion. A node that missed an update, joined late, or lost a record is brought level by the next sync round rather than the next write. -**Not built yet: commitments and audits.** Pointers are not yet leaves of the -storage commitment, so they do not count toward a node's quoted price and are not -spot-checked by the subtree audit. +**Built: commitments and audits** (see Storage commitments and audits above): +responsible pointers are leaves of the storage commitment, count toward the +quoted price, are spot-checked by the subtree audit in both rounds, survive a +restart in the persisted retention, and are kept from pruning while a retained +commitment holds them. **Not built: browser clients.** ADR-0015's WebRTC-direct transport admits, sanitizes and classifies message kinds by an explicit list, and pointer requests @@ -277,6 +320,17 @@ the browser client the same quorum and corroboration rules the native one uses. refused; the possession check penalises only the member that dropped the record; pruning deletes only once the close group proves it holds the record, and a node far outside the group prunes without asking. +- Storage audits, through the live responders and judged by the auditor's own + checks: a node holding committed pointers and chunks passes both rounds; an + update between the rounds does not fail it; a node that lost a committed + pointer fails round 1, and one that loses it after round 1 is caught in round + 2; another owner's valid record is not proof of the committed one; a pointer + leaf at any other length is refused. Over a live network, a node holding its + committed pointers passes the audit and one that dropped them fails it. +- A commitment holding pointers survives a restart with its exact pin; one + without is written in the pre-pointer format, which the old layout reads. +- Pruning keeps a pointer a retained commitment holds, and drops it once none + does. - Fresh offers, hints, fetches and state queries are covered by the replication protocol's per-variant tests: family, size ceiling, round-trip. - End to end against a live testnet with real settlement: create, update, read diff --git a/src/replication/commitment.rs b/src/replication/commitment.rs index 61082510..714c2034 100644 --- a/src/replication/commitment.rs +++ b/src/replication/commitment.rs @@ -65,6 +65,27 @@ pub fn leaf_hash(key: &XorName, bytes_hash: &[u8; 32]) -> [u8; 32] { *h.finalize().as_bytes() } +/// Key-derivation context for a pointer's commitment leaf (ADR-0016). +const POINTER_LEAF_CONTEXT: &str = "autonomi.pointer.commitment-leaf.v1"; + +/// The `bytes_hash` a pointer at `address` is committed under (ADR-0016). +/// +/// A chunk is committed as `(key, key)`: its address is the hash of its bytes. +/// A pointer's address is not, and its bytes change with every update, so it +/// is committed under a value derived from the address alone. The root then +/// moves only when a pointer is added or dropped, never on an update, and an +/// auditor tells the two kinds apart from the leaf itself: `bytes_hash == key` +/// for a chunk, `bytes_hash == pointer_leaf_hash(key)` for a pointer. Derive-key +/// keeps them apart — a chunk leaf cannot equal a pointer leaf without a BLAKE3 +/// preimage across modes. +/// +/// Possession of a pointer is proved in round 2 by serving the whole signed +/// record, which the auditor verifies; nothing about the bytes is bound here. +#[must_use] +pub fn pointer_leaf_hash(address: &XorName) -> [u8; 32] { + blake3::derive_key(POINTER_LEAF_CONTEXT, address) +} + /// Combine two child hashes into a Merkle internal-node hash. #[must_use] pub fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { @@ -106,6 +127,9 @@ pub struct MerkleTree { /// `levels[0].len() == leaves.len()`; `levels[L].len() == 1` where L /// is the root level. levels: Vec>, + /// Sorted indices of the leaves that commit a pointer, found once at build + /// so the audit and persistence paths never re-derive them per leaf. + pointer_leaves: Vec, } impl MerkleTree { @@ -137,6 +161,15 @@ impl MerkleTree { } } + // A chunk leaf is `(key, key)`, so the pointer check only runs on the + // leaves that are not. + let pointer_leaves: Vec = entries + .iter() + .enumerate() + .filter(|(_, (k, bh))| bh != k && *bh == pointer_leaf_hash(k)) + .map(|(idx, _)| idx) + .collect(); + let leaves: Vec<(XorName, [u8; 32])> = entries .into_iter() .map(|(k, bh)| { @@ -152,7 +185,11 @@ impl MerkleTree { levels.push(level.clone()); } - Ok(Self { leaves, levels }) + Ok(Self { + leaves, + levels, + pointer_leaves, + }) } /// The Merkle root of this tree. @@ -241,6 +278,34 @@ impl MerkleTree { self.leaves.get(idx).map(|(k, _)| *k) } + /// Whether the leaf at `idx` commits a pointer rather than a chunk. + #[must_use] + pub fn is_pointer_leaf(&self, idx: usize) -> bool { + self.pointer_leaves.binary_search(&idx).is_ok() + } + + /// Whether `key` is committed, as a pointer. + #[must_use] + pub fn commits_pointer(&self, key: &XorName) -> bool { + self.key_index(key) + .is_some_and(|idx| self.is_pointer_leaf(idx)) + } + + /// The keys committed as pointers, in the tree's sorted order. + #[must_use] + pub fn pointer_leaf_keys(&self) -> Vec { + self.pointer_leaves + .iter() + .filter_map(|idx| self.key_at(*idx)) + .collect() + } + + /// How many leaves commit a pointer. + #[must_use] + pub fn pointer_count(&self) -> usize { + self.pointer_leaves.len() + } + /// The sorted leaf index of `key`, if committed. `O(log n)` binary search /// over the (key-sorted) leaves — no separate key list needed, so callers /// don't have to keep a duplicate `sorted_keys` Vec alongside the tree. @@ -439,6 +504,30 @@ mod tests { assert!(matches!(result, Err(CommitmentError::EmptyKeySet))); } + /// A pointer leaf (ADR-0016) is told apart from a chunk leaf by its hash + /// alone, and a leaf hashed any other way is neither. + #[test] + fn pointer_leaves_are_told_apart_from_chunk_leaves() { + let chunk = [1u8; 32]; + let pointer = [2u8; 32]; + let other = [3u8; 32]; + let tree = MerkleTree::build(vec![ + (chunk, chunk), + (pointer, pointer_leaf_hash(&pointer)), + (other, [9u8; 32]), + ]) + .unwrap(); + assert!(tree.commits_pointer(&pointer)); + assert!(!tree.commits_pointer(&chunk)); + assert!(!tree.commits_pointer(&other)); + assert!( + !tree.commits_pointer(&[4u8; 32]), + "an uncommitted key is not" + ); + assert_eq!(tree.pointer_leaf_keys(), vec![pointer]); + assert_eq!(tree.pointer_count(), 1); + } + #[test] fn duplicate_keys_rejected() { let result = MerkleTree::build(vec![(xn(1), bh(1)), (xn(1), bh(2))]); diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 62b965f9..55349897 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -38,8 +38,8 @@ use serde::{Deserialize, Serialize}; use crate::ant_protocol::XorName; use crate::replication::commitment::{ - commitment_hash, sign_commitment, verify_commitment_signature, CommitmentError, MerkleTree, - StorageCommitment, + commitment_hash, pointer_leaf_hash, sign_commitment, verify_commitment_signature, + CommitmentError, MerkleTree, StorageCommitment, }; /// Auditor-side per-peer commitment state. @@ -280,6 +280,12 @@ impl BuiltCommitment { self.tree.leaf_keys() } + /// The subset of [`Self::leaf_keys`] committed as pointers (ADR-0016). + #[must_use] + pub fn pointer_leaf_keys(&self) -> Vec { + self.tree.pointer_leaf_keys() + } + /// Reconstruct a `BuiltCommitment` from a persisted signed commitment and a /// `tree` rebuilt from its leaf keys — WITHOUT re-signing, so the pin /// (`commitment_hash`) is preserved exactly across a restart (ML-DSA @@ -362,13 +368,74 @@ struct PersistedSlot { /// slot as already expired. `None` if the slot was never gossiped — then it /// survives reload only while it is the current slot. expires_at_unix: Option, + /// The subset of `leaf_keys` committed as pointers (ADR-0016). A pointer + /// leaf is `(key, pointer_leaf_hash(key))`, not `(key, key)`, so without + /// this the rebuilt tree would miss the signed root and the slot be lost. + pointer_keys: Vec, } /// Persisted-format version. Bump on any layout OR semantic change so an /// incompatible on-disk snapshot is rejected (→ empty retention, which self-heals /// via re-gossip) rather than silently misinterpreted (e.g. an old field read /// under new semantics). -const RETENTION_FORMAT_VERSION: u32 = 1; +/// +/// Format 2 added each slot's pointer keys. Format 1 is still read, and still +/// written whenever no slot commits a pointer (see [`PersistedRetention::to_bytes`]). +const RETENTION_FORMAT_VERSION: u32 = 2; + +/// The format that predates pointers: the same slots with no pointer keys. +const POINTERLESS_RETENTION_FORMAT_VERSION: u32 = 1; + +/// A format-1 slot, as read back from disk. +#[derive(Deserialize)] +struct PointerlessSlot { + commitment: StorageCommitment, + leaf_keys: Vec, + expires_at_unix: Option, +} + +/// A format-1 snapshot, as read back from disk. +#[derive(Deserialize)] +struct PointerlessRetention { + version: u32, + slots: Vec, + has_current: bool, +} + +/// A format-1 slot, as written: borrowed, so writing format 1 copies nothing. +#[derive(Serialize)] +struct PointerlessSlotRef<'a> { + commitment: &'a StorageCommitment, + leaf_keys: &'a [XorName], + expires_at_unix: Option, +} + +/// A format-1 snapshot, as written. +#[derive(Serialize)] +struct PointerlessRetentionRef<'a> { + version: u32, + slots: Vec>, + has_current: bool, +} + +impl From for PersistedRetention { + fn from(old: PointerlessRetention) -> Self { + Self { + version: old.version, + slots: old + .slots + .into_iter() + .map(|slot| PersistedSlot { + commitment: slot.commitment, + leaf_keys: slot.leaf_keys, + expires_at_unix: slot.expires_at_unix, + pointer_keys: Vec::new(), + }) + .collect(), + has_current: old.has_current, + } + } +} /// The persisted responder retention. Slots are newest-first; `has_current` /// says whether `slots[0]` was the live advertised commitment. @@ -384,18 +451,52 @@ impl PersistedRetention { /// Serialize for durable persistence (caller writes it atomically). `None` /// on a serialization error, so the caller can refuse to overwrite the /// durable file rather than truncate it. + /// + /// A snapshot with no pointer in any slot is written in format 1, which it + /// can say exactly. A node rolled back to a release that predates pointers + /// then still reloads its retention, instead of dropping every pin a peer + /// holds on it. #[must_use] pub fn to_bytes(&self) -> Option> { - postcard::to_allocvec(self).ok() + let pointerless = self.version == RETENTION_FORMAT_VERSION + && self.slots.iter().all(|slot| slot.pointer_keys.is_empty()); + if !pointerless { + return postcard::to_allocvec(self).ok(); + } + let old = PointerlessRetentionRef { + version: POINTERLESS_RETENTION_FORMAT_VERSION, + slots: self + .slots + .iter() + .map(|slot| PointerlessSlotRef { + commitment: &slot.commitment, + leaf_keys: &slot.leaf_keys, + expires_at_unix: slot.expires_at_unix, + }) + .collect(), + has_current: self.has_current, + }; + postcard::to_allocvec(&old).ok() } /// Decode a persisted snapshot. `None` on a corrupt blob OR a version /// mismatch — the caller then fails open LOCALLY (empty retention; the node /// re-gossips a fresh root), which never grants a remote grace. + /// + /// The version leads the blob in every format, so it is read first and + /// picks the layout the rest is decoded with. #[must_use] pub fn from_bytes(bytes: &[u8]) -> Option { - let this: Self = postcard::from_bytes(bytes).ok()?; - (this.version == RETENTION_FORMAT_VERSION).then_some(this) + let (version, _) = postcard::take_from_bytes::(bytes).ok()?; + match version { + RETENTION_FORMAT_VERSION => postcard::from_bytes(bytes).ok(), + POINTERLESS_RETENTION_FORMAT_VERSION => { + postcard::from_bytes::(bytes) + .ok() + .map(Self::from) + } + _ => None, + } } } @@ -769,6 +870,7 @@ impl ResponderCommitmentState { commitment: c.commitment().clone(), leaf_keys: c.leaf_keys(), expires_at_unix, + pointer_keys: c.pointer_leaf_keys(), } }) .collect(); @@ -796,7 +898,18 @@ impl ResponderCommitmentState { // else a later slot would be wrongly promoted to current. let mut first_slot_restored = false; for (i, slot) in persisted.slots.iter().enumerate() { - let entries: Vec<_> = slot.leaf_keys.iter().map(|k| (*k, *k)).collect(); + let pointer_keys: HashSet<&XorName> = slot.pointer_keys.iter().collect(); + let entries: Vec<_> = slot + .leaf_keys + .iter() + .map(|k| { + if pointer_keys.contains(k) { + (*k, pointer_leaf_hash(k)) + } else { + (*k, *k) + } + }) + .collect(); let Ok(tree) = MerkleTree::build(entries) else { continue; }; @@ -1051,6 +1164,77 @@ mod tests { ); } + /// A commitment holding pointers (ADR-0016) survives a restart like any + /// other. A pointer leaf is not `(key, key)`, so a snapshot that kept only + /// the key set would rebuild a different root, fail the signature check and + /// drop the slot, and every peer pinning it would then fail this node. + #[test] + fn a_commitment_holding_pointers_survives_a_restart() { + let (pk, sk) = keypair(); + let pk_bytes = pk.to_bytes(); + let mut entries: Vec<_> = (1..=4u8).map(|i| (key(i), key(i))).collect(); + entries.extend((10..=12u8).map(|i| (key(i), pointer_leaf_hash(&key(i))))); + let built = BuiltCommitment::build(entries, &[0xAB; 32], &sk, &pk_bytes).unwrap(); + let pin = built.hash(); + let state = ResponderCommitmentState::new(); + state.rotate(built); + state.mark_gossiped(pin); + + let bytes = state.snapshot().to_bytes().expect("serialize"); + assert_eq!( + postcard::take_from_bytes::(&bytes) + .map(|(v, _)| v) + .ok(), + Some(RETENTION_FORMAT_VERSION), + "a snapshot holding pointers needs the format that can say so" + ); + let fresh = ResponderCommitmentState::new(); + fresh.restore(&PersistedRetention::from_bytes(&bytes).expect("deserialize")); + + let got = fresh.lookup_by_hash(&pin).expect("pin survives restart"); + assert_eq!(got.hash(), pin); + assert!(fresh.is_held(&key(11)), "a committed pointer is still held"); + assert!(fresh.is_held(&key(2)), "a committed chunk is still held"); + assert_eq!( + got.pointer_leaf_keys(), + vec![key(10), key(11), key(12)], + "the pointers are still told apart from the chunks" + ); + } + + /// A snapshot with no pointer in it is written in format 1, byte for byte + /// what a release before pointers writes. Rolling such a node back then + /// keeps its retention; format 2 would read as an unknown version there and + /// drop every pin its peers hold. + #[test] + fn a_snapshot_without_pointers_is_written_in_the_old_format() { + let (pk, sk) = keypair(); + let pk_bytes = pk.to_bytes(); + let entries: Vec<_> = (1..=5u8).map(|i| (key(i), key(i))).collect(); + let built = BuiltCommitment::build(entries, &[0xAB; 32], &sk, &pk_bytes).unwrap(); + let pin = built.hash(); + let state = ResponderCommitmentState::new(); + state.rotate(built); + state.mark_gossiped(pin); + + let bytes = state.snapshot().to_bytes().expect("serialize"); + let old = postcard::from_bytes::(&bytes) + .expect("the old layout decodes it"); + assert_eq!(old.version, POINTERLESS_RETENTION_FORMAT_VERSION); + assert_eq!( + old.slots.first().map(|slot| slot.leaf_keys.len()), + Some(5), + "the old layout reads the same key set" + ); + + let fresh = ResponderCommitmentState::new(); + fresh.restore(&PersistedRetention::from_bytes(&bytes).expect("deserialize")); + assert!( + fresh.lookup_by_hash(&pin).is_some(), + "and this release reads it back too" + ); + } + /// A corrupt snapshot blob decodes to `None`, so the caller fails open with /// empty retention rather than trusting garbage. #[test] @@ -1071,6 +1255,7 @@ mod tests { commitment: built.commitment().clone(), leaf_keys: vec![key(1)], expires_at_unix: None, + pointer_keys: Vec::new(), }], has_current: true, }; @@ -1102,6 +1287,7 @@ mod tests { commitment: built.commitment().clone(), leaf_keys: leaf_keys.clone(), expires_at_unix, + pointer_keys: Vec::new(), }], has_current: false, // not current -> retention depends on the stamp }; diff --git a/src/replication/config.rs b/src/replication/config.rs index 9150d584..276ba1bb 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -426,7 +426,14 @@ pub const REPLICATION_PROTOCOL_ID: &str = "autonomi.ant.replication.v2"; /// possession/repair/commitment-fetch) with no per-peer limiter. A truly /// zero-penalty rollout needs an upstream `send_request` that does not /// auto-report trust; tracked as a saorsa-core follow-up. -pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v1"; +/// +/// `v2` (ADR-0016): commitments carry pointer leaves, committed as +/// `(address, pointer_leaf_hash(address))`, and round 2 answers one with the +/// whole signed record. A `v1` auditor would reject a pointer leaf in round 1 +/// as a content-address mismatch and penalise an honest holder, so the two +/// versions must not audit each other: the same bounded pause as the `v1` +/// introduction, for the same reason. +pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v2"; /// 10 MiB — maximum replication wire message size (accommodates hint batches). const REPLICATION_MESSAGE_SIZE_MIB: usize = 10; @@ -1487,7 +1494,7 @@ mod tests { assert_eq!(REPLICATION_PROTOCOL_ID, "autonomi.ant.replication.v2"); assert_eq!( SUBTREE_AUDIT_PROTOCOL_ID, - "autonomi.ant.replication.subtree-audit.v1" + "autonomi.ant.replication.subtree-audit.v2" ); assert_ne!(REPLICATION_PROTOCOL_ID, SUBTREE_AUDIT_PROTOCOL_ID); } diff --git a/src/replication/mod.rs b/src/replication/mod.rs index c4da9831..8e867ebf 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -67,6 +67,7 @@ use crate::payment::{ PaymentVerifier, VerificationContext, MAX_PAYMENT_PROOF_SIZE_BYTES, MIN_PAYMENT_PROOF_SIZE_BYTES, }; +use crate::pointer::store::PointerStore; use crate::replication::audit::AuditTickResult; use crate::replication::audit_coordinator::AuditChallengeCoordinator; use crate::replication::audit_metrics::{ @@ -2038,6 +2039,7 @@ impl ReplicationEngine { pub async fn rebuild_commitment_now(&self) -> Result<()> { rebuild_and_rotate_commitment( &self.storage, + self.pointers.as_ref().map(|p| p.store()), &self.identity, &self.commitment_state, &self.p2p_node, @@ -2218,7 +2220,7 @@ impl ReplicationEngine { /// Call before [`Self::start`]. pub fn with_pointers( &mut self, - store: crate::pointer::store::PointerStore, + store: PointerStore, fresh_writes: mpsc::UnboundedReceiver, ) { self.pointers = Some(Arc::new(pointer::PointerReplication::new( @@ -3404,6 +3406,7 @@ impl ReplicationEngine { let config = Arc::clone(&self.config); let sync_trigger = Arc::clone(&self.sync_trigger); let recent_provers = Arc::clone(&self.recent_provers); + let pointer_store = self.pointers.as_ref().map(|p| p.store().clone()); let handle = tokio::spawn(async move { // Build the first commitment immediately on startup so a @@ -3426,9 +3429,15 @@ impl ReplicationEngine { // unchanged — preserving the reloaded current pin; otherwise the // reloaded roots stay answerable as retained slots until their gossip // TTL lapses. Persistence is handled by the retention-persist loop. - if let Err(e) = - rebuild_and_rotate_commitment(&storage, &identity, &commitment_state, &p2p, &config) - .await + if let Err(e) = rebuild_and_rotate_commitment( + &storage, + pointer_store.as_ref(), + &identity, + &commitment_state, + &p2p, + &config, + ) + .await { warn!("Initial commitment build failed: {e}"); } else { @@ -3442,6 +3451,7 @@ impl ReplicationEngine { ) => { if let Err(e) = rebuild_and_rotate_commitment( &storage, + pointer_store.as_ref(), &identity, &commitment_state, &p2p, @@ -5183,6 +5193,7 @@ async fn handle_replication_message( let storage = Arc::clone(&ctx.storage); let p2p_node = Arc::clone(&ctx.p2p_node); let my_commitment_state = Arc::clone(&ctx.my_commitment_state); + let pointer_store = ctx.pointers.as_ref().map(|p| p.store().clone()); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -5195,9 +5206,10 @@ async fn handle_replication_message( let storage_commitment_audit::Round1Work { response, content_bytes, - } = storage_commitment_audit::handle_subtree_challenge_measured( + } = storage_commitment_audit::handle_subtree_challenge_measured_with_pointers( &challenge, &storage, + pointer_store.as_ref(), p2p_node.peer_id(), bootstrapping, Some(&my_commitment_state), @@ -5369,6 +5381,7 @@ async fn handle_replication_message( let storage = Arc::clone(&ctx.storage); let p2p_node = Arc::clone(&ctx.p2p_node); let my_commitment_state = Arc::clone(&ctx.my_commitment_state); + let pointer_store = ctx.pointers.as_ref().map(|p| p.store().clone()); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -5377,14 +5390,16 @@ async fn handle_replication_message( let _guard = guard; // global permit + per-peer slot, held until done let worker_started = Instant::now(); let processing_started = Instant::now(); - let response = storage_commitment_audit::handle_subtree_slice_challenge( - &challenge, - &storage, - p2p_node.peer_id(), - bootstrapping, - Some(&my_commitment_state), - ) - .await; + let response = + storage_commitment_audit::handle_subtree_slice_challenge_with_pointers( + &challenge, + &storage, + pointer_store.as_ref(), + p2p_node.peer_id(), + bootstrapping, + Some(&my_commitment_state), + ) + .await; let processing = processing_started.elapsed(); let response_kind = subtree_slice_response_kind(&response); let response_send_started = Instant::now(); @@ -7534,7 +7549,9 @@ async fn run_neighbor_sync_round( }) .await; if let Some(pointers) = pointers { - pointers.prune_pass(allow_remote_prune_audits).await; + pointers + .prune_pass(allow_remote_prune_audits, Some(commitment_state)) + .await; } // Take fresh close-neighbor snapshot (DHT query, no lock held). @@ -10060,11 +10077,16 @@ async fn write_retention_atomic(path: &Path, bytes: Vec) -> bool { /// BLAKE3(content)`, so `bytes_hash := key` and we don't have to /// re-read each chunk's bytes to compute the leaf hash. /// +/// The pointers this node is responsible for are committed alongside, each as +/// `(address, pointer_leaf_hash(address))` (ADR-0016), so they are audited and +/// priced exactly as chunks are. +/// /// Skips (returns `Ok(())`) if the key set is empty — no commitment to /// rotate. The auditor side handles "no commitment for this peer" by /// falling back to the legacy plain-digest audit path. async fn rebuild_and_rotate_commitment( storage: &Arc, + pointers: Option<&PointerStore>, identity: &Arc, state: &Arc, p2p: &Arc, @@ -10097,8 +10119,16 @@ async fn rebuild_and_rotate_commitment( keys.push(k); } } + // The same rule for pointers: an out-of-range pointer leaves the next + // commitment, and the pruner reclaims it once no retained slot holds it. + let mut pointer_keys = Vec::new(); + for state in pointers.map(PointerStore::held_states).unwrap_or_default() { + if admission::is_responsible(&self_id, &state.address, p2p, config.close_group_size).await { + pointer_keys.push(state.address); + } + } - if keys.is_empty() { + if keys.is_empty() && pointer_keys.is_empty() { // There used to be a second branch here that dropped every retained root outright // when the node looked empty. It is gone, and the reason is worth keeping. // @@ -10146,12 +10176,11 @@ async fn rebuild_and_rotate_commitment( // to more than the protocol limit; auditor would reject the // commitment otherwise). let cap = commitment::MAX_COMMITMENT_KEY_COUNT as usize; - if keys.len() > cap { + let total = keys.len().saturating_add(pointer_keys.len()); + if total > cap { warn!( - "Commitment rotation: key set ({}) exceeds MAX_COMMITMENT_KEY_COUNT ({}); \ - truncating — investigate as this likely means a misconfiguration", - keys.len(), - cap + "Commitment rotation: key set ({total}) exceeds MAX_COMMITMENT_KEY_COUNT ({cap}); \ + truncating — investigate as this likely means a misconfiguration" ); } @@ -10173,7 +10202,28 @@ async fn rebuild_and_rotate_commitment( // earn credit for `key`. If this module is ever reused for // non-content-addressed records, that `(k, k)` shortcut AND the verifier // gate must be replaced with `(key, BLAKE3(bytes))` computed from real bytes. - let entries: Vec<_> = keys.into_iter().take(cap).map(|k| (k, k)).collect(); + // + // Pointers are that case, and are handled by being told apart rather than + // hashed: a pointer leaf is `(address, pointer_leaf_hash(address))`, which + // the verifier accepts only for a leaf of exactly a pointer's size, and + // round 2 then demands the whole signed record in place of a Bao slice. + // The root binds which pointers are held, never their current state, so an + // update does not move it. + // + // A pointer address that is also a chunk key would need a BLAKE3 preimage; + // the dedup only keeps such a collision from failing the whole build. + let mut entries: Vec<_> = keys + .into_iter() + .map(|k| (k, k)) + .chain( + pointer_keys + .into_iter() + .map(|address| (address, commitment::pointer_leaf_hash(&address))), + ) + .collect(); + entries.sort_by_key(|(k, _)| *k); + entries.dedup_by_key(|(k, _)| *k); + entries.truncate(cap); // No-op-rotation guard: compute just the Merkle root from `entries` // and compare against the currently-advertised commitment's root. diff --git a/src/replication/pointer.rs b/src/replication/pointer.rs index ef14321f..69dcd20a 100644 --- a/src/replication/pointer.rs +++ b/src/replication/pointer.rs @@ -56,6 +56,7 @@ use crate::payment::{ }; use crate::pointer::store::{Inspected, PointerStore}; use crate::replication::admission; +use crate::replication::commitment_state::ResponderCommitmentState; use crate::replication::config::{ storage_admission_width, ReplicationConfig, FRESH_REPLICATION_DELIVERY_MAX_RETRIES, REPLICATION_PROTOCOL_ID, @@ -255,6 +256,11 @@ impl PointerReplication { } } + /// The pointer store this replication serves from. + pub(crate) fn store(&self) -> &PointerStore { + &self.store + } + // ----------------------------------------------------------------------- // Capability // ----------------------------------------------------------------------- @@ -1031,7 +1037,16 @@ impl PointerReplication { /// /// `allow_remote` is false while bootstrapping: candidacy is still tracked, /// but nothing that needs other peers' proof is decided. - pub async fn prune_pass(&self, allow_remote: bool) { + /// + /// A record a retained storage commitment still commits to is never + /// deleted, exactly as for a chunk: an auditor pinning that commitment may + /// still open it, and a node that deleted it would fail that audit. + pub async fn prune_pass( + &self, + allow_remote: bool, + commitment_state: Option<&ResponderCommitmentState>, + ) { + let committed = |address: &XorName| commitment_state.is_some_and(|cs| cs.is_held(address)); let self_id = *self.p2p.peer_id(); let retention = storage_admission_width(self.config.close_group_size); let now = Instant::now(); @@ -1060,6 +1075,9 @@ impl PointerReplication { if self.shutdown.is_cancelled() { return; } + if committed(&state.address) { + continue; + } let wide = self .p2p .dht_manager() @@ -1079,6 +1097,7 @@ impl PointerReplication { }; // Revalidate just before deleting: the group may have moved back. if confirmed + && !committed(&state.address) && !admission::is_responsible(&self_id, &state.address, &self.p2p, retention).await && self.delete(&state.address).await { diff --git a/src/replication/protocol.rs b/src/replication/protocol.rs index 59afd178..4f7415b0 100644 --- a/src/replication/protocol.rs +++ b/src/replication/protocol.rs @@ -1366,6 +1366,17 @@ pub enum SubtreeSliceItem { /// The committed key the responder could not serve. key: XorName, }, + /// The responder holds this committed pointer (ADR-0016) and serves the + /// whole signed record. A pointer's address is not a content hash, so no + /// slice of it could be authenticated against the address; the record can + /// be, by its signature. One per requested pointer key, whatever blocks + /// were named. + PointerRecord { + /// The requested key: the pointer's address. + key: XorName, + /// The pointer record, in its canonical encoding. + record: Vec, + }, } /// Response to a [`SubtreeSliceChallenge`] (round 2). diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index f99a4e70..39f3b396 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -17,7 +17,8 @@ use crate::logging::{debug, info, warn}; use rand::Rng; use crate::ant_protocol::XorName; -use crate::replication::commitment::{commitment_hash, StorageCommitment}; +use crate::pointer::store::PointerStore; +use crate::replication::commitment::{commitment_hash, pointer_leaf_hash, StorageCommitment}; use crate::replication::commitment_state::ResponderCommitmentState; use crate::replication::config::{ ReplicationConfig, MAX_SLICE_OPENINGS, SUBTREE_AUDIT_PROTOCOL_ID, @@ -30,10 +31,12 @@ use crate::replication::protocol::{ }; use crate::replication::recent_provers::RecentProvers; use crate::replication::subtree::{ - select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeProof, + select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeLeaf, + SubtreeProof, }; use crate::replication::types::{AuditFailureReason, AuditFailureSummary, FailureEvidence}; use crate::storage::ChunkStore; +use ant_protocol::pointer::{Pointer, POINTER_WIRE_LEN}; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; use tokio::sync::RwLock; @@ -723,12 +726,34 @@ pub(crate) fn evaluate_subtree_structure( // for honest content-addressed data the two are identical, so this never fails // an honest holder, and it re-binds Chain 1's `bytes_hash` check to the credited // `key`. - if proof.leaves.iter().any(|l| l.bytes_hash != l.key) { + // + // A pointer (ADR-0016) is the one other honest leaf shape: its address is not + // a content hash, so it is committed as `(key, pointer_leaf_hash(key))` at the + // fixed record length, and round 2 proves possession with the whole signed + // record instead of a slice. Anything else is still rejected. + if proof + .leaves + .iter() + .any(|l| l.bytes_hash != l.key && !is_pointer_leaf(l)) + { return Err(AuditFailureReason::DigestMismatch); } Ok(()) } +/// Whether a round-1 leaf commits a pointer (ADR-0016): committed under +/// [`pointer_leaf_hash`] of its key, at the fixed record length. +fn is_pointer_leaf(leaf: &SubtreeLeaf) -> bool { + leaf.bytes_hash == pointer_leaf_hash(&leaf.key) + && usize::try_from(leaf.content_len).ok() == Some(POINTER_WIRE_LEN) +} + +/// Whether `record` is a valid pointer record at `key`: its signature verifies +/// and it belongs at that address. +fn serves_pointer_at(key: &XorName, record: &[u8]) -> bool { + Pointer::from_bytes(record).is_ok_and(|pointer| pointer.address() == *key) +} + /// The auditor's **freshly-randomised** spot-check sample of the round-1 proof: /// `count` distinct leaves (deduplicated, in increasing-index order) whose /// original bytes the auditor will demand in round 2. @@ -870,6 +895,7 @@ pub(crate) fn verify_slice_response( let requested_keys: HashSet = openings.iter().map(|(leaf, _)| leaf.key).collect(); let mut present: HashSet<(XorName, u32)> = HashSet::new(); let mut absent: HashSet = HashSet::new(); + let mut records: HashSet = HashSet::new(); for it in items { let ok = match it { SubtreeSliceItem::Present { @@ -878,11 +904,19 @@ pub(crate) fn verify_slice_response( requested_blocks.contains(&(*key, *block_index)) && present.insert((*key, *block_index)) && !absent.contains(key) + && !records.contains(key) } SubtreeSliceItem::Absent { key } => { requested_keys.contains(key) && absent.insert(*key) && !present.iter().any(|(k, _)| k == key) + && !records.contains(key) + } + SubtreeSliceItem::PointerRecord { key, .. } => { + requested_keys.contains(key) + && records.insert(*key) + && !absent.contains(key) + && !present.iter().any(|(k, _)| k == key) } }; if !ok { @@ -892,6 +926,26 @@ pub(crate) fn verify_slice_response( let mut checked = 0usize; for (leaf, block_index) in openings { + // A pointer leaf is proved by the whole signed record: it must verify and + // belong at the committed address. Any valid record there passes, so an + // update between the rounds cannot fail an honest holder. + if is_pointer_leaf(leaf) { + let served = items.iter().find_map(|it| match it { + SubtreeSliceItem::PointerRecord { key, record } if key == &leaf.key => { + Some(Some(record.as_slice())) + } + SubtreeSliceItem::Absent { key } if key == &leaf.key => Some(None), + _ => None, + }); + match served { + Some(Some(record)) if serves_pointer_at(&leaf.key, record) => { + checked += 1; + continue; + } + Some(None) => return AuditVerdict::Fail(AuditFailureReason::KeyAbsent), + _ => return AuditVerdict::Fail(AuditFailureReason::DigestMismatch), + } + } let block_index = *block_index; // Match the responder's item for exactly this (key, block_index). A // missing item, an explicit Absent, or a different block is a provable lie. @@ -1020,7 +1074,13 @@ async fn verify_subtree_response( .iter() .flat_map(|leaf| { let leaf = (*leaf).clone(); - block_indices_for_leaf(leaf.content_len) + // A pointer is served whole, so it takes one opening, at block 0. + let indices = if is_pointer_leaf(&leaf) { + vec![0] + } else { + block_indices_for_leaf(leaf.content_len) + }; + indices .into_iter() .map(move |block_index| (leaf.clone(), block_index)) }) @@ -1271,6 +1331,27 @@ pub async fn handle_subtree_challenge_measured( self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, +) -> Round1Work { + handle_subtree_challenge_measured_with_pointers( + challenge, + storage, + None, + self_peer_id, + is_bootstrapping, + commitment_state, + ) + .await +} + +/// [`handle_subtree_challenge_measured`] for a node that also commits pointers +/// (ADR-0016): committed pointer leaves are answered from `pointers`. +pub async fn handle_subtree_challenge_measured_with_pointers( + challenge: &SubtreeAuditChallenge, + storage: &ChunkStore, + pointers: Option<&PointerStore>, + self_peer_id: &PeerId, + is_bootstrapping: bool, + commitment_state: Option<&Arc>, ) -> Round1Work { // The accumulator is threaded in rather than returned per-arm so that every // exit reports its work by construction: a new early return cannot forget to @@ -1279,6 +1360,7 @@ pub async fn handle_subtree_challenge_measured( let response = subtree_challenge_response( challenge, storage, + pointers, self_peer_id, is_bootstrapping, commitment_state, @@ -1298,6 +1380,7 @@ pub async fn handle_subtree_challenge_measured( async fn subtree_challenge_response( challenge: &SubtreeAuditChallenge, storage: &ChunkStore, + pointers: Option<&PointerStore>, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1361,7 +1444,29 @@ async fn subtree_challenge_response( // Read chunk bytes one leaf at a time so peak memory is bounded regardless // of subtree size, hashing each into its plain + nonced leaf. let mut leaves = Vec::with_capacity(plan.leaf_keys.len()); - for key in &plan.leaf_keys { + for (position, key) in plan.leaf_keys.iter().enumerate() { + // A pointer leaf (ADR-0016) commits no bytes: round 2 asks for the whole + // signed record. Round 1 only says whether it is still held, from the + // index, and admits a loss exactly as a missing chunk is admitted. + if plan.leaf_is_pointer.get(position).copied().unwrap_or(false) { + *content_bytes = content_bytes.saturating_add(SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES); + if pointers.and_then(|store| store.state(key)).is_none() { + let key_hex = hex::encode(key); + warn!("Subtree audit: committed pointer {key_hex} is not held"); + return SubtreeAuditResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Protocol, + reason: format!("missing bytes for committed key: {key_hex}"), + }; + } + leaves.push(SubtreeLeaf { + key: *key, + bytes_hash: pointer_leaf_hash(key), + content_len: u32::try_from(POINTER_WIRE_LEN).unwrap_or(u32::MAX), + nonced_root: [0u8; 32], + }); + continue; + } // Charge the fixed cost of ATTEMPTING a leaf before the read, because // it is owed whether or not the read succeeds: the LMDB lookup and its // retries, and the blocking-task round trip below. Charging only @@ -1551,6 +1656,28 @@ pub async fn handle_subtree_slice_challenge( self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, +) -> SubtreeSliceResponse { + handle_subtree_slice_challenge_with_pointers( + challenge, + storage, + None, + self_peer_id, + is_bootstrapping, + commitment_state, + ) + .await +} + +/// [`handle_subtree_slice_challenge`] for a node that also commits pointers +/// (ADR-0016): a committed pointer is answered with its whole signed record. +#[allow(clippy::too_many_lines)] +pub async fn handle_subtree_slice_challenge_with_pointers( + challenge: &SubtreeSliceChallenge, + storage: &ChunkStore, + pointers: Option<&PointerStore>, + self_peer_id: &PeerId, + is_bootstrapping: bool, + commitment_state: Option<&Arc>, ) -> SubtreeSliceResponse { if is_bootstrapping { return SubtreeSliceResponse::Bootstrapping { @@ -1682,6 +1809,29 @@ pub async fn handle_subtree_slice_challenge( let mut items = Vec::with_capacity(challenge.openings.len()); for key in key_order { let indices = indices_by_key.remove(&key).unwrap_or_default(); + if built.tree().commits_pointer(&key) { + // `get` verifies the signature before serving, so a record damaged + // on this disk is admitted as absent rather than served as proof. + let served = match pointers { + Some(store) => store.get(&key).await, + None => Ok(None), + }; + match served { + Ok(Some(record)) => items.push(SubtreeSliceItem::PointerRecord { + key, + record: record.to_bytes(), + }), + Ok(None) => items.push(SubtreeSliceItem::Absent { key }), + Err(e) => { + return SubtreeSliceResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Transient, + reason: format!("pointer read error: {e}"), + } + } + } + continue; + } match serve_committed_key_openings(challenge, storage, key, indices).await { KeyServe::Items(mut built_items) => items.append(&mut built_items), KeyServe::Absent => items.push(SubtreeSliceItem::Absent { key }), @@ -2514,3 +2664,367 @@ mod tests { }; } } + +/// Pointers in the storage audit (ADR-0016), driven through the live responders +/// against real stores and judged by the auditor's own checks, so a pass here +/// is a pass on the network. +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod pointer_audit_tests { + use super::*; + use crate::replication::commitment::MerkleTree; + use crate::replication::commitment_state::BuiltCommitment; + use crate::storage::ChunkStoreConfig; + use ant_protocol::pointer::{PointerTarget, PointerTargetKind}; + use saorsa_pqc::api::sig::ml_dsa_65; + use tempfile::TempDir; + + const CHALLENGE_ID: u64 = 7; + + /// A one-block chunk, distinct per `i`. + fn chunk(i: u8) -> Vec { + (0..=255u8).cycle().take(1024).map(|b| b ^ i).collect() + } + + /// `owner`'s pointer at `counter`. + fn pointer(owner: u8, counter: u64) -> Pointer { + let (pk, sk) = ml_dsa_65().generate_keypair_from_seed(&[owner; 32]); + let target = PointerTarget::new(PointerTargetKind::Chunk, [owner; 32]); + Pointer::sign(&sk, &pk, counter, target).expect("sign") + } + + /// A responder that holds and has committed to `chunks` chunks and + /// `pointers` pointers. + struct Responder { + storage: ChunkStore, + pointers: PointerStore, + state: Arc, + peer: PeerId, + peer_bytes: [u8; 32], + _dirs: (TempDir, TempDir), + } + + impl Responder { + async fn new(chunks: u8, pointers: u8) -> Self { + let chunk_dir = TempDir::new().expect("temp dir"); + let storage = ChunkStore::new(ChunkStoreConfig { + root_dir: chunk_dir.path().to_path_buf(), + ..ChunkStoreConfig::test_default() + }) + .await + .expect("chunk store"); + let pointer_dir = TempDir::new().expect("temp dir"); + let pointer_store = PointerStore::new(pointer_dir.path()) + .await + .expect("pointer store"); + + let mut entries = Vec::new(); + for i in 0..chunks { + let content = chunk(i); + let address = ChunkStore::compute_address(&content); + storage.put(&address, &content).await.expect("put chunk"); + entries.push((address, address)); + } + for owner in 0..pointers { + let record = pointer(owner, 1); + pointer_store + .put_bytes(&record.to_bytes()) + .await + .expect("put pointer"); + entries.push((record.address(), pointer_leaf_hash(&record.address()))); + } + + let (pk, sk) = ml_dsa_65().generate_keypair().expect("keypair"); + let peer_bytes = *blake3::hash(&pk.to_bytes()).as_bytes(); + let built = + BuiltCommitment::build(entries, &peer_bytes, &sk, &pk.to_bytes()).expect("build"); + let state = Arc::new(ResponderCommitmentState::new()); + state.rotate(built); + Self { + storage, + pointers: pointer_store, + state, + peer: PeerId::from_bytes(peer_bytes), + peer_bytes, + _dirs: (chunk_dir, pointer_dir), + } + } + + fn committed(&self) -> Arc { + self.state.current().expect("a current commitment") + } + + async fn round1(&self, nonce: [u8; 32]) -> SubtreeAuditResponse { + let challenge = SubtreeAuditChallenge { + challenge_id: CHALLENGE_ID, + nonce, + challenged_peer_id: self.peer_bytes, + expected_commitment_hash: self.committed().hash(), + }; + handle_subtree_challenge_measured_with_pointers( + &challenge, + &self.storage, + Some(&self.pointers), + &self.peer, + false, + Some(&self.state), + ) + .await + .response + } + + async fn round2( + &self, + nonce: [u8; 32], + openings: &[(SubtreeLeaf, u32)], + ) -> Vec { + let challenge = SubtreeSliceChallenge { + challenge_id: CHALLENGE_ID, + nonce, + challenged_peer_id: self.peer_bytes, + expected_commitment_hash: self.committed().hash(), + openings: openings + .iter() + .map(|(leaf, block_index)| SubtreeSliceOpening { + key: leaf.key, + block_index: *block_index, + }) + .collect(), + }; + match handle_subtree_slice_challenge_with_pointers( + &challenge, + &self.storage, + Some(&self.pointers), + &self.peer, + false, + Some(&self.state), + ) + .await + { + SubtreeSliceResponse::Items { items, .. } => items, + other => panic!("expected items, got {other:?}"), + } + } + + /// Round 1 as the auditor sees it: the proof, checked against the pin. + async fn proved_leaves(&self, nonce: [u8; 32]) -> Vec { + let committed = self.committed(); + match self.round1(nonce).await { + SubtreeAuditResponse::Proof { + commitment, proof, .. + } => { + assert_eq!( + evaluate_subtree_structure( + &commitment, + &proof, + &nonce, + &committed.hash(), + &self.peer_bytes, + ), + Ok(()), + "the auditor must accept the round-1 proof" + ); + proof.leaves + } + other => panic!("expected a proof, got {other:?}"), + } + } + } + + /// A nonce whose audited subtree holds both a pointer and a chunk. + fn mixed_nonce(tree: &MerkleTree) -> [u8; 32] { + (0..=255u8) + .map(|b| [b; 32]) + .find(|nonce| { + subtree_plan(tree, nonce).is_ok_and(|plan| { + plan.leaf_is_pointer.contains(&true) && plan.leaf_is_pointer.contains(&false) + }) + }) + .expect("some nonce audits a mixed subtree") + } + + /// What the auditor opens: every pointer leaf, then chunks, up to the + /// spot-check cap, with the block indices production draws. + fn openings(leaves: &[SubtreeLeaf]) -> Vec<(SubtreeLeaf, u32)> { + let (pointers, chunks): (Vec<_>, Vec<_>) = + leaves.iter().partition(|leaf| is_pointer_leaf(leaf)); + pointers + .into_iter() + .take(3) + .chain(chunks) + .take(BYTE_SPOTCHECK_MAX as usize) + .flat_map(|leaf| { + let indices = if is_pointer_leaf(leaf) { + vec![0] + } else { + block_indices_for_leaf(leaf.content_len) + }; + indices.into_iter().map(|i| (leaf.clone(), i)) + }) + .collect() + } + + fn first_pointer(openings: &[(SubtreeLeaf, u32)]) -> XorName { + openings + .iter() + .map(|(leaf, _)| leaf) + .find(|leaf| is_pointer_leaf(leaf)) + .map(|leaf| leaf.key) + .expect("a pointer is opened") + } + + #[tokio::test] + async fn a_committed_pointer_is_proved_by_its_signed_record() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let leaves = responder.proved_leaves(nonce).await; + let openings = openings(&leaves); + let items = responder.round2(nonce, &openings).await; + + assert!( + items + .iter() + .any(|item| matches!(item, SubtreeSliceItem::PointerRecord { .. })), + "a pointer is proved by its record" + ); + assert!( + items + .iter() + .any(|item| matches!(item, SubtreeSliceItem::Present { .. })), + "and the chunks beside it by their slices" + ); + assert!( + matches!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Pass { .. } + ), + "the auditor must pass an honest holder of both" + ); + } + + /// The commitment binds which pointers are held, not their state, so an + /// owner updating a pointer mid-audit cannot fail the node holding it. + #[tokio::test] + async fn an_update_between_the_rounds_does_not_fail_an_honest_holder() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let updated = first_pointer(&openings); + + let owner = (0..24u8) + .find(|owner| pointer(*owner, 1).address() == updated) + .expect("the opened pointer is one of ours"); + responder + .pointers + .put_bytes(&pointer(owner, 2).to_bytes()) + .await + .expect("update"); + + let items = responder.round2(nonce, &openings).await; + assert!(matches!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Pass { .. } + )); + } + + #[tokio::test] + async fn a_node_that_lost_a_committed_pointer_fails_round_one() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let plan = subtree_plan(responder.committed().tree(), &nonce).expect("plan"); + let lost = plan + .leaf_keys + .iter() + .zip(&plan.leaf_is_pointer) + .find_map(|(key, is_pointer)| is_pointer.then_some(*key)) + .expect("a pointer in the subtree"); + assert!(responder.pointers.delete(&lost).await.expect("delete")); + + match responder.round1(nonce).await { + SubtreeAuditResponse::Rejected { kind, .. } => { + assert_eq!( + grade_reject(kind), + RejectGrade::Confirmed, + "a lost pointer is a confirmed failure, as a lost chunk is" + ); + } + other => panic!("expected a rejection, got {other:?}"), + } + } + + #[tokio::test] + async fn a_pointer_lost_after_round_one_is_admitted_absent() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + assert!(responder + .pointers + .delete(&first_pointer(&openings)) + .await + .expect("delete")); + + let items = responder.round2(nonce, &openings).await; + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::KeyAbsent) + ); + } + + /// A node cannot prove it holds one pointer with another, genuinely signed + /// record: the record must belong at the committed address. + #[tokio::test] + async fn another_pointer_is_not_proof_of_the_one_committed() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let target = first_pointer(&openings); + + let mut items = responder.round2(nonce, &openings).await; + let substitute = pointer(200, 1).to_bytes(); + for item in &mut items { + if let SubtreeSliceItem::PointerRecord { key, record } = item { + if *key == target { + *record = substitute.clone(); + } + } + } + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::DigestMismatch) + ); + } + + /// The pointer leaf shape is accepted only at a pointer's exact size, so it + /// cannot stand in for a chunk of another length. + #[tokio::test] + async fn a_pointer_leaf_of_any_other_length_is_refused() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let committed = responder.committed(); + let SubtreeAuditResponse::Proof { + commitment, + mut proof, + .. + } = responder.round1(nonce).await + else { + panic!("expected a proof"); + }; + let leaf = proof + .leaves + .iter_mut() + .find(|leaf| is_pointer_leaf(leaf)) + .expect("a pointer leaf"); + leaf.content_len = leaf.content_len.saturating_add(1); + + assert_eq!( + evaluate_subtree_structure( + &commitment, + &proof, + &nonce, + &committed.hash(), + &responder.peer_bytes, + ), + Err(AuditFailureReason::DigestMismatch) + ); + } +} diff --git a/src/replication/subtree.rs b/src/replication/subtree.rs index a50a628f..65a3e62d 100644 --- a/src/replication/subtree.rs +++ b/src/replication/subtree.rs @@ -499,6 +499,9 @@ pub fn build_subtree_proof( pub struct SubtreePlan { /// The selected leaves' keys, in ascending leaf-index order. pub leaf_keys: Vec, + /// For each of `leaf_keys`, whether it is committed as a pointer + /// (ADR-0016) rather than a chunk. + pub leaf_is_pointer: Vec, /// One sibling cut-hash per level on the path to the subtree root, /// root-first. pub sibling_cut_hashes: Vec<[u8; 32]>, @@ -528,11 +531,13 @@ pub fn subtree_plan( } let mut leaf_keys = Vec::with_capacity(path.real_leaf_count() as usize); + let mut leaf_is_pointer = Vec::with_capacity(path.real_leaf_count() as usize); for idx in path.leaf_start..path.leaf_end { let key = tree .key_at(idx as usize) .ok_or(BuildProofError::MissingKey { leaf_index: idx })?; leaf_keys.push(key); + leaf_is_pointer.push(tree.is_pointer_leaf(idx as usize)); } // Sibling cut-hashes, root-first. The fixed-depth slot selection no longer @@ -557,6 +562,7 @@ pub fn subtree_plan( Ok(SubtreePlan { leaf_keys, + leaf_is_pointer, sibling_cut_hashes, }) } diff --git a/tests/e2e/pointer_replication.rs b/tests/e2e/pointer_replication.rs index 4992ccc5..c031d719 100644 --- a/tests/e2e/pointer_replication.rs +++ b/tests/e2e/pointer_replication.rs @@ -14,6 +14,9 @@ use ant_node::ant_protocol::chunk::{ ChunkMessage, ChunkMessageBody, PointerPutRequest, PointerPutResponse, }; use ant_node::pointer::PointerStore; +use ant_node::replication::audit::AuditTickResult; +use ant_node::replication::commitment::pointer_leaf_hash; +use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; use ant_node::replication::pointer::{PointerFreshWrite, PointerReplication}; use ant_node::ReplicationConfig; use ant_protocol::pointer::{Pointer, PointerState, PointerTarget, PointerTargetKind}; @@ -66,6 +69,13 @@ fn replication(node: &TestNode) -> &Arc { .expect("pointer replication") } +fn commitments(node: &TestNode) -> &ResponderCommitmentState { + node.replication_engine + .as_ref() + .expect("engine") + .commitment_state() +} + fn peer(node: &TestNode) -> PeerId { *node.p2p_node.as_ref().expect("p2p").peer_id() } @@ -502,8 +512,11 @@ async fn pruning_deletes_only_once_the_close_group_proves_it_holds_the_record() // Nobody else holds it yet: nothing proves it is safe to drop. let everyone: Vec = (0..harness.node_count()).collect(); exchange_hints(&harness, &everyone, &[pruner]).await; - let pruning = replication(harness.test_node(pruner).expect("node")); - pruning.prune_pass(true).await; + let pruner_node = harness.test_node(pruner).expect("node"); + let pruning = replication(pruner_node); + pruning + .prune_pass(true, Some(commitments(pruner_node))) + .await; assert!( holds(harness.test_node(pruner).expect("node"), &record), "the only copy was pruned" @@ -516,7 +529,9 @@ async fn pruning_deletes_only_once_the_close_group_proves_it_holds_the_record() .await .expect("put"); } - pruning.prune_pass(true).await; + pruning + .prune_pass(true, Some(commitments(pruner_node))) + .await; assert!( held(harness.test_node(pruner).expect("node"), &record).is_none(), "the record was not pruned though the close group holds it" @@ -543,13 +558,182 @@ async fn a_node_far_outside_the_group_prunes_without_asking() { .await .expect("put"); - replication(harness.test_node(pruner).expect("node")) - .prune_pass(false) + let pruner_node = harness.test_node(pruner).expect("node"); + replication(pruner_node) + .prune_pass(false, Some(commitments(pruner_node))) .await; assert!( - held(harness.test_node(pruner).expect("node"), &record).is_none(), + held(pruner_node, &record).is_none(), "a far-away record was kept" ); harness.teardown().await.expect("teardown"); } + +/// A pointer a retained storage commitment still holds is never pruned, as a +/// chunk is not: a peer pinning that commitment may yet audit it, and a node +/// that had deleted it would fail. Once no retained commitment holds it, it +/// goes. +#[tokio::test] +#[serial] +async fn pruning_keeps_a_pointer_a_retained_commitment_still_holds() { + let harness = TestHarness::setup_with_config(prune_network(3)) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let (pk, sk) = owner(); + let record = signed(&pk, &sk, 1, 1); + let pruner = out_of_range_node(&harness, &record).await; + let pruner_node = harness.test_node(pruner).expect("node"); + store(pruner_node) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + + // The commitment the node gossiped while it was still responsible. + let (node_pk, node_sk) = owner(); + let committed = BuiltCommitment::build( + vec![(record.address(), pointer_leaf_hash(&record.address()))], + peer(pruner_node).as_bytes(), + &node_sk, + &node_pk.to_bytes(), + ) + .expect("commitment"); + let state = commitments(pruner_node); + state.rotate(committed); + + replication(pruner_node) + .prune_pass(false, Some(state)) + .await; + assert!( + holds(pruner_node, &record), + "a pointer a retained commitment holds was pruned" + ); + + state.clear_all(); + replication(pruner_node) + .prune_pass(false, Some(state)) + .await; + assert!( + held(pruner_node, &record).is_none(), + "the pointer was kept after no commitment held it" + ); + + harness.teardown().await.expect("teardown"); +} + +/// Store `count` pointers on node `holder`, have it commit to them, and hand +/// that commitment to node `auditor`, as its gossip would. Returns what the +/// holder committed to. +async fn commit_pointers( + harness: &TestHarness, + holder: usize, + auditor: usize, + count: usize, +) -> Vec { + let holder_node = harness.test_node(holder).expect("holder"); + let records: Vec = (0..count) + .map(|_| { + let (pk, sk) = owner(); + signed(&pk, &sk, 1, 1) + }) + .collect(); + for record in &records { + store(holder_node) + .put_bytes(&record.to_bytes()) + .await + .expect("put"); + } + + let engine = holder_node.replication_engine.as_ref().expect("engine"); + engine.rebuild_commitment_now().await.expect("rebuild"); + let committed = engine + .commitment_state() + .current() + .expect("a current commitment"); + let pointers = committed.pointer_leaf_keys(); + assert!( + !pointers.is_empty(), + "the holder committed to none of the pointers it is responsible for" + ); + assert_eq!( + committed.leaf_keys(), + pointers, + "the holder has no chunks, so every leaf audited is a pointer" + ); + + harness + .test_node(auditor) + .expect("auditor") + .replication_engine + .as_ref() + .expect("engine") + .inject_peer_commitment_for_test(&peer(holder_node), committed.commitment().clone()) + .await; + records + .into_iter() + .filter(|record| pointers.contains(&record.address())) + .collect() +} + +/// A node holding the pointers it committed to passes the storage audit over +/// the wire, proving each opened one with its signed record. +#[tokio::test] +#[serial] +async fn a_node_holding_its_committed_pointers_passes_the_storage_audit() { + let harness = TestHarness::setup_small().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + let (holder, auditor) = (3, 4); + commit_pointers(&harness, holder, auditor, 48).await; + + let holder_peer = peer(harness.test_node(holder).expect("holder")); + let result = harness + .test_node(auditor) + .expect("auditor") + .replication_engine + .as_ref() + .expect("engine") + .audit_peer_now(&holder_peer) + .await; + assert!( + matches!(result, AuditTickResult::Passed { keys_checked, .. } if keys_checked >= 1), + "an honest pointer holder must pass, got {result:?}" + ); + + harness.teardown().await.expect("teardown"); +} + +/// A node that dropped the pointers it committed to fails the storage audit, +/// exactly as a node that dropped its chunks does. +#[tokio::test] +#[serial] +async fn a_node_that_dropped_its_committed_pointers_fails_the_storage_audit() { + let harness = TestHarness::setup_small().await.expect("setup"); + harness.warmup_dht().await.expect("warmup"); + let (holder, auditor) = (5, 6); + let committed = commit_pointers(&harness, holder, auditor, 48).await; + + let holder_node = harness.test_node(holder).expect("holder"); + for record in &committed { + assert!(store(holder_node) + .delete(&record.address()) + .await + .expect("delete")); + } + + let result = harness + .test_node(auditor) + .expect("auditor") + .replication_engine + .as_ref() + .expect("engine") + .audit_peer_now(&peer(holder_node)) + .await; + assert!( + matches!(result, AuditTickResult::Failed { .. }), + "a node that dropped its committed pointers must fail, got {result:?}" + ); + + harness.teardown().await.expect("teardown"); +} diff --git a/tests/poc_audit_handler_live.rs b/tests/poc_audit_handler_live.rs index 5f970a08..9200e62b 100644 --- a/tests/poc_audit_handler_live.rs +++ b/tests/poc_audit_handler_live.rs @@ -614,7 +614,8 @@ async fn slice_challenge_opens_valid_blocks_for_committed_keys() { "nonced opening must fold to the honest nonced root" ); } - other @ SubtreeSliceItem::Absent { .. } => { + other @ (SubtreeSliceItem::Absent { .. } + | SubtreeSliceItem::PointerRecord { .. }) => { panic!("expected Present for stored committed key, got {other:?}") } } @@ -703,7 +704,8 @@ async fn slice_challenge_coalesces_duplicate_and_interleaved_openings() { ); seen.push((*key, *block_index)); } - other @ SubtreeSliceItem::Absent { .. } => { + other @ (SubtreeSliceItem::Absent { .. } + | SubtreeSliceItem::PointerRecord { .. }) => { panic!("expected Present for a stored committed key, got {other:?}") } } diff --git a/tests/pointer_convergence.rs b/tests/pointer_convergence.rs index a842f4ff..4e1864aa 100644 --- a/tests/pointer_convergence.rs +++ b/tests/pointer_convergence.rs @@ -472,9 +472,13 @@ async fn sixty_four_signatures_buy_exactly_one_write() { ); assert_eq!(store.len(), 1, "one address, one record"); + // Records live one level down, in shard directories. let files: Vec<_> = std::fs::read_dir(store.dir()) .expect("read dir") .filter_map(std::result::Result::ok) + .filter(|shard| shard.path().is_dir()) + .flat_map(|shard| std::fs::read_dir(shard.path()).expect("read shard")) + .filter_map(std::result::Result::ok) .map(|e| e.file_name().to_string_lossy().into_owned()) .filter(|name| !name.starts_with('.')) .collect(); From b8bf379679c8f9e6dbdb8dcfd93f29840b2a75e6 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Sep 2026 15:02:00 +0900 Subject: [PATCH 26/32] feat(pointer): serve pointers to browsers, and log them like chunks Browser clients (ADR-0015) can now read and write pointers: - The WebRTC-direct listener admits a pointer GET and a paid pointer PUT through chunk_protocol, each bounded by the small response size; a full record fits well inside it. - Every pointer reply passes the sanitizer; errors and refused payments are redacted as for chunks. - HELLO advertises pointer_protocol, so a browser never asks a node that would refuse. Telemetry parity with chunks: pointer_put_rpc and pointer_get_rpc latency events on the rpc_latency target, the disk pre-check refusal on the disk_precheck target, and the pointer count on commitment rotation. --- docs/adr/ADR-0016-pointers-immutable-owner.md | 45 ++++++++++--- src/pointer/service.rs | 10 ++- src/replication/mod.rs | 3 +- src/storage/handler.rs | 52 +++++++++++++-- src/web_rtc.rs | 63 ++++++++++++++++++- src/web_rtc/errors.rs | 53 ++++++++++++++-- 6 files changed, 204 insertions(+), 22 deletions(-) diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index 6d0a9f57..9d0dac98 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -254,8 +254,11 @@ verified when it was committed and every read verifies it again. Built: the record and wire messages (`ant-protocol`); the store with merge-on-put; request dispatch; payment routed at `state_id` with the close -group of `A`; admission gates; cross-kind refusal; and the client — create, -update, quorum store, merged reads and chain resolution. +group of `A`; admission gates; cross-kind refusal; per-request latency events +beside the chunk ones (`pointer_put_rpc`, `pointer_get_rpc`); and the client — +create, update, quorum store, merged reads and chain resolution, a write that +falls short retried with the proof it already paid for, a split payment for an +external signer, and the `ant pointer` commands. **Built: replication** (see Replication above): fresh offers with payment, neighbour-sync repair by quorum over exact states, pruning with possession @@ -270,14 +273,30 @@ quoted price, are spot-checked by the subtree audit in both rounds, survive a restart in the persisted retention, and are kept from pruning while a retained commitment holds them. -**Not built: browser clients.** ADR-0015's WebRTC-direct transport admits, -sanitizes and classifies message kinds by an explicit list, and pointer requests -are in none of them. The client's pointer API is therefore native-only rather -than compiled for a transport that would reject it. Reaching a pointer from a -browser needs four things, each a deliberate decision at a security boundary: -admit the two request kinds, let the response sanitizer pass their replies, -classify a pointer GET as a read and a pointer PUT as paid-exclusive, and give -the browser client the same quorum and corroboration rules the native one uses. +**Built: browser clients** (ADR-0015). Each of the four decisions at that +security boundary is made explicitly: + +- **Admission.** The WebRTC-direct listener admits a pointer GET and a paid + pointer PUT through `chunk_protocol`, each bounded by the small response size + a quote gets. One record is 5,303 bytes, well inside it. +- **Sanitizing.** Replies pass the sanitizer. A pointer outcome names only an + address and a state identifier, and a record is owner-signed public data. An + error is redacted as for a chunk, and a refused payment is reported without + its detail. +- **Classification.** A pointer PUT is a paid write: it takes the data lane, + holds its connection exclusively as a chunk PUT does, and goes only to nodes + on the page's payment network. A pointer GET returns one small record, so it + stays on the RPC lane, like a quote, and needs no bulk read slot. +- **Rules.** The browser runs the native client's own read quorum, + corroboration, write quorum and chain resolution. Only the wallet is the + page's. + +A node advertises `pointer_protocol` in HELLO when it admits pointers. A browser +client never sends a pointer request to a node that does not, so an older node's +refusal is never mistaken for a failed peer. The owner key crosses into the page +as its 32-byte FIPS 204 seed. A pointer payment is one quote and, unlike a file +upload, keeps no recovery journal: a payment interrupted between broadcast and +receipt is paid again on retry. ## Validation @@ -320,6 +339,12 @@ the browser client the same quorum and corroboration rules the native one uses. refused; the possession check penalises only the member that dropped the record; pruning deletes only once the close group proves it holds the record, and a node far outside the group prunes without asking. +- From a browser: a node admits pointer requests at its WebRTC boundary, bounds + a full record inside the small response limit, and passes every pointer reply + through the sanitizer with errors redacted. In real Chromium against a live + local network with on-chain payment, a pointer is created, updated, pointed + at a second pointer, read back by a client that wrote nothing and resolved + down the chain to its chunk. - Storage audits, through the live responders and judged by the auditor's own checks: a node holding committed pointers and chunks passes both rounds; an update between the rounds does not fail it; a node that lost a committed diff --git a/src/pointer/service.rs b/src/pointer/service.rs index db1c5a28..fd63e32d 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -40,7 +40,7 @@ use saorsa_core::P2PNode; use tokio::sync::mpsc; use crate::error::{Error, Result}; -use crate::logging::{debug, warn}; +use crate::logging::{debug, info, warn}; use crate::payment::PaymentVerifier; use crate::pointer::store::{Inspected, PointerStore, PutOutcome}; use crate::replication::admission; @@ -286,7 +286,13 @@ impl PointerService { // charge is taken at the commit; this only avoids paying to find // out the disk is full. if let Err(e) = chunks.check_capacity_for(POINTER_WIRE_LEN as u64) { - debug!("Rejecting pointer PUT for {}: {e}", hex::encode(address)); + let addr = hex::encode(address); + info!( + target: "ant_node::storage::disk_precheck", + addr = %addr, + kind = "pointer", + "Rejecting pointer PUT before payment verification: {e}" + ); return Some(PointerPutResponse::Error(ProtocolError::StorageFailed( e.to_string(), ))); diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 8e867ebf..652f0090 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -10278,8 +10278,9 @@ async fn rebuild_and_rotate_commitment( let hash = hex::encode(built.hash()); let key_count = built.commitment().key_count; + let pointer_count = built.tree().pointer_count(); state.rotate(built); - info!("Storage commitment rotated: hash={hash} key_count={key_count}"); + info!("Storage commitment rotated: hash={hash} key_count={key_count} pointers={pointer_count}"); // Counted only on the paths where the advertised commitment now genuinely reflects // the committable set, never merely on having read it. The retirement gate is what // consumes this, and it authorises deleting the legacy store. diff --git a/src/storage/handler.rs b/src/storage/handler.rs index d380200e..840a5ef9 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -32,7 +32,7 @@ use crate::ant_protocol::{ settlement_compatibility, ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, ChunkQuoteResponse, MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, - ProtocolError, SettlementCompatibility, CHUNK_PROTOCOL_ID, CURRENT_SETTLEMENT_VERSION, + ProtocolError, SettlementCompatibility, XorName, CHUNK_PROTOCOL_ID, CURRENT_SETTLEMENT_VERSION, MAX_CHUNK_SIZE, MIN_SUPPORTED_SETTLEMENT_VERSION, }; use crate::client::compute_address; @@ -216,6 +216,46 @@ impl Drop for GetRequestTelemetry { } } +/// One latency event per pointer PUT, on the target the chunk `put_rpc` uses, +/// so the same store-latency dashboards cover both kinds. +fn log_pointer_put_rpc(elapsed: Duration, record_size: usize, response: &PointerPutResponse) { + let duration_ms = duration_ms(elapsed); + let (outcome, address): (&'static str, Option<&XorName>) = match response { + PointerPutResponse::Success { address, .. } => ("success", Some(address)), + PointerPutResponse::Unchanged { address, .. } => ("unchanged", Some(address)), + PointerPutResponse::Stale { address, .. } => ("stale", Some(address)), + PointerPutResponse::PaymentRequired { .. } => ("payment_required", None), + PointerPutResponse::Error(_) => ("error", None), + }; + let addr = address.map(hex::encode).unwrap_or_default(); + info!( + target: "ant_node::storage::rpc_latency", + duration_ms, + record_size, + outcome, + addr = %addr, + "pointer_put_rpc" + ); +} + +/// One latency event per pointer GET, beside the chunk `get_rpc`. +fn log_pointer_get_rpc(elapsed: Duration, address: &XorName, response: &PointerGetResponse) { + let duration_ms = duration_ms(elapsed); + let outcome: &'static str = match response { + PointerGetResponse::Success { .. } => "success", + PointerGetResponse::NotFound { .. } => "not_found", + PointerGetResponse::Error(_) => "error", + }; + let addr = hex::encode(address); + info!( + target: "ant_node::storage::rpc_latency", + duration_ms, + outcome, + addr = %addr, + "pointer_get_rpc" + ); +} + /// How many unversioned quote requests to receive between adoption log lines. /// /// One line per request would drown the log at production quote rates, and one @@ -600,22 +640,26 @@ impl AntProtocol { ChunkResponseKey::MerkleQuoteV2, ), ChunkMessageBody::PointerPutRequest(req) => { + let started = Instant::now(); + let record_size = req.record.len(); let response = match &self.pointers { Some(service) => service.handle_put(req).await, None => PointerPutResponse::Error(ProtocolError::StorageFailed( "this node does not store pointers".to_string(), )), }; + log_pointer_put_rpc(started.elapsed(), record_size, &response); let key = ChunkResponseKey::of_pointer_put(&response); (ChunkMessageBody::PointerPutResponse(response), key) } ChunkMessageBody::PointerGetRequest(req) => { + let started = Instant::now(); + let address = req.address; let response = match &self.pointers { Some(service) => service.handle_get(req).await, - None => PointerGetResponse::NotFound { - address: req.address, - }, + None => PointerGetResponse::NotFound { address }, }; + log_pointer_get_rpc(started.elapsed(), &address, &response); let key = ChunkResponseKey::of_pointer_get(&response); (ChunkMessageBody::PointerGetResponse(response), key) } diff --git a/src/web_rtc.rs b/src/web_rtc.rs index 8c421ca3..152fc33e 100644 --- a/src/web_rtc.rs +++ b/src/web_rtc.rs @@ -58,6 +58,11 @@ const MAX_FIND_NODE_RESULTS: usize = 20; // target; clients still obey their global memory/CPU admission budgets. const MAX_CHANNEL_REQUESTS: usize = 4; const RPC_MULTIPLEX_CAPABILITY: &str = "rpc-multiplex-4"; + +/// Advertised when `chunk_protocol` admits pointer reads and paid pointer +/// writes (ADR-0016), so a browser client can tell a node that would refuse +/// them from one that failed. +const POINTER_PROTOCOL_CAPABILITY: &str = "pointer_protocol"; // Browser dials use a 10-second channel-open timeout. Give successful clients // modest server-side headroom while bounding associations that never open one. const FIRST_DATA_CHANNEL_TIMEOUT: Duration = Duration::from_secs(15); @@ -1732,7 +1737,14 @@ async fn process_request( // PUT acknowledgements and signed quotes contain no chunk payload. 64 KiB // covers the ML-DSA public key/signature, quote and signed commitment, including // worst-case MessagePack integer encoding. GET alone needs a full wire buffer. +// +// A pointer read returns one record of `POINTER_WIRE_LEN` bytes, so it is small +// too and needs no bulk slot (see `request_is_get`). const SMALL_BINARY_RESPONSE_BYTES: usize = 64 * 1024; +const _: () = assert!( + ant_protocol::pointer::POINTER_WIRE_LEN < SMALL_BINARY_RESPONSE_BYTES / 2, + "a pointer read must fit the small response bound with room for its envelope" +); fn binary_response_limit(body: &ChunkMessageBody) -> ServerResult { match body { @@ -1741,7 +1753,9 @@ fn binary_response_limit(body: &ChunkMessageBody) -> ServerResult { | ChunkMessageBody::QuoteRequest(_) | ChunkMessageBody::QuoteRequestV2(_) | ChunkMessageBody::MerkleCandidateQuoteRequest(_) - | ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), + | ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) + | ChunkMessageBody::PointerGetRequest(_) + | ChunkMessageBody::PointerPutRequest(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), _ => Err("unsupported chunk protocol request".to_string()), } } @@ -1784,6 +1798,7 @@ fn hello_response(request_id: u64, state: &ServerState) -> Response { "get_chunk".into(), "quote_chunk".into(), "put_chunk".into(), + POINTER_PROTOCOL_CAPABILITY.into(), ], }, 0, @@ -2257,6 +2272,10 @@ struct ServerState { )] mod tests { use super::*; + use ant_protocol::chunk::{ + PointerGetRequest, PointerGetResponse, PointerPutRequest, PointerPutResponse, + }; + use ant_protocol::pointer::POINTER_WIRE_LEN; use std::net::Ipv4Addr; #[test] @@ -2745,6 +2764,48 @@ mod tests { } } + /// Pointer reads and paid pointer writes are admitted (ADR-0016), and a + /// full record fits the small bound they are given. + #[test] + fn pointer_requests_are_admitted_and_a_full_record_fits() { + let get = ChunkMessageBody::PointerGetRequest(PointerGetRequest::new([0xff; 32])); + let put = ChunkMessageBody::PointerPutRequest(PointerPutRequest::with_payment( + vec![0xff; POINTER_WIRE_LEN].into(), + vec![0xff; 16 * 1024], + )); + assert_eq!( + binary_response_limit(&get).expect("pointer GET admitted"), + SMALL_BINARY_RESPONSE_BYTES + ); + assert_eq!( + binary_response_limit(&put).expect("pointer PUT admitted"), + SMALL_BINARY_RESPONSE_BYTES + ); + + let record = ChunkMessage { + request_id: u64::MAX, + body: ChunkMessageBody::PointerGetResponse(PointerGetResponse::Success { + record: vec![0xff; POINTER_WIRE_LEN].into(), + }), + }; + let bytes = encode_binary_response(&record, SMALL_BINARY_RESPONSE_BYTES) + .expect("a full record fits"); + assert_eq!(bytes, record.encode().expect("shared wire encoding")); + + let stored = ChunkMessage { + request_id: u64::MAX, + body: ChunkMessageBody::PointerPutResponse(PointerPutResponse::Success { + address: [0xff; 32], + state_id: [0xff; 32], + }), + }; + assert!(encode_binary_response(&stored, SMALL_BINARY_RESPONSE_BYTES).is_ok()); + + // Replies are never admitted as requests. + assert!(binary_response_limit(&record.body).is_err()); + assert!(binary_response_limit(&stored.body).is_err()); + } + #[test] fn maximum_binary_get_fits_and_oversized_responses_are_rejected() { let request = ChunkMessageBody::GetRequest(ant_protocol::ChunkGetRequest::new([0xff; 32])); diff --git a/src/web_rtc/errors.rs b/src/web_rtc/errors.rs index a24d20d3..37227f00 100644 --- a/src/web_rtc/errors.rs +++ b/src/web_rtc/errors.rs @@ -5,6 +5,7 @@ use super::ServerResult; use crate::logging::warn; +use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; use ant_protocol::{ ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutResponse, ChunkQuoteResponse, MerkleCandidateQuoteResponse, ProtocolError, @@ -45,10 +46,13 @@ pub(super) fn sanitize_response(message: &mut ChunkMessage) -> ServerResult<()> | ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(error)) | ChunkMessageBody::MerkleCandidateQuoteResponse(MerkleCandidateQuoteResponse::Error( error, - )) => { + )) + | ChunkMessageBody::PointerPutResponse(PointerPutResponse::Error(error)) + | ChunkMessageBody::PointerGetResponse(PointerGetResponse::Error(error)) => { sanitize_protocol_error(error); } - ChunkMessageBody::PutResponse(ChunkPutResponse::PaymentRequired { message }) => { + ChunkMessageBody::PutResponse(ChunkPutResponse::PaymentRequired { message }) + | ChunkMessageBody::PointerPutResponse(PointerPutResponse::PaymentRequired { message }) => { warn!(detail = %message, "Browser payment verification failed"); *message = "valid payment is required".to_string(); } @@ -61,7 +65,17 @@ pub(super) fn sanitize_response(message: &mut ChunkMessage) -> ServerResult<()> | ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Success { .. }) | ChunkMessageBody::MerkleCandidateQuoteResponse(MerkleCandidateQuoteResponse::Success { .. - }) => {} + }) + // A pointer outcome names only the address and a state identifier, + // and a record is owner-signed public data. + | ChunkMessageBody::PointerPutResponse( + PointerPutResponse::Success { .. } + | PointerPutResponse::Unchanged { .. } + | PointerPutResponse::Stale { .. }, + ) + | ChunkMessageBody::PointerGetResponse( + PointerGetResponse::Success { .. } | PointerGetResponse::NotFound { .. }, + ) => {} // New response variants must explicitly opt into the browser boundary. _ => { return Err(public_error( @@ -121,6 +135,8 @@ mod tests { ChunkMessageBody::MerkleCandidateQuoteResponse( MerkleCandidateQuoteResponse::Error(error.clone()), ), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Error(error.clone())), + ChunkMessageBody::PointerGetResponse(PointerGetResponse::Error(error.clone())), ]; for body in responses { let original = ChunkMessage { @@ -139,7 +155,10 @@ mod tests { | ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(sanitized)) | ChunkMessageBody::MerkleCandidateQuoteResponse( MerkleCandidateQuoteResponse::Error(sanitized), - )) = &decoded.body + ) + | ChunkMessageBody::PointerPutResponse(PointerPutResponse::Error(sanitized)) + | ChunkMessageBody::PointerGetResponse(PointerGetResponse::Error(sanitized))) = + &decoded.body else { unreachable!(); }; @@ -166,6 +185,16 @@ mod tests { assert!( matches!(decoded.body, ChunkMessageBody::PutResponse(ChunkPutResponse::PaymentRequired { ref message }) if message == "valid payment is required") ); + let pointer = ChunkMessage { + request_id: 7, + body: ChunkMessageBody::PointerPutResponse(PointerPutResponse::PaymentRequired { + message: PRIVATE_DETAIL.into(), + }), + }; + let decoded = decode_response(pointer.encode().expect("encode").into()).expect("redact"); + assert!( + matches!(decoded.body, ChunkMessageBody::PointerPutResponse(PointerPutResponse::PaymentRequired { ref message }) if message == "valid payment is required") + ); for code in [ "storage_error", "quote_failed", @@ -186,6 +215,22 @@ mod tests { address: [1; 32], content: vec![2; 128], }), + ChunkMessageBody::PointerGetResponse(PointerGetResponse::Success { + record: vec![3; 64].into(), + }), + ChunkMessageBody::PointerGetResponse(PointerGetResponse::NotFound { address: [4; 32] }), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Success { + address: [5; 32], + state_id: [6; 32], + }), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Unchanged { + address: [5; 32], + state_id: [6; 32], + }), + ChunkMessageBody::PointerPutResponse(PointerPutResponse::Stale { + address: [5; 32], + state_id: [7; 32], + }), ChunkMessageBody::PutResponse(ChunkPutResponse::Error( ProtocolError::AddressMismatch { expected: [1; 32], From 61bd8816264c734a0a0c6c0958f04b1c8c2652c4 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Sep 2026 15:03:30 +0900 Subject: [PATCH 27/32] docs(pointer): drop a public doc link to a private method --- src/replication/pointer.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/replication/pointer.rs b/src/replication/pointer.rs index 69dcd20a..49cdcd69 100644 --- a/src/replication/pointer.rs +++ b/src/replication/pointer.rs @@ -386,8 +386,9 @@ impl PointerReplication { .spawn(async move { this.push_hints(&peers).await }); } - /// Push hints to `peers` and wait until they are sent. What - /// [`Self::push_hints_detached`] runs; tests call it to drive a round. + /// Push hints to `peers` and wait until they are sent. This is what each + /// neighbour-sync round runs in the background; tests call it to drive a + /// round. pub async fn push_hints(&self, peers: &[PeerId]) { let self_id = *self.p2p.peer_id(); let mut by_peer: HashMap> = From 41f10d92d6a2a83da3a4fb212c26206d4b801b80 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Sep 2026 15:17:16 +0900 Subject: [PATCH 28/32] docs(adr-0016): say plainly that a pointer audit does not resist relaying --- docs/adr/ADR-0016-pointers-immutable-owner.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index 9d0dac98..ee84d999 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -222,6 +222,7 @@ verified when it was committed and every read verifies it again. | One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | | Two peers decide it | **Not defended against, at the read.** Two colluding close-group peers clear a read's bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Replication does not spread such a state: the rest of the group adopts only what a quorum of it holds, so the honest members keep the paid state, but a reader that happens to hear from both colluders still sees theirs. Raising the read's bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold | | Commit to pointers it does not hold, for price or audit credit | A committed pointer is proved in round 2 by the whole signed record, which the auditor verifies and checks belongs at the address; the pointer leaf shape is accepted only at the fixed record length | +| Relay pointers instead of storing them | **Weaker than for a chunk, by nature.** A chunk audit makes a relay expensive: round 1 binds a nonce over the bytes of every chunk in the audited subtree, which a node that does not hold them has to fetch within the deadline. Records are public and 5 KB each, so even a nonced round 1 would cost a relay a few hundred kilobytes, and binding bytes the owner may replace between the rounds would fail honest holders. So round 1 binds no bytes for a pointer, and a node that fetches the few sampled records on demand passes. What it gains is small: it still serves only real, signed records that exist on the network, and it saves 5 KB of disk each | | Get a group to adopt a state nobody paid for | Repair adopts only a state a quorum of the close group hold exactly, counted over the whole group, and a fresh offer is stored only after the receiver verifies its payment itself | | Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, so the lost state is taken back and nothing older is: an address nothing is known about admits any record, and a replay could otherwise roll the node back. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | From 3c70728b1bc70e37219bd99b53e7243e72c18550 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Sep 2026 15:43:38 +0900 Subject: [PATCH 29/32] fix(pointer): bind pointer bytes in audits, keep rollbacks safe From an adversarial review of the pointer audit and browser work: - A pointer audit could be passed by a node holding nothing, fetching the few sampled records once round 2 named them. Round 1 now binds a nonced root over each pointer record's bytes, as a chunk leaf does, and round 2 must reproduce it; another replica's copy of the same state does not. The store keeps the record each update replaced for five minutes (bounded), and round 2 serves it beside the current one, so an owner updating mid-audit does not fail an honest holder. - Retention wrote a new format whenever a slot held a pointer, which a rolled-back node could not read at all. The retention file is back to the pre-pointer format; pointer leaves go in a sidecar keyed by commitment hash, so an older release keeps every chunk-only commitment. - HELLO advertised pointer_protocol even on a node with no pointer store. --- docs/adr/ADR-0016-pointers-immutable-owner.md | 69 +++-- src/pointer/store.rs | 116 ++++++++ src/replication/commitment_state.rs | 262 +++++++++--------- src/replication/mod.rs | 42 ++- src/replication/protocol.rs | 15 +- src/replication/storage_commitment_audit.rs | 257 +++++++++++++---- src/web_rtc.rs | 43 ++- 7 files changed, 581 insertions(+), 223 deletions(-) diff --git a/docs/adr/ADR-0016-pointers-immutable-owner.md b/docs/adr/ADR-0016-pointers-immutable-owner.md index ee84d999..ac1a1909 100644 --- a/docs/adr/ADR-0016-pointers-immutable-owner.md +++ b/docs/adr/ADR-0016-pointers-immutable-owner.md @@ -172,25 +172,35 @@ chunks are. can fail an honest holder for having taken one mid-audit. - **Price.** A quote is priced from the key count of the commitment it pins, so every committed pointer counts toward it exactly as a chunk does. -- **Round 1.** A pointer leaf is reported at its address with that hash and the - fixed record length. The auditor accepts that one leaf shape besides - `(key, key)`, and only at exactly a pointer's length, so it cannot stand in for - a chunk of any other size. A node that has lost a committed pointer refuses - round 1, which is a confirmed failure, as a lost chunk is. +- **Round 1.** A pointer leaf is reported at its address with that hash, the + fixed record length, and a nonced root over the bytes of the record the node + holds, under the audit's fresh nonce, as a chunk leaf's is. So the node has to + read every pointer in the audited subtree before it learns which few will be + sampled. The auditor accepts that one leaf shape besides `(key, key)`, and only + at exactly a pointer's length, so it cannot stand in for a chunk of any other + size. A node that has lost a committed pointer refuses round 1, which is a + confirmed failure, as a lost chunk is. - **Round 2.** Where a chunk is proved by a Bao slice and a nonced opening, a pointer is proved by its whole signed record. The auditor checks the - signature and that the record belongs at the committed address. A peer signs - its own commitment, so without this step it could commit any key under the - hash of cheap bytes it holds; a record the auditor verifies itself cannot be - forged that way. Any valid record at the address passes, whatever its state. - At most five records are opened, about 26 KB, well under the audit message - ceiling. + signature, that the record belongs at the committed address, and that its + nonced root is the one round 1 gave. A peer signs its own commitment, so + without the signature check it could commit any key under the hash of cheap + bytes it holds; without the nonced root it could hold nothing and fetch the + sampled records on demand. At most five records are opened, a few tens of + kilobytes, well under the audit message ceiling. +- **Updates between the rounds.** The owner may update a pointer after round 1 + bound it. The store keeps the record an update replaced for five minutes, + longer than an audit session lives, and round 2 serves it beside the new one; + the auditor accepts whichever matches. Two updates to one pointer inside the + same audit would fail an honest holder, and would take the owner two paid + updates within seconds of each other. - **Retention and pruning.** A retained commitment that still holds a pointer vetoes its deletion, as for a chunk, so a peer pinning that commitment can - still audit it. The persisted retention records which leaves are pointers, so - a restart rebuilds the exact signed root. It is written in format 2 only when - some slot holds a pointer, and in the pre-pointer format 1 otherwise, so a node - rolled back to an earlier release still reloads its retention. + still audit it. The persisted retention keeps which leaves are pointers in a + file beside it, keyed by commitment hash, so a restart rebuilds the exact + signed root. The retention file itself is unchanged, so a node rolled back to + an earlier release still reloads every commitment it can answer for, and + drops only those that commit a pointer. Round 2 carries a new slice item, so the subtree audit's protocol id moves to `v2`, as ADR-0009 did before. Nodes on different ids do not audit each other @@ -222,7 +232,7 @@ verified when it was committed and every read verifies it again. | One peer decides what a pointer says | A read returns a state only if two of the answering peers name it, and a write must reach a majority **plus one** so that two always do. Otherwise a single close-group peer serving an owner-signed state nobody paid to store would be believed by every reader: the record verifies, belongs at the address, and wins the merge. It cannot make a second peer agree. The read counts each state separately, so a state one peer names cannot bury the one the rest agree on — that would be denial of service in place of forgery, and it is also what an ordinary read during an update looks like | | Two peers decide it | **Not defended against, at the read.** Two colluding close-group peers clear a read's bar, and only the owner can sign, so what this buys is the owner's own updates unpaid. Replication does not spread such a state: the rest of the group adopts only what a quorum of it holds, so the honest members keep the paid state, but a reader that happens to hear from both colluders still sees theirs. Raising the read's bar only raises the number of nodes to grind: both the pointer's address and a node's id are choosable, so an owner determined to sit beside their own pointer can reach any fixed threshold | | Commit to pointers it does not hold, for price or audit credit | A committed pointer is proved in round 2 by the whole signed record, which the auditor verifies and checks belongs at the address; the pointer leaf shape is accepted only at the fixed record length | -| Relay pointers instead of storing them | **Weaker than for a chunk, by nature.** A chunk audit makes a relay expensive: round 1 binds a nonce over the bytes of every chunk in the audited subtree, which a node that does not hold them has to fetch within the deadline. Records are public and 5 KB each, so even a nonced round 1 would cost a relay a few hundred kilobytes, and binding bytes the owner may replace between the rounds would fail honest holders. So round 1 binds no bytes for a pointer, and a node that fetches the few sampled records on demand passes. What it gains is small: it still serves only real, signed records that exist on the network, and it saves 5 KB of disk each | +| Relay pointers instead of storing them | Round 1 binds a nonced root over the bytes of every pointer in the audited subtree before the sample is drawn, and round 2 must reproduce one. Another replica's copy of the same state does not match: every signature is randomised. Still weaker than for a chunk by nature: a record is 5 KB, so fetching a whole subtree of them on demand costs a relay far less than a subtree of chunks would | | Get a group to adopt a state nobody paid for | Repair adopts only a state a quorum of the close group hold exactly, counted over the whole group, and a fresh offer is stored only after the receiver verifies its payment itself | | Node claims a record it no longer holds | An index entry is only a claim about a file. Before answering "unchanged" or "stale" the node reads the record back and checks it is still the one the index names; if it is not, the node stops answering for that address and the arrival becomes a repair. It keeps what it lost, so the lost state is taken back and nothing older is: an address nothing is known about admits any record, and a replay could otherwise roll the node back. That check parses the body, so a signature corrupted in place passes it and is caught on the next read instead — verifying there would put ML-DSA in front of the payment gate, which is the one place it must not be | @@ -245,9 +255,11 @@ verified when it was committed and every read verifies it again. correctly signed value and cannot tell. - Replicas may hold different valid signatures of one state; nothing compares record bytes across replicas. -- A pointer audit opens a whole 5,303-byte record where a chunk audit opens one - 1 KiB block, so a round 2 over pointers is a few times larger. It stays - bounded by the five-leaf cap. +- A pointer audit opens a whole 5,303-byte record, and up to two of them after + an update, where a chunk audit opens one 1 KiB block, so a round 2 over + pointers is several times larger. It stays bounded by the five-leaf cap. +- A node keeps the record each update replaced in memory for five minutes, up + to 2,048 of them, about 11 MB. - The subtree audit's protocol id moved to `v2`, so across the upgrade the old and new releases do not audit each other. @@ -292,7 +304,8 @@ security boundary is made explicitly: corroboration, write quorum and chain resolution. Only the wallet is the page's. -A node advertises `pointer_protocol` in HELLO when it admits pointers. A browser +A node advertises `pointer_protocol` in HELLO when it admits pointers and has a +pointer store to serve them from. A browser client never sends a pointer request to a node that does not, so an older node's refusal is never mistaken for a failed peer. The owner key crosses into the page as its 32-byte FIPS 204 seed. A pointer payment is one quote and, unlike a file @@ -348,13 +361,17 @@ receipt is paid again on retry. down the chain to its chunk. - Storage audits, through the live responders and judged by the auditor's own checks: a node holding committed pointers and chunks passes both rounds; an - update between the rounds does not fail it; a node that lost a committed - pointer fails round 1, and one that loses it after round 1 is caught in round - 2; another owner's valid record is not proof of the committed one; a pointer - leaf at any other length is refused. Over a live network, a node holding its + update between the rounds does not fail it, and does if the replaced record + is not kept; a node that lost a committed pointer fails round 1, and one that + loses it after round 1 is caught in round 2; another owner's valid record is + not proof of the committed one; a relay that answers round 1 without the + bytes fails, as does one serving another replica's copy of the same state; a + pointer leaf at any other length is refused. Over a live network, a node holding its committed pointers passes the audit and one that dropped them fails it. -- A commitment holding pointers survives a restart with its exact pin; one - without is written in the pre-pointer format, which the old layout reads. +- A commitment holding pointers survives a restart with its exact pin. An + older release reading the same retention keeps the chunk-only commitment a + peer pinned before the upgrade, and a pointer-leaves file from another + snapshot attaches nothing. - Pruning keeps a pointer a retained commitment holds, and drops it once none does. - Fresh offers, hints, fetches and state queries are covered by the replication diff --git a/src/pointer/store.rs b/src/pointer/store.rs index 2664f9c6..3628d173 100644 --- a/src/pointer/store.rs +++ b/src/pointer/store.rs @@ -51,6 +51,7 @@ use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use std::time::{Duration, Instant}; use fs2::FileExt; use parking_lot::Mutex; @@ -76,6 +77,20 @@ const LOCK_FILE_NAME: &str = ".pointer-store-lock"; /// address's last byte. const SHARD_COUNT: u16 = 256; +/// How long a replaced record is kept after an update, in memory. +/// +/// A storage audit binds the record a node holds in its first round and asks +/// for it in the second (ADR-0016). An owner updating the pointer between the +/// two would otherwise fail the honest node that took the update, so the +/// record the update replaced stays servable for longer than an audit session +/// lives. +pub const SUPERSEDED_RETENTION: Duration = Duration::from_mins(5); + +/// Most replaced records kept at once, about 11 MB at the cap. Past it the +/// oldest goes first; reaching it inside [`SUPERSEDED_RETENTION`] takes that +/// many paid updates to pointers this node holds. +const MAX_SUPERSEDED: usize = 2048; + /// The name of the shard directory `address` lives in: its last byte in hex. fn shard_name(address: &XorName) -> String { let last = address.last().copied().unwrap_or_default(); @@ -243,6 +258,9 @@ struct Inner { generation: AtomicU64, /// What the store has done, for telemetry. counters: Counters, + /// The record each recent update replaced, by address, with when (see + /// [`SUPERSEDED_RETENTION`]). + superseded: Mutex)>>, /// Held for the store's lifetime; releasing it releases the directory. _lock_file: File, } @@ -295,6 +313,7 @@ impl PointerStore { write_seq: AtomicU64::new(0), generation: AtomicU64::new(next_generation), counters: Counters::default(), + superseded: Mutex::new(HashMap::new()), _lock_file: lock_file, }), }) @@ -500,6 +519,36 @@ impl PointerStore { .map(|entry| entry.state.state_id) } + /// The record an update at `address` replaced within the last + /// [`SUPERSEDED_RETENTION`], if any: what a storage audit that bound it + /// before the update is still owed. + #[must_use] + pub fn superseded(&self, address: &XorName) -> Option> { + self.inner + .superseded + .lock() + .get(address) + .filter(|(at, _)| at.elapsed() < SUPERSEDED_RETENTION) + .map(|(_, bytes)| bytes.clone()) + } + + /// The bytes of the record held at `address`, read from disk without + /// verifying them. + /// + /// For a caller that only binds the bytes and has them verified later, by + /// whoever they are served to: verifying here would cost a signature check + /// per record for nothing. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file exists but cannot be read. + pub async fn record_bytes(&self, address: &XorName) -> Result>> { + let path = self.path_for(address); + spawn_blocking(move || read_record_file(&path)) + .await + .map_err(|e| Error::Storage(format!("pointer read panicked: {e}")))? + } + /// The state held at `address`, as the index records it, if this node can /// serve one. /// @@ -689,6 +738,25 @@ impl PointerStore { } impl Inner { + /// Keep `bytes` as the record just replaced at `address`, dropping what + /// has aged out and, past the cap, the oldest. + fn keep_superseded(&self, address: XorName, bytes: Vec) { + let now = Instant::now(); + let mut superseded = self.superseded.lock(); + superseded.retain(|_, (at, _)| now.duration_since(*at) < SUPERSEDED_RETENTION); + while superseded.len() >= MAX_SUPERSEDED { + let Some(oldest) = superseded + .iter() + .min_by_key(|(_, (at, _))| *at) + .map(|(address, _)| *address) + else { + break; + }; + superseded.remove(&oldest); + } + superseded.insert(address, (now, bytes)); + } + /// Remove the file and the index entry for `address` under one lock, so a /// commit can never land between the two and be forgotten on disk. fn delete_blocking(&self, address: &XorName) -> Result { @@ -775,6 +843,7 @@ impl Inner { // Re-check: staging is not instantaneous and a newer state may // have committed while it ran. + let replacing = index.get(&address).is_some_and(|entry| entry.on_disk); let outcome = match index.get(&address) { // Nothing held, or a record this node lost: either way the // write must happen, whatever state it carries. @@ -793,6 +862,16 @@ impl Inner { return Ok(outcome); } + // What this replaces, read under the lock so it is the record the + // index names. Kept for an audit that bound it; a record that + // cannot be read is not kept, and an audit owed it fails as it + // would have on the lost file. + let previous = if replacing { + read_record_file(&path).ok().flatten() + } else { + None + }; + // The rename is the commit point: nothing fallible happens between // it and the index update, and both are under this one lock. if let Err(e) = rename_with_retry(&temp, &path) { @@ -807,6 +886,9 @@ impl Inner { } let generation = self.generation.fetch_add(1, Ordering::Relaxed); index.insert(address, IndexEntry::of(record, generation)); + if let Some(previous) = previous { + self.keep_superseded(address, previous); + } // A file is on the disk now. Charge it whether or not this replaced // one: telling those apart would mean trusting an observation taken // before the rename, and that observation can be wrong in the one @@ -1187,6 +1269,40 @@ mod tests { (store, dir) } + /// The record an update replaces stays servable for a while, byte for + /// byte, so an audit that bound it before the update is not failed by it. + #[tokio::test] + async fn an_update_keeps_the_record_it_replaced_for_a_while() { + let (store, _dir) = store().await; + let first = signed(1, 1, 1); + store.put_bytes(&first.to_bytes()).await.expect("put"); + assert_eq!( + store.superseded(&first.address()), + None, + "a creation replaces nothing" + ); + + let second = signed(1, 2, 2); + assert_eq!( + store.put_bytes(&second.to_bytes()).await.expect("put"), + PutOutcome::Changed + ); + assert_eq!( + store.superseded(&first.address()), + Some(first.to_bytes()), + "the replaced record, exactly as it was held" + ); + assert_eq!( + store.record_bytes(&first.address()).await.expect("read"), + Some(second.to_bytes()), + "and the new one is what is held" + ); + + // A stale arrival replaces nothing, so it keeps nothing. + store.put_bytes(&first.to_bytes()).await.expect("put"); + assert_eq!(store.superseded(&first.address()), Some(first.to_bytes())); + } + #[tokio::test] async fn stores_then_reads_back_the_same_bytes() { let (store, _dir) = store().await; diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 55349897..08d3ce46 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -28,7 +28,7 @@ //! `2 × (key_count × ~64 bytes + signature_size)` — for 10k keys, ~1.3 MB. use saorsa_core::identity::PeerId; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -371,6 +371,11 @@ struct PersistedSlot { /// The subset of `leaf_keys` committed as pointers (ADR-0016). A pointer /// leaf is `(key, pointer_leaf_hash(key))`, not `(key, key)`, so without /// this the rebuilt tree would miss the signed root and the slot be lost. + /// + /// Not part of this file: it is kept in the pointer-leaves sidecar (see + /// [`PersistedRetention::pointer_leaves_bytes`]), so the retention file + /// stays exactly what a release before pointers reads. + #[serde(skip)] pointer_keys: Vec, } @@ -379,62 +384,26 @@ struct PersistedSlot { /// via re-gossip) rather than silently misinterpreted (e.g. an old field read /// under new semantics). /// -/// Format 2 added each slot's pointer keys. Format 1 is still read, and still -/// written whenever no slot commits a pointer (see [`PersistedRetention::to_bytes`]). -const RETENTION_FORMAT_VERSION: u32 = 2; - -/// The format that predates pointers: the same slots with no pointer keys. -const POINTERLESS_RETENTION_FORMAT_VERSION: u32 = 1; - -/// A format-1 slot, as read back from disk. -#[derive(Deserialize)] -struct PointerlessSlot { - commitment: StorageCommitment, - leaf_keys: Vec, - expires_at_unix: Option, -} - -/// A format-1 snapshot, as read back from disk. -#[derive(Deserialize)] -struct PointerlessRetention { - version: u32, - slots: Vec, - has_current: bool, -} +/// Pointers did not change it. Their leaves are persisted beside this file, so +/// a node rolled back to a release that predates pointers still reloads every +/// slot it can answer for, and drops only the ones that commit a pointer, which +/// it could not answer for anyway. +const RETENTION_FORMAT_VERSION: u32 = 1; -/// A format-1 slot, as written: borrowed, so writing format 1 copies nothing. -#[derive(Serialize)] -struct PointerlessSlotRef<'a> { - commitment: &'a StorageCommitment, - leaf_keys: &'a [XorName], - expires_at_unix: Option, -} +/// Version of the pointer-leaves sidecar. +const POINTER_LEAVES_FORMAT_VERSION: u32 = 1; -/// A format-1 snapshot, as written. -#[derive(Serialize)] -struct PointerlessRetentionRef<'a> { +/// The pointer leaves of each retained slot, keyed by the slot's commitment +/// hash (ADR-0016). +/// +/// Keyed by hash rather than by position so that a sidecar left behind by a +/// different snapshot, as after a rollback and upgrade, can only ever describe +/// the commitment it names: a hash binds the root, and the root binds which +/// leaves are pointers. +#[derive(Serialize, Deserialize)] +struct PersistedPointerLeaves { version: u32, - slots: Vec>, - has_current: bool, -} - -impl From for PersistedRetention { - fn from(old: PointerlessRetention) -> Self { - Self { - version: old.version, - slots: old - .slots - .into_iter() - .map(|slot| PersistedSlot { - commitment: slot.commitment, - leaf_keys: slot.leaf_keys, - expires_at_unix: slot.expires_at_unix, - pointer_keys: Vec::new(), - }) - .collect(), - has_current: old.has_current, - } - } + slots: Vec<([u8; 32], Vec)>, } /// The persisted responder retention. Slots are newest-first; `has_current` @@ -451,52 +420,60 @@ impl PersistedRetention { /// Serialize for durable persistence (caller writes it atomically). `None` /// on a serialization error, so the caller can refuse to overwrite the /// durable file rather than truncate it. - /// - /// A snapshot with no pointer in any slot is written in format 1, which it - /// can say exactly. A node rolled back to a release that predates pointers - /// then still reloads its retention, instead of dropping every pin a peer - /// holds on it. #[must_use] pub fn to_bytes(&self) -> Option> { - let pointerless = self.version == RETENTION_FORMAT_VERSION - && self.slots.iter().all(|slot| slot.pointer_keys.is_empty()); - if !pointerless { - return postcard::to_allocvec(self).ok(); - } - let old = PointerlessRetentionRef { - version: POINTERLESS_RETENTION_FORMAT_VERSION, - slots: self - .slots - .iter() - .map(|slot| PointerlessSlotRef { - commitment: &slot.commitment, - leaf_keys: &slot.leaf_keys, - expires_at_unix: slot.expires_at_unix, - }) - .collect(), - has_current: self.has_current, - }; - postcard::to_allocvec(&old).ok() + postcard::to_allocvec(self).ok() } /// Decode a persisted snapshot. `None` on a corrupt blob OR a version /// mismatch — the caller then fails open LOCALLY (empty retention; the node /// re-gossips a fresh root), which never grants a remote grace. - /// - /// The version leads the blob in every format, so it is read first and - /// picks the layout the rest is decoded with. #[must_use] pub fn from_bytes(bytes: &[u8]) -> Option { - let (version, _) = postcard::take_from_bytes::(bytes).ok()?; - match version { - RETENTION_FORMAT_VERSION => postcard::from_bytes(bytes).ok(), - POINTERLESS_RETENTION_FORMAT_VERSION => { - postcard::from_bytes::(bytes) - .ok() - .map(Self::from) + let this: Self = postcard::from_bytes(bytes).ok()?; + (this.version == RETENTION_FORMAT_VERSION).then_some(this) + } + + /// Serialize the pointer leaves of every slot, for the sidecar written + /// beside the retention file. `None` on a serialization error. + #[must_use] + pub fn pointer_leaves_bytes(&self) -> Option> { + let slots = self + .slots + .iter() + .filter(|slot| !slot.pointer_keys.is_empty()) + .filter_map(|slot| { + commitment_hash(&slot.commitment).map(|hash| (hash, slot.pointer_keys.clone())) + }) + .collect(); + postcard::to_allocvec(&PersistedPointerLeaves { + version: POINTER_LEAVES_FORMAT_VERSION, + slots, + }) + .ok() + } + + /// Give each slot the pointer leaves the sidecar records for its + /// commitment. Returns `false`, attaching nothing, on a corrupt sidecar or + /// an unknown version; a slot left without its pointer leaves then fails to + /// rebuild and is dropped, which fails open locally as a corrupt snapshot + /// does. + pub fn attach_pointer_leaves(&mut self, bytes: &[u8]) -> bool { + let Ok(sidecar) = postcard::from_bytes::(bytes) else { + return false; + }; + if sidecar.version != POINTER_LEAVES_FORMAT_VERSION { + return false; + } + let by_hash: HashMap<[u8; 32], Vec> = sidecar.slots.into_iter().collect(); + for slot in &mut self.slots { + if let Some(keys) = + commitment_hash(&slot.commitment).and_then(|hash| by_hash.get(&hash)) + { + slot.pointer_keys.clone_from(keys); } - _ => None, } + true } } @@ -1164,32 +1141,37 @@ mod tests { ); } + /// Build a signed commitment over `chunks` and `pointers`. + fn mixed(chunks: &[u8], pointers: &[u8]) -> BuiltCommitment { + let (pk, sk) = keypair(); + let mut entries: Vec<_> = chunks.iter().map(|i| (key(*i), key(*i))).collect(); + entries.extend( + pointers + .iter() + .map(|i| (key(*i), pointer_leaf_hash(&key(*i)))), + ); + BuiltCommitment::build(entries, &[0xAB; 32], &sk, &pk.to_bytes()).unwrap() + } + /// A commitment holding pointers (ADR-0016) survives a restart like any /// other. A pointer leaf is not `(key, key)`, so a snapshot that kept only /// the key set would rebuild a different root, fail the signature check and /// drop the slot, and every peer pinning it would then fail this node. #[test] fn a_commitment_holding_pointers_survives_a_restart() { - let (pk, sk) = keypair(); - let pk_bytes = pk.to_bytes(); - let mut entries: Vec<_> = (1..=4u8).map(|i| (key(i), key(i))).collect(); - entries.extend((10..=12u8).map(|i| (key(i), pointer_leaf_hash(&key(i))))); - let built = BuiltCommitment::build(entries, &[0xAB; 32], &sk, &pk_bytes).unwrap(); + let built = mixed(&[1, 2, 3, 4], &[10, 11, 12]); let pin = built.hash(); let state = ResponderCommitmentState::new(); state.rotate(built); state.mark_gossiped(pin); - let bytes = state.snapshot().to_bytes().expect("serialize"); - assert_eq!( - postcard::take_from_bytes::(&bytes) - .map(|(v, _)| v) - .ok(), - Some(RETENTION_FORMAT_VERSION), - "a snapshot holding pointers needs the format that can say so" - ); + let snapshot = state.snapshot(); + let bytes = snapshot.to_bytes().expect("serialize"); + let sidecar = snapshot.pointer_leaves_bytes().expect("serialize sidecar"); + let mut reloaded = PersistedRetention::from_bytes(&bytes).expect("deserialize"); + assert!(reloaded.attach_pointer_leaves(&sidecar)); let fresh = ResponderCommitmentState::new(); - fresh.restore(&PersistedRetention::from_bytes(&bytes).expect("deserialize")); + fresh.restore(&reloaded); let got = fresh.lookup_by_hash(&pin).expect("pin survives restart"); assert_eq!(got.hash(), pin); @@ -1202,36 +1184,68 @@ mod tests { ); } - /// A snapshot with no pointer in it is written in format 1, byte for byte - /// what a release before pointers writes. Rolling such a node back then - /// keeps its retention; format 2 would read as an unknown version there and - /// drop every pin its peers hold. + /// A node rolled back to a release that predates pointers keeps every + /// commitment it can still answer for. The retention file is exactly the + /// format such a release reads; it just knows nothing of the sidecar, so + /// the slot committing pointers is dropped and the chunk-only one a peer + /// pinned before the upgrade is not. #[test] - fn a_snapshot_without_pointers_is_written_in_the_old_format() { - let (pk, sk) = keypair(); - let pk_bytes = pk.to_bytes(); - let entries: Vec<_> = (1..=5u8).map(|i| (key(i), key(i))).collect(); - let built = BuiltCommitment::build(entries, &[0xAB; 32], &sk, &pk_bytes).unwrap(); - let pin = built.hash(); + fn a_rollback_keeps_every_chunk_only_commitment() { + let legacy = mixed(&[1, 2, 3], &[]); + let legacy_pin = legacy.hash(); + let current = mixed(&[1, 2, 3], &[10]); + let current_pin = current.hash(); let state = ResponderCommitmentState::new(); - state.rotate(built); - state.mark_gossiped(pin); + state.rotate(legacy); + state.mark_gossiped(legacy_pin); + state.rotate(current); + state.mark_gossiped(current_pin); let bytes = state.snapshot().to_bytes().expect("serialize"); - let old = postcard::from_bytes::(&bytes) - .expect("the old layout decodes it"); - assert_eq!(old.version, POINTERLESS_RETENTION_FORMAT_VERSION); assert_eq!( - old.slots.first().map(|slot| slot.leaf_keys.len()), - Some(5), - "the old layout reads the same key set" + postcard::take_from_bytes::(&bytes) + .map(|(v, _)| v) + .ok(), + Some(RETENTION_FORMAT_VERSION) + ); + // What an older release does: read the file and nothing beside it. + let older = ResponderCommitmentState::new(); + older.restore(&PersistedRetention::from_bytes(&bytes).expect("deserialize")); + assert!( + older.lookup_by_hash(&legacy_pin).is_some(), + "the chunk-only commitment is still answerable" + ); + assert!( + older.lookup_by_hash(¤t_pin).is_none(), + "the one committing a pointer cannot be rebuilt without the sidecar" ); + } + + /// A sidecar that describes other commitments, as one left by a snapshot + /// from before a rollback might, attaches nothing to these. + #[test] + fn a_sidecar_only_describes_the_commitments_it_names() { + let other = ResponderCommitmentState::new(); + other.rotate(mixed(&[1], &[10])); + let sidecar = other.snapshot().pointer_leaves_bytes().expect("sidecar"); + let built = mixed(&[1], &[10]); + let pin = built.hash(); + let state = ResponderCommitmentState::new(); + state.rotate(built); + let mut reloaded = + PersistedRetention::from_bytes(&state.snapshot().to_bytes().expect("bytes")) + .expect("deserialize"); + assert!(reloaded.attach_pointer_leaves(&sidecar)); let fresh = ResponderCommitmentState::new(); - fresh.restore(&PersistedRetention::from_bytes(&bytes).expect("deserialize")); + fresh.restore(&reloaded); + assert!( + fresh.lookup_by_hash(&pin).is_none(), + "another commitment's pointer leaves were applied to this one" + ); assert!( - fresh.lookup_by_hash(&pin).is_some(), - "and this release reads it back too" + !reloaded.attach_pointer_leaves(&[0xff; 7]), + "garbage is refused" ); } diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 652f0090..73d91f45 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -9996,7 +9996,24 @@ async fn load_commitment_retention(state: &ResponderCommitmentState, path: &Path return; } }; - if let Some(persisted) = PersistedRetention::from_bytes(&bytes) { + if let Some(mut persisted) = PersistedRetention::from_bytes(&bytes) { + let sidecar = pointer_leaves_path(path); + match tokio::fs::read(&sidecar).await { + Ok(leaves) => { + if !persisted.attach_pointer_leaves(&leaves) { + warn!( + "Commitment retention: corrupt pointer leaves at {}; \ + commitments holding pointers will not be restored", + sidecar.display() + ); + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!( + "Commitment retention: failed to read {}: {e}", + sidecar.display() + ), + } state.restore(&persisted); info!( "Commitment retention: reloaded {} slot(s) from {}", @@ -10016,23 +10033,38 @@ async fn load_commitment_retention(state: &ResponderCommitmentState, path: &Path /// needless disk writes on idle nodes. On success updates `last` to the bytes /// written; on a serialization/write error the existing on-disk snapshot is left /// intact (never truncated). +/// +/// The pointer leaves go to a sidecar written first (ADR-0016), so the +/// retention file never names a commitment whose pointer leaves are not yet +/// on disk. A crash between the two leaves a sidecar ahead of the file, which +/// is harmless: it is keyed by commitment hash. async fn persist_retention_if_changed( state: &ResponderCommitmentState, path: &Path, last: &mut Option>, ) { - let Some(bytes) = state.snapshot().to_bytes() else { + let snapshot = state.snapshot(); + let (Some(bytes), Some(leaves)) = (snapshot.to_bytes(), snapshot.pointer_leaves_bytes()) else { warn!("Commitment retention: serialization failed; keeping previous snapshot"); return; }; - if last.as_deref() == Some(bytes.as_slice()) { + let mut combined = leaves.clone(); + combined.extend_from_slice(&bytes); + if last.as_deref() == Some(combined.as_slice()) { return; } - if write_retention_atomic(path, bytes.clone()).await { - *last = Some(bytes); + if write_retention_atomic(&pointer_leaves_path(path), leaves).await + && write_retention_atomic(path, bytes).await + { + *last = Some(combined); } } +/// Where the pointer leaves of the retention at `path` are kept. +fn pointer_leaves_path(path: &Path) -> PathBuf { + path.with_file_name("commitment_retention_pointers.bin") +} + /// Durably write `bytes` to `path`: temp file → fsync temp → atomic rename → /// fsync parent dir (so the rename itself survives a crash). Returns `true` on /// success. Only the retention-persist loop writes this path, so a fixed temp diff --git a/src/replication/protocol.rs b/src/replication/protocol.rs index 4f7415b0..b2ca7962 100644 --- a/src/replication/protocol.rs +++ b/src/replication/protocol.rs @@ -1369,16 +1369,23 @@ pub enum SubtreeSliceItem { /// The responder holds this committed pointer (ADR-0016) and serves the /// whole signed record. A pointer's address is not a content hash, so no /// slice of it could be authenticated against the address; the record can - /// be, by its signature. One per requested pointer key, whatever blocks - /// were named. + /// be, by its signature, and against the nonced root round 1 bound over + /// its bytes. One per requested pointer key, whatever blocks were named. PointerRecord { /// The requested key: the pointer's address. key: XorName, - /// The pointer record, in its canonical encoding. - record: Vec, + /// The record held now, and the one an update replaced since round 1 + /// if there was one, each in its canonical encoding. At most + /// [`MAX_POINTER_RECORDS_PER_ITEM`]: round 1 bound one of them, and + /// the responder cannot tell which without keeping round 1's answer. + records: Vec>, }, } +/// Most records one [`SubtreeSliceItem::PointerRecord`] may carry: the one +/// held now and the one it replaced. +pub const MAX_POINTER_RECORDS_PER_ITEM: usize = 2; + /// Response to a [`SubtreeSliceChallenge`] (round 2). /// /// The contract is **coalesced and order-independent**: the responder groups the diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index 39f3b396..c20b8e60 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -17,19 +17,20 @@ use crate::logging::{debug, info, warn}; use rand::Rng; use crate::ant_protocol::XorName; -use crate::pointer::store::PointerStore; +use crate::pointer::store::{PointerStore, SUPERSEDED_RETENTION}; use crate::replication::commitment::{commitment_hash, pointer_leaf_hash, StorageCommitment}; use crate::replication::commitment_state::ResponderCommitmentState; use crate::replication::config::{ ReplicationConfig, MAX_SLICE_OPENINGS, SUBTREE_AUDIT_PROTOCOL_ID, - SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES, + SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES, SUBTREE_SESSION_TTL, }; use crate::replication::protocol::{ RejectKind, ReplicationMessage, ReplicationMessageBody, SubtreeAuditChallenge, SubtreeAuditResponse, SubtreeSliceChallenge, SubtreeSliceItem, SubtreeSliceOpening, - SubtreeSliceResponse, + SubtreeSliceResponse, MAX_POINTER_RECORDS_PER_ITEM, }; use crate::replication::recent_provers::RecentProvers; +use crate::replication::slice::nonced_block_root; use crate::replication::subtree::{ select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeLeaf, SubtreeProof, @@ -741,6 +742,12 @@ pub(crate) fn evaluate_subtree_structure( Ok(()) } +// Round 2 may be owed the record an update replaced after round 1 bound it. +const _: () = assert!( + SUPERSEDED_RETENTION.as_secs() > SUBTREE_SESSION_TTL.as_secs(), + "a replaced pointer record must outlive the audit session that may be owed it" +); + /// Whether a round-1 leaf commits a pointer (ADR-0016): committed under /// [`pointer_leaf_hash`] of its key, at the fixed record length. fn is_pointer_leaf(leaf: &SubtreeLeaf) -> bool { @@ -748,10 +755,46 @@ fn is_pointer_leaf(leaf: &SubtreeLeaf) -> bool { && usize::try_from(leaf.content_len).ok() == Some(POINTER_WIRE_LEN) } -/// Whether `record` is a valid pointer record at `key`: its signature verifies -/// and it belongs at that address. -fn serves_pointer_at(key: &XorName, record: &[u8]) -> bool { - Pointer::from_bytes(record).is_ok_and(|pointer| pointer.address() == *key) +/// Check the round-2 item for a pointer leaf: one of the records served must +/// prove it. An admitted absence is `KeyAbsent`; anything else that does not +/// prove it, including no item at all, is `DigestMismatch`. +fn verify_pointer_item( + nonce: &[u8; 32], + challenged_peer_bytes: &[u8; 32], + leaf: &SubtreeLeaf, + items: &[SubtreeSliceItem], +) -> Result<(), AuditFailureReason> { + let served = items.iter().find_map(|it| match it { + SubtreeSliceItem::PointerRecord { key, records } if key == &leaf.key => { + Some(Some(records.as_slice())) + } + SubtreeSliceItem::Absent { key } if key == &leaf.key => Some(None), + _ => None, + }); + match served { + Some(Some(records)) + if records + .iter() + .any(|record| proves_pointer(nonce, challenged_peer_bytes, leaf, record)) => + { + Ok(()) + } + Some(None) => Err(AuditFailureReason::KeyAbsent), + _ => Err(AuditFailureReason::DigestMismatch), + } +} + +/// Whether `record` proves the pointer `leaf` commits: its signature verifies, +/// it belongs at the leaf's address, and it is the record round 1 bound the +/// leaf's nonced root over. +fn proves_pointer( + nonce: &[u8; 32], + challenged_peer_bytes: &[u8; 32], + leaf: &SubtreeLeaf, + record: &[u8], +) -> bool { + nonced_block_root(nonce, challenged_peer_bytes, &leaf.key, record) == leaf.nonced_root + && Pointer::from_bytes(record).is_ok_and(|pointer| pointer.address() == leaf.key) } /// The auditor's **freshly-randomised** spot-check sample of the round-1 proof: @@ -912,8 +955,12 @@ pub(crate) fn verify_slice_response( && !present.iter().any(|(k, _)| k == key) && !records.contains(key) } - SubtreeSliceItem::PointerRecord { key, .. } => { + SubtreeSliceItem::PointerRecord { + key, + records: served, + } => { requested_keys.contains(key) + && (1..=MAX_POINTER_RECORDS_PER_ITEM).contains(&served.len()) && records.insert(*key) && !absent.contains(key) && !present.iter().any(|(k, _)| k == key) @@ -926,25 +973,17 @@ pub(crate) fn verify_slice_response( let mut checked = 0usize; for (leaf, block_index) in openings { - // A pointer leaf is proved by the whole signed record: it must verify and - // belong at the committed address. Any valid record there passes, so an - // update between the rounds cannot fail an honest holder. + // A pointer leaf is proved by the whole signed record: it must verify, + // belong at the committed address, and be the record round 1 bound its + // nonced root over, which the responder had to read before it knew what + // would be sampled. An update between the rounds does not fail an + // honest holder: it serves the record it held then beside the new one. if is_pointer_leaf(leaf) { - let served = items.iter().find_map(|it| match it { - SubtreeSliceItem::PointerRecord { key, record } if key == &leaf.key => { - Some(Some(record.as_slice())) - } - SubtreeSliceItem::Absent { key } if key == &leaf.key => Some(None), - _ => None, - }); - match served { - Some(Some(record)) if serves_pointer_at(&leaf.key, record) => { - checked += 1; - continue; - } - Some(None) => return AuditVerdict::Fail(AuditFailureReason::KeyAbsent), - _ => return AuditVerdict::Fail(AuditFailureReason::DigestMismatch), + if let Err(reason) = verify_pointer_item(nonce, challenged_peer_bytes, leaf, items) { + return AuditVerdict::Fail(reason); } + checked += 1; + continue; } let block_index = *block_index; // Match the responder's item for exactly this (key, block_index). A @@ -1445,25 +1484,58 @@ async fn subtree_challenge_response( // of subtree size, hashing each into its plain + nonced leaf. let mut leaves = Vec::with_capacity(plan.leaf_keys.len()); for (position, key) in plan.leaf_keys.iter().enumerate() { - // A pointer leaf (ADR-0016) commits no bytes: round 2 asks for the whole - // signed record. Round 1 only says whether it is still held, from the - // index, and admits a loss exactly as a missing chunk is admitted. + // A pointer leaf (ADR-0016) commits no bytes in the tree, since its + // bytes change with every update. Round 1 binds them here instead: the + // nonced root over the record held now, which round 2 must reproduce. + // So a node answers from records it holds, not ones it could fetch once + // it learns which few are sampled. A loss is admitted exactly as a + // missing chunk is, and a read error is transient as for a chunk. if plan.leaf_is_pointer.get(position).copied().unwrap_or(false) { *content_bytes = content_bytes.saturating_add(SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES); - if pointers.and_then(|store| store.state(key)).is_none() { - let key_hex = hex::encode(key); - warn!("Subtree audit: committed pointer {key_hex} is not held"); - return SubtreeAuditResponse::Rejected { - challenge_id: challenge.challenge_id, - kind: RejectKind::Protocol, - reason: format!("missing bytes for committed key: {key_hex}"), - }; - } + let key_hex = hex::encode(key); + let read = match pointers { + Some(store) => store.record_bytes(key).await, + None => Ok(None), + }; + let record = match read { + Ok(Some(record)) => record, + Ok(None) => { + warn!("Subtree audit: committed pointer {key_hex} is not held"); + return SubtreeAuditResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Protocol, + reason: format!("missing bytes for committed key: {key_hex}"), + }; + } + Err(e) => { + warn!( + "Subtree audit: read error for committed pointer {key_hex}: {e} \ + (rejecting as transient, not a confirmed failure)" + ); + return SubtreeAuditResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Transient, + reason: format!("transient storage read error: {e}"), + }; + } + }; + // Top up to the record read, as for a chunk. + let content = i64::try_from(record.len()).unwrap_or(i64::MAX); + *content_bytes = content_bytes.saturating_add( + content + .saturating_sub(SUBTREE_ROUND1_LEAF_WORK_FLOOR_BYTES) + .max(0), + ); leaves.push(SubtreeLeaf { key: *key, bytes_hash: pointer_leaf_hash(key), content_len: u32::try_from(POINTER_WIRE_LEN).unwrap_or(u32::MAX), - nonced_root: [0u8; 32], + nonced_root: nonced_block_root( + &challenge.nonce, + &challenge.challenged_peer_id, + key, + &record, + ), }); continue; } @@ -1810,18 +1882,16 @@ pub async fn handle_subtree_slice_challenge_with_pointers( for key in key_order { let indices = indices_by_key.remove(&key).unwrap_or_default(); if built.tree().commits_pointer(&key) { - // `get` verifies the signature before serving, so a record damaged - // on this disk is admitted as absent rather than served as proof. - let served = match pointers { - Some(store) => store.get(&key).await, - None => Ok(None), + // The bytes held now, exactly as round 1 read them, and the record + // an update replaced since, if any: round 1 bound one of the two. + // The auditor verifies whichever it checks, so nothing is verified + // here. + let Some(store) = pointers else { + items.push(SubtreeSliceItem::Absent { key }); + continue; }; - match served { - Ok(Some(record)) => items.push(SubtreeSliceItem::PointerRecord { - key, - record: record.to_bytes(), - }), - Ok(None) => items.push(SubtreeSliceItem::Absent { key }), + let current = match store.record_bytes(&key).await { + Ok(current) => current, Err(e) => { return SubtreeSliceResponse::Rejected { challenge_id: challenge.challenge_id, @@ -1829,6 +1899,12 @@ pub async fn handle_subtree_slice_challenge_with_pointers( reason: format!("pointer read error: {e}"), } } + }; + let records: Vec> = current.into_iter().chain(store.superseded(&key)).collect(); + if records.is_empty() { + items.push(SubtreeSliceItem::Absent { key }); + } else { + items.push(SubtreeSliceItem::PointerRecord { key, records }); } continue; } @@ -2980,20 +3056,93 @@ mod pointer_audit_tests { let target = first_pointer(&openings); let mut items = responder.round2(nonce, &openings).await; - let substitute = pointer(200, 1).to_bytes(); - for item in &mut items { - if let SubtreeSliceItem::PointerRecord { key, record } = item { - if *key == target { - *record = substitute.clone(); + replace_records(&mut items, &target, &[pointer(200, 1).to_bytes()]); + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::DigestMismatch) + ); + } + + fn replace_records(items: &mut [SubtreeSliceItem], target: &XorName, with: &[Vec]) { + for item in items { + if let SubtreeSliceItem::PointerRecord { key, records } = item { + if key == target { + *records = with.to_vec(); } } } + } + + /// A node that holds no pointers cannot pass by fetching the few sampled + /// records once round 2 names them: round 1 had to bind each record's + /// bytes under the nonce before anything was sampled, and it had none. + #[tokio::test] + async fn a_relay_that_fetches_records_only_in_round_two_fails() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let mut leaves = responder.proved_leaves(nonce).await; + // What a relay can say in round 1 without the bytes. + for leaf in leaves.iter_mut().filter(|leaf| is_pointer_leaf(leaf)) { + leaf.nonced_root = [0u8; 32]; + } + let openings = openings(&leaves); + // And in round 2 it serves the genuine records, fetched on demand. + let items = responder.round2(nonce, &openings).await; + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::DigestMismatch) + ); + } + + /// Nor can it pass with another replica's copy of the very state it + /// committed: every signature is randomised, so each replica's bytes, and + /// the nonced root over them, are its own. + #[tokio::test] + async fn another_replicas_copy_of_the_same_state_is_not_proof() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let target = first_pointer(&openings); + let owner = (0..24u8) + .find(|owner| pointer(*owner, 1).address() == target) + .expect("the opened pointer is one of ours"); + let replica = pointer(owner, 1); + assert_eq!( + replica.state_id(), + responder.pointers.state(&target).expect("held").state_id, + "the same state" + ); + + let mut items = responder.round2(nonce, &openings).await; + replace_records(&mut items, &target, &[replica.to_bytes()]); assert_eq!( verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), AuditVerdict::Fail(AuditFailureReason::DigestMismatch) ); } + /// A pointer item may carry the record held now and the one it replaced, + /// never more. + #[tokio::test] + async fn a_pointer_item_with_more_than_two_records_is_malformed() { + let responder = Responder::new(24, 24).await; + let nonce = mixed_nonce(responder.committed().tree()); + let openings = openings(&responder.proved_leaves(nonce).await); + let target = first_pointer(&openings); + let mut items = responder.round2(nonce, &openings).await; + let held = responder + .pointers + .record_bytes(&target) + .await + .expect("read") + .expect("held"); + replace_records(&mut items, &target, &[held.clone(), held.clone(), held]); + assert_eq!( + verify_slice_response(&openings, &nonce, &responder.peer_bytes, &items), + AuditVerdict::Fail(AuditFailureReason::MalformedResponse) + ); + } + /// The pointer leaf shape is accepted only at a pointer's exact size, so it /// cannot stand in for a chunk of another length. #[tokio::test] diff --git a/src/web_rtc.rs b/src/web_rtc.rs index 152fc33e..6bb3e8e0 100644 --- a/src/web_rtc.rs +++ b/src/web_rtc.rs @@ -1790,21 +1790,35 @@ fn hello_response(request_id: u64, state: &ServerState) -> Response { max_chunk_size: MAX_CHUNK_SIZE, endpoint, payment: state.payment.clone(), - capabilities: vec![ - "chunk_protocol".into(), - RPC_MULTIPLEX_CAPABILITY.into(), - "find_node".into(), - ant_protocol::transport::ADDRESS_V2_CAPABILITY.into(), - "get_chunk".into(), - "quote_chunk".into(), - "put_chunk".into(), - POINTER_PROTOCOL_CAPABILITY.into(), - ], + capabilities: hello_capabilities(serves_pointers(state.ant_protocol.as_deref())), }, 0, ) } +/// Whether requests reach a pointer store. A node without one refuses pointer +/// writes and has none to read, so it must not invite them: a browser could +/// otherwise pay a quote before learning the write goes nowhere. +fn serves_pointers(protocol: Option<&AntProtocol>) -> bool { + protocol.is_some_and(|protocol| protocol.pointer_service().is_some()) +} + +fn hello_capabilities(pointers: bool) -> Vec { + let mut capabilities: Vec = vec![ + "chunk_protocol".into(), + RPC_MULTIPLEX_CAPABILITY.into(), + "find_node".into(), + ant_protocol::transport::ADDRESS_V2_CAPABILITY.into(), + "get_chunk".into(), + "quote_chunk".into(), + "put_chunk".into(), + ]; + if pointers { + capabilities.push(POINTER_PROTOCOL_CAPABILITY.into()); + } + capabilities +} + async fn process_find_node( request_id: u64, target: String, @@ -2764,6 +2778,15 @@ mod tests { } } + /// Pointers are advertised only by a node that serves them. + #[test] + fn only_a_node_that_serves_pointers_advertises_them() { + let advertises = |caps: &[String]| caps.iter().any(|c| c == POINTER_PROTOCOL_CAPABILITY); + assert!(advertises(&hello_capabilities(true))); + assert!(!advertises(&hello_capabilities(false))); + assert!(!serves_pointers(None), "no storage, no pointers"); + } + /// Pointer reads and paid pointer writes are admitted (ADR-0016), and a /// full record fits the small bound they are given. #[test] From 627f2eb53ae59f08db25ec21746a21b4888ec38b Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 25 Sep 2026 18:44:03 +0900 Subject: [PATCH 30/32] test(e2e): bring a network up on fresh ports when a node cannot bind A node already retries its own port, which clears a socket still being released. It cannot clear a port the host will not hand out at all, as happens on Windows runners, where "Failed to create transport" for node 0 has failed runs on main and on this branch in whichever test drew that range. More tests starting networks means more draws, so the harness now moves a network that cannot create a node to a fresh random range, up to three times, and a test holds a port to prove it does. --- tests/e2e/harness.rs | 47 +++++++++++++++++++++++++++++----- tests/e2e/integration_tests.rs | 22 ++++++++++++++++ tests/e2e/testnet.rs | 14 ++++++++++ 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs index 39db7c5f..eaa560f9 100644 --- a/tests/e2e/harness.rs +++ b/tests/e2e/harness.rs @@ -4,13 +4,50 @@ //! both the ant node network and optional Anvil EVM testnet. use super::anvil::TestAnvil; -use super::testnet::{TestNetwork, TestNetworkConfig, TestNode}; +use super::testnet::{TestNetwork, TestNetworkConfig, TestNode, TestnetError}; use ant_node::client::XorName; use evmlib::common::TxHash; use saorsa_core::P2PNode; use std::collections::HashMap; use std::sync::{Arc, Mutex}; -use tracing::info; +use tracing::{info, warn}; + +/// How many port ranges a network is brought up on before its bind failure +/// is reported. +/// +/// A node retries its own port, which clears a socket still being released. +/// It cannot clear a port the host will not hand out at all, as happens on +/// Windows runners whose reserved port ranges move between runs, so a network +/// that cannot create a node moves to a fresh random range instead. +const NETWORK_START_ATTEMPTS: u32 = 3; + +/// Create and start a network from `config`, moving it to a fresh port range +/// if a node cannot be created. +async fn start_network(mut config: TestNetworkConfig) -> Result { + let mut attempt = 1; + loop { + let mut network = TestNetwork::new(config.clone()).await?; + match network.start().await { + Ok(()) => return Ok(network), + Err(TestnetError::Startup(reason)) + if attempt < NETWORK_START_ATTEMPTS + && reason.starts_with("Failed to create node ") => + { + let base_port = config.base_port; + warn!( + "Test network on base port {base_port} could not create a node \ + ({reason}); retrying on a fresh port range" + ); + if let Err(e) = network.shutdown().await { + warn!("Cleanup after a failed bring-up failed: {e}"); + } + config = config.with_fresh_ports(); + attempt += 1; + } + Err(e) => return Err(e.into()), + } + } +} /// Error type for test harness operations. #[derive(Debug, thiserror::Error)] @@ -192,8 +229,7 @@ impl TestHarness { pub async fn setup_with_config(config: TestNetworkConfig) -> Result { info!("Setting up test harness with {} nodes", config.node_count); - let mut network = TestNetwork::new(config).await?; - network.start().await?; + let network = start_network(config).await?; Ok(Self { network, @@ -257,8 +293,7 @@ impl TestHarness { config.node_count ); - let mut network = TestNetwork::new(config).await?; - network.start().await?; + let network = start_network(config).await?; // Warm up DHT routing tables (essential for quote collection) info!("Warming up DHT routing tables..."); diff --git a/tests/e2e/integration_tests.rs b/tests/e2e/integration_tests.rs index 8369d445..3f65ba0c 100644 --- a/tests/e2e/integration_tests.rs +++ b/tests/e2e/integration_tests.rs @@ -11,6 +11,7 @@ use super::testnet::{ use super::{NetworkState, TestHarness, TestNetwork, TestNetworkConfig}; use saorsa_core::P2PEvent; use serial_test::serial; +use std::net::UdpSocket; use std::time::Duration; /// Test that a minimal network (5 nodes) can form and stabilize. @@ -273,3 +274,24 @@ async fn test_node_to_node_messaging() { .await .expect("Failed to teardown test harness"); } + +/// A network whose first node cannot bind its port is brought up on a fresh +/// port range rather than failing the test that asked for it. +#[tokio::test] +#[serial] +async fn a_network_whose_port_is_taken_moves_to_a_fresh_range() { + let config = TestNetworkConfig::minimal(); + let taken = config.base_port; + let _holder = UdpSocket::bind(("127.0.0.1", taken)).expect("hold the first node's port"); + + let harness = TestHarness::setup_with_config(config) + .await + .expect("the network comes up on another range"); + assert_ne!( + harness.network().config().base_port, + taken, + "the network moved off the port it could not bind" + ); + + harness.teardown().await.expect("teardown"); +} diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index d3e64036..6e878709 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -236,6 +236,20 @@ pub struct TestNetworkConfig { pub storage_disk_reserve_overrides: HashMap, } +impl TestNetworkConfig { + /// The same configuration on a freshly drawn port range and data + /// directory, for a network whose first bring-up could not bind. + #[must_use] + pub fn with_fresh_ports(self) -> Self { + let fresh = Self::default(); + Self { + base_port: fresh.base_port, + test_data_dir: fresh.test_data_dir, + ..self + } + } +} + impl Default for TestNetworkConfig { fn default() -> Self { let mut rng = rand::thread_rng(); From a61abe870ffc15c2a263aa8086ff1aac65520dff Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sat, 26 Sep 2026 15:41:45 +0100 Subject: [PATCH 31/32] chore(logging): expose pointer rejection outcomes at info --- src/pointer/service.rs | 2 +- src/replication/pointer.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pointer/service.rs b/src/pointer/service.rs index fd63e32d..bdda6c54 100644 --- a/src/pointer/service.rs +++ b/src/pointer/service.rs @@ -264,7 +264,7 @@ impl PointerService { // covers a record the node lost, which that comparison cannot see, so // a replay cannot roll it back. if !self.store.admits(state) { - debug!( + info!( "Rejecting pointer PUT for {}: counter {} does not beat the state this node knows", hex::encode(address), state.counter diff --git a/src/replication/pointer.rs b/src/replication/pointer.rs index 49cdcd69..df62a0f4 100644 --- a/src/replication/pointer.rs +++ b/src/replication/pointer.rs @@ -580,7 +580,7 @@ impl PointerReplication { } Verdict::Undecided => (address, false), Verdict::Refused => { - debug!( + info!( "No quorum backs a newer state for pointer {}", hex::encode(address) ); @@ -877,7 +877,7 @@ impl PointerReplication { ) { self.mark_capable(&source); let Ok(permit) = Arc::clone(&self.offer_permits).try_acquire_owned() else { - debug!("Dropping a fresh pointer offer from {source}: too many in flight"); + info!("Dropping a fresh pointer offer from {source}: too many in flight"); return; }; let this = Arc::clone(self); @@ -947,7 +947,7 @@ impl PointerReplication { ) .await { - debug!( + info!( "Fresh pointer offer for {} from {source} is not paid for: {e}", hex::encode(state.address) ); From 7353c9ea2740de33495e78a3c36ff75b75116d4b Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 27 Sep 2026 16:24:35 +0100 Subject: [PATCH 32/32] chore: pin pointer protocol without a release version bump --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b852f2af..27a074bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -882,8 +882,8 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "3.1.0" -source = "git+https://github.com/WithAutonomi/ant-protocol?rev=4412b6af4fab2ceb0a4e1efea1efca08e68c6e4e#4412b6af4fab2ceb0a4e1efea1efca08e68c6e4e" +version = "3.0.0" +source = "git+https://github.com/WithAutonomi/ant-protocol?rev=d11d1010d93958fee6e63e4aace8770f39a0678b#d11d1010d93958fee6e63e4aace8770f39a0678b" dependencies = [ "blake3", "bytes", diff --git a/Cargo.toml b/Cargo.toml index cddbaf58..03efec06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -232,8 +232,8 @@ webrtc-direct = [ # Pointers (ADR-0016) add the `Pointer` record and its wire messages on top of # the published 3.0.0, so this is the only entry that has to leave the release # baseline. A rev, not a branch, so the pin is immutable. Drop it once the -# pointer PR lands and 3.1.0 is published. -ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", rev = "4412b6af4fab2ceb0a4e1efea1efca08e68c6e4e" } +# pointer PR lands and a release includes it. Versions are bumped at release. +ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", rev = "d11d1010d93958fee6e63e4aace8770f39a0678b" } [profile.release] lto = true