diff --git a/AGENTS.md b/AGENTS.md index fc2d47f1..11a1e554 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # GitNexus — Code Intelligence -This project is indexed by GitNexus as **dn-3127** (11540 symbols, 31404 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **dn-365-366** (11798 symbols, 32150 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. > If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. @@ -17,7 +17,7 @@ This project is indexed by GitNexus as **dn-3127** (11540 symbols, 31404 relatio 1. `gitnexus_query({query: ""})` — find execution flows related to the issue 2. `gitnexus_context({name: ""})` — see all callers, callees, and process participation -3. `READ gitnexus://repo/dn-3127/process/{processName}` — trace the full execution flow step by step +3. `READ gitnexus://repo/dn-365-366/process/{processName}` — trace the full execution flow step by step 4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed ## When Refactoring @@ -56,10 +56,10 @@ This project is indexed by GitNexus as **dn-3127** (11540 symbols, 31404 relatio | Resource | Use for | |----------|---------| -| `gitnexus://repo/dn-3127/context` | Codebase overview, check index freshness | -| `gitnexus://repo/dn-3127/clusters` | All functional areas | -| `gitnexus://repo/dn-3127/processes` | All execution flows | -| `gitnexus://repo/dn-3127/process/{name}` | Step-by-step execution trace | +| `gitnexus://repo/dn-365-366/context` | Codebase overview, check index freshness | +| `gitnexus://repo/dn-365-366/clusters` | All functional areas | +| `gitnexus://repo/dn-365-366/processes` | All execution flows | +| `gitnexus://repo/dn-365-366/process/{name}` | Step-by-step execution trace | ## Self-Check Before Finishing diff --git a/CLAUDE.md b/CLAUDE.md index fc2d47f1..11a1e554 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,7 +1,7 @@ # GitNexus — Code Intelligence -This project is indexed by GitNexus as **dn-3127** (11540 symbols, 31404 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **dn-365-366** (11798 symbols, 32150 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. > If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. @@ -17,7 +17,7 @@ This project is indexed by GitNexus as **dn-3127** (11540 symbols, 31404 relatio 1. `gitnexus_query({query: ""})` — find execution flows related to the issue 2. `gitnexus_context({name: ""})` — see all callers, callees, and process participation -3. `READ gitnexus://repo/dn-3127/process/{processName}` — trace the full execution flow step by step +3. `READ gitnexus://repo/dn-365-366/process/{processName}` — trace the full execution flow step by step 4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed ## When Refactoring @@ -56,10 +56,10 @@ This project is indexed by GitNexus as **dn-3127** (11540 symbols, 31404 relatio | Resource | Use for | |----------|---------| -| `gitnexus://repo/dn-3127/context` | Codebase overview, check index freshness | -| `gitnexus://repo/dn-3127/clusters` | All functional areas | -| `gitnexus://repo/dn-3127/processes` | All execution flows | -| `gitnexus://repo/dn-3127/process/{name}` | Step-by-step execution trace | +| `gitnexus://repo/dn-365-366/context` | Codebase overview, check index freshness | +| `gitnexus://repo/dn-365-366/clusters` | All functional areas | +| `gitnexus://repo/dn-365-366/processes` | All execution flows | +| `gitnexus://repo/dn-365-366/process/{name}` | Step-by-step execution trace | ## Self-Check Before Finishing diff --git a/Cargo.lock b/Cargo.lock index 13d5f885..daaf56de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2818,9 +2818,9 @@ dependencies = [ [[package]] name = "dig-ipc-protocol" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5558929d2823b73a9ead6d617cc96ea4fe2284949fdb2cbe2309a160e01b60cc" +checksum = "5e020b45104158b247370416b35b4386efdc201854b969254cfe6cc693bdb011" dependencies = [ "base64", "chia-bls 0.26.0", @@ -2950,7 +2950,7 @@ dependencies = [ [[package]] name = "dig-node-core" -version = "0.58.0" +version = "0.59.1" dependencies = [ "async-trait", "axum", @@ -3012,7 +3012,7 @@ dependencies = [ [[package]] name = "dig-node-service" -version = "0.156.0" +version = "0.156.1" dependencies = [ "async-trait", "axum", @@ -3138,9 +3138,9 @@ dependencies = [ [[package]] name = "dig-rpc-protocol" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40447a4cb8f3c7bd6aa40e734e56646a637e720061d6bfb71c22a2b7510ef5d" +checksum = "94ebacf438d821c3b22027a2ea6a532a92ef7c4a1a5797d836540432bdfab45b" dependencies = [ "serde", "serde_json", @@ -3274,7 +3274,7 @@ dependencies = [ [[package]] name = "dig-wallet" -version = "0.37.1" +version = "0.37.2" dependencies = [ "async-trait", "axum", diff --git a/Cargo.toml b/Cargo.toml index f3b9574d..b9ea8b51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ edition = "2021" # the ROOT manifest (`[workspace.package].version`), so it MUST be set here for a # release to fire (§3.6). The library crates (dig-node-core/dig-runtime/dig-wallet) # keep their own independent versions — only the released binary tracks the workspace version. -version = "0.156.0" +version = "0.156.1" # Release hardening, matching digstore: keep integer-overflow checks ON in release. # The node parses untrusted serialized input and does offset/length arithmetic over diff --git a/DEVELOPMENT_LOG.md b/DEVELOPMENT_LOG.md index 811ce54d..200b691f 100644 --- a/DEVELOPMENT_LOG.md +++ b/DEVELOPMENT_LOG.md @@ -1507,3 +1507,93 @@ processes contend for one port the loser is exactly the party who needs to be to "does not stop the node", never "is not reported": the outcome is now logged at WARN and published on `control.status`, so `dign info` says `wallet mTLS UNAVAILABLE (port 9776 held by another process ...)`. Any other best-effort bind in this repo should be read the same way. + +## A "quorum" is only as independent as what its members can REACH (dig-node#365) + +Corroboration rules fail in one characteristic way here: the sources are counted, not weighed. On +PR#354 a 2-of-2 "independent-group" rule in `dig-wallet` was satisfied by ONE HTTPS endpoint, +because the second group's peer tier was configured with `max_peers: 0` and could reach nothing at +all. The rule was correct about its own definition of a group and wrong about the world. + +So `seams/chia_peer/endpoints.rs` derives independence from RESOLVED ADDRESSES: two endpoints are +one voice when their address sets intersect, transitively. Not their type, and not their host name +— a CNAME costs an attacker nothing, and two names for one machine is exactly the shape a name-based +rule waves through. + +Two consequences worth knowing before touching it: + +* **Set INTERSECTION, not equality.** A dual-stack host answers with several addresses (§5.2 makes + IPv6 the ordinary case), and a resolver returning different subsets on two lookups would make one + machine read as two voices under an equality rule. +* **The merge must be transitive.** An endpoint bridging two previously-disjoint groups makes all + three one voice; stopping at the first matching group leaves the second counted separately, and a + third endpoint then turns one host into a passing quorum. + +## Refusing on a default install is not a safe default (dig-node#365) + +The anchored root decides which bytes a user is served, so a single-source resolution of it is a +real gap. Refusing to resolve it without corroboration would close the gap and stop every +unconfigured node serving any content at all — including the surfaces on which an operator would +then configure a second endpoint. The shipped rule therefore degrades to the single source, states +that in `SPEC.md` §14.4b as an accepted limitation with its blast radius, and refuses only where two +independent voices exist and DISAGREE. "State it honestly" is a legitimate outcome; implying +corroboration the code does not perform is not. + +## Widening a source-scanning guard widens its blind spots (dig-node#366) + +The NC-12 sole-owner sweep is a line classifier over Rust source, and it took five gate rounds to +make honest within one crate. Adding `dig-node-core` to its haystack surfaced two failure modes that +a wider scope creates rather than inherits: + +* **File names repeat across a workspace.** Sites were reported as bare basenames and the owner was + `"sources.rs"`, so a second fabric built in a `sources.rs` in ANY newly-swept crate would have + been accepted as the owner. Sites are now crate-qualified (`dig-wallet/sources.rs:NNN`). +* **A crate with none of the needle cannot tell you it was read.** `dig-node-core` contains no + `ChiaQuery::new(` at all, so a total taken across both roots stays positive from `dig-wallet` + alone — a typo in the second path leaves the new scope reading zero files while the guard still + reports a real haystack. The haystack test now asserts a FILE count per root, which separates + "this crate is clean" from "this crate was never opened". + +Both known-silent shapes were re-measured against the new haystack. The column-0 `#[cfg(test)]` as +string content is absent from both crates. The trailing-comment terminator occurs ONCE, at +`dig-node-core/src/lib.rs:195` (`const DEFAULT_CACHE_CAP: … ; // 1 GiB`) — harmless only because no +column-0 `#[cfg(test)]` precedes it in that file, so the latch is never set when it is reached. That +is a property of where the line sits, not of the line, and nothing would report it if it moved. + +## A yes/no answer cannot carry a dissent, and a threshold hides that (dig-node#365) + +`Result<(), String>` has no value channel, so a source saying *"that root is not current"* and one +saying *"I could not reach the chain"* arrive as the SAME `Err`. An agreement rule reading that +cannot tell dissent from silence, and the natural implementation — drop the errors, count the `Ok`s, +require two — is a flat *k*-of-*N* threshold whose **bar does not rise with `N`**. 2-of-3 and 2-of-10 +are the same bar. + +This shipped. `verify_pinned_root` and `verify_lineage_root` had it while `anchored_state`, which +carries a value, was correct on byte-identical input — so the two calls the read-path pin actually +makes were the two without the property, and three endpoints with one a generation behind served +stale content with no attacker involved. + +Three things worth carrying forward: + +* **A tri-state at the source beats classifying an error string.** `Verdict::{Confirmed, Rejected, + Unreachable}` is decided where the evidence exists: the lineage walk already separated the cases + structurally (a completed walk missing the root is a rejection; a failed walk is unreachable), and + the bounded pin needed one extra reachability probe on the failure path only. +* **Arrange the remaining ambiguity to fail in the refusing direction.** That probe races the call it + classifies. If the chain drops in between, a genuine unreachability is recorded as a rejection — + which refuses. The opposite error fails OPEN, and is the defect being removed. +* **The dangerous half was the COMPOSITION.** `content_serve.rs`, `dig_rpc/dispatch.rs` and + `module_reshare.rs` all treat a failed tip resolution as the #747 broken-walk case and fall back to + `verify_pinned_root`. Widening what the tip's `Err` MEANS routed the strongest signal the feature + produces onto the one check that could not hear it. Whenever an existing error value gains a new + meaning, re-read every arm that already matches on it — the arm was written against the old set. + +## Test the configuration that separates the semantics, not the one that is easiest to script + +Every verification test here scripted only an unreachable source. That fixture cannot distinguish a +dissent rule from a threshold rule, because under both an unreachable voice is dropped — so the +defect above was untested rather than tested-and-wrong. The missing fixture was one sentence long: a +voice that is REACHED and says no. + +Ask what the nearest wrong implementation is, then ask which input it would answer differently on. If +no fixture in the suite is that input, the property is undefended however many tests surround it. diff --git a/SPEC.md b/SPEC.md index 0f56358f..6dd1b20a 100644 --- a/SPEC.md +++ b/SPEC.md @@ -216,6 +216,7 @@ does not own them (except `DIG_NODE_UPSTREAM`, which the shell SETS — see belo |---|---|---|---| | `DIG_NODE_CACHE_CAP` | LRU cache size cap, in bytes | `1073741824` (1 GiB) | Parsed as `u64`. Consulted ONLY when the persisted `cache_cap_bytes` key in `config.json` is absent or `0` (the persisted value wins). Unparsable/unset ⇒ default. | | `DIG_NODE_COINSET` | override the coinset API base used for chain-anchored-root resolution | `https://api.coinset.org` (mainnet) | Blank/unset ⇒ mainnet default. Used for tests / alternate endpoints. | +| `DIG_NODE_CHAIN_ENDPOINTS` | comma-separated coinset-protocol endpoints the anchored root is CORROBORATED across (§14.4b) | unset (single-source) | Two or more INDEPENDENTLY-HOSTED endpoints enable the agreement rule; independence is derived from resolved addresses, so two names for one machine stay ONE voice. Unparseable entries are DROPPED, never defaulted. Prefer three or more. Takes precedence over `DIG_NODE_COINSET`. | | `DIG_NODE_PIN` | read-path anchored-root pin enforcement (§14.4) | `on` (ENFORCED, fail-closed) | ONLY `off`/`0`/`false` disable the node-side pin (a named offline/local-dev escape hatch); any other value or unset ENFORCES. Clients still verify proofs against their own trust root regardless. | | `DIG_NODE_WATCH_INTERVAL` | chain-watch poll interval, in seconds (§14.2) | `30` | Parsed as `u64`; `0`/unparsable/unset ⇒ default `30`; floored at `1` s so a mis-set value cannot flood coinset. | | `DIG_NODE_UPSTREAM` | **INTERNAL** — the effective upstream the node library reads | *(unset — NO default upstream)* | NOT a user knob. The shell resolves the upstream (§3.4) and writes this via `Config::apply_to_env()` (§3.5); the shell's public knob is `DIG_RPC_UPSTREAM`. Empty ⇒ the library makes no upstream request at all. | @@ -3850,6 +3851,113 @@ against the on-chain current root or fails closed — it NEVER trusts an upstrea a NODE-side gate; clients still verify the returned proof against their own trust root regardless, so the opt-out only relaxes the node's serve gate for local dev. +#### 14.4b. Chain corroboration — how many voices decide the anchored root (NC-12) + +The anchored root is the chain fact that decides WHICH BYTES A USER IS SERVED, so NC-12's +"agreement across several concurrently-queried untrusted sources" applies to it directly. This +section states exactly how much corroboration the node performs, and — as precisely — how much it +does not. A SPEC that implied more than the code performs would be worse than one that admits the +gap. + +**The endpoints.** The node resolves the anchored root from the coinset-protocol endpoints named by +`DIG_NODE_CHAIN_ENDPOINTS` (a comma-separated list). When that is unset, `DIG_NODE_COINSET` names a +single endpoint; when neither is set the node uses `https://api.coinset.org`. An unparseable entry +is DROPPED, never defaulted — a typo MUST NOT be able to masquerade as an additional source. + +**Independence is derived from REACH.** Two endpoints are ONE voice whenever their resolved address +sets intersect, and the relation is transitive. Independence is NOT derived from an endpoint's type, +its URL, or its host name: a CNAME costs an attacker nothing, and a "quorum" satisfied by two names +for one machine is the defect this rule exists to prevent. With two or more endpoints configured, an +endpoint that resolves to no address contributes NO voice — the lookup is the evidence that it is a +separate machine, and a source that cannot be shown independent MUST NOT be counted as one. + +A SINGLE configured endpoint is a voice whether or not its name resolves. Independence is a relation +BETWEEN endpoints, so with one endpoint there is nothing for a lookup to decide, and requiring one +would make name resolution a gate on READING: a resolver failure would deny a read the HTTP client +would have served, on the default install, over a path that performed no name resolution at all +before this rule existed. Failing closed is required when the CHAIN ANSWER is in doubt; a name +lookup is not that. + +**The agreement rule, with two or more independent voices.** Every voice that answered MUST give the +SAME answer, and at least two MUST have answered; otherwise the resolution FAILS CLOSED and the pin +rejects the serve (§14.4, `-32005 ROOT_NOT_ANCHORED`). Specifically: + +- One dissenting voice is a REFUSAL, never a repaired value. There is NO majority vote and no + tie-break — a majority rule would hand the answer to whoever can field the most endpoints. +- Presence and absence are different answers: one source reporting a root while another reports no + confirmed generation is a DISAGREEMENT, so a single source cannot conjure a store into being. +- A source that could not be REACHED is dropped rather than counted as dissent, because an outage + and an attack demand opposite remedies. Dropping is still fail-closed: fewer answers means the + agreement rule has less evidence, and fewer than two answers is a refusal. +- The rule covers all three resolution calls — the tip state, the bounded pinned-root verification, + and lineage membership — because all three are consulted by the serve decision. + +**A reached source that REJECTS a root vetoes the resolution; it is never outvoted.** This is stated +separately because the two verification calls answer a yes/no question and so have no value channel +in which to disagree. A source that is reached and says *"that root is not current"* MUST be +distinguished from one that could not be asked, and its rejection MUST refuse the whole resolution +regardless of how many other sources confirmed. A flat *k*-of-*N* threshold does NOT satisfy this +clause: under one, 2-of-3 and 2-of-10 are the same bar, so adding endpoints would not raise an +attacker's cost — and, with no attacker at all, three endpoints of which one is a generation behind +would serve stale content. **The bar rises with the number of REACHED sources, because every source +that answers can veto — not because more must agree.** + +State that precisely, because the difference is the whole security property. A source can only veto +a resolution it was reached for; an UNREACHABLE source neither confirms nor vetoes, and adding an +endpoint therefore raises the bar only while that endpoint is answering. So an attacker who can +SILENCE a source — degrade its reachability rather than change its answer — removes that source's +veto, and a root two lagging endpoints still confirm is then served. Two consequences follow and +both are normative: + +- A source that has ANSWERED a read MUST NOT be reclassified as unreachable on the strength of a + later failed probe. Reachability is established BEFORE a verification, never re-tested after one: + an endpoint that answered HAS been reached, and a subsequent failure describes its future rather + than its past. The residual misclassification MUST run the other way — a chain that drops between + the probe and the verification is recorded as a REJECTION, which refuses. +- The remedy available to an operator is the same one §14.4b already recommends and is stated here + for a different reason: prefer **three or more** endpoints, so that silencing ONE still leaves a + reached source able to veto. + +**Latency and disclosure, both consequences of asking more than one source.** Endpoint independence +is recomputed per resolution, so the endpoint set is consulted on the content-serve request path. +Recomputing is NOT re-resolving: resolved address sets are CACHED for 60 seconds, so an ordinary read +costs a map lookup rather than a name lookup, and an independence verdict can be at most that stale. +A lookup that fails while a previously-resolved answer is still held (within 10 minutes) reuses that +answer rather than dropping the voice — a resolver blip is not evidence that an endpoint moved, and +silently changing the VOICE COUNT on that evidence would report a DNS hiccup as a corroboration +failure. Lookups that are performed are CONCURRENT and each is bounded (3 seconds); an endpoint that +exceeds it and has no cached answer counts as unreachable — the fail-closed direction. And resolving +discloses the STORE ID to every configured endpoint rather than to one. An operator adding endpoints +buys corroboration and widens that disclosure; both halves are real. + +**ACCEPTED LIMITATION — the DEFAULT INSTALL resolves the anchored root from ONE third party.** With +fewer than two independent voices configured, the node answers from its single source, exactly as it +did before this rule existed. It does not claim corroboration in that state, and it does not refuse: +refusing would stop every unconfigured node serving any content at all, removing the surface on +which an operator could configure a second endpoint. + +*Blast radius of that limitation:* a default install trusts `api.coinset.org` for the root every +content read is pinned to. A source that lied about it would redirect reads on every such install, +and the pin would fail closed against the WRONG root rather than the right one — indistinguishable, +from outside, from the store having moved on. It does NOT let that source forge content: bytes are +still accepted only because they verify against the merkle root (§21.2, §22.3). The remedy available +to an operator today is to name several independently-hosted endpoints. + +*How many:* prefer **three or more**, not two. Two is the minimum the rule accepts and the most +fragile count it accepts, because with two ANY single source being unreachable drops the node to one +answer — below the corroboration floor — so every read refuses until it returns. A third endpoint is +what makes the guarantee survive one outage rather than converting one outage into a serve failure. +Note the veto is unconditional at every count: a third source raises availability, and it also gains +a third party able to refuse the read. + +**ALSO SINGLE-SOURCED, and NOT corroborated (stated, not fixed).** These paths go to one endpoint +regardless of configuration: `coin_records_by_puzzle_hashes`, `coin_records_by_hints` and +`coin_records_by_parent` (the wallet's chain-fallback tier), mempool submission, the melt +confirmation, and the direct singleton walks the RPC surface performs. They are enumerated here so a +reader does not infer from the rule above that every chain fact is corroborated. By contrast +`peak_height`, `coin_record_by_id`, `coin_spend` and their cached variants ARE corroborated across +the node's own dialled Chia peers (§18). + ### 14.4a. Per-path generation resolution (#2088) — serve TIP-AUTHORITATIVE, redirect only on a genuine tip miss (#2211) A resource UNCHANGED since an earlier commit lives in an OLDER capsule whose own root ≠ the tip; diff --git a/crates/dig-node-core/Cargo.toml b/crates/dig-node-core/Cargo.toml index 0636b7d9..8ae9df48 100644 --- a/crates/dig-node-core/Cargo.toml +++ b/crates/dig-node-core/Cargo.toml @@ -30,7 +30,12 @@ name = "dig-node-core" # dig-node#276/#296). Changing a public return type is BREAKING for an out-of-workspace implementor; # this crate is consumed in-workspace only and is pre-1.0, so it is a MINOR bump under SemVer's 0.x # rule -- recorded here rather than letting the number imply the locator surface held still. -version = "0.58.0" +# 0.59.0 makes the anchored-root resolver a CORROBORATING one: `default_anchored_resolver` now +# returns a resolver over the endpoints named by `DIG_NODE_CHAIN_ENDPOINTS`, believing the root only +# when independent voices agree and REFUSING on dissent (dig-node#365, NC-12). No public type +# changes -- `CoinsetResolver` keeps its shape and its single-endpoint behaviour -- but the BEHAVIOUR +# of the node's default resolver does, which is a new capability rather than a fix, hence MINOR. +version = "0.59.1" edition = "2021" license = "GPL-2.0-only" description = "The canonical DIG node ENGINE library (crate `dig_node_core`): the JSON-RPC dispatch (`handle_rpc`, the same contract as rpc.dig.net), local-first content serve/fetch/redirect from LOCAL .dig store modules (via digstore_host::serve_blind), chain-anchored-root resolution, chain-watch + subscriptions + generation gap-fill, the LRU cache, and the full P2P stack. Shared UNCHANGED by both host shells: the `dig-node` OS-service binary (dig-node-service) and the DIG Browser's in-process cdylib (dig-runtime). Native Rust so the compiled-module serve path works." @@ -127,7 +132,10 @@ chia-traits = "0.26" dig-ip = "0.1.2" # The canonical dig-app ⇄ dig-node IPC session/signing contract (engine role-half + wire types + # domain builders + frame bounds). SSOT shared with dig-app so the two halves can never drift (#1080). -dig-ipc-protocol = "0.2" +# Pinned EXACTLY: 0.3.0 is the first release on this line and the caret range would silently accept +# a future 0.3.x. The crate reaches this one through the IPC surface, so an unreviewed minor is a +# wire-shape change arriving without a PR (dig-node#369 gate). +dig-ipc-protocol = "=0.3.0" # On-chain DID resolution (WU3, #778): the honest chain-authenticated read path the production # DidSigningKeyResolver delegates to. # @@ -165,7 +173,7 @@ serde_json = "1" # per-method tier) and the mTLS peer-reachability allowlist. dig-node-core reads its # method names + the peer allowlist from HERE (never hand-rolled) so the contract # cannot drift from the other node implementation or the discovery document (#1075). -dig-rpc-protocol = "0.10.1" +dig-rpc-protocol = "0.10.2" # The directed-message base protocol (epic #793/#796): the e2e seal/open pipeline + the typed envelope # the chat subsystem seals into. dig-node is the TRANSPORT — it seals an app-supplied opaque DIGCHAT1 # envelope to the recipient's 0x0010 BLS identity key and dig-gossip directed-sends the sealed bytes. diff --git a/crates/dig-node-core/SPEC.md b/crates/dig-node-core/SPEC.md index 81619198..fa272212 100644 --- a/crates/dig-node-core/SPEC.md +++ b/crates/dig-node-core/SPEC.md @@ -394,10 +394,17 @@ the current root. `async fn anchored_root(store_id) -> Result, _>` — `Ok(Some(root))` = the confirmed unspent-singleton tip's `metadata.root_hash`; `Ok(None)` = the store has no confirmed generation; `Err` = the chain is unreachable. -- Production uses `CoinsetResolver`, which walks the singleton via - `digstore_chain::singleton::sync_datastore` over a coinset endpoint (`Coinset::mainnet()` / - `api.coinset.org`, overridable by `DIG_NODE_COINSET`). Tests inject a deterministic resolver so the - fail-closed gate is unit-testable without a chain. +- Production uses `CorroboratedResolver`, which holds one `EndpointResolver` per configured chain + endpoint — each walking the singleton via `digstore_chain::singleton::sync_datastore` over its own + coinset endpoint — and serves an answer only when independent endpoints AGREE. Endpoints come from + `DIG_NODE_CHAIN_ENDPOINTS` (comma-separated); `DIG_NODE_COINSET` names a single endpoint when that + is unset, and the default is `api.coinset.org`. Independence is derived from RESOLVED ADDRESSES, + never from a name: two endpoints whose address sets intersect are ONE voice. A reached endpoint + that rejects a root vetoes the resolution; fewer than two independent answers refuses. With one + configured endpoint — the default install — resolution is single-sourced and does not claim + corroboration; the full rule and its accepted limitations are the repo `SPEC.md` §14.4b. + `CoinsetResolver` remains as the single-endpoint resolver against the process-wide default. Tests + inject a deterministic resolver so the fail-closed gate is unit-testable without a chain. ### 4.2 The read-path pin (fail-closed, `-32005`) @@ -1196,6 +1203,7 @@ cache cap is `config.json` > env > default). | `DIG_PEER_PORT` | mTLS peer-RPC listen port (dual-stack `[::]`) | `9444` | | `DIG_NODE_UPSTREAM` | §21 host base for sync + read proxy fallback (OPTIONAL) | *(unset — no upstream)* | | `DIG_NODE_COINSET` | coinset API base for chain reads | `Coinset::mainnet()` (api.coinset.org) | +| `DIG_NODE_CHAIN_ENDPOINTS` | comma-separated endpoints the anchored root is corroborated across | unset (single-source; see repo `SPEC.md` §14.4b) | | `DIG_NODE_CACHE` | override the shared cache dir | per-OS `DigNode/cache` (§3.4) | | `DIG_NODE_CACHE_CAP` | on-disk cache cap (bytes) | `DEFAULT_CACHE_CAP` = 1 GiB (floor 64 MiB) | | `DIG_NODE_PIN` | anchored-root pin enforcement (`off`/`0`/`false` disables) | enforced (fail-closed) | diff --git a/crates/dig-node-core/src/seams/chia_peer/coinset_resolver.rs b/crates/dig-node-core/src/seams/chia_peer/coinset_resolver.rs index 3b05b63b..7aadd0a5 100644 --- a/crates/dig-node-core/src/seams/chia_peer/coinset_resolver.rs +++ b/crates/dig-node-core/src/seams/chia_peer/coinset_resolver.rs @@ -5,10 +5,12 @@ use std::sync::Arc; -use digstore_chain::coinset::Coinset; +use digstore_chain::coinset::{ChainReads, Coinset}; use digstore_chain::singleton::{sync_datastore, sync_datastore_with_history, verify_pinned_root}; use digstore_core::Bytes32; +use super::corroborated_resolver::{ChainVoice, CorroboratedResolver, Verdict}; +use super::endpoints::{CachedReach, ChainEndpoint, DnsReach}; use crate::shared::chain_view::{AnchoredRootResolver, AnchoredStoreState}; /// Coinset client used to resolve chain-anchored roots. `DIG_NODE_COINSET` @@ -26,20 +28,178 @@ pub(crate) fn resolution_coinset() -> Coinset { /// and returns its metadata root — exactly the source `dig.getAnchoredRoot` and /// `dig-resolver` already use, and the same authority the CLI clone/pull pin /// resolves against (`current_root`). NEVER consults the serving node. +/// +/// This speaks to the ONE endpoint [`resolution_coinset`] names. It is the voice, not the +/// verdict: [`CorroboratedResolver`] holds several of these and serves an answer only when +/// independent ones agree (dig-node#365). pub struct CoinsetResolver; #[async_trait::async_trait] impl AnchoredRootResolver for CoinsetResolver { async fn anchored_root(&self, store_id: &[u8; 32]) -> Result, String> { - Ok(self.anchored_state(store_id).await?.map(|s| s.root)) + AnchoredRootResolver::anchored_root(&EndpointResolver::new(resolution_coinset()), store_id) + .await + } + + async fn anchored_state( + &self, + store_id: &[u8; 32], + ) -> Result, String> { + AnchoredRootResolver::anchored_state(&EndpointResolver::new(resolution_coinset()), store_id) + .await + } + + async fn verify_pinned_root( + &self, + store_id: &[u8; 32], + pinned_root: Bytes32, + ) -> Result<(), String> { + AnchoredRootResolver::verify_pinned_root( + &EndpointResolver::new(resolution_coinset()), + store_id, + pinned_root, + ) + .await + } + + async fn verify_lineage_root(&self, store_id: &[u8; 32], root: Bytes32) -> Result<(), String> { + AnchoredRootResolver::verify_lineage_root( + &EndpointResolver::new(resolution_coinset()), + store_id, + root, + ) + .await + } +} + +/// The same walk, against ONE named endpoint rather than the process-wide default. +/// +/// Split out from [`CoinsetResolver`] so corroboration has something to hold: a rule that needs +/// several voices needs a resolver that can be pointed at a specific one, and reading the endpoint +/// from a process-global environment variable inside the walk makes every instance the same voice +/// no matter how many are constructed. +pub(crate) struct EndpointResolver { + /// The coinset-protocol client for this endpoint. + /// + /// Held behind `dyn ChainReads` rather than as a concrete [`Coinset`] so the classification + /// below can be exercised against a chain that answers one read and then stops. That sequence + /// is the entire difference between the two possible orderings of the probe, and no live + /// endpoint can be asked to produce it on demand. + chain: Arc, +} + +impl EndpointResolver { + /// A resolver that walks `chain` and nothing else. + pub fn new(chain: Coinset) -> Self { + Self::over(Arc::new(chain)) + } + + /// The same resolver over any chain reader. + pub fn over(chain: Arc) -> Self { + Self { chain } + } + + /// Could this endpoint be reached AT ALL, right now? + /// + /// `unspent_coins_by_hint` is the read `digstore_chain::singleton::verify_pinned_root` itself + /// starts with, so an endpoint that answers this is one the verification below will be talking + /// to. Answering is the whole signal: the coins are not inspected, only the fact that a + /// response came back. + async fn reached(&self, store_id: &[u8; 32]) -> Result<(), String> { + self.chain + .unspent_coins_by_hint(chia_protocol::Bytes32::new(*store_id)) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) } +} +/// One endpoint speaking for itself, able to say NO as distinct from saying nothing. +/// +/// # Where the classification comes from, and which way it errs +/// +/// * `verify_lineage_root` needs no extra call: the walk ALREADY separates the two cases +/// structurally — a completed walk whose history lacks the root is a rejection, and a failed +/// walk is an unreachable chain. +/// * `verify_pinned_root` delegates to a digstore function that performs its own read and collapses +/// both outcomes into one error, so reachability has to be established by a separate probe. +/// +/// # The probe runs BEFORE the verification, and the order is the property +/// +/// Reachability is a fact that, once established, cannot be withdrawn: an endpoint that answered a +/// read HAS been reached, and a later failure says something about the endpoint's future, never +/// about its past. Probing first encodes that. Probing afterwards does not — it re-tests a fact +/// already settled, and a rejection whose follow-up probe happens to fail is then filed as +/// `Unreachable`, DROPPED by the agreement rule, and casts no veto. +/// +/// That is not a theoretical ordering preference. `Unreachable` is the one verdict an attacker +/// gains by producing, because [`CorroboratedResolver`] discards it while a `Rejected` refuses the +/// whole resolution — so a probe placed after the verification is a mechanism for converting the +/// dissent of a source one can degrade into silence, and two lagging endpoints then carry the +/// serve. With the probe first, degrading a source's reachability can only cost its voice, never +/// convert it. +/// +/// The residual misclassification runs the other way and is deliberate: a chain that drops BETWEEN +/// the probe and the verification records a genuine unreachability as a REJECTION, which refuses. +/// So the classification is wrong only in the direction that refuses, and the ordering above is +/// what makes that a property of the code rather than a claim made about it. +/// +/// The cost is one extra `unspent_coins_by_hint` on the confirming path, beside the several reads +/// the verification already performs (hint, parent record, parent spend, and the launcher descent). +#[async_trait::async_trait] +impl ChainVoice for EndpointResolver { async fn anchored_state( &self, store_id: &[u8; 32], ) -> Result, String> { + AnchoredRootResolver::anchored_state(self, store_id).await + } + + async fn verify_pinned_root(&self, store_id: &[u8; 32], pinned_root: Bytes32) -> Verdict { + // Establish "this source was reached" FIRST. Everything after this point is this + // endpoint's OPINION, and an opinion is not un-formed by the chain going away afterwards. + if let Err(why) = self.reached(store_id).await { + return Verdict::Unreachable(why); + } + match AnchoredRootResolver::verify_pinned_root(self, store_id, pinned_root).await { + Ok(()) => Verdict::Confirmed, + Err(why) => Verdict::Rejected(why), + } + } + + async fn verify_lineage_root(&self, store_id: &[u8; 32], root: Bytes32) -> Verdict { let launcher = chia_protocol::Bytes32::new(*store_id); - match sync_datastore(&resolution_coinset(), launcher).await { + match sync_datastore_with_history(self.chain.as_ref(), launcher).await { + // The walk COMPLETED, so this endpoint has a real opinion about the lineage. + Ok((_store, history)) => { + if history.history.iter().any(|c| c.root_hash == root) { + Verdict::Confirmed + } else { + Verdict::Rejected(format!( + "root {} is not in the store's on-chain lineage (chain is the authority)", + root.to_hex() + )) + } + } + Err(e) => Verdict::Unreachable(e.to_string()), + } + } +} + +#[async_trait::async_trait] +impl AnchoredRootResolver for EndpointResolver { + async fn anchored_root(&self, store_id: &[u8; 32]) -> Result, String> { + Ok(AnchoredRootResolver::anchored_state(self, store_id) + .await? + .map(|s| s.root)) + } + + async fn anchored_state( + &self, + store_id: &[u8; 32], + ) -> Result, String> { + let launcher = chia_protocol::Bytes32::new(*store_id); + match sync_datastore(self.chain.as_ref(), launcher).await { Ok(store) => { // Convert chia_protocol::Bytes32 → digstore_core::Bytes32 (the // node's content-root type), mirroring the CLI clone/pull pin. @@ -79,7 +239,7 @@ impl AnchoredRootResolver for CoinsetResolver { ) -> Result<(), String> { let launcher = chia_protocol::Bytes32::new(*store_id); let pinned = chia_protocol::Bytes32::new(pinned_root.0); - verify_pinned_root(&resolution_coinset(), launcher, pinned) + verify_pinned_root(self.chain.as_ref(), launcher, pinned) .await .map_err(|e| e.to_string()) } @@ -94,7 +254,7 @@ impl AnchoredRootResolver for CoinsetResolver { /// redirect the serve to `root`". async fn verify_lineage_root(&self, store_id: &[u8; 32], root: Bytes32) -> Result<(), String> { let launcher = chia_protocol::Bytes32::new(*store_id); - match sync_datastore_with_history(&resolution_coinset(), launcher).await { + match sync_datastore_with_history(self.chain.as_ref(), launcher).await { Ok((_store, history)) => { if history .history @@ -116,9 +276,57 @@ impl AnchoredRootResolver for CoinsetResolver { } } -/// The default anchored-root resolver (production coinset walk). +/// The coinset-protocol endpoint the mainnet default speaks to. +/// +/// Named here rather than left implicit inside `Coinset::mainnet()` because the independence rule +/// needs an authority to resolve, and a default endpoint with no URL cannot be compared against an +/// operator's second one. +const MAINNET_ENDPOINT: &str = "https://api.coinset.org"; + +/// Every chain endpoint the node may ask, in configuration order. +/// +/// `DIG_NODE_CHAIN_ENDPOINTS` is a comma-separated list and is what turns single-source resolution +/// into corroborated resolution — an operator who names two independently-hosted coinset-protocol +/// endpoints gets the agreement rule; one who names none gets today's behaviour. +/// +/// `DIG_NODE_COINSET` keeps its existing meaning (a single override, used by tests and by +/// operators pointing at one alternate endpoint) and is honoured when the list is unset, so no +/// existing configuration changes meaning. Unparseable entries are DROPPED rather than defaulted: +/// silently substituting the mainnet endpoint for a typo would let a misconfiguration masquerade +/// as a second voice. +pub(crate) fn resolution_endpoints() -> Vec { + let configured = match std::env::var("DIG_NODE_CHAIN_ENDPOINTS") { + Ok(list) if !list.trim().is_empty() => list, + _ => std::env::var("DIG_NODE_COINSET") + .ok() + .filter(|url| !url.trim().is_empty()) + .unwrap_or_else(|| MAINNET_ENDPOINT.to_string()), + }; + configured + .split(',') + .filter_map(ChainEndpoint::parse) + .collect() +} + +/// The default anchored-root resolver: the configured endpoints, believed only on agreement. +/// +/// With one endpoint configured — the default install — this resolves exactly as +/// [`CoinsetResolver`] always did, from a single third party. That limitation is REAL and is +/// recorded in `SPEC.md` rather than dressed up: see [`CorroboratedResolver`] for why +/// refusing instead was rejected, and dig-node#365 for the blast radius it leaves. pub(crate) fn default_anchored_resolver() -> Arc { - Arc::new(CoinsetResolver) + Arc::new(CorroboratedResolver::new( + resolution_endpoints(), + // Cached, because independence is recomputed on every content read and an uncached reach + // makes that a `getaddrinfo` per read — twice on a read that falls back from the tip to + // the bounded pinned-root check. + Arc::new(CachedReach::new(DnsReach)), + Arc::new(|endpoint: &ChainEndpoint| { + Arc::new(EndpointResolver::new(Coinset::with_url( + endpoint.url.clone(), + ))) as Arc + }), + )) } #[cfg(test)] @@ -167,4 +375,197 @@ mod tests { "an unanchorable pinned root MUST fail closed (do not serve), never Ok: {outcome:?}" ); } + + // -- The probe ordering ------------------------------------------------------------------ + + use chia_protocol::{Bytes32 as ChiaBytes32, Coin, CoinSpend, SpendBundle}; + use digstore_chain::coinset::{CoinInfo, CoinRecord}; + use digstore_chain::error::{ChainError, Result as ChainResult}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::super::endpoints::{Authority, EndpointReach}; + + /// A chain that answers a BOUNDED number of hint reads and then stops answering. + /// + /// This is the one fixture shape that can tell the two probe orderings apart, and it took a + /// deliberate design to build: a chain that fails EVERY read is `Unreachable` under both + /// orderings, and a chain that answers every read is `Rejected` under both. Only a chain that + /// is reachable for the FIRST read and gone by the second distinguishes "reachability + /// established before the verification" from "reachability re-tested after it". + /// + /// `Ok(vec![])` is a real answer, not a shrug: `digstore_chain::singleton::verify_pinned_root` + /// finds no candidate tip in it and returns its "no current unspent singleton" error — a + /// determination made by a source that ANSWERED. + struct DegradingChain { + /// How many hint reads still succeed. + answers_left: AtomicUsize, + } + + impl DegradingChain { + /// A chain that answers `answers` hint reads and fails every one after them. + fn answering(answers: usize) -> Arc { + Arc::new(Self { + answers_left: AtomicUsize::new(answers), + }) + } + } + + #[async_trait::async_trait] + impl ChainReads for DegradingChain { + async fn unspent_coins_by_hint(&self, _hint: ChiaBytes32) -> ChainResult> { + match self + .answers_left + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |left| { + left.checked_sub(1) + }) { + Ok(_) => Ok(Vec::new()), + Err(_) => Err(ChainError::Chain("coinset unreachable".into())), + } + } + async fn unspent_coins(&self, _ph: ChiaBytes32) -> ChainResult> { + unimplemented!("the pinned-root check reads by hint") + } + async fn coin_records_by_puzzle_hash( + &self, + _ph: ChiaBytes32, + _include_spent: bool, + ) -> ChainResult> { + unimplemented!("the pinned-root check reads by hint") + } + async fn coin_record(&self, _name: ChiaBytes32) -> ChainResult> { + unimplemented!("no candidate tip is ever reached in this fixture") + } + async fn coin_spend( + &self, + _coin_id: ChiaBytes32, + _spent_height: u32, + ) -> ChainResult> { + unimplemented!("no candidate tip is ever reached in this fixture") + } + async fn peak_height(&self) -> ChainResult { + unimplemented!("the pinned-root check does not read the peak") + } + async fn push(&self, _bundle: SpendBundle) -> ChainResult<()> { + unimplemented!("resolution is read-only") + } + async fn estimate_fee(&self, _bundle: &SpendBundle, _target: u64) -> ChainResult { + unimplemented!("resolution is read-only") + } + } + + /// A voice that confirms whatever it is asked — the honest majority a veto must survive. + struct AlwaysConfirms; + + #[async_trait::async_trait] + impl ChainVoice for AlwaysConfirms { + async fn anchored_state( + &self, + _store_id: &[u8; 32], + ) -> Result, String> { + Ok(Some(AnchoredStoreState { + root: Bytes32([0xAA; 32]), + owner_puzzle_hash: None, + })) + } + async fn verify_pinned_root(&self, _store_id: &[u8; 32], _pinned: Bytes32) -> Verdict { + Verdict::Confirmed + } + async fn verify_lineage_root(&self, _store_id: &[u8; 32], _root: Bytes32) -> Verdict { + Verdict::Confirmed + } + } + + /// One address per authority, in configuration order, so every endpoint is its own voice. + struct DistinctReach; + + #[async_trait::async_trait] + impl EndpointReach for DistinctReach { + async fn addrs( + &self, + authority: &Authority, + ) -> Result, String> { + let last = u8::try_from(authority.host.len() % 251).map_err(|e| e.to_string())?; + Ok(std::collections::BTreeSet::from([std::net::IpAddr::from( + [203, 0, 113, last], + )])) + } + } + + /// A source that ANSWERED and then rejected still vetoes, even if its chain has since gone. + /// + /// # Why this is the composition and not a unit of the classifier + /// + /// `Unreachable` is the verdict [`CorroboratedResolver`] DISCARDS; `Rejected` is the one that + /// refuses. So an ordering that re-tests reachability after a verification is not merely + /// untidy — it is a way to turn the dissent of a source one can degrade into silence, and two + /// lagging endpoints then carry the serve unopposed. Asserting the classifier's verdict alone + /// would pin the shape without pinning the consequence, so the assertion below is made through + /// the agreement rule, with two honest voices present to be outvoted by. + #[tokio::test] + async fn a_source_that_answered_then_rejected_still_vetoes_when_its_chain_drops() { + let root = Bytes32([0xAA; 32]); + let store = [7u8; 32]; + // One answer: the probe consumes it, and the verification's own hint read then fails. Under + // the reverse ordering the verification consumes it (determining a rejection) and the probe + // fails — which is exactly the state that used to be filed as silence. + let degrading = DegradingChain::answering(1); + + let endpoints: Vec = ["https://a.example.org", "https://bb.example.org"] + .iter() + .map(|url| ChainEndpoint::parse(url).expect("a parseable fixture url")) + .collect(); + let dissenter = ChainEndpoint::parse("https://ccc.example.org").expect("parses"); + let all = [endpoints.clone(), vec![dissenter.clone()]].concat(); + + let honest_only = CorroboratedResolver::new( + endpoints, + Arc::new(DistinctReach), + Arc::new(|_| Arc::new(AlwaysConfirms) as Arc), + ); + assert_eq!( + honest_only.verify_pinned_root(&store, root).await, + Ok(()), + "the control: two honest voices confirm the pin. Without it every assertion below is \ + satisfied by a resolver that refuses unconditionally" + ); + + let dissent_url = dissenter.url.clone(); + let with_dissenter = CorroboratedResolver::new( + all, + Arc::new(DistinctReach), + Arc::new(move |endpoint: &ChainEndpoint| { + if endpoint.url == dissent_url { + Arc::new(EndpointResolver::over(degrading.clone())) as Arc + } else { + Arc::new(AlwaysConfirms) as Arc + } + }), + ); + let refusal = with_dissenter + .verify_pinned_root(&store, root) + .await + .expect_err("a source that answered and then rejected must refuse the resolution"); + assert!( + refusal.contains("REJECTED"), + "the refusal must be the DISSENT rule. Classifying that source as UNREACHABLE drops \ + it, leaves two confirmations standing, and SERVES — which is the whole defect: \ + {refusal}" + ); + } + + /// A chain that never answers at all is genuinely silent, and is dropped rather than vetoing. + /// + /// The control for the ordering above: without it, an implementation that returned + /// [`Verdict::Rejected`] for every failure would satisfy the veto assertion while converting + /// every endpoint outage into a total refusal to serve. + #[tokio::test] + async fn a_chain_that_never_answers_is_silence_not_dissent() { + let voice = EndpointResolver::over(DegradingChain::answering(0)); + let verdict = ChainVoice::verify_pinned_root(&voice, &[7u8; 32], Bytes32([0xAA; 32])).await; + assert!( + matches!(verdict, Verdict::Unreachable(_)), + "an endpoint that could not be reached AT ALL has said nothing, and a rule that read \ + it as dissent would take the node down on any single outage: {verdict:?}" + ); + } } diff --git a/crates/dig-node-core/src/seams/chia_peer/corroborated_resolver.rs b/crates/dig-node-core/src/seams/chia_peer/corroborated_resolver.rs new file mode 100644 index 00000000..899f3333 --- /dev/null +++ b/crates/dig-node-core/src/seams/chia_peer/corroborated_resolver.rs @@ -0,0 +1,861 @@ +//! [`CorroboratedResolver`] — the anchored root, agreed across independent voices or REFUSED. +//! +//! # Why this fact in particular (dig-node#365) +//! +//! The anchored root is the fact that decides WHICH BYTES A USER IS SERVED. Every other chain +//! fact the node holds is a number on a status surface; this one selects content. A source that +//! lies about it redirects a read on every install that trusts it, and the read-path pin then +//! fails closed against the WRONG root rather than the right one — which looks, from the outside, +//! exactly like the store having moved on. +//! +//! NC-12 exists to stop one voice determining a chain fact, so this resolver asks several and +//! serves an answer only when they AGREE. +//! +//! # The rule, stated as the caller sees it +//! +//! * **Two or more independent voices** ([`super::endpoints::independent_voices`]) — every voice +//! that answered must give the SAME answer, and at least two must have answered. One dissenter +//! is a REFUSAL, never a repaired value: there is no majority vote and no tie-break here, +//! because a majority rule hands the answer to whoever can field the most endpoints, and the +//! caller's failure mode on a refusal (do not serve) is survivable while its failure mode on a +//! wrong root is not. +//! * **Fewer than two independent voices** — the node has ONE source and says so. The answer is +//! that source's, exactly as before this resolver existed. This is the DEFAULT INSTALL, and it +//! is a named limitation rather than a corroboration claim: see `SPEC.md` §14.4b and +//! dig-node#365. +//! +//! Refusing on a default install was considered and rejected: it would stop every unconfigured +//! node serving any content at all, which trades a corroboration gap for a total outage and +//! removes the only surface on which an operator could then configure a second endpoint. +//! +//! # All THREE calls take that rule, and one of them had to be rebuilt to +//! +//! `anchored_state` carries a value, so disagreement is expressible in its return type. +//! `verify_pinned_root` and `verify_lineage_root` answer a yes/no question and originally returned +//! `Result<(), String>` per voice — in which a rejection and an unreachable chain are the same +//! `Err`. A rule reading that cannot tell dissent from silence, and the version that shipped in +//! this file counted the `Ok`s and passed at two, so **the bar did not rise with `N`** and three +//! endpoints with two a generation behind served stale content. [`Verdict`] is what fixed it. +//! +//! # What this resolver discloses, which is new +//! +//! Resolving now sends the STORE ID to every configured endpoint rather than one. Corroboration is +//! not free: the price of not trusting a single third party is telling several of them what this +//! node is looking for. An operator adding endpoints is widening that disclosure, and `SPEC.md` +//! §14.4b says so rather than leaving it to be discovered. + +use std::sync::Arc; + +use digstore_core::Bytes32; + +use super::endpoints::{independent_voices, ChainEndpoint, EndpointReach}; +use crate::shared::chain_view::{AnchoredRootResolver, AnchoredStoreState}; + +/// What ONE voice said about a yes/no chain question. +/// +/// # Why a yes/no question needs THREE answers +/// +/// `Result<(), String>` cannot carry a dissent. With no value channel, the only way a voice can +/// say *"that root is NOT current"* is `Err` — which is also the only way it can say *"I could not +/// reach the chain"*. An agreement rule reading that `Result` must therefore treat a rejection as +/// an absence of evidence, and a flat count of the `Ok`s passes however many voices objected. +/// +/// That is not hypothetical: it shipped in this file. `verify_pinned_root` and +/// `verify_lineage_root` were a flat 2-of-N threshold that did not rise with `N`, so 2-of-3 and +/// 2-of-10 were the same bar, and three endpoints with two a generation behind served STALE +/// content with no attacker involved — while the same fixture on `anchored_root` correctly refused. +/// The two calls the read-path pin actually makes were the two without the property. +/// +/// So dissent gets its own variant, and the distinction is drawn where the evidence exists rather +/// than by inspecting an error string. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Verdict { + /// The voice was reached and it AGREES. + Confirmed, + /// The voice was reached and it says NO. This is DISSENT, and it refuses the whole resolution. + Rejected(String), + /// The voice could not be asked, so it has said nothing either way. + Unreachable(String), +} + +/// One independent source, able to express dissent on every question it is asked. +/// +/// Separate from [`AnchoredRootResolver`] deliberately: that trait is the node-wide CONTRACT and +/// its `Result<(), String>` verification calls are consumed all over the crate, so widening it +/// would be a breaking change to a surface that does not need one. This is the narrower +/// vocabulary the agreement rule needs from a single voice, and [`CorroboratedResolver`] is what +/// translates a set of these back into the node-wide contract. +#[async_trait::async_trait] +pub(crate) trait ChainVoice: Send + Sync { + /// The store's tip state as THIS voice sees it. Already has a value channel, so it needs no + /// tri-state: a differing root is expressible directly. + async fn anchored_state( + &self, + store_id: &[u8; 32], + ) -> Result, String>; + + /// Is `pinned_root` the store's CURRENT on-chain generation, as this voice sees it? + async fn verify_pinned_root(&self, store_id: &[u8; 32], pinned_root: Bytes32) -> Verdict; + + /// Is `root` anywhere in the store's authenticated lineage, as this voice sees it? + async fn verify_lineage_root(&self, store_id: &[u8; 32], root: Bytes32) -> Verdict; +} + +/// Builds the voice that speaks to ONE endpoint. +/// +/// Injected rather than hardcoded so the corroboration rule can be exercised against sources that +/// answer from a fixture. Without this seam the only fixture available is a live network, and the +/// dissent case — the entire point of this module — cannot be expressed at all. +pub(crate) type SourceFactory = Arc Arc + Send + Sync>; + +/// An [`AnchoredRootResolver`] that believes a chain fact only when independent voices agree. +pub(crate) struct CorroboratedResolver { + /// Every endpoint the operator configured, in configuration order. + endpoints: Vec, + /// How an endpoint's independence is measured — its reachable addresses (§5.2 dual-stack; + /// an IPv6 and an IPv4 address of the SAME host are one voice, which is the reason this + /// compares address SETS rather than a single address). + reach: Arc, + /// How a per-endpoint resolver is built. + source: SourceFactory, +} + +impl CorroboratedResolver { + /// A resolver over `endpoints`, measuring independence with `reach` and building each voice + /// with `source`. + pub fn new( + endpoints: Vec, + reach: Arc, + source: SourceFactory, + ) -> Self { + Self { + endpoints, + reach, + source, + } + } + + /// The independent voices available right now — ONE speaker per voice. + /// + /// Recomputed per resolution rather than fixed at start-up: an endpoint's addresses change, and + /// a voice count decided once would keep claiming corroboration long after two endpoints had + /// converged on one host — a stale independence verdict is worse than none, because it is the + /// verdict the refusal rule trusts. Recomputing is not the same as re-RESOLVING, and the + /// production reach caches its lookups on a short TTL + /// ([`super::endpoints::CachedReach`]) so this costs a map read rather than a DNS round trip + /// on every content read. + /// + /// # A SINGLE configured endpoint is a voice whether or not its name resolves + /// + /// Independence is a relation BETWEEN endpoints, so with one endpoint there is nothing for a + /// lookup to decide. Asking anyway turns name resolution into a gate on reading: a resolver + /// failure would deny a read the HTTP client — with its own resolver, its own cache, and + /// possibly a proxy — would have served, and it would do so on the DEFAULT INSTALL, which had + /// no name resolution on this path at all before corroboration existed. Failing closed is + /// right when the CHAIN ANSWER is in doubt; a name lookup is not that. + /// + /// This does not soften the quorum rule. With two or more endpoints an unresolvable one still + /// contributes NO voice, because there it is exactly the evidence that the endpoint is a + /// separate machine — and a voice that cannot be shown independent must not be counted. + /// + /// # Why the first member speaks, and no failover between members + /// + /// A group's members are, by construction, endpoints that land on the SAME machine. Trying a + /// second one after the first fails is a retry against the host that just failed, not a + /// fallback to another source — so it would buy availability only in the narrow case of one + /// name of a multi-homed host going bad, and it would cost a second round trip on the request + /// path in every other case. + /// + /// It would also blur the very distinction this module rests on: a voice is a machine, and a + /// voice that is really "whichever of these names answered" is harder to reason about when a + /// rejection has to be attributed to a source. Recorded because an earlier version of this + /// comment promised per-group failover that the code did not perform, which is a worse state + /// than either choice. + async fn voices(&self) -> Vec> { + if let [only] = self.endpoints.as_slice() { + return vec![(self.source)(only)]; + } + independent_voices(&self.endpoints, self.reach.as_ref()) + .await + .iter() + .filter_map(|group| group.first()) + .map(|&ix| (self.source)(&self.endpoints[ix])) + .collect() + } + + /// Ask every voice, then apply the agreement rule to whatever came back. + /// + /// `ask` returns `Err` for a voice that could not be reached; such a voice is DROPPED rather + /// than treated as dissent, because "I could not ask" and "I was told something else" demand + /// opposite remedies and conflating them makes a network blip indistinguishable from an + /// attack. Dropping is still fail-closed: the agreement rule then has fewer answers, and too + /// few answers is a refusal. + async fn agreed(&self, what: &str, ask: F) -> Result + where + T: PartialEq + Clone, + F: Fn(Arc) -> Fut, + Fut: std::future::Future>, + { + let voices = self.voices().await; + let Some((first, rest)) = voices.split_first() else { + return Err(format!( + "{what}: no configured chain endpoint could be reached (chain is the authority)" + )); + }; + + // ONE voice is the default install: answer as the single source, and do not dress that up + // as agreement. `SPEC.md` records the limitation this leaves. + if rest.is_empty() { + return ask(first.clone()).await; + } + + let mut answers: Vec = Vec::new(); + let mut refusals: Vec = Vec::new(); + for voice in &voices { + match ask(voice.clone()).await { + Ok(answer) => answers.push(answer), + Err(e) => refusals.push(e), + } + } + + let Some(candidate) = answers.first().cloned() else { + return Err(format!( + "{what}: no independent chain source answered ({}) — refusing rather than \ + guessing (chain is the authority)", + refusals.join("; ") + )); + }; + if answers.len() < 2 { + return Err(format!( + "{what}: only ONE independent chain source answered, so nothing corroborates it \ + ({}) — refusing (chain is the authority)", + refusals.join("; ") + )); + } + if answers.iter().any(|answer| *answer != candidate) { + return Err(format!( + "{what}: independent chain sources DISAGREE — refusing rather than picking one \ + (chain is the authority)" + )); + } + Ok(candidate) + } + + /// Ask every voice a yes/no question and apply the SAME rule [`Self::agreed`] applies. + /// + /// The rule, in the order it is decided — and the order matters, because dissent must not be + /// outvoted by counting: + /// + /// 1. **Any reached voice that REJECTS refuses the whole resolution.** No majority, no + /// threshold. One source saying the pinned root is not current is exactly the signal this + /// feature exists to surface, and the read-path pin fails closed on it. + /// 2. Otherwise at least TWO voices must have CONFIRMED. An unreachable voice is dropped + /// rather than counted as dissent, so too few confirmations is thin evidence, not a + /// disagreement — the two demand opposite remedies. + /// 3. One voice is the default install and answers alone, exactly as [`Self::agreed`] does. + /// + /// Rule 1 is what makes the bar rise with `N`: every added endpoint is another source that can + /// veto, where the flat 2-of-N threshold this replaces made 2-of-3 and 2-of-10 the same bar. + async fn all_confirmed(&self, what: &str, ask: F) -> Result<(), String> + where + F: Fn(Arc) -> Fut, + Fut: std::future::Future, + { + let voices = self.voices().await; + let Some((first, rest)) = voices.split_first() else { + return Err(format!( + "{what}: no configured chain endpoint could be reached (chain is the authority)" + )); + }; + + if rest.is_empty() { + return match ask(first.clone()).await { + Verdict::Confirmed => Ok(()), + Verdict::Rejected(why) | Verdict::Unreachable(why) => Err(why), + }; + } + + let mut confirmations = 0usize; + let mut unreachable: Vec = Vec::new(); + for voice in &voices { + match ask(voice.clone()).await { + Verdict::Confirmed => confirmations += 1, + // Decided the moment it is seen: a rejection is not weighed against the + // confirmations, it ends the resolution. + Verdict::Rejected(why) => { + return Err(format!( + "{what}: an independent chain source REJECTED this root ({why}) — \ + refusing rather than outvoting it (chain is the authority)" + )) + } + Verdict::Unreachable(why) => unreachable.push(why), + } + } + + if confirmations < 2 { + return Err(format!( + "{what}: fewer than two independent chain sources could confirm this root ({}) — \ + refusing (chain is the authority)", + unreachable.join("; ") + )); + } + Ok(()) + } +} + +#[async_trait::async_trait] +impl AnchoredRootResolver for CorroboratedResolver { + async fn anchored_root(&self, store_id: &[u8; 32]) -> Result, String> { + Ok(self.anchored_state(store_id).await?.map(|s| s.root)) + } + + /// The store's tip state, agreed or refused. + /// + /// `Option` is part of what must agree: one voice saying "not minted" while another names a + /// root is a disagreement about whether the store exists, and serving the root would let a + /// single source conjure a store into being. + async fn anchored_state( + &self, + store_id: &[u8; 32], + ) -> Result, String> { + let store_id = *store_id; + self.agreed("anchored state", move |voice| async move { + voice.anchored_state(&store_id).await + }) + .await + } + + /// A pinned root is confirmed only when no reached voice rejects it and at least two confirm. + /// + /// This is the call the read-path pin makes on an explicit-root request, so it is the one that + /// decides which bytes a user is served. It takes [`CorroboratedResolver::all_confirmed`] — + /// the SAME rule [`Self::anchored_state`] takes, expressed for a question whose answer has no + /// value channel of its own. + async fn verify_pinned_root( + &self, + store_id: &[u8; 32], + pinned_root: Bytes32, + ) -> Result<(), String> { + let store_id = *store_id; + self.all_confirmed("pinned-root verification", move |voice| async move { + voice.verify_pinned_root(&store_id, pinned_root).await + }) + .await + } + + /// Lineage membership, on the same terms as [`Self::verify_pinned_root`]. + async fn verify_lineage_root(&self, store_id: &[u8; 32], root: Bytes32) -> Result<(), String> { + let store_id = *store_id; + self.all_confirmed("lineage-root verification", move |voice| async move { + voice.verify_lineage_root(&store_id, root).await + }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::seams::chia_peer::endpoints::Authority; + use std::collections::{BTreeMap, BTreeSet}; + use std::net::IpAddr; + + const STORE: [u8; 32] = [9u8; 32]; + + /// What one endpoint says about the store, keyed by URL. + /// + /// `Rejects` is the variant whose ABSENCE let a defect ship. Every earlier fixture scripted + /// only `Unreachable`, so no test could express the one configuration that distinguishes + /// "reached and objecting" from "not reached" — and the verification calls, which had only the + /// latter meaning available to them, counted a rejection as silence. + #[derive(Clone)] + enum Voice { + Root(u8), + NotMinted, + Unreachable, + /// Reached, and it says the root is NOT current. A genuine dissent. + Rejects, + } + + /// A resolver standing in for ONE endpoint. + struct Scripted(Voice); + + #[async_trait::async_trait] + impl ChainVoice for Scripted { + async fn anchored_state( + &self, + _store_id: &[u8; 32], + ) -> Result, String> { + match self.0 { + Voice::Root(byte) => Ok(Some(AnchoredStoreState { + root: Bytes32([byte; 32]), + owner_puzzle_hash: None, + })), + Voice::NotMinted => Ok(None), + Voice::Unreachable => Err("chain unreachable".into()), + // A voice that rejects a PIN still has a tip it believes in; it simply is not the + // one being asked about. Modelling it as a differing root keeps the double + // consistent across all three calls rather than able to lie on one of them. + Voice::Rejects => Ok(Some(AnchoredStoreState { + root: Bytes32([0xEE; 32]), + owner_puzzle_hash: None, + })), + } + } + + async fn verify_pinned_root(&self, _store_id: &[u8; 32], _pinned: Bytes32) -> Verdict { + match self.0 { + Voice::Root(_) => Verdict::Confirmed, + Voice::NotMinted => Verdict::Rejected("no confirmed generation".into()), + Voice::Unreachable => Verdict::Unreachable("chain unreachable".into()), + Voice::Rejects => Verdict::Rejected("pinned root is not the current root".into()), + } + } + + async fn verify_lineage_root(&self, _store_id: &[u8; 32], _root: Bytes32) -> Verdict { + match self.0 { + Voice::Root(_) => Verdict::Confirmed, + Voice::NotMinted => Verdict::Rejected("no confirmed generation".into()), + Voice::Unreachable => Verdict::Unreachable("chain unreachable".into()), + Voice::Rejects => Verdict::Rejected("root is not in the lineage".into()), + } + } + } + + /// Reach answers built from an explicit address per URL, so a fixture can make two endpoints + /// one voice or two INDEPENDENTLY of what each of them says about the store. Keeping the two + /// dimensions separate is what lets the dissent tests hold independence fixed while varying + /// exactly one voice. + struct ScriptedReach(BTreeMap); + + #[async_trait::async_trait] + impl EndpointReach for ScriptedReach { + async fn addrs(&self, authority: &Authority) -> Result, String> { + self.0 + .get(authority) + .map(|ip| BTreeSet::from([*ip])) + .ok_or_else(|| "does not resolve".to_string()) + } + } + + /// Build a resolver from `(url, address, what it says)` rows. + fn resolver(rows: &[(&str, &str, Voice)]) -> CorroboratedResolver { + let endpoints: Vec = rows + .iter() + .map(|(url, ..)| ChainEndpoint::parse(url).expect("a parseable fixture url")) + .collect(); + let reach = ScriptedReach( + rows.iter() + .zip(&endpoints) + .map(|((_, addr, _), endpoint)| { + (endpoint.authority.clone(), addr.parse().expect("an ip")) + }) + .collect(), + ); + let script: BTreeMap = rows + .iter() + .map(|(url, _, voice)| ((*url).to_string(), voice.clone())) + .collect(); + CorroboratedResolver::new( + endpoints, + Arc::new(reach), + Arc::new(move |endpoint: &ChainEndpoint| { + Arc::new(Scripted( + script.get(&endpoint.url).cloned().expect("a scripted url"), + )) as Arc + }), + ) + } + + /// Independent voices that agree are believed; ONE dissenter is a refusal, not a repair. + /// + /// The fixture varies exactly ONE actor between the two halves and keeps two honest voices in + /// both, which is what makes each half load-bearing. An all-hostile fixture would be blind + /// here: with no honest majority left it cannot distinguish "refused because they disagreed" + /// from "refused because nothing answered", and a majority-vote implementation — the nearest + /// wrong one — passes it. + /// + /// Measured against the compiled resolver: replacing the unanimity check with a MAJORITY vote + /// (two of three say `0xAA`, so serve `0xAA`) fails this test, and fails + /// [`one_source_reporting_not_minted_against_a_named_root_is_a_disagreement`] and + /// [`two_endpoints_on_one_machine_do_not_corroborate_each_other`] with it. That is the revert + /// this test is named for. + #[tokio::test] + async fn a_single_dissenting_source_refuses_rather_than_repairing_the_root() { + let agreeing = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Root(0xAA)), + ("https://c.example.org", "192.0.2.3", Voice::Root(0xAA)), + ]); + assert_eq!( + agreeing.anchored_root(&STORE).await, + Ok(Some(Bytes32([0xAA; 32]))), + "three independent voices agreeing must YIELD the root — the control that kills an \ + implementation which refuses unconditionally, which would satisfy every assertion \ + below without corroborating anything" + ); + + let dissenting = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Root(0xAA)), + ("https://c.example.org", "192.0.2.3", Voice::Root(0xBB)), + ]); + let refusal = dissenting + .anchored_root(&STORE) + .await + .expect_err("one dissenter among three must refuse"); + assert!( + refusal.contains("DISAGREE"), + "the refusal must be the AGREEMENT rule. A majority vote — two of three say 0xAA — \ + would serve 0xAA here, which is a repaired value and hands the answer to whoever \ + fields the most endpoints: {refusal}" + ); + } + + /// Disagreement about whether the store EXISTS is disagreement. + /// + /// `Ok(None)` is a legitimate answer, so an implementation comparing only the roots it was + /// given would drop the `None` and report a unanimous root — letting one source conjure a + /// store into being for a node whose other sources have never seen it. + #[tokio::test] + async fn one_source_reporting_not_minted_against_a_named_root_is_a_disagreement() { + let split = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Root(0xAA)), + ("https://c.example.org", "192.0.2.3", Voice::NotMinted), + ]); + let refusal = split + .anchored_state(&STORE) + .await + .expect_err("a not-minted answer beside a named root is dissent"); + assert!( + refusal.contains("DISAGREE"), + "absence and presence must compare as different answers: {refusal}" + ); + + let unanimous_absence = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::NotMinted), + ("https://b.example.org", "198.51.100.2", Voice::NotMinted), + ]); + assert_eq!( + unanimous_absence.anchored_state(&STORE).await, + Ok(None), + "unanimous absence is a corroborated answer, not a failure — without this control the \ + assertion above passes against an implementation that refuses on any `None` at all" + ); + } + + /// Two endpoints on ONE machine are one voice, so their agreement corroborates nothing. + /// + /// This is the PR#354 trap at the layer that acts on it: both endpoints say the same thing, so + /// an implementation counting CONFIGURED sources — or counting sources by type — sees a 2-of-2 + /// agreement and serves the root. The control below moves ONE endpoint to a second machine + /// and nothing else, so the fixture cannot pass by refusing everything. + /// + /// Which revert this catches, measured rather than assumed: it fires on the MAJORITY-vote + /// revert (a and b are one voice, so the second half is 1-against-1 and a majority rule serves + /// `0xAA`). It does NOT fire on a name-based independence revert — under that rule a, b and c + /// are three voices, two agree and one dissents, and unanimity refuses anyway. The name-based + /// revert is caught in [`super::super::endpoints`] by + /// `two_names_for_one_machine_are_one_voice_and_two_machines_are_two`, which is where the + /// grouping rule lives. Recorded because a test whose comment claims a revert it does not + /// catch is how the revert that IS uncaught goes unnoticed. + #[tokio::test] + async fn two_endpoints_on_one_machine_do_not_corroborate_each_other() { + let one_machine = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "203.0.113.1", Voice::Root(0xAA)), + ]); + assert_eq!( + one_machine.anchored_root(&STORE).await, + Ok(Some(Bytes32([0xAA; 32]))), + "one voice answers as a single source — the documented default-install path. What it \ + must NOT do is claim corroboration, which the dissent case below measures" + ); + + // The same two endpoints, still agreeing, but now genuinely two machines — and a third + // that dissents. Under a rule that counts CONFIGURED endpoints the two-machine and + // one-machine cases are indistinguishable, so this pair is what separates them. + let one_machine_plus_dissenter = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://c.example.org", "192.0.2.3", Voice::Root(0xBB)), + ]); + let refusal = one_machine_plus_dissenter + .anchored_root(&STORE) + .await + .expect_err("two voices that disagree must refuse"); + assert!( + refusal.contains("DISAGREE"), + "a and b are ONE voice, so this is 1-against-1 and not 2-against-1; an implementation \ + counting endpoints sees a majority for 0xAA and serves it: {refusal}" + ); + } + + /// A source that could not be reached is dropped, and too few answers is a refusal. + /// + /// The two halves separate "was not asked" from "was asked and refused": with only one voice + /// left answering there is nothing to corroborate against, and answering anyway would make an + /// outage at one endpoint silently restore single-source resolution. + #[tokio::test] + async fn an_unreachable_source_leaves_too_little_evidence_to_serve_on() { + let one_down = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Unreachable), + ]); + let refusal = one_down + .anchored_root(&STORE) + .await + .expect_err("one answer out of two independent voices corroborates nothing"); + assert!( + refusal.contains("only ONE independent chain source answered"), + "the refusal must say the evidence was too THIN, not that the sources disagreed — a \ + transient outage and an attack demand opposite remedies: {refusal}" + ); + + let both_up = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Root(0xAA)), + ]); + assert_eq!( + both_up.anchored_root(&STORE).await, + Ok(Some(Bytes32([0xAA; 32]))), + "the same two endpoints both answering DO corroborate — the control proving the \ + refusal above came from the thin-evidence rule and not from a resolver that never \ + answers" + ); + } + + /// The pinned-root and lineage checks take the same rule, and a lone `Ok` is not enough. + /// + /// These two are the calls the read-path pin makes, so a corroboration rule applied to + /// `anchored_state` alone would leave the actual serve decision single-sourced while the SPEC + /// claimed otherwise. + #[tokio::test] + async fn the_verification_calls_need_two_agreeing_voices_too() { + let root = Bytes32([0xAA; 32]); + let one_down = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Unreachable), + ]); + for outcome in [ + one_down.verify_pinned_root(&STORE, root).await, + one_down.verify_lineage_root(&STORE, root).await, + ] { + let refusal = outcome.expect_err("one voice cannot corroborate a serve decision"); + assert!( + refusal.contains("fewer than two independent chain sources could confirm"), + "both verification calls must refuse on THIN EVIDENCE and say so, rather than \ + reporting dissent — an unreachable source has said nothing, and a message \ + blaming disagreement would send an operator hunting a hostile endpoint during \ + a plain outage: {refusal}" + ); + } + + let both_up = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Root(0xAA)), + ]); + assert_eq!( + both_up.verify_pinned_root(&STORE, root).await, + Ok(()), + "two agreeing voices confirm the pin — the control that kills an always-refuse \ + implementation of the verification path" + ); + assert_eq!( + both_up.verify_lineage_root(&STORE, root).await, + Ok(()), + "and the same for lineage membership" + ); + } + + /// A REACHED voice that rejects the root refuses the resolution — it is not outvoted. + /// + /// # This is the fixture whose absence let a defect ship + /// + /// Every earlier verification test scripted only `Voice::Unreachable`, so none of them could + /// express the one configuration that separates the two possible semantics. The shipped code + /// was a flat 2-of-N threshold: `verify_pinned_root` and `verify_lineage_root` returned + /// `Result<(), String>` per voice, so a rejection and an unreachable chain arrived as the same + /// `Err` and both were discarded as "could not ask". Two confirmations then passed regardless + /// of how many voices objected. + /// + /// Both halves below are load-bearing, and the second is the one a threshold cannot satisfy: + /// + /// * **2 confirm, 1 rejects** — a flat 2-of-N threshold returns `Ok(())` here. It must refuse. + /// * **9 confirm, 1 rejects** — proves the bar RISES with `N`. Under a threshold, 2-of-3 and + /// 2-of-10 are the same bar, so adding endpoints bought nothing on the two calls the + /// read-path pin actually makes. + /// + /// And no attacker is required for the first: three endpoints with one a generation behind is + /// an ordinary operational state, and under the old rule it served stale content. + #[tokio::test] + async fn a_reached_voice_that_rejects_refuses_and_is_never_outvoted() { + let two_against_one = resolver(&[ + ("https://a.example.org", "203.0.113.1", Voice::Root(0xAA)), + ("https://b.example.org", "198.51.100.2", Voice::Root(0xAA)), + ("https://c.example.org", "192.0.2.3", Voice::Rejects), + ]); + let root = Bytes32([0xAA; 32]); + + let pinned = two_against_one + .verify_pinned_root(&STORE, root) + .await + .expect_err("a reached rejection must refuse the pin"); + assert!( + pinned.contains("REJECTED"), + "the refusal must be the DISSENT rule, not the thin-evidence one — two voices did \ + confirm, so a thin-evidence message would mean the rejection was still being counted \ + as silence: {pinned}" + ); + let lineage = two_against_one + .verify_lineage_root(&STORE, root) + .await + .expect_err("a reached rejection must refuse the lineage redirect"); + assert!( + lineage.contains("REJECTED"), + "and the same for lineage membership: {lineage}" + ); + + // THE COMPOSITION, and it is the half of this defect that made it dangerous rather than + // merely wrong. `content_serve.rs`, `dig_rpc/dispatch.rs` and `module_reshare.rs` all + // treat a failed tip resolution as the #747 broken-walk case and FALL BACK to + // `verify_pinned_root`. Widening what `anchored_state`'s `Err` means — it now also means + // "sources DISAGREE" — routes the strongest signal this feature produces onto that + // fallback. So the fallback must refuse on the same fixture, or a disagreement is + // laundered into a serve by the ladder that exists for a different failure entirely. + assert!( + two_against_one.anchored_state(&STORE).await.is_err(), + "the tip resolution must refuse on dissent — this is the input to the fallback below" + ); + assert!( + two_against_one + .verify_pinned_root(&STORE, root) + .await + .is_err(), + "and the BOUNDED fallback the serve path drops to must refuse on the SAME fixture. If \ + it does not, a disagreement about the tip is answered by the #747 escape hatch and \ + the read is served" + ); + + // Nine confirmations against one rejection. A threshold rule passes this trivially; the + // dissent rule refuses it, which is what "the bar rises with N" means. + let mut many: Vec<(&str, &str, Voice)> = Vec::new(); + const URLS: [(&str, &str); 9] = [ + ("https://n1.example.org", "203.0.113.11"), + ("https://n2.example.org", "203.0.113.12"), + ("https://n3.example.org", "203.0.113.13"), + ("https://n4.example.org", "203.0.113.14"), + ("https://n5.example.org", "203.0.113.15"), + ("https://n6.example.org", "203.0.113.16"), + ("https://n7.example.org", "203.0.113.17"), + ("https://n8.example.org", "203.0.113.18"), + ("https://n9.example.org", "203.0.113.19"), + ]; + for (url, addr) in URLS { + many.push((url, addr, Voice::Root(0xAA))); + } + many.push(("https://dissent.example.org", "192.0.2.99", Voice::Rejects)); + let nine_against_one = resolver(&many); + + let refusal = nine_against_one + .verify_pinned_root(&STORE, root) + .await + .expect_err("one rejection among ten must still refuse"); + assert!( + refusal.contains("REJECTED"), + "with nine confirmations a threshold rule is satisfied nine times over; only a rule \ + where any reached rejection vetoes makes an added endpoint raise the bar: {refusal}" + ); + + // The control, and it is what stops all of the above passing against an always-refuse + // implementation: the same ten voices with the dissenter merely UNREACHABLE still serve. + let mut nine_and_a_silent_one = many.clone(); + nine_and_a_silent_one.pop(); + nine_and_a_silent_one.push(( + "https://dissent.example.org", + "192.0.2.99", + Voice::Unreachable, + )); + assert_eq!( + resolver(&nine_and_a_silent_one) + .verify_pinned_root(&STORE, root) + .await, + Ok(()), + "an UNREACHABLE tenth voice is silence, not dissent — it must not refuse, or a single \ + endpoint outage takes the node down and the two conditions become indistinguishable" + ); + } + + /// A reach that resolves NOTHING — a DNS outage, or a resolver this process cannot use. + struct NeverResolves; + + #[async_trait::async_trait] + impl EndpointReach for NeverResolves { + async fn addrs(&self, _authority: &Authority) -> Result, String> { + Err("does not resolve".into()) + } + } + + /// Build a resolver over `rows` whose reach answers nothing at all. + fn unresolvable(rows: &[(&str, Voice)]) -> CorroboratedResolver { + let endpoints: Vec = rows + .iter() + .map(|(url, _)| ChainEndpoint::parse(url).expect("a parseable fixture url")) + .collect(); + let script: BTreeMap = rows + .iter() + .map(|(url, voice)| ((*url).to_string(), voice.clone())) + .collect(); + CorroboratedResolver::new( + endpoints, + Arc::new(NeverResolves), + Arc::new(move |endpoint: &ChainEndpoint| { + Arc::new(Scripted( + script.get(&endpoint.url).cloned().expect("a scripted url"), + )) as Arc + }), + ) + } + + /// A name lookup must not become a gate on READING when there is only one endpoint. + /// + /// Independence is a relation BETWEEN endpoints, so a single configured endpoint has nothing + /// for a lookup to decide — and the DEFAULT INSTALL is exactly that case. Asking anyway turns + /// a resolver failure into a refusal to serve content the HTTP client (its own resolver, its + /// own cache, possibly a proxy) would have fetched, on a path that performed no name + /// resolution at all before corroboration existed. + /// + /// # The second half is the load-bearing one + /// + /// With TWO endpoints an unresolvable one must STILL contribute no voice, because there the + /// lookup is the evidence that they are separate machines. Without that control this test + /// would be satisfied by simply deleting the reach check, which would let two names for one + /// host corroborate each other whenever DNS was unavailable — the PR#354 defect, restored by + /// an outage. + #[tokio::test] + async fn one_endpoint_answers_without_a_name_lookup_but_two_still_need_one() { + let alone = unresolvable(&[("https://a.example.org", Voice::Root(0xAA))]); + assert_eq!( + alone.anchored_root(&STORE).await, + Ok(Some(Bytes32([0xAA; 32]))), + "one configured endpoint is one voice whether or not its NAME resolves; refusing here \ + denies a read on the strength of a DNS failure that says nothing about the chain" + ); + + let pair = unresolvable(&[ + ("https://a.example.org", Voice::Root(0xAA)), + ("https://b.example.org", Voice::Root(0xAA)), + ]); + let refusal = pair + .anchored_root(&STORE) + .await + .expect_err("two endpoints that cannot be told apart corroborate nothing"); + assert!( + refusal.contains("no configured chain endpoint could be reached"), + "two endpoints whose independence cannot be measured must NOT be counted as two \ + voices — an unmeasured pair may be one machine, which is the quorum this rule exists \ + to refuse: {refusal}" + ); + } +} diff --git a/crates/dig-node-core/src/seams/chia_peer/endpoints.rs b/crates/dig-node-core/src/seams/chia_peer/endpoints.rs new file mode 100644 index 00000000..366bab95 --- /dev/null +++ b/crates/dig-node-core/src/seams/chia_peer/endpoints.rs @@ -0,0 +1,605 @@ +//! Which chain endpoints the node may ask, and which of them are genuinely DIFFERENT VOICES. +//! +//! # Independence is derived from REACH, never from type or from a name +//! +//! NC-12 asks that a chain fact be agreed by several *independent* sources. The failure this +//! module exists to prevent is a "quorum" that is really one voice wearing two hats — the exact +//! defect found in `dig-wallet` on dig-node PR#354, where a 2-of-2 "independent-group" rule was +//! satisfied by ONE HTTPS endpoint because the second group's peer tier was configured with +//! `max_peers: 0` and could reach nothing at all. +//! +//! So independence here is not a property of a source's TYPE, and not a property of its URL +//! either: two hostnames are one voice whenever they land on the same machine, and a CNAME costs +//! an attacker nothing. It is derived from what each endpoint can actually be REACHED at — its +//! resolved addresses — and two endpoints whose address sets INTERSECT are treated as a single +//! voice, however different they look on paper. +//! +//! Among SEVERAL endpoints, one that cannot be resolved contributes NO voice. That is the +//! fail-closed direction: a source whose independence cannot be measured has not been shown to be +//! a second machine, and counting it would let a typo inflate a quorum. With a SINGLE endpoint +//! there is no independence to measure and the lookup is skipped entirely — see +//! [`super::corroborated_resolver::CorroboratedResolver`], where that carve-out lives, for why a +//! name lookup must not become a gate on reading. + +use std::collections::BTreeSet; +use std::net::IpAddr; + +/// A chain endpoint's network identity: the host and port a client would connect to. +/// +/// Held separately from the URL because the URL is what an operator wrote and the authority is +/// what the network sees — `https://api.example.org/v1/` and `https://API.example.org` are one +/// authority, and the resolved-address comparison below only makes sense per authority. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct Authority { + /// Lowercased host, with no brackets around an IPv6 literal. + pub host: String, + /// The port a client dials — the URL's explicit port, else the scheme's default. + pub port: u16, +} + +/// One configured chain endpoint: the URL to query, and the authority it dials. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ChainEndpoint { + /// The URL as configured, passed through to the coinset client unchanged. + pub url: String, + /// Where that URL lands on the network. + pub authority: Authority, +} + +impl ChainEndpoint { + /// Parse `url` into an endpoint, or `None` when it names no host this node could dial. + /// + /// Deliberately a small hand-rolled parse rather than a URL crate: the only fields that matter + /// here are the host and the port, and pulling a parser in for two fields would add a + /// dependency whose own failure modes then need answers of their own. + pub fn parse(url: &str) -> Option { + let url = url.trim(); + let (scheme, rest) = url.split_once("://")?; + let default_port = match scheme.to_ascii_lowercase().as_str() { + "http" => 80, + "https" => 443, + _ => return None, + }; + // Keep only what a client would dial: no path, no query, no fragment, no userinfo. + let hostport = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let hostport = hostport.rsplit_once('@').map_or(hostport, |(_, host)| host); + + let (host, port) = split_host_port(hostport, default_port)?; + Some(Self { + url: url.to_string(), + authority: Authority { host, port }, + }) + } +} + +/// Split `hostport` into a lowercased host and a port, honouring the `[::1]:8555` IPv6 form. +/// +/// IPv6 literals are bracketed precisely because a bare `::1:8555` is ambiguous, so the bracketed +/// form is handled first and a `:` split is only ever applied to a name or an IPv4 literal. +fn split_host_port(hostport: &str, default_port: u16) -> Option<(String, u16)> { + if let Some(rest) = hostport.strip_prefix('[') { + let (host, after) = rest.split_once(']')?; + let port = match after.strip_prefix(':') { + Some(explicit) => explicit.parse().ok()?, + None if after.is_empty() => default_port, + None => return None, + }; + return non_empty(host.to_ascii_lowercase()).map(|host| (host, port)); + } + match hostport.rsplit_once(':') { + Some((host, port)) => { + let port = port.parse().ok()?; + non_empty(host.to_ascii_lowercase()).map(|host| (host, port)) + } + None => non_empty(hostport.to_ascii_lowercase()).map(|host| (host, default_port)), + } +} + +fn non_empty(value: String) -> Option { + (!value.is_empty()).then_some(value) +} + +/// What a chain endpoint can actually be reached at. +/// +/// A trait so the independence rule can be pinned against address sets chosen to express the +/// same-machine case, which no live DNS lookup can be relied upon to produce on demand. +#[async_trait::async_trait] +pub(crate) trait EndpointReach: Send + Sync { + /// The addresses `authority` resolves to, or `Err` when it resolves to nothing. + async fn addrs(&self, authority: &Authority) -> Result, String>; +} + +/// How long one endpoint's name resolution may take before it counts as unreachable. +/// +/// Chosen to bound the request path rather than to be generous: independence is recomputed per +/// resolution, so this is paid on reads, and an endpoint slower than this is not one the node can +/// usefully corroborate against anyway. +const LOOKUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); + +/// Production reach: ordinary DNS, through the same resolver the HTTP client will use. +pub(crate) struct DnsReach; + +#[async_trait::async_trait] +impl EndpointReach for DnsReach { + /// Resolve `authority` by handing the host and the port to the resolver SEPARATELY. + /// + /// The `(host, port)` tuple form is not stylistic. Rendering an address as + /// `format!("{host}:{port}")` is invalid for every IPv6 literal — the grammar requires brackets + /// — and this module strips those brackets when it parses, so re-joining the two would produce + /// `::1:8555` and fail to resolve every v6 endpoint. §5.2 makes IPv6 the first-class case here, + /// so that would not be an edge; it would be the common path. `crates/dig-node-core/tests/ + /// banned_address_patterns.rs` bans the concatenation at the source for exactly this reason, + /// and it caught this function. + async fn addrs(&self, authority: &Authority) -> Result, String> { + let Authority { host, port } = authority; + // BOUNDED, because this runs on the content-serve request path. A resolver that never + // answers would otherwise hold a read open indefinitely, and it would do so while the node + // looks healthy — the endpoint set is consulted per resolution, so one black-holed DNS + // server would stall every read rather than costing one voice. + // + // A timeout is an UNREACHABLE verdict, which is the fail-closed direction: the endpoint + // contributes no voice, and too few voices refuses. + let addrs: BTreeSet = tokio::time::timeout( + LOOKUP_TIMEOUT, + tokio::net::lookup_host((host.as_str(), *port)), + ) + .await + .map_err(|_| { + format!( + "host {host} port {port} did not resolve within {}s", + LOOKUP_TIMEOUT.as_secs() + ) + })? + .map_err(|e| format!("host {host} port {port} does not resolve: {e}"))? + .map(|socket| socket.ip()) + .collect(); + if addrs.is_empty() { + return Err(format!("host {host} port {port} resolves to no address")); + } + Ok(addrs) + } +} + +/// How long a resolved address set is reused before the name is looked up again. +/// +/// Independence must not be decided from a verdict fixed at start-up — an endpoint's addresses +/// change, and a permanently-cached grouping would keep claiming corroboration long after two +/// endpoints had converged on one host. A short TTL keeps that from happening while removing the +/// per-READ lookup: the verdict can be at most this stale, which is the same order as the DNS TTLs +/// the resolver is honouring anyway. +const REACH_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +/// How long a previously-resolved address set is still served AFTER the name stops resolving. +/// +/// A resolver blip is not evidence that an endpoint moved, and treating it as one silently changes +/// the VOICE COUNT — the number the refusal rule trusts — on the basis of a failure that has +/// nothing to do with the chain. Serving the last known answer for a bounded window keeps a DNS +/// hiccup from being reported as a corroboration failure. Beyond the window the entry is abandoned: +/// a name that has not resolved for ten minutes has genuinely stopped resolving. +const REACH_STALE_GRACE: std::time::Duration = std::time::Duration::from_secs(600); + +/// What one authority last resolved to, and when. +struct CachedAddrs { + /// The addresses that lookup returned. + addrs: BTreeSet, + /// When they were learned. + learned: std::time::Instant, +} + +/// An [`EndpointReach`] that remembers, so name resolution is not paid on every read. +/// +/// Wrapping rather than folding the cache into [`DnsReach`] keeps the lookup and the caching +/// policy separately testable: a cache tested through a live resolver can only be tested against +/// whatever that resolver happens to do. +pub(crate) struct CachedReach { + /// The reach this one is a cache over. + inner: R, + /// One entry per authority. A `std::sync::Mutex` is deliberate — the critical section is a map + /// lookup with no `await` in it, so an async mutex would buy nothing and cost a task wake. + entries: std::sync::Mutex>, +} + +impl CachedReach { + /// A cache over `inner`. + pub fn new(inner: R) -> Self { + Self { + inner, + entries: std::sync::Mutex::new(std::collections::HashMap::new()), + } + } + + /// The cached answer for `authority`, if one is fresh enough to be used under `limit`. + fn cached( + &self, + authority: &Authority, + limit: std::time::Duration, + ) -> Option> { + let entries = self.entries.lock().ok()?; + let entry = entries.get(authority)?; + (entry.learned.elapsed() <= limit).then(|| entry.addrs.clone()) + } +} + +#[async_trait::async_trait] +impl EndpointReach for CachedReach { + async fn addrs(&self, authority: &Authority) -> Result, String> { + if let Some(fresh) = self.cached(authority, REACH_TTL) { + return Ok(fresh); + } + match self.inner.addrs(authority).await { + Ok(addrs) => { + if let Ok(mut entries) = self.entries.lock() { + entries.insert( + authority.clone(), + CachedAddrs { + addrs: addrs.clone(), + learned: std::time::Instant::now(), + }, + ); + } + Ok(addrs) + } + // A failed lookup falls back to the last known answer rather than dropping the voice. + Err(why) => self.cached(authority, REACH_STALE_GRACE).ok_or(why), + } + } +} + +/// Partition `endpoints` into groups that are each ONE voice, dropping every endpoint that could +/// not be reached. +/// +/// Returned as indices into `endpoints` so a caller can try each member of a group in turn: a +/// group is a voice, and any of its endpoints may speak for it. +/// +/// Two endpoints join the same group when their resolved address sets INTERSECT. The relation is +/// made transitive on purpose — if A and B share an address and B and C share another, all three +/// are one machine's worth of evidence, and treating A and C as independent because they happen +/// not to overlap directly is how a three-way "quorum" becomes one host. +pub(crate) async fn independent_voices( + endpoints: &[ChainEndpoint], + reach: &dyn EndpointReach, +) -> Vec> { + // CONCURRENTLY, because this is on the content-serve request path and the lookups are + // independent of one another: resolving N endpoints one after another makes the cost of + // configuring another source a latency penalty on every read, which is a reason not to + // configure one. `join_all` keeps the results in endpoint order, so the grouping below stays + // deterministic — the merge is order-sensitive, and a set of voices that varied run to run + // would make the refusal rule itself nondeterministic. + let looked_up = futures::future::join_all( + endpoints + .iter() + .map(|endpoint| reach.addrs(&endpoint.authority)), + ) + .await; + let reachable: Vec<(usize, BTreeSet)> = looked_up + .into_iter() + .enumerate() + .filter_map(|(ix, addrs)| addrs.ok().map(|addrs| (ix, addrs))) + .collect(); + + let mut groups: Vec<(Vec, BTreeSet)> = Vec::new(); + for (ix, addrs) in reachable { + // Merge into EVERY group this endpoint touches, not merely the first — an endpoint that + // bridges two previously-disjoint groups makes them one voice, and stopping at the first + // match would leave the other counted as a second. + let mut members = vec![ix]; + let mut merged = addrs; + groups.retain(|(existing, existing_addrs)| { + if existing_addrs.is_disjoint(&merged) { + return true; + } + members.extend(existing.iter().copied()); + merged.extend(existing_addrs.iter().copied()); + false + }); + members.sort_unstable(); + groups.push((members, merged)); + } + + let mut voices: Vec> = groups.into_iter().map(|(members, _)| members).collect(); + voices.sort(); + voices +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + use std::sync::Arc; + + /// A reach table written by the test: authority -> the addresses it lands on, or absence for + /// "does not resolve". A map rather than a single-answer double, because every property here + /// is about how the answers for DIFFERENT authorities relate to each other, and a double that + /// can only vary one field cannot express a disagreement between two of them. + struct TableReach(BTreeMap>); + + impl TableReach { + fn new(rows: &[(&str, &[&str])]) -> Arc { + let mut table = BTreeMap::new(); + for (url, addrs) in rows { + let endpoint = ChainEndpoint::parse(url).expect("a parseable fixture url"); + table.insert( + endpoint.authority, + addrs.iter().map(|a| a.parse().expect("an ip")).collect(), + ); + } + Arc::new(Self(table)) + } + } + + #[async_trait::async_trait] + impl EndpointReach for TableReach { + async fn addrs(&self, authority: &Authority) -> Result, String> { + match self.0.get(authority) { + Some(addrs) => Ok(addrs.iter().copied().collect()), + // Rendered with the host and port kept APART, matching the production reach and + // the source-level ban -- an IPv6 literal has no valid bare form. + None => Err(format!( + "host {} port {} does not resolve", + authority.host, authority.port + )), + } + } + } + + fn endpoints(urls: &[&str]) -> Vec { + urls.iter() + .map(|u| ChainEndpoint::parse(u).expect("a parseable fixture url")) + .collect() + } + + #[test] + fn an_authority_is_the_host_and_the_port_a_client_would_dial() { + let parsed = ChainEndpoint::parse("https://API.Coinset.ORG/v1/?x=1").expect("parses"); + assert_eq!( + parsed.authority, + Authority { + host: "api.coinset.org".into(), + port: 443 + }, + "case, path and query are not part of what the network sees; folding them is what \ + makes two spellings of one host compare equal before any lookup happens" + ); + assert_eq!( + parsed.url, "https://API.Coinset.ORG/v1/?x=1", + "the URL is passed to the client UNCHANGED — an endpoint rewritten to its authority \ + would drop the path an operator deliberately configured" + ); + + assert_eq!( + ChainEndpoint::parse("http://[::1]:8555") + .expect("parses") + .authority, + Authority { + host: "::1".into(), + port: 8555 + }, + "a bracketed IPv6 literal keeps its colons and yields the explicit port (§5.2 makes \ + IPv6 the ordinary case here, not the exotic one)" + ); + assert_eq!( + ChainEndpoint::parse("http://example.org") + .expect("parses") + .authority + .port, + 80, + "an http URL with no port dials 80, so it is NOT the same authority as the https one" + ); + + for rejected in ["", "api.coinset.org", "ftp://api.coinset.org", "https://"] { + assert!( + ChainEndpoint::parse(rejected).is_none(), + "{rejected:?} names no host this node could dial over http(s), and admitting it \ + would let an unusable entry count toward a quorum" + ); + } + } + + /// Two endpoints that land on the same machine are ONE voice, however different they look. + /// + /// This is the PR#354 trap in its purest form. The fixture varies exactly one thing — the + /// second endpoint's address — between the two halves, so a grouping rule that keys off the + /// URL, the host name, or the source's type reports two voices in BOTH halves and this test + /// fails on the first. A control that keys off reach reports one and two respectively. + #[tokio::test] + async fn two_names_for_one_machine_are_one_voice_and_two_machines_are_two() { + let endpoints = endpoints(["https://a.example.org", "https://b.example.org"].as_slice()); + + let shared = TableReach::new(&[ + ("https://a.example.org", &["203.0.113.7"]), + ("https://b.example.org", &["203.0.113.7"]), + ]); + assert_eq!( + independent_voices(&endpoints, shared.as_ref()).await, + vec![vec![0, 1]], + "two hostnames resolving to ONE address are one voice — a CNAME must not manufacture \ + a second, which is precisely how a 2-of-2 rule was satisfied by one endpoint" + ); + + let distinct = TableReach::new(&[ + ("https://a.example.org", &["203.0.113.7"]), + ("https://b.example.org", &["198.51.100.9"]), + ]); + assert_eq!( + independent_voices(&endpoints, distinct.as_ref()).await, + vec![vec![0], vec![1]], + "the SAME two names on genuinely different machines are two voices — without this \ + control an implementation that always answers one voice passes the assertion above" + ); + } + + /// A partial overlap is still one voice, and the merge is transitive. + /// + /// A dual-stack host answers with several addresses (§5.2 makes IPv6 the ordinary case here), + /// so requiring set EQUALITY would call one machine two voices whenever a resolver returned + /// its addresses in different combinations. + /// + /// # The bridging endpoint is deliberately LAST, and that ordering is the whole test + /// + /// The property is that an endpoint joining two ALREADY-DISJOINT groups merges both. An + /// earlier version of this fixture listed the bridge in the middle, so at every step there was + /// only ever ONE existing group to consider and merge-into-all and merge-into-the-first agreed + /// on every input — measured: reverting the merge to stop at the first match left this test + /// GREEN. The fixture asserted a property the code has, on an input that could not exhibit it. + /// + /// With the bridge last, `a` and `c` are two separate voices by the time it arrives, and only + /// a rule that keeps merging collapses all three. + #[tokio::test] + async fn an_endpoint_bridging_two_disjoint_groups_merges_all_of_them() { + let endpoints = endpoints( + [ + "https://a.example.org", + "https://c.example.org", + "https://bridge.example.org", + ] + .as_slice(), + ); + let bridged = TableReach::new(&[ + ("https://a.example.org", &["203.0.113.7"]), + ("https://c.example.org", &["2001:db8::1"]), + ( + "https://bridge.example.org", + &["203.0.113.7", "2001:db8::1"], + ), + ]); + + assert_eq!( + independent_voices(&endpoints, bridged.as_ref()).await, + vec![vec![0, 1, 2]], + "a and c share no address directly and are two groups when the bridge arrives; a rule \ + that merged only into the FIRST match leaves c standing alone, so ONE machine \ + reachable under three names reports two independent voices and satisfies the quorum" + ); + + // The control: the same three endpoints with the bridge reaching a THIRD machine are + // three voices. Without it an implementation that collapsed everything into one group + // would satisfy the assertion above while destroying the property it is named for. + let unbridged = TableReach::new(&[ + ("https://a.example.org", &["203.0.113.7"]), + ("https://c.example.org", &["2001:db8::1"]), + ("https://bridge.example.org", &["192.0.2.5"]), + ]); + assert_eq!( + independent_voices(&endpoints, unbridged.as_ref()).await, + vec![vec![0], vec![1], vec![2]], + "three genuinely separate machines must stay three voices" + ); + } + + /// An endpoint that resolves to nothing is dropped, not counted. + /// + /// The control is the point: the same fixture with the second endpoint resolvable yields two + /// voices, so this cannot pass against an implementation that simply never counts anything. + #[tokio::test] + async fn an_unresolvable_endpoint_contributes_no_voice() { + let endpoints = endpoints(["https://a.example.org", "https://typo.example.org"].as_slice()); + + let one_missing = TableReach::new(&[("https://a.example.org", &["203.0.113.7"])]); + assert_eq!( + independent_voices(&endpoints, one_missing.as_ref()).await, + vec![vec![0]], + "an endpoint nothing can reach has agreed with nothing; counting it would let a typo \ + inflate the quorum to a size the network cannot actually supply" + ); + + let both_present = TableReach::new(&[ + ("https://a.example.org", &["203.0.113.7"]), + ("https://typo.example.org", &["198.51.100.9"]), + ]); + assert_eq!( + independent_voices(&endpoints, both_present.as_ref()).await, + vec![vec![0], vec![1]], + "with the second endpoint reachable the same input yields TWO voices — the control \ + that kills an implementation which drops every endpoint" + ); + } + + /// A reach that counts its calls and can be switched to failing, so a cache is observable. + /// + /// Cloned rather than shared behind a pointer so a test can hold a handle AND hand one to the + /// cache: the counters live behind their own `Arc`s, so every clone observes the same calls. + #[derive(Clone)] + struct CountingReach { + /// How many lookups actually reached this reach. + calls: Arc, + /// Whether lookups currently succeed. + answering: Arc, + } + + impl CountingReach { + fn new() -> Self { + Self { + calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + answering: Arc::new(std::sync::atomic::AtomicBool::new(true)), + } + } + fn calls(&self) -> usize { + self.calls.load(std::sync::atomic::Ordering::SeqCst) + } + fn stop_answering(&self) { + self.answering + .store(false, std::sync::atomic::Ordering::SeqCst); + } + } + + #[async_trait::async_trait] + impl EndpointReach for CountingReach { + async fn addrs(&self, _authority: &Authority) -> Result, String> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.answering.load(std::sync::atomic::Ordering::SeqCst) { + Ok(BTreeSet::from(["203.0.113.7".parse().expect("an ip")])) + } else { + Err("does not resolve".into()) + } + } + } + + /// The lookup is paid once per TTL, not once per read, and a blip does not drop the voice. + /// + /// Independence is recomputed on every content read, so an uncached reach makes name + /// resolution a per-READ `getaddrinfo` on a caller-drivable path — and a resolver failure then + /// denies a read the HTTP client would have served. The two halves below pin both. + #[tokio::test] + async fn a_cached_reach_looks_up_once_per_ttl_and_survives_a_resolver_blip() { + let inner = CountingReach::new(); + let cached = CachedReach::new(inner.clone()); + let authority = ChainEndpoint::parse("https://a.example.org") + .expect("parses") + .authority; + + let first = cached.addrs(&authority).await.expect("resolves"); + let second = cached.addrs(&authority).await.expect("resolves"); + assert_eq!( + first, second, + "the cached answer must be the answer that was learned, not an empty stand-in" + ); + assert_eq!( + inner.calls(), + 1, + "the second resolution must be served from the cache. A per-read lookup is a DNS gate \ + on the content-serve path, paid twice on a read that falls back from the tip to the \ + bounded pinned-root check" + ); + + // The resolver now fails. The endpoint has not moved, and the voice count must not change + // on that evidence. + inner.stop_answering(); + assert_eq!( + cached.addrs(&authority).await, + Ok(first), + "a lookup failure with a known-good answer in hand must serve that answer: failing \ + closed is right when the CHAIN ANSWER is in doubt, and wrong when only the NAME \ + lookup is" + ); + + // The control, and it is what stops the above passing against a reach that never fails: + // with nothing ever learned, a failing resolver is still a failure. + let cold = CachedReach::new(CountingReach::new()); + cold.inner.stop_answering(); + assert!( + cold.addrs(&authority).await.is_err(), + "an authority that has NEVER resolved has no last-known answer to fall back on, and \ + inventing one would let a typo contribute a voice" + ); + } +} diff --git a/crates/dig-node-core/src/seams/chia_peer/mod.rs b/crates/dig-node-core/src/seams/chia_peer/mod.rs index ec643c35..f4e932eb 100644 --- a/crates/dig-node-core/src/seams/chia_peer/mod.rs +++ b/crates/dig-node-core/src/seams/chia_peer/mod.rs @@ -24,6 +24,8 @@ //! existing pool rather than dialling a set of full nodes of its own. mod coinset_resolver; +mod corroborated_resolver; +mod endpoints; pub use coinset_resolver::CoinsetResolver; pub(crate) use coinset_resolver::{default_anchored_resolver, resolution_coinset}; diff --git a/crates/dig-node-core/src/seams/dig_peer/profile_sync.rs b/crates/dig-node-core/src/seams/dig_peer/profile_sync.rs index f9fc97c2..54080cdd 100644 --- a/crates/dig-node-core/src/seams/dig_peer/profile_sync.rs +++ b/crates/dig-node-core/src/seams/dig_peer/profile_sync.rs @@ -1227,10 +1227,34 @@ mod tests { } } + /// A fresh directory for one test. + /// + /// # Why the clock is in the name + /// + /// `(process id, unique_suffix())` is unique WITHIN a run and repeats ACROSS runs: the counter + /// restarts at zero every process, and the OS recycles process ids. These tests never remove + /// what they create, so a later run that draws a recycled pid inherits an earlier run's + /// directory — already populated — and a test asserting on the FULL contents of its own + /// temp tree fails on somebody else's leftovers. + /// + /// That is not hypothetical: `held_pairs_skips_names_this_module_did_not_write` failed once in + /// a full-suite run against 223 leaked `dig-profile-sync-test--` directories, passed + /// alone, and passed on a re-run — the signature of a name collision rather than a defect in + /// the code under test. A monotonic component makes the name unrepeatable across runs, which + /// is what the tests actually need from it. + /// + /// The LEAK itself is untouched here and is still real (dig-node#365 lane finding): the fix + /// for it is a drop guard per test, which is a larger change than this file's share of the + /// work. fn tempdir() -> PathBuf { + let since_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("a clock after 1970") + .as_nanos(); let dir = std::env::temp_dir().join(format!( - "dig-profile-sync-test-{}-{}", + "dig-profile-sync-test-{}-{}-{}", std::process::id(), + since_epoch, unique_suffix() )); std::fs::create_dir_all(&dir).expect("temp dir"); diff --git a/crates/dig-wallet/Cargo.toml b/crates/dig-wallet/Cargo.toml index 0fd120c1..cf739f6d 100644 --- a/crates/dig-wallet/Cargo.toml +++ b/crates/dig-wallet/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dig-wallet" -version = "0.37.1" +version = "0.37.2" edition = "2021" license = "GPL-2.0-only" description = "DIG Browser built-in Chia wallet sidecar: a local axum server (using digstore-chain + chia-wallet-sdk over coinset.org) that serves a Sage-mirroring wallet UI. Native Rust so BLS signing works; the browser opens it at 127.0.0.1." diff --git a/crates/dig-wallet/src/sage/sources.rs b/crates/dig-wallet/src/sage/sources.rs index c7915348..df5dfac8 100644 --- a/crates/dig-wallet/src/sage/sources.rs +++ b/crates/dig-wallet/src/sage/sources.rs @@ -412,8 +412,14 @@ mod independence_tests { mod sole_owner_tests { use std::path::{Path, PathBuf}; - /// The file that is allowed to construct the fabric. - const OWNER: &str = "sources.rs"; + /// The file that is allowed to construct the fabric, CRATE-QUALIFIED. + /// + /// The crate prefix is load-bearing now that the sweep walks more than one crate: file names + /// repeat across a workspace (both swept crates already contain a `chain.rs`), so an + /// unqualified `"sources.rs"` would accept a second fabric built in a file of that name in + /// ANY swept crate — the guard widening its haystack and losing its property in the same + /// change. + const OWNER: &str = "dig-wallet/sources.rs"; /// The call that constructs a chia peer fabric. const CONSTRUCTOR: &str = "ChiaQuery::new("; @@ -542,7 +548,24 @@ mod sole_owner_tests { /// Both drop a production `CONSTRUCTOR` and leave [`Swept::ended_inside_a_test_item`] `false`, /// so nothing reports them. Both are left unhandled DELIBERATELY: each would need literal or /// comment tracking, which is the brace-counting mistake in another costume, and a fifth round - /// of heuristic would buy less than an honest bound does. Neither occurs in this crate today. + /// of heuristic would buy less than an honest bound does. + /// + /// Re-measured across the WIDENED haystack when `dig-node-core` was added (dig-node#366), + /// because widening a heuristic's input widens its blind spots and a bound measured on the + /// old scope says nothing about the new one: + /// + /// * A column-0 `#[cfg(test)]` as string content: **absent from both crates.** Every + /// occurrence of that text inside a literal here is INDENTED, and indentation is what the + /// latch requires. + /// * A terminator carrying trailing content: **present once**, at `dig-node-core`'s + /// `lib.rs:195` (`const DEFAULT_CACHE_CAP: … ; // 1 GiB`). It is HARMLESS today for a + /// reason worth writing down rather than trusting: the shape only bites when the latch is + /// already set, and no column-0 `#[cfg(test)]` precedes it in that file. The same line + /// moved below one would go silent, and nothing would say so. + /// + /// No file in either crate ends inside a test item, so the fail-closed refusal is not + /// currently firing anywhere — which is the state that makes the two shapes above the whole + /// of the residual risk. /// /// * **A `#[cfg(test)]` at column 0 as STRING CONTENT** — inside a raw or multi-line literal — /// latches the sweep on text that is not code. It needs a source file that quotes Rust @@ -637,37 +660,67 @@ mod sole_owner_tests { } } - /// Every PRODUCTION call site of `CONSTRUCTOR` in this crate, as `file:line`. + /// The crate source trees this sweep walks, as `(label, path)`. /// - /// # Scope, stated so it is not overstated + /// `dig-node-core` is here because NC-12's clause is about the NODE, not about one crate + /// (dig-node#366). It holds no `CONSTRUCTOR` today — its `chia-peer` light client was removed + /// (`seams/chia_peer/mod.rs`) — and that is exactly why it needs sweeping: an unguarded crate + /// with a chain seam in it is where the next fabric gets built. /// - /// It walks `dig-wallet/src` ONLY. A fabric constructed in another crate — `dig-node-core`, a - /// bin target — is invisible to it. No such site exists today, and this guard is not what - /// proves that; what it proves is that within the crate owning the wallet's chain access, one - /// file builds the fabric. - fn production_call_sites() -> (Vec, Vec) { - let mut files = Vec::new(); - rust_files( - Path::new(env!("CARGO_MANIFEST_DIR")).join("src").as_path(), - &mut files, - ); - files.sort(); + /// Paths are relative to THIS crate's manifest, which is what makes them stable under a + /// worktree, a checkout at any location, and a `cargo test` run from any directory. + fn swept_roots() -> Vec<(String, PathBuf)> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("the crates directory") + .to_path_buf(); + ["dig-wallet", "dig-node-core"] + .into_iter() + .map(|crate_name| { + ( + crate_name.to_string(), + workspace.join(crate_name).join("src"), + ) + }) + .collect() + } + /// Every PRODUCTION call site of `CONSTRUCTOR` under `roots`, as `crate/file:line`, plus the + /// files the sweep could not finish classifying. + /// + /// # Scope, stated so it is not overstated + /// + /// It walks the `src` tree of `dig-wallet` and `dig-node-core`, and nothing else. A fabric + /// constructed in any OTHER crate of this workspace — `dig-node-service`, `dig-runtime`, + /// `dig-chat-protocol` — or in a `tests/`, `benches/` or `examples/` target of any crate, + /// INCLUDING the two it does walk, is invisible to it. So is one built by a dependency. + /// + /// What a green here means is therefore narrower than "the node holds one fabric": it is + /// "within the two crates that own the node's chain access, one file builds the fabric, and + /// no file in either was unreadable". That is the strongest claim a source sweep over a named + /// list of directories can make, and naming the list is what keeps it from reading as the + /// wider one. + fn production_call_sites_in(roots: &[(String, PathBuf)]) -> (Vec, Vec) { let mut sites = Vec::new(); let mut unread = Vec::new(); - for file in files { - let name = file - .file_name() - .expect("a file name") - .to_string_lossy() - .into_owned(); - let text = std::fs::read_to_string(&file).expect("read a source file"); - let swept = sweep(&text); - if swept.ended_inside_a_test_item { - unread.push(name.clone()); - } - for line_no in swept.sites { - sites.push(format!("{name}:{line_no}")); + for (label, root) in roots { + let mut files = Vec::new(); + rust_files(root, &mut files); + files.sort(); + for file in files { + let name = file + .file_name() + .expect("a file name") + .to_string_lossy() + .into_owned(); + let text = std::fs::read_to_string(&file).expect("read a source file"); + let swept = sweep(&text); + if swept.ended_inside_a_test_item { + unread.push(format!("{label}/{name}")); + } + for line_no in swept.sites { + sites.push(format!("{label}/{name}:{line_no}")); + } } } (sites, unread) @@ -927,32 +980,114 @@ mod sole_owner_tests { ); } - /// The haystack is real: the sweep can still SEE a call site. + /// The haystack is real: the sweep can still SEE a call site, and it reaches EVERY root. /// - /// Without this the sweep below passes just as well against a broken file walk, a renamed - /// constructor, or a source tree it never read — the failure mode where a guard reports clean - /// because it looked at nothing. + /// Without the first assertion the sweep passes just as well against a broken file walk, a + /// renamed constructor, or a source tree it never read — the failure mode where a guard + /// reports clean because it looked at nothing. + /// + /// The PER-ROOT assertion is what the widening to `dig-node-core` needs, and it is a + /// different property. `dig-node-core` contains NO `CONSTRUCTOR` today, so a total taken + /// across both roots stays positive from `dig-wallet` alone: a typo in the second path, or a + /// crate rename, would leave the new scope reading zero files while the guard still reported + /// a real haystack. Counting FILES per root separates "this crate is clean" from "this crate + /// was never opened", which is the same distinction [`Swept::ended_inside_a_test_item`] draws + /// within a file. + /// + /// Both misconfigurations were measured against the compiled walk, because they fail through + /// different mechanisms and only one of them is this assertion: + /// + /// * A root that does NOT EXIST panics inside `rust_files` (`read_dir(…).expect(…)`) before + /// this assertion is reached. Loud, and fine — but it is the panic doing the work, not the + /// count. + /// * A root that EXISTS and holds no Rust — a `src` renamed, a crate reorganised, a path that + /// silently stopped being the source tree — reaches this assertion, and it is what fires. + /// Verified by pointing both roots at an empty sibling directory: this test fails naming the + /// root it read zero files from. #[test] - fn the_sweep_can_find_a_construction_site_at_all() { - let mut files = Vec::new(); - rust_files( - Path::new(env!("CARGO_MANIFEST_DIR")).join("src").as_path(), - &mut files, - ); - let total: usize = files - .iter() - .map(|f| { - std::fs::read_to_string(f) - .expect("read a source file") - .matches(CONSTRUCTOR) - .count() - }) - .sum(); + fn the_sweep_reaches_every_root_and_can_find_a_construction_site_at_all() { + let roots = swept_roots(); + let mut total = 0usize; + for (label, root) in &roots { + let mut files = Vec::new(); + rust_files(root, &mut files); + assert!( + !files.is_empty(), + "the sweep read ZERO files under {label} ({}), so that crate contributes nothing \ + and its half of the guard is vacuous however green it looks", + root.display() + ); + total += files + .iter() + .map(|f| { + std::fs::read_to_string(f) + .expect("read a source file") + .matches(CONSTRUCTOR) + .count() + }) + .sum::(); + } assert!( total > 0, - "the sweep found NO `{CONSTRUCTOR}` anywhere in {} files — it is measuring nothing, \ - so the sole-owner assertion below would pass no matter what the code did", - files.len() + "the sweep found NO `{CONSTRUCTOR}` anywhere under {} roots — it is measuring \ + nothing, so the sole-owner assertion below would pass no matter what the code did", + roots.len() + ); + } + + /// A second fabric introduced in `dig-node-core` FAILS the sweep — demonstrated, not asserted. + /// + /// `dig-node-core` holds no `CONSTRUCTOR` today, so extending the scope to it proves nothing + /// on its own: the new root would report clean whether it was swept or skipped. This drives + /// the SAME classifier over a stand-in root whose only content is the violation, and checks + /// both halves of the verdict the real assertion acts on — that the site is SEEN, and that it + /// is judged a STRAY rather than accepted as the owner. + /// + /// The stray test is the half that would rot silently. `dig-node-core` could perfectly well + /// gain a `sources.rs` of its own, and against the unqualified `OWNER` this fixture's site + /// would then be accepted as the owner's — a second fabric passing the guard because it was + /// filed under a familiar name. Measured: reverting `OWNER` and the site format to their + /// unqualified single-crate forms fails THIS test and leaves every other test in the module + /// green, including [`only_the_registry_owner_constructs_a_peer_fabric`] — which is precisely + /// why the demonstration is needed rather than the real assertion alone. + #[test] + fn a_second_fabric_in_the_new_root_is_seen_and_judged_a_stray() { + let root = std::env::temp_dir().join(format!( + "dig-wallet-sole-owner-{}-{}", + std::process::id(), + line!() + )); + let nested = root.join("seams"); + std::fs::create_dir_all(&nested).expect("a temporary root"); + // Built from `CONSTRUCTOR` rather than written out, so this file never contains the + // needle it sweeps for; and named `sources.rs` because that is the name the unqualified + // owner check would wave through. + std::fs::write( + nested.join("sources.rs"), + format!("pub fn rogue() {{ let _ = {CONSTRUCTOR}cfg); }}\n"), + ) + .expect("write the fixture"); + + let roots = vec![("dig-node-core".to_string(), root.clone())]; + let (sites, unread) = production_call_sites_in(&roots); + std::fs::remove_dir_all(&root).expect("clean up the temporary root"); + + assert!( + unread.is_empty(), + "the fixture is ordinary production code and must classify cleanly: {unread:?}" + ); + assert_eq!( + sites, + vec!["dig-node-core/sources.rs:1".to_string()], + "the sweep must SEE a fabric built in the new root, CRATE-QUALIFIED — an unqualified \ + site string is what lets a second crate's `sources.rs` impersonate the owner" + ); + let strays: Vec<&String> = sites.iter().filter(|s| !s.starts_with(OWNER)).collect(); + assert_eq!( + strays.len(), + 1, + "and must judge it a STRAY: this is the assertion the real sweep makes, run against \ + a root that actually contains the violation" ); } @@ -970,7 +1105,7 @@ mod sole_owner_tests { /// second owner exists" and is the strongest one a source sweep can make. #[test] fn only_the_registry_owner_constructs_a_peer_fabric() { - let (sites, unread) = production_call_sites(); + let (sites, unread) = production_call_sites_in(&swept_roots()); assert!( unread.is_empty(), "the sweep never saw a `#[cfg(test)]` item END in these files, so everything below \