From f03a2d96f597661b2fdf49b403e8d11378831035 Mon Sep 17 00:00:00 2001 From: ael Date: Thu, 13 Aug 2026 22:11:17 +0200 Subject: [PATCH] feat: add secure Hypersnap HTTP failover --- .env.example | 20 + README.md | 29 +- apps/collector/src/adapter.test.ts | 4 +- apps/collector/src/cli.ts | 15 +- apps/collector/src/collector.test.ts | 369 ++++++++- apps/collector/src/collector.ts | 336 +++++++- apps/collector/src/config.test.ts | 70 ++ apps/collector/src/config.ts | 137 ++- apps/collector/src/database.test.ts | 178 +++- apps/collector/src/database.ts | 186 ++++- apps/collector/src/delivery.test.ts | 10 +- apps/collector/src/doctor.test.ts | 63 ++ apps/collector/src/doctor.ts | 95 ++- apps/collector/src/hypersnap-http-rpc.test.ts | 321 +++++++ apps/collector/src/hypersnap-http-rpc.ts | 782 ++++++++++++++++++ apps/collector/src/rpc.ts | 18 +- apps/dashboard/worker/index.worker.test.ts | 13 +- docs/architecture.md | 25 +- docs/data-sources.md | 42 + docs/local-reconstruction.md | 14 +- docs/security.md | 26 +- docs/troubleshooting.md | 16 + docs/windows-runbook.md | 32 +- packages/protocol/src/classifier.test.ts | 9 +- packages/protocol/src/classifier.ts | 1 + packages/protocol/src/rpc.test.ts | 82 ++ packages/protocol/src/rpc.ts | 139 +++- scripts/SnapMeter.Common.psm1 | 81 ++ scripts/bootstrap.ps1 | 18 +- scripts/check-health.ps1 | 38 +- vitest.config.ts | 4 +- 31 files changed, 3062 insertions(+), 111 deletions(-) create mode 100644 apps/collector/src/hypersnap-http-rpc.test.ts create mode 100644 apps/collector/src/hypersnap-http-rpc.ts diff --git a/.env.example b/.env.example index de4b192..b0cdc11 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,10 @@ SNAPCHAIN_GRPC_URL=127.0.0.1:3383 HYPERSNAP_GRPC_URL=127.0.0.1:4383 SNAPCHAIN_GRPC_TLS=false HYPERSNAP_GRPC_TLS=false +# Optional exact identity pins for the preferred local Hypersnap endpoint. Once +# first accepted, endpoint identity is also enrolled in the collector database. +HYPERSNAP_EXPECTED_PEER_ID= +HYPERSNAP_EXPECTED_VERSION= # Optional complete authorization metadata values for authenticated proxies. SNAPCHAIN_GRPC_AUTHORIZATION= HYPERSNAP_GRPC_AUTHORIZATION= @@ -13,10 +17,26 @@ HYPERSNAP_GRPC_API_KEY= # Minimum delay between GetEvents request starts. Use 250ms (4 RPS) for Neynar Starter. SNAPCHAIN_RPC_MIN_INTERVAL_MS=0 HYPERSNAP_RPC_MIN_INTERVAL_MS=0 +# Keep local/public Hypersnap probes bounded even if Snapchain needs a longer +# hosted-provider timeout. +HYPERSNAP_RPC_TIMEOUT_MS=5000 SNAPCHAIN_SOURCE_MODE=verified # Keep derived unless a dedicated verified Hyper-write source is implemented. HYPERSNAP_SOURCE_MODE=derived +# Ordered Hypersnap replica policy: prefer local gRPC, use the reviewed public +# HTTPS canonical-event API only while local is unavailable, then return local +# after consecutive healthy probes. The public peer/version are not secrets. +HYPERSNAP_FALLBACK_HTTP_URL=https://haatz.quilibrium.com +HYPERSNAP_FALLBACK_EXPECTED_PEER_ID=12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc +HYPERSNAP_FALLBACK_EXPECTED_VERSION=0.13.3 +HYPERSNAP_FALLBACK_POLL_INTERVAL_MS=5000 +HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS=1000 +HYPERSNAP_FAILOVER_AFTER_FAILURES=3 +HYPERSNAP_PREFERRED_RECOVERY_INTERVAL_MS=60000 +HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES=3 +HYPERSNAP_MAX_BLOCK_DELAY_SECONDS=30 + # Public Worker ingest endpoint and shared HMAC secret. SNAPMETER_INGEST_URL= SNAPMETER_INGEST_SECRET= diff --git a/README.md b/README.md index 2b6f9af..a02c841 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ The public origin, API, authenticated ingest, D1 persistence, WebSocket hydratio - **Snapchain** uses successful canonical `MERGE_MESSAGE` HubEvents, so its evidence mode is `verified`; a separate status becomes stale, degraded, partial, or disconnected when coverage, freshness, or reconciliation is unhealthy. - **Hypersnap** is currently reported as **Hypersnap observed active FIDs** with a visible `DERIVED` state. The value is inferred from successful canonical merges seen through the configured Hypersnap node whose message types are eligible for its Hyper shadow stores. Upstream exposes no per-write Hyper success stream, so SnapMeter does not claim those shadow writes were independently verified. +- **Hypersnap endpoint failover** keeps one canonical source active at a time: the preferred local gRPC node first, then an identity-pinned HTTPS canonical-event replica if local is unavailable, and back to local only after repeated healthy probes and cursor/fingerprint continuity checks. A fallback does not make the metric verified or complete. - **Unavailable, stale, degraded, and partial** states stay visible. Loading the website does not make a source live. See [Data sources](docs/data-sources.md), [upstream pins](docs/upstream-sources.md), and the [complete metric policy](docs/metrics.md). @@ -47,7 +48,7 @@ pnpm dev Open the URL printed by Vite with `?demo=1`, normally `http://127.0.0.1:5173/?demo=1`. Demo mode is deterministic, synthetic, clearly labelled, and does not contact a collector or require private data. -To connect the collector, copy the environment template only after the demo works, keep the resulting file untracked, and configure one or two private Snapchain-compatible gRPC endpoints: +To connect the collector, copy the environment template only after the demo works and keep the resulting file untracked. The template prefers private local gRPC endpoints and includes an identity-pinned public Hypersnap HTTPS fallback: ```powershell Copy-Item .env.example .env @@ -56,18 +57,18 @@ Copy-Item .env.example .env ./scripts/run-collector.ps1 -EnvFile .env -Mode run ``` -The convenience endpoint defaults are `127.0.0.1:3383` for Snapchain and `127.0.0.1:4383` for Hypersnap. Upstream Hypersnap also listens on internal port `3383`; `4383` is only the documented host remap when both nodes share one machine. +The convenience endpoint defaults are `127.0.0.1:3383` for Snapchain and `127.0.0.1:4383` for the preferred local Hypersnap node. Upstream Hypersnap also listens on internal port `3383`; `4383` is only the documented host remap when both nodes share one machine. The checked-in fallback points to the public node currently listed by the [official Hypersnap portal](https://hypersnap.org/). That listing proves neither node age nor historical uptime, and the exact peer/version pins deliberately fail closed when the operator changes the endpoint. The complete clean-room procedure, including local D1 migration, optional upstream-node checkouts, storage layout, validation, and the boundary around intentionally excluded private data, is in [Local reconstruction](docs/local-reconstruction.md). ## Architecture ```text -Snapchain HubService ----+ - +--> Windows collector --> SQLite + durable outbox -Hypersnap HubService ----+ | - | signed batches - v +Snapchain HubService -------------------+ + +--> Windows collector --> SQLite + durable outbox +Hypersnap local gRPC (preferred) -------+ | +Hypersnap HTTPS events (fallback only) -+ | signed batches + v React/Vite assets <-- Cloudflare Worker API <-- D1 + hibernating Durable Object ^ | +---------------- WebSocket live fan-out --------+ @@ -84,6 +85,16 @@ SNAPCHAIN_GRPC_URL=127.0.0.1:3383 HYPERSNAP_GRPC_URL=127.0.0.1:4383 SNAPCHAIN_GRPC_TLS=false HYPERSNAP_GRPC_TLS=false +HYPERSNAP_EXPECTED_PEER_ID= +HYPERSNAP_EXPECTED_VERSION= +HYPERSNAP_RPC_TIMEOUT_MS=5000 +HYPERSNAP_FALLBACK_HTTP_URL=https://haatz.quilibrium.com +HYPERSNAP_FALLBACK_EXPECTED_PEER_ID=12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc +HYPERSNAP_FALLBACK_EXPECTED_VERSION=0.13.3 +HYPERSNAP_FAILOVER_AFTER_FAILURES=3 +HYPERSNAP_PREFERRED_RECOVERY_INTERVAL_MS=60000 +HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES=3 +HYPERSNAP_MAX_BLOCK_DELAY_SECONDS=30 SNAPCHAIN_GRPC_API_KEY= SNAPCHAIN_RPC_MIN_INTERVAL_MS=0 SNAPMETER_INGEST_URL= @@ -93,7 +104,9 @@ SNAPMETER_DATA_DIR=C:\ProgramData\SnapMeter After deployment, set `SNAPMETER_INGEST_URL` to the smoke-tested origin plus `/api/v1/ingest/batch` and set the secret locally to the value stored with Wrangler. Never commit either value. -For a hosted Neynar Snapchain source, use `SNAPCHAIN_GRPC_URL=snapchain-grpc-api.neynar.com:443`, enable `SNAPCHAIN_GRPC_TLS=true`, place the Neynar credential in `SNAPCHAIN_GRPC_API_KEY`, and set `SNAPCHAIN_RPC_MIN_INTERVAL_MS=250` to pace shared two-shard replay below the Starter-plan request ceiling. Set `HYPERSNAP_SOURCE_MODE=unavailable` when no separate Hypersnap endpoint is connected. Keep the API key only in the ignored local environment file. +For a hosted Neynar Snapchain source, use `SNAPCHAIN_GRPC_URL=snapchain-grpc-api.neynar.com:443`, enable `SNAPCHAIN_GRPC_TLS=true`, place the Neynar credential in `SNAPCHAIN_GRPC_API_KEY`, and set `SNAPCHAIN_RPC_MIN_INTERVAL_MS=250` to pace shared two-shard replay below the Starter-plan request ceiling. Set `HYPERSNAP_SOURCE_MODE=unavailable` when neither a local node nor an accepted HTTPS fallback is connected. Keep the API key only in the ignored local environment file. + +The public Hypersnap fallback exposes canonical HubEvents over HTTPS and is still `derived`; it does not expose an independently verified Hyper-write stream. A live retention probe during implementation reached only about three days, so it cannot supply an exact 30-day cold start. Keep the dashboard partial until prospective coverage reaches the full window. See [Data sources](docs/data-sources.md) for the trust, enrollment, and switching rules. ```powershell ./scripts/run-collector.ps1 -EnvFile .env -Mode doctor # endpoints, shards, storage, clock, ingest auth, cursors, disk diff --git a/apps/collector/src/adapter.test.ts b/apps/collector/src/adapter.test.ts index 6b558e0..045da85 100644 --- a/apps/collector/src/adapter.test.ts +++ b/apps/collector/src/adapter.test.ts @@ -12,14 +12,14 @@ describe("source activity adapters", () => { shardIndex: 1, type: "HUB_EVENT_TYPE_MERGE_MESSAGE", timestamp, - mergeMessageBody: { message: { data: { type: "MESSAGE_TYPE_CAST_ADD", fid: "1", timestamp } } } + mergeMessageBody: { message: { data: { type: "MESSAGE_TYPE_CAST_ADD", fid: "1", timestamp, network: "FARCASTER_NETWORK_MAINNET" } } } }, Date.now(), false)?.action).toBe("cast"); expect(adapter.normalize({ id: "2", shardIndex: 1, type: "HUB_EVENT_TYPE_MERGE_MESSAGE", timestamp, - mergeMessageBody: { message: { data: { type: "MESSAGE_TYPE_CHANNEL_UPDATE", fid: "1", timestamp } } } + mergeMessageBody: { message: { data: { type: "MESSAGE_TYPE_CHANNEL_UPDATE", fid: "1", timestamp, network: 1 } } } }, Date.now(), false)).toBeNull(); }); diff --git a/apps/collector/src/cli.ts b/apps/collector/src/cli.ts index ea3018a..89940ec 100644 --- a/apps/collector/src/cli.ts +++ b/apps/collector/src/cli.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node import { statSync } from "node:fs"; -import { loadConfig } from "./config.js"; +import { loadConfig, type RpcEndpointConfig } from "./config.js"; import { CollectorRuntime } from "./collector.js"; import { CollectorDatabase } from "./database.js"; import { runDoctor } from "./doctor.js"; @@ -82,14 +82,23 @@ function assertSupportedNode(): void { } } -function endpointSummary(endpoint: { url: string; tls: boolean; sourceMode: string; authorization?: string; apiKey?: string; getEventsMinIntervalMs: number }): Record { +function endpointSummary(endpoint: RpcEndpointConfig): Record { return { url: endpoint.url, + transport: endpoint.transport, tls: endpoint.tls, sourceMode: endpoint.sourceMode, authorizationConfigured: Boolean(endpoint.authorization), apiKeyConfigured: Boolean(endpoint.apiKey), - getEventsMinIntervalMs: endpoint.getEventsMinIntervalMs + getEventsMinIntervalMs: endpoint.getEventsMinIntervalMs, + identityPinned: Boolean(endpoint.expectedPeerId || endpoint.expectedVersion), + fallback: endpoint.fallback ? { + transport: endpoint.fallback.transport, + tls: endpoint.fallback.tls, + identityPinned: Boolean(endpoint.fallback.expectedPeerId && endpoint.fallback.expectedVersion), + getEventsMinIntervalMs: endpoint.fallback.getEventsMinIntervalMs, + pollIntervalMs: endpoint.fallback.pollIntervalMs + } : null }; } diff --git a/apps/collector/src/collector.test.ts b/apps/collector/src/collector.test.ts index f6106f6..2dd15f7 100644 --- a/apps/collector/src/collector.test.ts +++ b/apps/collector/src/collector.test.ts @@ -1,14 +1,20 @@ import { describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; import type { IngestBatch } from "@snapmeter/contracts"; -import { FARCASTER_EPOCH_MS, type NodeInfo, type RawHubEvent } from "@snapmeter/protocol"; -import { CollectorRuntime, discoveredShardIds } from "./collector.js"; +import { FARCASTER_EPOCH_MS, rawHubEventFingerprint, type NodeInfo, type RawHubEvent } from "@snapmeter/protocol"; +import { CollectorRuntime, discoveredShardIds, validateCandidateInfo } from "./collector.js"; import { loadConfig } from "./config.js"; import { CollectorDatabase } from "./database.js"; import { createLogger } from "./logger.js"; import { rawEventShard, type CollectorRpc, type RpcFactory, type RpcSubscription } from "./rpc.js"; +const TEST_FARCASTER_SECONDS = Math.floor((Date.now() - FARCASTER_EPOCH_MS) / 1_000); + function mergeEvent(id: string, fid: string): RawHubEvent { - const seconds = Math.floor((Date.now() - FARCASTER_EPOCH_MS) / 1_000); + const seconds = TEST_FARCASTER_SECONDS; return { id, shardIndex: 7, @@ -21,20 +27,24 @@ function mergeEvent(id: string, fid: string): RawHubEvent { class FakeRpc implements CollectorRpc { readonly info: NodeInfo = { version: "fake-snapchain/1", + peerId: "12D3KooWTestPrimaryPeerIdentity", numShards: 99, shardInfos: [{ shardId: 7, maxHeight: 100, blockDelay: 0, mempoolSize: 0xffff_ffff }] }; readonly subscriptions: Array<{ emit(event: RawHubEvent): void; end(): void }> = []; readonly subscribeFromIds: Array = []; subscribeCalls = 0; + cancelCalls = 0; + getEventCalls = 0; getEventsCalls: Array<{ shard: number; startId: string; stopId?: string }> = []; async getInfo(): Promise { return this.info; } - async getEvent(): Promise { - return mergeEvent("1", "1"); + async getEvent(_shard: number, id: string): Promise { + this.getEventCalls += 1; + return mergeEvent(id, id); } async getEvents(shard: number, startId: string, _token?: Uint8Array, stopId?: string) { @@ -51,7 +61,14 @@ class FakeRpc implements CollectorRpc { this.subscriptions.push(control); // Simulates a subscription event racing with initial historical replay. onEvent(mergeEvent("3", "3")); - return { cancel: control.end, ready: Promise.resolve(), done }; + return { + cancel: () => { + this.cancelCalls += 1; + control.end(); + }, + ready: Promise.resolve(), + done + }; } close(): void {} @@ -79,7 +96,14 @@ class LiveOnlyRpc extends FakeRpc { const done = new Promise((resolve) => { finish = resolve; }); const control = { emit: onEvent, end: () => finish?.() }; this.subscriptions.push(control); - return { cancel: control.end, ready: Promise.resolve(), done }; + return { + cancel: () => { + this.cancelCalls += 1; + control.end(); + }, + ready: Promise.resolve(), + done + }; } } @@ -243,6 +267,48 @@ class PendingDiscoveryRpc implements CollectorRpc { } } +class ToggleRpc extends LiveOnlyRpc { + available = true; + closed = false; + getInfoCalls = 0; + + override async getInfo(): Promise { + this.getInfoCalls += 1; + if (!this.available) throw new Error("endpoint unavailable"); + return this.info; + } + + override close(): void { + this.closed = true; + } +} + +class FailAfterFirstInfoRpc extends ToggleRpc { + override async getInfo(): Promise { + this.getInfoCalls += 1; + if (this.getInfoCalls > 1) throw new Error("active endpoint failed"); + return this.info; + } +} + +class DriftAfterOpenRpc extends ToggleRpc { + override async getInfo(): Promise { + this.getInfoCalls += 1; + if (this.getInfoCalls === 1) return this.info; + return { + ...this.info, + peerId: "12D3KooWChangedMidSessionPeer", + shardInfos: [{ shardId: 8, maxHeight: 100, blockDelay: 0, mempoolSize: 0 }] + }; + } +} + +class PersistentReplayFailureRpc extends ToggleRpc { + override async getEvents(): Promise<{ events: RawHubEvent[] }> { + throw new Error("persistent historical replay failure"); + } +} + describe("collector runtime integration", () => { it("discovers only explicit positive data shards and normalizes event shard zero to its subscription", () => { expect(discoveredShardIds({ @@ -260,6 +326,295 @@ describe("collector runtime integration", () => { expect(() => rawEventShard({}, 0)).toThrow(/positive data shard/); }); + it("requires an exact, duplicate-free Hypersnap data-shard topology", () => { + const transport = loadConfig({ SNAPCHAIN_SOURCE_MODE: "unavailable" }).endpoints.hypersnap; + const shard = (shardId: number) => ({ shardId, maxHeight: 100, blockDelay: 0, mempoolSize: 0 }); + const valid = { version: "0.13.4", peerId: "12D3KooWPeer", numShards: 2, shardInfos: [shard(0), shard(1), shard(2)] }; + expect(() => validateCandidateInfo("hypersnap", transport, valid, 30)).not.toThrow(); + expect(() => validateCandidateInfo("hypersnap", transport, { ...valid, shardInfos: [shard(0), shard(1)] }, 30)) + .toThrow(/topology/); + expect(() => validateCandidateInfo("hypersnap", transport, { ...valid, shardInfos: [shard(0), shard(1), shard(2), shard(3)] }, 30)) + .toThrow(/topology/); + expect(() => validateCandidateInfo("hypersnap", transport, { ...valid, shardInfos: [shard(0), shard(1), shard(1), shard(2)] }, 30)) + .toThrow(/topology/); + }); + + it("activates the pinned HTTP fallback while local Hypersnap is unavailable", async () => { + const database = new CollectorDatabase(":memory:"); + const primary = new ToggleRpc(); + primary.available = false; + primary.info.numShards = 1; + const fallback = new ToggleRpc(); + fallback.info.numShards = 1; + fallback.info.peerId = "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc"; + const config = loadConfig({ + SNAPCHAIN_SOURCE_MODE: "unavailable", + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: fallback.info.peerId, + HYPERSNAP_FALLBACK_EXPECTED_VERSION: fallback.info.version + }); + config.discoveryIntervalMs = 20; + config.pulseIntervalMs = 10; + const runtime = new CollectorRuntime({ + config, + database, + rpcFactory: (candidate) => candidate.url.startsWith("https://") ? fallback : primary, + logger: createLogger({ write() {} }), + random: () => 0 + }); + const running = runtime.run(); + try { + await waitFor(() => fallback.subscribeCalls > 0); + expect(primary.closed).toBe(true); + expect(database.sourceHealth("hypersnap")[0]).toMatchObject({ + sourceMode: "derived", + node: { coveredShards: 1 } + }); + } finally { + runtime.stop(); + await running; + database.close(); + } + }); + + it.each(["blank", "mismatch"] as const)("rejects a fallback with a %s durable cursor fingerprint", async (state) => { + const directory = mkdtempSync(join(tmpdir(), `snapmeter-fallback-${state}-test-`)); + const path = join(directory, "state.sqlite3"); + let database = new CollectorDatabase(path); + const stored = mergeEvent("1", "99"); + database.recordEvent({ + source: "hypersnap", + shard: 7, + eventId: "1", + eventType: "HUB_EVENT_TYPE_MERGE_MESSAGE", + eventFingerprint: rawHubEventFingerprint(stored, 7), + receivedAtMs: Date.now(), + activity: null + }); + database.checkpointCursor("hypersnap", 7, "1", Date.now()); + if (state === "blank") { + database.close(); + const legacy = new DatabaseSync(path); + legacy.prepare("UPDATE event_dedupe SET event_fingerprint = '' WHERE source = 'hypersnap'").run(); + legacy.close(); + database = new CollectorDatabase(path); + } + const primary = new ToggleRpc(); + primary.available = false; + primary.info.numShards = 1; + const fallback = new ToggleRpc(); + fallback.info.numShards = 1; + fallback.info.peerId = "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc"; + const config = loadConfig({ + SNAPCHAIN_SOURCE_MODE: "unavailable", + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: fallback.info.peerId, + HYPERSNAP_FALLBACK_EXPECTED_VERSION: fallback.info.version + }); + config.discoveryIntervalMs = 10; + config.pulseIntervalMs = 10; + const runtime = new CollectorRuntime({ + config, + database, + rpcFactory: (candidate) => candidate.url.startsWith("https://") ? fallback : primary, + logger: createLogger({ write() {} }), + random: () => 0 + }); + const running = runtime.run(); + try { + await waitFor(() => fallback.getInfoCalls > 0); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(fallback.subscribeCalls).toBe(0); + expect(fallback.getEventCalls).toBeGreaterThan(0); + } finally { + runtime.stop(); + await running; + database.close(); + const resolved = resolve(directory); + if (resolved.startsWith(resolve(tmpdir()))) rmSync(resolved, { recursive: true, force: true }); + } + }); + + it("switches the complete source session to fallback after repeated active-endpoint failure", async () => { + const database = new CollectorDatabase(":memory:"); + const primary = new FailAfterFirstInfoRpc(); + primary.info.numShards = 1; + const fallback = new ToggleRpc(); + fallback.info.numShards = 1; + fallback.info.peerId = "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc"; + const config = loadConfig({ + SNAPCHAIN_SOURCE_MODE: "unavailable", + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: fallback.info.peerId, + HYPERSNAP_FALLBACK_EXPECTED_VERSION: fallback.info.version, + HYPERSNAP_FAILOVER_AFTER_FAILURES: "1" + }); + config.discoveryIntervalMs = 20; + config.pulseIntervalMs = 10; + const cursorEvent = mergeEvent("1", "1"); + database.recordEvent({ + source: "hypersnap", + shard: 7, + eventId: "1", + eventType: "HUB_EVENT_TYPE_MERGE_MESSAGE", + eventFingerprint: rawHubEventFingerprint(cursorEvent, 7), + receivedAtMs: Date.now(), + activity: null + }); + database.checkpointCursor("hypersnap", 7, "1", Date.now()); + const runtime = new CollectorRuntime({ + config, + database, + rpcFactory: (candidate) => candidate.url.startsWith("https://") ? fallback : primary, + logger: createLogger({ write() {} }), + random: () => 0 + }); + const running = runtime.run(); + try { + await waitFor(() => fallback.subscribeCalls > 0); + expect(primary.closed).toBe(true); + expect(primary.subscriptions.length).toBe(1); + expect(primary.cancelCalls).toBeGreaterThan(0); + expect(fallback.getEventsCalls.some((call) => call.startId === "1")).toBe(true); + expect(database.eventFingerprint("hypersnap", 7, "1")).toBe(rawHubEventFingerprint(cursorEvent, 7)); + } finally { + runtime.stop(); + await running; + database.close(); + } + }); + + it("quarantines a mid-session identity and shard-set drift before activating new workers", async () => { + const database = new CollectorDatabase(":memory:"); + const primary = new DriftAfterOpenRpc(); + primary.info.numShards = 1; + const fallback = new ToggleRpc(); + fallback.info.numShards = 1; + fallback.info.peerId = "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc"; + const config = loadConfig({ + SNAPCHAIN_SOURCE_MODE: "unavailable", + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: fallback.info.peerId, + HYPERSNAP_FALLBACK_EXPECTED_VERSION: fallback.info.version, + HYPERSNAP_FAILOVER_AFTER_FAILURES: "3" + }); + config.discoveryIntervalMs = 10; + config.pulseIntervalMs = 10; + const runtime = new CollectorRuntime({ + config, + database, + rpcFactory: (candidate) => candidate.url.startsWith("https://") ? fallback : primary, + logger: createLogger({ write() {} }), + random: () => 0 + }); + const running = runtime.run(); + try { + await waitFor(() => fallback.subscribeCalls > 0); + expect(primary.getInfoCalls).toBe(2); + expect(primary.subscribeCalls).toBe(1); + expect(primary.cancelCalls).toBeGreaterThan(0); + expect(primary.closed).toBe(true); + } finally { + runtime.stop(); + await running; + database.close(); + } + }); + + it("switches the whole source when GetInfo stays healthy but shard replay repeatedly fails", async () => { + const database = new CollectorDatabase(":memory:"); + const primary = new PersistentReplayFailureRpc(); + primary.info.numShards = 1; + const fallback = new ToggleRpc(); + fallback.info.numShards = 1; + fallback.info.peerId = "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc"; + const config = loadConfig({ + SNAPCHAIN_SOURCE_MODE: "unavailable", + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: fallback.info.peerId, + HYPERSNAP_FALLBACK_EXPECTED_VERSION: fallback.info.version, + HYPERSNAP_FAILOVER_AFTER_FAILURES: "2" + }); + config.discoveryIntervalMs = 10; + config.pulseIntervalMs = 10; + const runtime = new CollectorRuntime({ + config, + database, + rpcFactory: (candidate) => candidate.url.startsWith("https://") ? fallback : primary, + logger: createLogger({ write() {} }), + random: () => 0 + }); + const running = runtime.run(); + try { + await waitFor(() => fallback.subscribeCalls > 0); + expect(primary.getInfoCalls).toBeGreaterThan(1); + expect(primary.subscribeCalls).toBeGreaterThanOrEqual(2); + expect(primary.cancelCalls).toBeGreaterThanOrEqual(2); + expect(primary.closed).toBe(true); + } finally { + runtime.stop(); + await running; + database.close(); + } + }); + + it("fails back to the preferred local endpoint only after stable compatible probes", async () => { + const database = new CollectorDatabase(":memory:"); + let primaryAvailable = false; + const primaryInstances: ToggleRpc[] = []; + const fallback = new ToggleRpc(); + fallback.info.numShards = 1; + fallback.info.peerId = "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc"; + const config = loadConfig({ + SNAPCHAIN_SOURCE_MODE: "unavailable", + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: fallback.info.peerId, + HYPERSNAP_FALLBACK_EXPECTED_VERSION: fallback.info.version, + HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES: "2" + }); + config.discoveryIntervalMs = 10; + config.endpoints.hypersnap.preferredRecoveryIntervalMs = 10; + config.pulseIntervalMs = 10; + const cursorEvent = mergeEvent("1", "1"); + database.recordEvent({ + source: "hypersnap", + shard: 7, + eventId: "1", + eventType: "HUB_EVENT_TYPE_MERGE_MESSAGE", + eventFingerprint: rawHubEventFingerprint(cursorEvent, 7), + receivedAtMs: Date.now(), + activity: null + }); + database.checkpointCursor("hypersnap", 7, "1", Date.now()); + const runtime = new CollectorRuntime({ + config, + database, + rpcFactory: (candidate) => { + if (candidate.url.startsWith("https://")) return fallback; + const rpc = new ToggleRpc(); + rpc.available = primaryAvailable; + rpc.info.numShards = 1; + primaryInstances.push(rpc); + return rpc; + }, + logger: createLogger({ write() {} }), + random: () => 0 + }); + const running = runtime.run(); + try { + await waitFor(() => fallback.subscribeCalls > 0); + primaryAvailable = true; + await waitFor(() => primaryInstances.some((rpc) => rpc.subscribeCalls > 0)); + expect(fallback.closed).toBe(true); + expect(fallback.cancelCalls).toBeGreaterThan(0); + expect(primaryInstances.filter((rpc) => rpc.getInfoCalls > 0).length).toBeGreaterThanOrEqual(3); + } finally { + runtime.stop(); + await running; + database.close(); + } + }); + it("discovers a nonzero shard, catches up transactionally, and later advances after fixed-bound reconciliation", async () => { const database = new CollectorDatabase(":memory:"); const rpc = new FakeRpc(); diff --git a/apps/collector/src/collector.ts b/apps/collector/src/collector.ts index d2a1297..b89dec8 100644 --- a/apps/collector/src/collector.ts +++ b/apps/collector/src/collector.ts @@ -1,8 +1,8 @@ import type { NodeInfo, RawHubEvent } from "@snapmeter/protocol"; -import { FARCASTER_EPOCH_MS } from "@snapmeter/protocol"; +import { FARCASTER_EPOCH_MS, rawHubEventFingerprint } from "@snapmeter/protocol"; import { shouldEmitPulse } from "@snapmeter/metrics"; import type { Source, SourceMode, SourceStatus } from "@snapmeter/contracts"; -import type { CollectorConfig } from "./config.js"; +import type { CollectorConfig, RpcEndpointConfig, RpcTransportConfig } from "./config.js"; import { defaultActivityAdapterFactory, type ActivityAdapterFactory, type SourceActivityAdapter } from "./adapter.js"; import { CollectorDatabase, compareEventIds, maxEventId, type SourceHealthRecord } from "./database.js"; import { @@ -35,6 +35,7 @@ interface SourceRuntimeState { synchronized: boolean; lastContactAtMs: number | null; lastError: string | null; + activeEndpointRole: EndpointRole | null; } interface ShardWorker { @@ -42,6 +43,15 @@ interface ShardWorker { promise: Promise; } +type EndpointRole = "primary" | "fallback"; + +interface OpenEndpoint { + role: EndpointRole; + transport: RpcTransportConfig; + rpc: CollectorRpc; + info: NodeInfo; +} + export interface CollectorRuntimeOptions { config: CollectorConfig; database: CollectorDatabase; @@ -131,9 +141,12 @@ export class CollectorRuntime { for (const source of sources()) { const endpoint = this.#config.endpoints[source]; if (endpoint.sourceMode === "unavailable") continue; - const rpc = this.#rpcFactory(endpoint); + const opened = await this.#openFirstAvailable(source, endpoint); + const rpc = opened.rpc; try { - const info = await rpc.getInfo(); + this.#state(source).activeEndpointRole = opened.role; + this.#database.setActiveSourceEndpoint(source, opened.role, this.#now()); + const info = opened.info; const shardIds = discoveredShardIds(info); if (shardIds.length === 0) throw new Error("node reported no shards"); for (const shard of shardIds) { @@ -156,6 +169,7 @@ export class CollectorRuntime { this.#publishHealth(source); } finally { rpc.close(); + this.#clients.delete(rpc); } } const snapshotAtMs = this.#now(); @@ -177,16 +191,96 @@ export class CollectorRuntime { await untilAborted(signal); return; } - const rpc = this.#rpcFactory(endpoint); - this.#clients.add(rpc); + let role: EndpointRole = "primary"; + let failures = 0; + while (!signal.aborted) { + const transport = endpointForRole(endpoint, role); + if (!transport) { + role = "primary"; + await abortableDelay(exponentialBackoffMs(failures++, { random: this.#random }), signal); + continue; + } + let opened: OpenEndpoint; + try { + opened = await this.#openEndpoint(source, role, transport, signal); + } catch (error) { + if (signal.aborted) break; + this.#markEndpointUnavailable(source); + this.#logger.warn("source.endpoint_rejected", { source, role, error }); + role = alternateRole(role, endpoint); + if (role === "primary") { + await abortableDelay(exponentialBackoffMs(failures++, { random: this.#random }), signal); + } + continue; + } + try { + this.#logger.info("source.endpoint_activated", { source, role, transport: transport.transport }); + this.#state(source).activeEndpointRole = role; + this.#database.setActiveSourceEndpoint(source, role, this.#now()); + failures = 0; + await this.#runSourceSession(source, endpoint, opened, signal); + return; + } catch (error) { + if (signal.aborted) break; + const preferredRecovered = error instanceof PreferredEndpointReady; + this.#logger.warn("source.endpoint_switching", { + source, + fromRole: role, + toRole: preferredRecovered ? "primary" : alternateRole(role, endpoint), + reason: preferredRecovered ? "preferred_recovered" : "active_endpoint_unavailable" + }); + role = preferredRecovered ? "primary" : alternateRole(role, endpoint); + } finally { + opened.rpc.close(); + this.#clients.delete(opened.rpc); + this.#resetSessionState(source); + } + } + } + + async #runSourceSession( + source: Source, + endpoint: RpcEndpointConfig, + opened: OpenEndpoint, + signal: AbortSignal + ): Promise { + const rpc = opened.rpc; const workers = new Map(); + const shardFailureCounts = new Map(); + let resolveDataPlaneFailure: ((error: ActiveEndpointUnavailable) => void) | undefined; + const dataPlaneFailure = new Promise((resolve) => { + resolveDataPlaneFailure = resolve; + }); + let dataPlaneFailed = false; + const onShardFailure = (shard: number): void => { + const failures = (shardFailureCounts.get(shard) ?? 0) + 1; + shardFailureCounts.set(shard, failures); + if (!dataPlaneFailed && failures >= endpoint.failoverAfterFailures) { + dataPlaneFailed = true; + resolveDataPlaneFailure?.(new ActiveEndpointUnavailable("active endpoint failed repeated shard data-plane operations")); + } + }; + const onShardHealthy = (shard: number): void => { + shardFailureCounts.set(shard, 0); + }; let discoveryFailures = 0; + let info: NodeInfo | null = opened.info; + let noCoverageSinceMs: number | null = null; + let nextPreferredProbeAtMs = this.#now() + endpoint.preferredRecoveryIntervalMs; + let preferredRecoverySuccesses = 0; try { while (!signal.aborted) { try { - const info = await rpc.getInfo(signal); - this.#updateInfo(source, info); - const shardIds = discoveredShardIds(info); + const currentInfo = info ?? await rpc.getInfo(signal); + info = null; + try { + validateCandidateInfo(source, opened.transport, currentInfo, endpoint.maximumBlockDelaySeconds); + this.#validateEndpointEnrollment(source, opened.role, opened.transport, currentInfo); + } catch (error) { + throw new ActiveEndpointUnavailable("active endpoint identity or topology changed", { cause: error }); + } + this.#updateInfo(source, currentInfo); + const shardIds = discoveredShardIds(currentInfo); if (shardIds.length === 0) throw new Error("node reported zero discoverable shards"); const desired = new Set(shardIds); for (const [shard, worker] of workers) { @@ -201,22 +295,54 @@ export class CollectorRuntime { for (const shard of shardIds) { if (workers.has(shard)) continue; const controller = linkedAbortController(signal); - const promise = this.#runShard(source, shard, rpc, controller.signal) + const promise = this.#runShard(source, shard, rpc, controller.signal, onShardFailure, onShardHealthy) .catch((error) => this.#logger.error("source.shard_worker_failed", { source, shard, error })); workers.set(shard, { controller, promise }); this.#logger.info("source.shard_discovered", { source, shard }); } discoveryFailures = 0; - this.#state(source).lastContactAtMs = this.#now(); + const state = this.#state(source); + state.lastContactAtMs = this.#now(); this.#publishHealth(source); - await abortableDelay(this.#config.discoveryIntervalMs, signal); + + if (state.connectedShards.size >= state.expectedShardCount) noCoverageSinceMs = null; + else if (noCoverageSinceMs === null) noCoverageSinceMs = this.#now(); + else if (this.#now() - noCoverageSinceMs >= this.#config.disconnectedAfterMs) { + throw new ActiveEndpointUnavailable("active endpoint lacks complete connected data-shard coverage"); + } + + if (opened.role === "fallback" && endpoint.fallback && this.#now() >= nextPreferredProbeAtMs) { + try { + await this.#probeAndClose(source, "primary", endpoint, signal); + preferredRecoverySuccesses += 1; + if (preferredRecoverySuccesses >= endpoint.preferredRecoverySuccesses) { + throw new PreferredEndpointReady(); + } + } catch (error) { + if (error instanceof PreferredEndpointReady) throw error; + if (signal.aborted) break; + preferredRecoverySuccesses = 0; + this.#logger.debug("source.preferred_probe_failed", { source }); + } finally { + nextPreferredProbeAtMs = this.#now() + endpoint.preferredRecoveryIntervalMs; + } + } + const dataPlaneError = await Promise.race([ + abortableDelay(this.#config.discoveryIntervalMs, signal).then(() => null), + dataPlaneFailure + ]); + if (dataPlaneError !== null) throw dataPlaneError; } catch (error) { if (signal.aborted) break; + if (error instanceof PreferredEndpointReady || error instanceof ActiveEndpointUnavailable) throw error; discoveryFailures += 1; const state = this.#state(source); state.lastError = errorMessage(error); state.synchronized = false; this.#publishHealth(source, "rpc_unavailable"); + if (discoveryFailures >= endpoint.failoverAfterFailures) { + throw new ActiveEndpointUnavailable("active endpoint failed repeated discovery probes"); + } const delayMs = exponentialBackoffMs(discoveryFailures - 1, { random: this.#random }); this.#logger.warn("source.discovery_failed", { source, attempt: discoveryFailures, delayMs, error }); await abortableDelay(delayMs, signal); @@ -225,12 +351,141 @@ export class CollectorRuntime { } finally { for (const worker of workers.values()) worker.controller.abort(); await Promise.all([...workers.values()].map((worker) => worker.promise)); + } + } + + async #openFirstAvailable(source: Source, endpoint: RpcEndpointConfig): Promise { + let lastError: unknown; + for (const role of ["primary", "fallback"] as const) { + const transport = endpointForRole(endpoint, role); + if (!transport) continue; + try { + return await this.#openEndpoint(source, role, transport); + } catch (error) { + lastError = error; + } + } + throw lastError instanceof Error ? lastError : new Error(`no compatible ${source} endpoint is available`); + } + + async #openEndpoint( + source: Source, + role: EndpointRole, + transport: RpcTransportConfig, + signal?: AbortSignal + ): Promise { + const rpc = this.#rpcFactory(transport); + this.#clients.add(rpc); + try { + const info = await rpc.getInfo(signal); + validateCandidateInfo(source, transport, info, this.#config.endpoints[source].maximumBlockDelaySeconds); + if (source === "hypersnap") { + await this.#verifyCursorContinuity(source, role, rpc, info, signal); + this.#validateEndpointEnrollment(source, role, transport, info); + } + return { role, transport, rpc, info }; + } catch (error) { rpc.close(); this.#clients.delete(rpc); + throw error; + } + } + + async #probeAndClose( + source: Source, + role: EndpointRole, + endpoint: RpcEndpointConfig, + signal: AbortSignal + ): Promise { + const transport = endpointForRole(endpoint, role); + if (!transport) throw new Error(`${role} endpoint is not configured`); + const opened = await this.#openEndpoint(source, role, transport, signal); + opened.rpc.close(); + this.#clients.delete(opened.rpc); + } + + #validateEndpointEnrollment( + source: Source, + role: EndpointRole, + transport: RpcTransportConfig, + info: NodeInfo + ): void { + if (source !== "hypersnap") return; + if (!info.peerId) throw new Error("Hypersnap endpoint did not expose a peer identity"); + this.#database.validateOrEnrollSourceEndpoint({ + source, + role, + transport: transport.transport, + canonicalUrl: canonicalTransportUrl(transport), + peerId: info.peerId, + version: info.version, + shardIds: discoveredShardIds(info) + }, this.#now()); + } + + async #verifyCursorContinuity( + source: Source, + role: EndpointRole, + rpc: CollectorRpc, + info: NodeInfo, + signal?: AbortSignal + ): Promise { + const shards = discoveredShardIds(info); + const advertised = new Set(shards); + for (const cursor of this.#database.getCursors().filter((item) => item.source === source)) { + if (!advertised.has(cursor.shard)) throw new Error("candidate endpoint is missing a durable cursor shard"); + } + for (const shard of shards) { + const cursor = this.#database.getCursor(source, shard); + if (cursor === "0") continue; + const event = await rpc.getEvent(shard, cursor, signal); + if (rawEventId(event) !== cursor || rawEventShard(event, shard) !== shard) { + throw new Error("candidate endpoint failed durable cursor continuity verification"); + } + const knownFingerprint = this.#database.eventFingerprint(source, shard, cursor); + if (knownFingerprint === null) { + if (role === "fallback") { + throw new Error("fallback endpoint cannot establish an unbound legacy cursor fingerprint"); + } + } else if (rawHubEventFingerprint(event, shard) !== knownFingerprint) { + throw new Error("candidate endpoint event fingerprint conflicted with the durable cursor"); + } } } - async #runShard(source: Source, shard: number, rpc: CollectorRpc, signal: AbortSignal): Promise { + #markEndpointUnavailable(source: Source): void { + const state = this.#state(source); + state.synchronized = false; + state.lastError = "endpoint_unavailable"; + this.#publishHealth(source, "rpc_unavailable"); + } + + #resetSessionState(source: Source): void { + const state = this.#state(source); + state.version = "unknown"; + state.shardIds.clear(); + state.expectedShardCount = 0; + state.connectedShards.clear(); + state.replayingShards.clear(); + state.height = null; + state.blockDelaySeconds = null; + state.mempoolSize = null; + state.synchronized = false; + state.lastContactAtMs = null; + state.reconciliationState = "gap"; + state.lastError = "endpoint_switching"; + state.activeEndpointRole = null; + this.#publishHealth(source, "endpoint_switching"); + } + + async #runShard( + source: Source, + shard: number, + rpc: CollectorRpc, + signal: AbortSignal, + onFailure: (shard: number) => void, + onHealthy: (shard: number) => void + ): Promise { const mode = this.#config.endpoints[source].sourceMode; let reconnectAttempt = 0; while (!signal.aborted) { @@ -293,6 +548,7 @@ export class CollectorRuntime { catchup = false; state.replayingShards.delete(shard); state.reconciliationState = "ok"; + onHealthy(shard); state.lastContactAtMs = this.#now(); this.#publishHealth(source); this.#logger.info("source.catchup_complete", { source, shard, cursor: catchupWatermark, events: initial.eventCount }); @@ -320,6 +576,7 @@ export class CollectorRuntime { state.connectedShards.delete(shard); state.replayingShards.delete(shard); state.reconnectCount += 1; + onFailure(shard); state.lastError = errorMessage(error); state.reconciliationState = "gap"; this.#publishHealth(source, "subscription_interrupted"); @@ -382,7 +639,6 @@ export class CollectorRuntime { if (!adapter || adapter.sourceMode !== mode) throw new Error(`missing ${source} activity adapter for ${mode} mode`); let activity = adapter.normalize(event, receivedAtMs, replay); const authoritativeAtMs = activity?.actionAtMs ?? eventTimeMs(event); - if (authoritativeAtMs !== null) this.#database.recordHistoryCoverage(source, shard, authoritativeAtMs); if (activity && activity.actionAtMs < cutoffMs) activity = null; if (!activity && authoritativeAtMs !== null && authoritativeAtMs < cutoffMs) return; const result = this.#database.recordEvent({ @@ -390,9 +646,11 @@ export class CollectorRuntime { shard, eventId, eventType: rawEventType(event), + eventFingerprint: rawHubEventFingerprint(event, shard), receivedAtMs, activity }); + if (authoritativeAtMs !== null) this.#database.recordHistoryCoverage(source, shard, authoritativeAtMs); if (result.actionInserted && activity && shouldEmitPulse(activity)) this.#pulse.add({ activity, eventId }); const state = this.#state(source); state.lastContactAtMs = receivedAtMs; @@ -490,7 +748,9 @@ export class CollectorRuntime { historyCoverageStartMs, historyComplete: completeHistory }, - message: message ?? (state.lastError ? "source_unavailable" : null) + message: message ?? (state.activeEndpointRole === "fallback" + ? "Public HTTPS fallback active; this remains a derived canonical-event view." + : state.lastError ? "source_unavailable" : null) }; this.#database.upsertSourceHealth(record); } @@ -535,7 +795,8 @@ function initialState(source: Source, sourceMode: SourceMode): SourceRuntimeStat mempoolSize: null, synchronized: false, lastContactAtMs: null, - lastError: null + lastError: null, + activeEndpointRole: null }; } @@ -597,3 +858,46 @@ function eventTimeMs(event: RawHubEvent): number | null { function historyComplete(startMs: number | null, nowMs: number): boolean { return startMs !== null && startMs <= nowMs - 30 * 86_400_000; } + +function endpointForRole(endpoint: RpcEndpointConfig, role: EndpointRole): RpcTransportConfig | undefined { + return role === "primary" ? endpoint : endpoint.fallback; +} + +function alternateRole(role: EndpointRole, endpoint: RpcEndpointConfig): EndpointRole { + return role === "primary" && endpoint.fallback ? "fallback" : "primary"; +} + +export function validateCandidateInfo( + source: Source, + transport: RpcTransportConfig, + info: NodeInfo, + maximumBlockDelaySeconds: number +): void { + if (transport.expectedPeerId && info.peerId !== transport.expectedPeerId) { + throw new Error("endpoint peer identity did not match the configured pin"); + } + if (transport.expectedVersion && info.version !== transport.expectedVersion) { + throw new Error("endpoint version did not match the reviewed version pin"); + } + const shards = discoveredShardIds(info); + const positiveDescriptors = info.shardInfos.filter((item) => Number.isSafeInteger(item.shardId) && item.shardId > 0); + if (!Number.isSafeInteger(info.numShards) || info.numShards <= 0 || shards.length === 0) { + throw new Error("endpoint did not expose positive data shards"); + } + if (source !== "hypersnap") return; + if (shards.length !== info.numShards || positiveDescriptors.length !== shards.length) { + throw new Error("endpoint positive data-shard topology did not exactly match its declared count"); + } + const shardInfo = info.shardInfos.filter((item) => shards.includes(item.shardId)); + if (shardInfo.some((item) => !Number.isSafeInteger(item.blockDelay) || item.blockDelay < 0 || item.blockDelay > maximumBlockDelaySeconds)) { + throw new Error("endpoint block delay exceeds the configured activation limit"); + } +} + +class ActiveEndpointUnavailable extends Error {} +class PreferredEndpointReady extends Error {} + +function canonicalTransportUrl(transport: RpcTransportConfig): string { + if (transport.transport === "https-json") return new URL(transport.url).toString(); + return transport.url.toLowerCase(); +} diff --git a/apps/collector/src/config.test.ts b/apps/collector/src/config.test.ts index 60d1107..693872f 100644 --- a/apps/collector/src/config.test.ts +++ b/apps/collector/src/config.test.ts @@ -17,6 +17,7 @@ describe("collector configuration", () => { expect(config.dataDir).toBe(join(localAppData, "SnapMeter")); expect(config.endpoints.snapchain).toMatchObject({ url: "snap.example:443", + transport: "grpc", tls: true, authorization: "Bearer private", apiKey: "neynar-private", @@ -24,9 +25,55 @@ describe("collector configuration", () => { sourceMode: "verified" }); expect(config.endpoints.hypersnap.sourceMode).toBe("derived"); + expect(config.endpoints.hypersnap.timeoutMs).toBe(5_000); expect(config.retentionDays).toBeGreaterThanOrEqual(31); }); + it("configures a peer-pinned HTTPS Hypersnap fallback independently", () => { + const config = loadConfig({ + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc", + HYPERSNAP_FALLBACK_EXPECTED_VERSION: "0.13.3", + HYPERSNAP_FALLBACK_POLL_INTERVAL_MS: "750", + HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS: "300", + HYPERSNAP_FAILOVER_AFTER_FAILURES: "2", + HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES: "4" + }); + expect(config.endpoints.hypersnap).toMatchObject({ + transport: "grpc", + failoverAfterFailures: 2, + preferredRecoverySuccesses: 4, + fallback: { + url: "https://public.example/", + transport: "https-json", + tls: true, + expectedPeerId: "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc", + expectedVersion: "0.13.3", + pollIntervalMs: 750, + getEventsMinIntervalMs: 300 + } + }); + }); + + it("fails closed for unpinned, plaintext, or malformed public fallbacks", () => { + expect(() => loadConfig({ + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example" + })).toThrow(/EXPECTED_PEER_ID is required/); + expect(() => loadConfig({ + HYPERSNAP_FALLBACK_HTTP_URL: "http://127.0.0.1:3381", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc", + HYPERSNAP_FALLBACK_EXPECTED_VERSION: "0.13.3" + })).toThrow(/must use HTTPS/); + expect(() => loadConfig({ + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc" + })).toThrow(/require HYPERSNAP_FALLBACK_HTTP_URL/); + expect(() => loadConfig({ + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: "not-a-peer!", + HYPERSNAP_FALLBACK_EXPECTED_VERSION: "0.13.3" + })).toThrow(/base58 peer identifier/); + }); + it("requires ingest URL and secret together", () => { expect(() => loadConfig({ SNAPMETER_INGEST_URL: "https://example.test/api/v1/ingest/batch" })).toThrow(/configured together/); expect(() => loadConfig({ @@ -71,6 +118,9 @@ describe("collector configuration", () => { }); it("never sends RPC credentials over a non-loopback plaintext endpoint", () => { + expect(() => loadConfig({ + SNAPCHAIN_GRPC_URL: "public.example:3383" + })).toThrow(/GRPC_TLS must be true/); expect(() => loadConfig({ SNAPCHAIN_GRPC_URL: "snap.example:3383", SNAPCHAIN_GRPC_API_KEY: "private-key" @@ -79,11 +129,31 @@ describe("collector configuration", () => { SNAPCHAIN_GRPC_URL: "127.0.0.2:3383", SNAPCHAIN_GRPC_API_KEY: "local-key" }).endpoints.snapchain.apiKey).toBe("local-key"); + expect(() => loadConfig({ + SNAPCHAIN_GRPC_URL: "10.0.0.2:3383", + SNAPCHAIN_GRPC_API_KEY: "private-key" + })).toThrow(/credentials are configured for a non-loopback endpoint/); + expect(() => loadConfig({ + SNAPCHAIN_GRPC_URL: "host.docker.internal:3383", + SNAPCHAIN_GRPC_AUTHORIZATION: "Bearer private" + })).toThrow(/credentials are configured for a non-loopback endpoint/); + expect(loadConfig({ SNAPCHAIN_GRPC_URL: "192.168.1.2:3383" }).endpoints.snapchain.tls).toBe(false); expect(() => loadConfig({ SNAPCHAIN_GRPC_AUTHORIZATION: "Bearer value\nInjected: value" })).toThrow(/control characters/); }); + it("accepts the checked-in Docker host-gateway endpoints as private plaintext transports", () => { + const config = loadConfig({ + SNAPCHAIN_GRPC_URL: "host.docker.internal:3383", + SNAPCHAIN_GRPC_TLS: "false", + HYPERSNAP_GRPC_URL: "host.docker.internal:4383", + HYPERSNAP_GRPC_TLS: "false" + }); + expect(config.endpoints.snapchain).toMatchObject({ url: "host.docker.internal:3383", tls: false }); + expect(config.endpoints.hypersnap).toMatchObject({ url: "host.docker.internal:4383", tls: false }); + }); + it("defaults RPC pacing to zero and rejects invalid minimum intervals", () => { expect(loadConfig({}).endpoints.snapchain.getEventsMinIntervalMs).toBe(0); expect(() => loadConfig({ SNAPCHAIN_RPC_MIN_INTERVAL_MS: "-1" })).toThrow(/SNAPCHAIN_RPC_MIN_INTERVAL_MS/); diff --git a/apps/collector/src/config.ts b/apps/collector/src/config.ts index 35094a1..0c0a9c6 100644 --- a/apps/collector/src/config.ts +++ b/apps/collector/src/config.ts @@ -2,14 +2,28 @@ import { homedir } from "node:os"; import { join, resolve } from "node:path"; import type { Source, SourceMode } from "@snapmeter/contracts"; -export interface RpcEndpointConfig { +export type RpcTransport = "grpc" | "https-json"; + +export interface RpcTransportConfig { url: string; + transport: RpcTransport; tls: boolean; authorization?: string; apiKey?: string; - sourceMode: SourceMode; timeoutMs: number; getEventsMinIntervalMs: number; + pollIntervalMs?: number; + expectedPeerId?: string; + expectedVersion?: string; +} + +export interface RpcEndpointConfig extends RpcTransportConfig { + sourceMode: SourceMode; + fallback?: RpcTransportConfig; + failoverAfterFailures: number; + preferredRecoveryIntervalMs: number; + preferredRecoverySuccesses: number; + maximumBlockDelaySeconds: number; } export interface CollectorConfig { @@ -83,6 +97,13 @@ function endpoint( defaultMode: SourceMode, timeoutMs: number ): RpcEndpointConfig { + const endpointTimeoutMs = integer( + env[`${prefix}_RPC_TIMEOUT_MS`], + prefix === "HYPERSNAP" ? Math.min(timeoutMs, 5_000) : timeoutMs, + 250, + 120_000, + `${prefix}_RPC_TIMEOUT_MS` + ); const url = clean(env[`${prefix}_GRPC_URL`]) ?? defaultUrl; const endpointMatch = /^(?:\[([^\]\s]+)\]|([^:\s]+)):(\d{1,5})$/.exec(url); const host = endpointMatch?.[1] ?? endpointMatch?.[2] ?? ""; @@ -107,26 +128,121 @@ function endpoint( const tls = booleanValue(env[`${prefix}_GRPC_TLS`], false, `${prefix}_GRPC_TLS`); const authorization = optionalCredential(env[`${prefix}_GRPC_AUTHORIZATION`], `${prefix}_GRPC_AUTHORIZATION`); const apiKey = optionalCredential(env[`${prefix}_GRPC_API_KEY`], `${prefix}_GRPC_API_KEY`); + const expectedPeerId = peerId(env[`${prefix}_EXPECTED_PEER_ID`], `${prefix}_EXPECTED_PEER_ID`); + const expectedVersion = optionalVersion(env[`${prefix}_EXPECTED_VERSION`], `${prefix}_EXPECTED_VERSION`); if (!tls && !isLoopbackHost(host) && (authorization !== undefined || apiKey !== undefined)) { throw new Error(`${prefix}_GRPC_TLS must be true when credentials are configured for a non-loopback endpoint`); } - return { + if (!tls && !isPrivateOrLoopbackHost(host)) { + throw new Error(`${prefix}_GRPC_TLS must be true for an endpoint outside a private or loopback network`); + } + const result: RpcEndpointConfig = { url, + transport: "grpc", tls, authorization, apiKey, + expectedPeerId, + expectedVersion, sourceMode, - timeoutMs, + timeoutMs: endpointTimeoutMs, getEventsMinIntervalMs: integer( env[`${prefix}_RPC_MIN_INTERVAL_MS`], 0, 0, 3_600_000, `${prefix}_RPC_MIN_INTERVAL_MS` - ) + ), + failoverAfterFailures: integer( + env[`${prefix}_FAILOVER_AFTER_FAILURES`], + 3, + 1, + 100, + `${prefix}_FAILOVER_AFTER_FAILURES` + ), + preferredRecoveryIntervalMs: integer( + env[`${prefix}_PREFERRED_RECOVERY_INTERVAL_MS`], + 60_000, + 5_000, + 3_600_000, + `${prefix}_PREFERRED_RECOVERY_INTERVAL_MS` + ), + preferredRecoverySuccesses: integer( + env[`${prefix}_PREFERRED_RECOVERY_SUCCESSES`], + 3, + 1, + 100, + `${prefix}_PREFERRED_RECOVERY_SUCCESSES` + ), + maximumBlockDelaySeconds: prefix === "HYPERSNAP" + ? integer( + env.HYPERSNAP_MAX_BLOCK_DELAY_SECONDS, + 30, + 0, + 86_400, + "HYPERSNAP_MAX_BLOCK_DELAY_SECONDS" + ) + : 30 + }; + if (prefix === "HYPERSNAP") result.fallback = hypersnapHttpFallback(env, endpointTimeoutMs); + return result; +} + +function hypersnapHttpFallback(env: EnvironmentLike, timeoutMs: number): RpcTransportConfig | undefined { + const url = optionalHttpUrl(env.HYPERSNAP_FALLBACK_HTTP_URL, "HYPERSNAP_FALLBACK_HTTP_URL"); + if (url === undefined) { + if (clean(env.HYPERSNAP_FALLBACK_EXPECTED_PEER_ID) !== undefined || clean(env.HYPERSNAP_FALLBACK_EXPECTED_VERSION) !== undefined) { + throw new Error("Hypersnap fallback identity pins require HYPERSNAP_FALLBACK_HTTP_URL"); + } + return undefined; + } + if (!url.startsWith("https://")) throw new Error("HYPERSNAP_FALLBACK_HTTP_URL must use HTTPS"); + const expectedPeerId = peerId(env.HYPERSNAP_FALLBACK_EXPECTED_PEER_ID, "HYPERSNAP_FALLBACK_EXPECTED_PEER_ID"); + if (!expectedPeerId) throw new Error("HYPERSNAP_FALLBACK_EXPECTED_PEER_ID is required for a public HTTP fallback"); + const expectedVersion = optionalVersion(env.HYPERSNAP_FALLBACK_EXPECTED_VERSION, "HYPERSNAP_FALLBACK_EXPECTED_VERSION"); + if (!expectedVersion) throw new Error("HYPERSNAP_FALLBACK_EXPECTED_VERSION is required for a public HTTP fallback"); + return { + url, + transport: "https-json", + tls: true, + timeoutMs, + getEventsMinIntervalMs: integer( + env.HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS, + 1_000, + 0, + 3_600_000, + "HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS" + ), + pollIntervalMs: integer( + env.HYPERSNAP_FALLBACK_POLL_INTERVAL_MS, + 5_000, + 250, + 60_000, + "HYPERSNAP_FALLBACK_POLL_INTERVAL_MS" + ), + expectedPeerId, + expectedVersion }; } +function peerId(value: string | undefined, name: string): string | undefined { + const normalized = clean(value); + if (normalized === undefined) return undefined; + if (normalized.length > 128 || !/^[1-9A-HJ-NP-Za-km-z]+$/.test(normalized)) { + throw new Error(`${name} must be a base58 peer identifier`); + } + return normalized; +} + +function optionalVersion(value: string | undefined, name: string): string | undefined { + const normalized = clean(value); + if (normalized === undefined) return undefined; + if (normalized.length > 64 || !/^[0-9A-Za-z][0-9A-Za-z._/+:-]*$/.test(normalized)) { + throw new Error(`${name} contains an invalid version identifier`); + } + return normalized; +} + function sourceModeValue(value: string | undefined, fallback: SourceMode, name: string): SourceMode { const normalized = clean(value)?.toLowerCase(); if (normalized === undefined) return fallback; @@ -168,6 +284,17 @@ function isLoopbackHost(value: string): boolean { return host.split(".").every((part) => Number(part) <= 255); } +function isPrivateOrLoopbackHost(value: string): boolean { + if (isLoopbackHost(value)) return true; + const host = value.replace(/^\[|\]$/g, "").toLowerCase(); + if (host === "host.docker.internal") return true; + const parts = host.split(".").map(Number); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; + return parts[0] === 10 + || (parts[0] === 172 && (parts[1] as number) >= 16 && (parts[1] as number) <= 31) + || (parts[0] === 192 && parts[1] === 168); +} + function hasControlCharacters(value: string): boolean { for (let index = 0; index < value.length; index += 1) { const code = value.charCodeAt(index); diff --git a/apps/collector/src/database.test.ts b/apps/collector/src/database.test.ts index 0118fb4..e2b3260 100644 --- a/apps/collector/src/database.test.ts +++ b/apps/collector/src/database.test.ts @@ -22,7 +22,15 @@ function action(source: "snapchain" | "hypersnap", fid: string, eventId: string, } function record(database: CollectorDatabase, activity: ActivityRecord, eventId: string, shard = 0) { - return database.recordEvent({ source: activity.source, shard, eventId, eventType: "MERGE_MESSAGE", receivedAtMs: activity.receivedAtMs, activity }); + return database.recordEvent({ + source: activity.source, + shard, + eventId, + eventType: "MERGE_MESSAGE", + eventFingerprint: "a".repeat(64), + receivedAtMs: activity.receivedAtMs, + activity + }); } describe("collector SQLite state", () => { @@ -41,6 +49,51 @@ describe("collector SQLite state", () => { } }); + it("fails closed when an endpoint supplies conflicting content for an existing event id", () => { + const database = new CollectorDatabase(":memory:"); + try { + record(database, action("hypersnap", "10", "100"), "100", 1); + expect(database.eventFingerprint("hypersnap", 1, "100")).toBe("a".repeat(64)); + expect(() => database.recordEvent({ + source: "hypersnap", + shard: 1, + eventId: "100", + eventType: "MERGE_MESSAGE", + eventFingerprint: "b".repeat(64), + receivedAtMs: NOW, + activity: null + })).toThrow(/fingerprint conflict/); + expect(database.status().dedupeEvents).toBe(1); + } finally { + database.close(); + } + }); + + it("durably pins endpoint identity and rejects silent peer, version, or topology changes", () => { + const database = new CollectorDatabase(":memory:"); + const enrollment = { + source: "hypersnap" as const, + role: "fallback" as const, + transport: "https-json" as const, + canonicalUrl: "https://public.example/", + peerId: "12D3KooWPeer", + version: "0.13.3", + shardIds: [2, 1] + }; + try { + database.validateOrEnrollSourceEndpoint(enrollment, NOW); + database.validateOrEnrollSourceEndpoint({ ...enrollment, shardIds: [1, 2] }, NOW + 1); + expect(() => database.validateOrEnrollSourceEndpoint({ ...enrollment, peerId: "12D3KooWChanged" }, NOW + 2)) + .toThrow(/identity changed/); + expect(() => database.validateOrEnrollSourceEndpoint({ ...enrollment, version: "0.13.4" }, NOW + 3)) + .toThrow(/identity changed/); + expect(() => database.validateOrEnrollSourceEndpoint({ ...enrollment, shardIds: [1] }, NOW + 4)) + .toThrow(/identity changed/); + } finally { + database.close(); + } + }); + it("never regresses a durable cursor under out-of-order input", () => { const database = new CollectorDatabase(":memory:"); try { @@ -148,7 +201,7 @@ describe("collector SQLite state", () => { const upgraded = new CollectorDatabase(path); try { - expect(upgraded.status().schemaVersion).toBe(3); + expect(upgraded.status().schemaVersion).toBe(4); expect(upgraded.loadActions("snapchain", 0).map((item) => item.fid)).toEqual(["77"]); expect(upgraded.getCursor("snapchain", 3)).toBe("700"); upgraded.recordHistoryCoverage("snapchain", 3, NOW - 31 * 86_400_000); @@ -162,6 +215,98 @@ describe("collector SQLite state", () => { } }); + it("migrates a genuine version-3 schema, preserves legacy rows, and reopens with durable enrollment", () => { + const directory = mkdtempSync(join(tmpdir(), "snapmeter-v3-migration-test-")); + const path = join(directory, "state.sqlite3"); + try { + const seeded = new CollectorDatabase(path); + record(seeded, action("hypersnap", "77", "700"), "700", 3); + seeded.checkpointCursor("hypersnap", 3, "700", NOW); + seeded.close(); + downgradeToVersion3(path); + + const upgraded = new CollectorDatabase(path); + expect(upgraded.status().schemaVersion).toBe(4); + expect(upgraded.eventFingerprint("hypersnap", 3, "700")).toBeNull(); + expect(upgraded.getCursor("hypersnap", 3)).toBe("700"); + expect(upgraded.loadActions("hypersnap", 0).map((item) => item.fid)).toEqual(["77"]); + expect(upgraded.recordEvent({ + source: "hypersnap", + shard: 3, + eventId: "700", + eventType: "MERGE_MESSAGE", + eventFingerprint: "b".repeat(64), + receivedAtMs: NOW, + activity: null + })).toMatchObject({ duplicate: true }); + const enrollment = { + source: "hypersnap" as const, + role: "fallback" as const, + transport: "https-json" as const, + canonicalUrl: "https://public.example/", + peerId: "12D3KooWPeer", + version: "0.13.3", + shardIds: [3] + }; + upgraded.validateOrEnrollSourceEndpoint(enrollment, NOW); + upgraded.close(); + + const reopened = new CollectorDatabase(path); + expect(reopened.eventFingerprint("hypersnap", 3, "700")).toBe("b".repeat(64)); + expect(reopened.checkSourceEndpointEnrollment(enrollment)).toBe("match"); + reopened.close(); + } finally { + const resolved = resolve(directory); + if (resolved.startsWith(resolve(tmpdir()))) rmSync(resolved, { recursive: true, force: true }); + } + }); + + it("rolls back every version-4 schema change when migration fails", () => { + const directory = mkdtempSync(join(tmpdir(), "snapmeter-v4-rollback-test-")); + const path = join(directory, "state.sqlite3"); + try { + const seeded = new CollectorDatabase(path); + seeded.close(); + downgradeToVersion3(path); + const blocked = new DatabaseSync(path); + blocked.exec(` + CREATE TRIGGER block_v4_migration + BEFORE INSERT ON schema_migrations + WHEN NEW.version = 4 + BEGIN + SELECT RAISE(ABORT, 'blocked version-4 migration'); + END; + `); + blocked.close(); + + expect(() => new CollectorDatabase(path)).toThrow(); + + const inspected = new DatabaseSync(path); + const columns = inspected.prepare("PRAGMA table_info(event_dedupe)").all() as Array<{ name: string }>; + const migration = inspected.prepare("SELECT COUNT(*) AS total FROM schema_migrations WHERE version = 4") + .get() as { total: number }; + const v4Objects = inspected.prepare(` + SELECT COUNT(*) AS total FROM sqlite_schema + WHERE type = 'table' AND name IN ('source_endpoint_enrollment', 'active_source_endpoint') + `).get() as { total: number }; + const metadata = inspected.prepare("SELECT value FROM collector_metadata WHERE key = 'schema_version'") + .get() as { value: string }; + expect(columns.some((column) => column.name === "event_fingerprint")).toBe(false); + expect(Number(migration.total)).toBe(0); + expect(Number(v4Objects.total)).toBe(0); + expect(metadata.value).toBe("3"); + inspected.exec("DROP TRIGGER block_v4_migration"); + inspected.close(); + + const recovered = new CollectorDatabase(path); + expect(recovered.status().schemaVersion).toBe(4); + recovered.close(); + } finally { + const resolved = resolve(directory); + if (resolved.startsWith(resolve(tmpdir()))) rmSync(resolved, { recursive: true, force: true }); + } + }); + it("transactionally provisions a 32-byte local pseudonym key when upgrading version 2", () => { const directory = mkdtempSync(join(tmpdir(), "snapmeter-key-migration-test-")); const path = join(directory, "state.sqlite3"); @@ -178,7 +323,7 @@ describe("collector SQLite state", () => { legacy.close(); const upgraded = new CollectorDatabase(path); - expect(upgraded.status().schemaVersion).toBe(3); + expect(upgraded.status().schemaVersion).toBe(4); expect(upgraded.actorDayPseudonym("snapchain", "2026-08-13", "77")).toMatch(/^[0-9a-f]{64}$/); expect(() => upgraded.setMetadata("actor_pseudonym_key_v1", "attacker-controlled")) .toThrow(/unsupported runtime metadata key/); @@ -234,7 +379,7 @@ describe("collector SQLite state", () => { legacy.close(); const upgraded = new CollectorDatabase(path); - expect(upgraded.status().schemaVersion).toBe(3); + expect(upgraded.status().schemaVersion).toBe(4); expect(upgraded.dueOutbox(NOW)[0]?.payloadJson).toBe(payload); expect(upgraded.pendingActorDays()).toEqual([]); upgraded.close(); @@ -278,3 +423,28 @@ describe("collector SQLite state", () => { } }); }); + +function downgradeToVersion3(path: string): void { + const database = new DatabaseSync(path); + database.exec(` + DROP TABLE source_endpoint_enrollment; + DROP TABLE active_source_endpoint; + DROP INDEX event_dedupe_retention_idx; + ALTER TABLE event_dedupe RENAME TO event_dedupe_v4; + CREATE TABLE event_dedupe ( + source TEXT NOT NULL CHECK (source IN ('snapchain', 'hypersnap')), + shard INTEGER NOT NULL CHECK (shard >= 0), + event_id TEXT NOT NULL, + event_type TEXT NOT NULL, + received_at_ms INTEGER NOT NULL, + PRIMARY KEY (source, shard, event_id) + ) STRICT; + INSERT INTO event_dedupe(source, shard, event_id, event_type, received_at_ms) + SELECT source, shard, event_id, event_type, received_at_ms FROM event_dedupe_v4; + DROP TABLE event_dedupe_v4; + CREATE INDEX event_dedupe_retention_idx ON event_dedupe(received_at_ms); + DELETE FROM schema_migrations WHERE version = 4; + UPDATE collector_metadata SET value = '3' WHERE key = 'schema_version'; + `); + database.close(); +} diff --git a/apps/collector/src/database.ts b/apps/collector/src/database.ts index c22382c..71f4764 100644 --- a/apps/collector/src/database.ts +++ b/apps/collector/src/database.ts @@ -20,10 +20,21 @@ export interface RecordedEvent { shard: number; eventId: string; eventType: string; + eventFingerprint: string; receivedAtMs: number; activity: ActivityRecord | null; } +export interface SourceEndpointEnrollment { + source: Source; + role: "primary" | "fallback"; + transport: "grpc" | "https-json"; + canonicalUrl: string; + peerId: string; + version: string; + shardIds: number[]; +} + export interface RecordEventResult { duplicate: boolean; actionInserted: boolean; @@ -61,7 +72,7 @@ export interface DatabaseStatus { lastCloudAckAtMs: number | null; } -const SCHEMA_VERSION = 3; +const SCHEMA_VERSION = 4; const ACTOR_PSEUDONYM_KEY_NAME = "actor_pseudonym_key_v1"; const ACTOR_PSEUDONYM_KEY_BYTES = 32; const ACTOR_PSEUDONYM_DOMAIN = "snapmeter-actor-day-v2\0"; @@ -112,13 +123,28 @@ export class CollectorDatabase { recordEvent(event: RecordedEvent): RecordEventResult { assertEventId(event.eventId); + assertFingerprint(event.eventFingerprint); if (!Number.isSafeInteger(event.shard) || event.shard < 0) throw new Error("invalid shard index"); return this.#transaction(() => { - const inserted = changes(this.#prepare(` - INSERT OR IGNORE INTO event_dedupe(source, shard, event_id, event_type, received_at_ms) - VALUES (?, ?, ?, ?, ?) - `).run(event.source, event.shard, event.eventId, event.eventType, event.receivedAtMs)) > 0; - if (!inserted) return { duplicate: true, actionInserted: false }; + const existingEvent = this.#prepare(` + SELECT event_fingerprint FROM event_dedupe WHERE source = ? AND shard = ? AND event_id = ? + `).get(event.source, event.shard, event.eventId) as { event_fingerprint?: unknown } | undefined; + if (existingEvent) { + const fingerprint = typeof existingEvent.event_fingerprint === "string" ? existingEvent.event_fingerprint : ""; + if (fingerprint && fingerprint !== event.eventFingerprint) { + throw new Error("event fingerprint conflict for an existing source/shard/id"); + } + if (!fingerprint) { + this.#prepare(` + UPDATE event_dedupe SET event_fingerprint = ? WHERE source = ? AND shard = ? AND event_id = ? + `).run(event.eventFingerprint, event.source, event.shard, event.eventId); + } + return { duplicate: true, actionInserted: false }; + } + this.#prepare(` + INSERT INTO event_dedupe(source, shard, event_id, event_type, event_fingerprint, received_at_ms) + VALUES (?, ?, ?, ?, ?, ?) + `).run(event.source, event.shard, event.eventId, event.eventType, event.eventFingerprint, event.receivedAtMs); if (!event.activity) return { duplicate: false, actionInserted: false }; const action = event.activity; @@ -221,6 +247,64 @@ export class CollectorDatabase { `).get(source, shard, eventId)); } + eventFingerprint(source: Source, shard: number, eventId: string): string | null { + assertEventId(eventId); + const row = this.#prepare(` + SELECT event_fingerprint FROM event_dedupe WHERE source = ? AND shard = ? AND event_id = ? + `).get(source, shard, eventId) as { event_fingerprint?: unknown } | undefined; + const value = typeof row?.event_fingerprint === "string" ? row.event_fingerprint : ""; + return /^[0-9a-f]{64}$/.test(value) ? value : null; + } + + validateOrEnrollSourceEndpoint(enrollment: SourceEndpointEnrollment, nowMs: number): void { + const normalized = normalizeEndpointEnrollment(enrollment); + this.#transaction(() => { + const row = this.#sourceEndpointEnrollmentRow(normalized.source, normalized.role); + if (!row) { + this.#prepare(` + INSERT INTO source_endpoint_enrollment( + source, role, transport, canonical_url, peer_id, version, shard_ids_json, enrolled_at_ms, last_verified_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + normalized.source, + normalized.role, + normalized.transport, + normalized.canonicalUrl, + normalized.peerId, + normalized.version, + normalized.shardsJson, + nowMs, + nowMs + ); + return; + } + assertEndpointEnrollmentMatches(row, normalized); + this.#prepare(` + UPDATE source_endpoint_enrollment SET last_verified_at_ms = ? WHERE source = ? AND role = ? + `).run(nowMs, normalized.source, normalized.role); + }); + } + + /** + * Read-only enrollment check used by doctor. An unenrolled endpoint is + * allowed because the runtime will atomically enroll it before activation; + * any change to an existing enrollment fails closed. + */ + checkSourceEndpointEnrollment(enrollment: SourceEndpointEnrollment): "unenrolled" | "match" { + const normalized = normalizeEndpointEnrollment(enrollment); + const row = this.#sourceEndpointEnrollmentRow(normalized.source, normalized.role); + if (!row) return "unenrolled"; + assertEndpointEnrollmentMatches(row, normalized); + return "match"; + } + + setActiveSourceEndpoint(source: Source, role: "primary" | "fallback", nowMs: number): void { + this.#prepare(` + INSERT INTO active_source_endpoint(source, role, switched_at_ms) VALUES (?, ?, ?) + ON CONFLICT(source) DO UPDATE SET role = excluded.role, switched_at_ms = excluded.switched_at_ms + `).run(source, role, nowMs); + } + loadActions(source: Source, sinceExclusiveMs: number): ActivityRecord[] { return (this.#prepare(` SELECT source, source_mode, shard, event_id, fid, action_family, action_at_ms, received_at_ms, is_replay @@ -511,6 +595,13 @@ export class CollectorDatabase { return this.#database.prepare(sql); } + #sourceEndpointEnrollmentRow(source: Source, role: "primary" | "fallback"): SqlRow | undefined { + return this.#prepare(` + SELECT transport, canonical_url, peer_id, version, shard_ids_json + FROM source_endpoint_enrollment WHERE source = ? AND role = ? + `).get(source, role) as SqlRow | undefined; + } + #transaction(operation: () => T): T { this.#database.exec("BEGIN IMMEDIATE"); try { @@ -532,19 +623,20 @@ export class CollectorDatabase { `); const current = numberColumn(this.#prepare("SELECT COALESCE(MAX(version), 0) AS value FROM schema_migrations").get(), "value"); if (current > SCHEMA_VERSION) throw new Error(`database schema ${current} is newer than collector schema ${SCHEMA_VERSION}`); - if (current < 1) { + const applied = new Set((this.#prepare("SELECT version FROM schema_migrations").all() as Array<{ version: number }>).map((row) => Number(row.version))); + if (!applied.has(1)) { this.#transaction(() => { this.#database.exec(MIGRATION_1); this.#prepare("INSERT INTO schema_migrations(version, applied_at_ms) VALUES (1, ?)").run(Date.now()); }); } - if (current < 2) { + if (!applied.has(2)) { this.#transaction(() => { this.#database.exec(MIGRATION_2); this.#prepare("INSERT INTO schema_migrations(version, applied_at_ms) VALUES (2, ?)").run(Date.now()); }); } - if (current < 3) { + if (!applied.has(3)) { this.#transaction(() => { this.#database.exec(MIGRATION_3); this.#prepare("INSERT INTO collector_secrets(name, value) VALUES (?, ?)") @@ -553,6 +645,19 @@ export class CollectorDatabase { this.#prepare("INSERT INTO schema_migrations(version, applied_at_ms) VALUES (3, ?)").run(Date.now()); }); } + if (!applied.has(4)) { + this.#transaction(() => { + const columns = this.#prepare("PRAGMA table_info(event_dedupe)").all() as Array<{ name?: unknown }>; + if (!columns.some((column) => column.name === "event_fingerprint")) { + this.#database.exec(` + ALTER TABLE event_dedupe ADD COLUMN event_fingerprint TEXT NOT NULL DEFAULT '' + CHECK (event_fingerprint = '' OR (length(event_fingerprint) = 64 AND event_fingerprint NOT GLOB '*[^0-9a-f]*')) + `); + } + this.#database.exec(MIGRATION_4); + this.#prepare("INSERT INTO schema_migrations(version, applied_at_ms) VALUES (4, ?)").run(Date.now()); + }); + } } } @@ -573,6 +678,48 @@ function assertEventId(eventId: string): void { if (!/^\d+$/.test(eventId)) throw new Error(`invalid event id: ${eventId}`); } +function assertFingerprint(value: string): void { + if (!/^[0-9a-f]{64}$/.test(value)) throw new Error("event fingerprint must be a lowercase SHA-256 value"); +} + +interface NormalizedSourceEndpointEnrollment extends Omit { + shardsJson: string; +} + +function normalizeEndpointEnrollment(enrollment: SourceEndpointEnrollment): NormalizedSourceEndpointEnrollment { + if (!enrollment.peerId || enrollment.peerId.length > 256 || hasControlCharacters(enrollment.peerId)) { + throw new Error("endpoint enrollment peer identity is invalid"); + } + if (!enrollment.version || enrollment.version.length > 128 || hasControlCharacters(enrollment.version)) { + throw new Error("endpoint enrollment version is invalid"); + } + if (!enrollment.canonicalUrl || enrollment.canonicalUrl.length > 2_048 || hasControlCharacters(enrollment.canonicalUrl)) { + throw new Error("endpoint enrollment canonical URL is invalid"); + } + const shardIds = [...new Set(enrollment.shardIds)].sort((left, right) => left - right); + if (shardIds.length === 0 || shardIds.some((shard) => !Number.isSafeInteger(shard) || shard <= 0)) { + throw new Error("endpoint enrollment requires positive data shards"); + } + return { ...enrollment, shardsJson: JSON.stringify(shardIds) }; +} + +function assertEndpointEnrollmentMatches(row: SqlRow, enrollment: NormalizedSourceEndpointEnrollment): void { + const matches = row.transport === enrollment.transport + && row.canonical_url === enrollment.canonicalUrl + && row.peer_id === enrollment.peerId + && row.version === enrollment.version + && row.shard_ids_json === enrollment.shardsJson; + if (!matches) throw new Error("endpoint identity changed from its durable enrollment"); +} + +function hasControlCharacters(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code <= 31 || code === 127) return true; + } + return false; +} + function changes(result: { changes: number | bigint }): number { return Number(result.changes); } @@ -725,3 +872,24 @@ const MIGRATION_3 = ` value BLOB NOT NULL CHECK (typeof(value) = 'blob' AND length(value) = 32) ) STRICT; `; + +const MIGRATION_4 = ` + CREATE TABLE IF NOT EXISTS source_endpoint_enrollment ( + source TEXT NOT NULL CHECK (source IN ('snapchain', 'hypersnap')), + role TEXT NOT NULL CHECK (role IN ('primary', 'fallback')), + transport TEXT NOT NULL CHECK (transport IN ('grpc', 'https-json')), + canonical_url TEXT NOT NULL, + peer_id TEXT NOT NULL, + version TEXT NOT NULL, + shard_ids_json TEXT NOT NULL, + enrolled_at_ms INTEGER NOT NULL, + last_verified_at_ms INTEGER NOT NULL, + PRIMARY KEY (source, role) + ) STRICT; + + CREATE TABLE IF NOT EXISTS active_source_endpoint ( + source TEXT PRIMARY KEY CHECK (source IN ('snapchain', 'hypersnap')), + role TEXT NOT NULL CHECK (role IN ('primary', 'fallback')), + switched_at_ms INTEGER NOT NULL + ) STRICT; +`; diff --git a/apps/collector/src/delivery.test.ts b/apps/collector/src/delivery.test.ts index 06ff847..1a9796f 100644 --- a/apps/collector/src/delivery.test.ts +++ b/apps/collector/src/delivery.test.ts @@ -42,7 +42,15 @@ function action(source: "snapchain" | "hypersnap", fid: string, eventId: string) } function record(database: CollectorDatabase, activity: ActivityRecord, eventId: string) { - database.recordEvent({ source: activity.source, shard: 0, eventId, eventType: "MERGE_MESSAGE", receivedAtMs: activity.receivedAtMs, activity }); + database.recordEvent({ + source: activity.source, + shard: 0, + eventId, + eventType: "MERGE_MESSAGE", + eventFingerprint: "a".repeat(64), + receivedAtMs: activity.receivedAtMs, + activity + }); } describe("pulse and batch delivery", () => { diff --git a/apps/collector/src/doctor.test.ts b/apps/collector/src/doctor.test.ts index 3fbbc99..44adf24 100644 --- a/apps/collector/src/doctor.test.ts +++ b/apps/collector/src/doctor.test.ts @@ -58,6 +58,69 @@ describe("collector doctor", () => { if (resolved.startsWith(resolve(tmpdir()))) rmSync(resolved, { recursive: true, force: true }); } }); + + it("accepts a healthy peer-pinned HTTPS fallback when the local Hypersnap endpoint is offline", async () => { + const directory = mkdtempSync(join(tmpdir(), "snapmeter-doctor-fallback-test-")); + const peerId = "12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc"; + const config = loadConfig({ + SNAPMETER_DATA_DIR: directory, + SNAPCHAIN_SOURCE_MODE: "unavailable", + HYPERSNAP_FALLBACK_HTTP_URL: "https://public.example", + HYPERSNAP_FALLBACK_EXPECTED_PEER_ID: peerId, + HYPERSNAP_FALLBACK_EXPECTED_VERSION: "0.13.3", + SNAPMETER_MIN_FREE_DISK_BYTES: "1" + }); + const database = new CollectorDatabase(config.databasePath); + const rpcFactory: RpcFactory = (candidate) => candidate.transport === "grpc" + ? { ...fakeRpc(), getInfo: async () => { throw new Error("offline"); } } + : { + ...fakeRpc(), + getInfo: async () => ({ + version: "0.13.3", + peerId, + numShards: 2, + shardInfos: [ + { shardId: 1, maxHeight: 20, blockDelay: 0, mempoolSize: 0 }, + { shardId: 2, maxHeight: 20, blockDelay: 0, mempoolSize: 0 } + ] + }) + }; + try { + const report = await runDoctor({ config, database, rpcFactory, now: () => Date.UTC(2026, 7, 13, 12) }); + expect(report.checks.find((check) => check.name === "hypersnap.rpc")).toMatchObject({ + status: "pass", + details: { role: "fallback", transport: "https-json" } + }); + expect(database.checkSourceEndpointEnrollment({ + source: "hypersnap", + role: "fallback", + transport: "https-json", + canonicalUrl: "https://public.example/", + peerId, + version: "0.13.3", + shardIds: [1, 2] + })).toBe("unenrolled"); + + database.validateOrEnrollSourceEndpoint({ + source: "hypersnap", + role: "fallback", + transport: "https-json", + canonicalUrl: "https://previous.example/", + peerId, + version: "0.13.3", + shardIds: [1, 2] + }, Date.UTC(2026, 7, 13, 11)); + const mismatched = await runDoctor({ config, database, rpcFactory, now: () => Date.UTC(2026, 7, 13, 12) }); + expect(mismatched.checks.find((check) => check.name === "hypersnap.rpc")).toMatchObject({ + status: "fail", + message: "no configured endpoint passed identity, health, and cursor-continuity checks" + }); + } finally { + database.close(); + const resolved = resolve(directory); + if (resolved.startsWith(resolve(tmpdir()))) rmSync(resolved, { recursive: true, force: true }); + } + }); }); function fakeRpc(): CollectorRpc { diff --git a/apps/collector/src/doctor.ts b/apps/collector/src/doctor.ts index 39fa19d..860f544 100644 --- a/apps/collector/src/doctor.ts +++ b/apps/collector/src/doctor.ts @@ -1,12 +1,12 @@ import { randomUUID } from "node:crypto"; import { mkdirSync, statfsSync } from "node:fs"; import { IngestBatchSchema, SCHEMA_VERSION, signIngest, type IngestBatch, type Source } from "@snapmeter/contracts"; -import { FARCASTER_EPOCH_MS, HYPERSNAP_UPSTREAM_SHA, SNAPCHAIN_UPSTREAM_SHA } from "@snapmeter/protocol"; -import type { CollectorConfig } from "./config.js"; +import { FARCASTER_EPOCH_MS, HYPERSNAP_UPSTREAM_SHA, SNAPCHAIN_UPSTREAM_SHA, rawHubEventFingerprint, type NodeInfo } from "@snapmeter/protocol"; +import type { CollectorConfig, RpcEndpointConfig, RpcTransportConfig } from "./config.js"; import { CollectorDatabase } from "./database.js"; import type { FetchLike } from "./delivery.js"; -import { defaultRpcFactory, type RpcFactory } from "./rpc.js"; -import { discoveredShardIds } from "./collector.js"; +import { defaultRpcFactory, rawEventId, rawEventShard, type CollectorRpc, type RpcFactory } from "./rpc.js"; +import { discoveredShardIds, validateCandidateInfo } from "./collector.js"; export type CheckStatus = "pass" | "warn" | "fail" | "skipped"; @@ -58,9 +58,19 @@ export async function runDoctor(options: DoctorOptions): Promise { checks.push({ name: `${source}.rpc`, status: "skipped", message: "source is explicitly unavailable" }); continue; } - const rpc = rpcFactory(endpoint); + let selected: DoctorEndpoint; + try { + selected = await selectDoctorEndpoint(source, endpoint, options.database, rpcFactory); + } catch { + checks.push({ + name: `${source}.rpc`, + status: "fail", + message: "no configured endpoint passed identity, health, and cursor-continuity checks" + }); + continue; + } + const { rpc, info, role, transport } = selected; try { - const info = await rpc.getInfo(); const shards = discoveredShardIds(info); expectedShards.set(source, shards); checks.push({ @@ -69,7 +79,7 @@ export async function runDoctor(options: DoctorOptions): Promise { message: info.version && shards.length > 0 ? `GetInfo succeeded; discovered ${shards.length} shard(s)` : "GetInfo returned an incompatible response", - details: { endpoint: endpoint.url, tls: endpoint.tls, version: info.version, shards } + details: { role, transport: transport.transport, tls: transport.tls, version: info.version, shards } }); checks.push({ name: `${source}.shards`, @@ -131,16 +141,16 @@ export async function runDoctor(options: DoctorOptions): Promise { } else { checks.push({ name: `${source}.clock`, status: "warn", message: "no near-head event timestamp was available for a clock comparison" }); } - } catch (error) { - checks.push({ name: `${source}.protocol`, status: "fail", message: errorMessage(error) }); + } catch { + checks.push({ name: `${source}.protocol`, status: "fail", message: "the selected endpoint failed the event protocol probe" }); } } - } catch (error) { + } catch { checks.push({ name: `${source}.rpc`, status: "fail", - message: errorMessage(error), - details: { endpoint: endpoint.url, tls: endpoint.tls } + message: "the selected endpoint failed a bounded protocol check", + details: { role, transport: transport.transport } }); } finally { rpc.close(); @@ -167,6 +177,67 @@ export async function runDoctor(options: DoctorOptions): Promise { }; } +interface DoctorEndpoint { + role: "primary" | "fallback"; + transport: RpcTransportConfig; + rpc: CollectorRpc; + info: NodeInfo; +} + +async function selectDoctorEndpoint( + source: Source, + endpoint: RpcEndpointConfig, + database: CollectorDatabase, + rpcFactory: RpcFactory +): Promise { + for (const role of ["primary", "fallback"] as const) { + const transport = role === "primary" ? endpoint : endpoint.fallback; + if (!transport) continue; + const rpc = rpcFactory(transport); + try { + const info = await rpc.getInfo(); + const shards = discoveredShardIds(info); + if (!info.version || shards.length === 0) throw new Error("incompatible GetInfo response"); + if (transport.expectedPeerId && info.peerId !== transport.expectedPeerId) throw new Error("peer identity mismatch"); + if (transport.expectedVersion && info.version !== transport.expectedVersion) throw new Error("version mismatch"); + if (source === "hypersnap") { + validateCandidateInfo(source, transport, info, endpoint.maximumBlockDelaySeconds); + if (!info.peerId) throw new Error("Hypersnap endpoint did not expose a peer identity"); + const advertised = new Set(shards); + for (const cursor of database.getCursors().filter((item) => item.source === source)) { + if (!advertised.has(cursor.shard)) throw new Error("candidate endpoint is missing a durable cursor shard"); + const known = database.eventFingerprint(source, cursor.shard, cursor.eventId); + if (role === "fallback" && known === null) throw new Error("fallback cursor fingerprint is not enrolled"); + const point = await rpc.getEvent(cursor.shard, cursor.eventId); + if (rawEventId(point) !== cursor.eventId + || rawEventShard(point, cursor.shard) !== cursor.shard + || (known !== null && rawHubEventFingerprint(point, cursor.shard) !== known)) { + throw new Error(`${role} cursor continuity mismatch`); + } + } + database.checkSourceEndpointEnrollment({ + source, + role, + transport: transport.transport, + canonicalUrl: canonicalTransportUrl(transport), + peerId: info.peerId, + version: info.version, + shardIds: shards + }); + } + return { role, transport, rpc, info }; + } catch { + rpc.close(); + } + } + throw new Error(`no compatible ${source} endpoint`); +} + +function canonicalTransportUrl(transport: RpcTransportConfig): string { + if (transport.transport === "https-json") return new URL(transport.url).toString(); + return transport.url.toLowerCase(); +} + function databaseCheck(database: CollectorDatabase, nowMs: number): DoctorCheck { try { database.setMetadata("doctor_last_write_at_ms", String(nowMs)); diff --git a/apps/collector/src/hypersnap-http-rpc.test.ts b/apps/collector/src/hypersnap-http-rpc.test.ts new file mode 100644 index 0000000..08a44bd --- /dev/null +++ b/apps/collector/src/hypersnap-http-rpc.test.ts @@ -0,0 +1,321 @@ +import { describe, expect, it, vi } from "vitest"; +import { FARCASTER_EPOCH_MS, normalizeMergeEvent, rawHubEventFingerprint, type RawHubEvent } from "@snapmeter/protocol"; +import { HypersnapHttpRpc, type HypersnapHttpFetch } from "./hypersnap-http-rpc.js"; +import type { RpcSubscription } from "./rpc.js"; + +const PEER_ID = "12D3KooWTestPeerIdentity123456789abcdefghijk"; + +describe("Hypersnap HTTP CollectorRpc adapter", () => { + it("requires an uncredentialed HTTPS base URL", () => { + expect(() => new HypersnapHttpRpc({ baseUrl: "http://node.example" })).toThrow(/use HTTPS/); + expect(() => new HypersnapHttpRpc({ baseUrl: "https://user:password@example.test/" })).toThrow(/must not contain credentials/); + expect(() => new HypersnapHttpRpc({ baseUrl: "https://node.example?token=secret" })).toThrow(/query or fragment/); + expect(() => new HypersnapHttpRpc({ baseUrl: "https://node.example", expectedPeerId: "bad peer" })).toThrow(/peer identifier/); + }); + + it("maps GetInfo, pins peer identity, and sends no credentials", async () => { + const fetcher = vi.fn(async (input, init) => { + const url = new URL(input.toString()); + expect(url.href).toBe("https://node.example/api/v1/info"); + expect(new Headers(init?.headers).get("authorization")).toBeNull(); + expect(new Headers(init?.headers).get("accept")).toBe("application/json"); + expect(init).toMatchObject({ method: "GET", credentials: "omit", redirect: "error", cache: "no-store" }); + return json({ + version: "0.13.4", + peer_id: PEER_ID, + numShards: 2, + shardInfos: [ + { shardId: 0, maxHeight: 20, blockDelay: 0, mempoolSize: 1 }, + { shardId: 1, maxHeight: 19, blockDelay: 2, mempoolSize: 3 } + ] + }); + }); + const rpc = new HypersnapHttpRpc({ + baseUrl: "https://node.example/api/", + expectedPeerId: PEER_ID, + fetcher + }); + + await expect(rpc.getInfo()).resolves.toEqual({ + version: "0.13.4", + peerId: PEER_ID, + numShards: 2, + shardInfos: [ + { shardId: 0, maxHeight: 20, blockDelay: 0, mempoolSize: 1 }, + { shardId: 1, maxHeight: 19, blockDelay: 2, mempoolSize: 3 } + ] + }); + expect(fetcher).toHaveBeenCalledTimes(1); + + const mismatched = new HypersnapHttpRpc({ + baseUrl: "https://node.example/api/", + expectedPeerId: "12D3KooWAnotherExpectedPeer", + fetcher + }); + await expect(mismatched.getInfo()).rejects.toThrow(/did not match the pinned identity/); + }); + + it("preserves unsafe uint64 values and hydrates canonical block time for a merge event", async () => { + const fetcher = vi.fn(async (input) => { + const url = new URL(input.toString()); + expect(url.pathname).toBe("/v1/eventById"); + expect(url.searchParams.get("shard_index")).toBe("1"); + if (url.searchParams.get("event_id") === "9007199254740992") { + return rawJson(`{ + "type":"HUB_EVENT_TYPE_BLOCK_CONFIRMED", + "id":9007199254740992, + "blockConfirmedBody":{"blockNumber":549755813888,"shardIndex":1,"timestamp":142732900,"totalEvents":2}, + "blockNumber":549755813888, + "shardIndex":1 + }`); + } + expect(url.searchParams.get("event_id")).toBe("9007199254740993"); + return rawJson(`{ + "type":"HUB_EVENT_TYPE_MERGE_MESSAGE", + "id":9007199254740993, + "mergeMessageBody":{"message":{"hash":"0xaabbccddeeff00112233445566778899aabbccdd","data":{"type":"MESSAGE_TYPE_CAST_ADD","fid":42,"timestamp":142732800,"network":"FARCASTER_NETWORK_MAINNET","custom":{"kept":true}}}}, + "blockNumber":549755813888, + "shardIndex":1, + "extension":{"large":18446744073709551615} + }`); + }); + const rpc = new HypersnapHttpRpc({ baseUrl: "https://node.example", fetcher }); + const event = await rpc.getEvent(1, "9007199254740993"); + + expect(event).toMatchObject({ + id: "9007199254740993", + timestamp: "142732900", + blockNumber: "549755813888", + shardIndex: 1, + mergeMessageBody: { + message: { data: { fid: "42", timestamp: "142732800", custom: { kept: true } } } + }, + extension: { large: "18446744073709551615" } + }); + const native: RawHubEvent = { + ...event, + timestamp: 142732900, + mergeMessageBody: { message: { + hash: Buffer.from("aabbccddeeff00112233445566778899aabbccdd", "hex"), + data: { type: 1, fid: 42, timestamp: 142732800, network: 1 } + } } + }; + expect(rawHubEventFingerprint(event, 1)).toBe(rawHubEventFingerprint(native, 1)); + expect(normalizeMergeEvent(event, "hypersnap", "derived", Date.now(), true)?.actionAtMs) + .toBe(FARCASTER_EPOCH_MS + 142732900_000); + expect(normalizeMergeEvent(native, "hypersnap", "derived", Date.now(), true)?.actionAtMs) + .toBe(FARCASTER_EPOCH_MS + 142732900_000); + expect(fetcher).toHaveBeenCalledTimes(2); + }); + + it("accepts the reviewed channel-owner hint event without treating it as activity", async () => { + const eventId = 9n << 14n; + const hintId = eventId + 1n; + const fetcher = vi.fn(async (input) => { + const requested = BigInt(new URL(input.toString()).searchParams.get("event_id") as string); + if (requested === eventId) return rawJson(eventJson(eventId, 1)); + return rawJson(`{ + "type":"HUB_EVENT_TYPE_CHANNEL_OWNER_CHANGE_HINT", + "id":${hintId}, + "blockNumber":9, + "shardIndex":1 + }`); + }); + const rpc = new HypersnapHttpRpc({ baseUrl: "https://node.example", fetcher }); + + const hint = await rpc.getEvent(1, hintId.toString()); + expect(hint).toMatchObject({ + type: "HUB_EVENT_TYPE_CHANNEL_OWNER_CHANGE_HINT", + id: hintId.toString(), + timestamp: "142732800", + shardIndex: 1 + }); + const native = { ...hint, type: 12, timestamp: 142732800 } satisfies RawHubEvent; + expect(rawHubEventFingerprint(hint, 1)).toBe(rawHubEventFingerprint(native, 1)); + }); + + it("keeps canonical block timestamps separate for equal heights on different shards", async () => { + const blockNumber = 9n; + const mergeId = (blockNumber << 14n) + 1n; + const fetcher = vi.fn(async (input) => { + const url = new URL(input.toString()); + const shard = Number(url.searchParams.get("shard_index")); + const eventId = BigInt(url.searchParams.get("event_id") as string); + if (eventId % 16_384n === 0n) { + return rawJson(eventJson(eventId, shard, 142732900 + shard)); + } + return rawJson(mergeEventJson(eventId, shard)); + }); + const rpc = new HypersnapHttpRpc({ baseUrl: "https://node.example", fetcher }); + + const [left, right] = await Promise.all([ + rpc.getEvent(1, mergeId.toString()), + rpc.getEvent(2, mergeId.toString()) + ]); + + expect(left.timestamp).toBe("142732901"); + expect(right.timestamp).toBe("142732902"); + }); + + it("rejects injected event timestamps that conflict with canonical block time", async () => { + const blockNumber = 9n; + const mergeId = (blockNumber << 14n) + 1n; + const mergeMismatch = new HypersnapHttpRpc({ + baseUrl: "https://node.example", + fetcher: async (input) => { + const eventId = BigInt(new URL(input.toString()).searchParams.get("event_id") as string); + if (eventId % 16_384n === 0n) return rawJson(eventJson(eventId, 1, 142732900)); + const merge = JSON.parse(mergeEventJson(eventId, 1)) as Record; + merge.timestamp = 142732901; + return json(merge); + } + }); + await expect(mergeMismatch.getEvent(1, mergeId.toString())).rejects.toThrow(/conflicted with its canonical block/); + + const confirmationMismatch = new HypersnapHttpRpc({ + baseUrl: "https://node.example", + fetcher: async () => { + const confirmation = JSON.parse(eventJson(blockNumber << 14n, 1, 142732900)) as Record; + confirmation.timestamp = 142732901; + return json(confirmation); + } + }); + await expect(confirmationMismatch.getEvent(1, (blockNumber << 14n).toString())) + .rejects.toThrow(/conflicted with its body timestamp/); + }); + + it("synthesizes an opaque forward page token from the last exact event id plus one", async () => { + const firstId = 9_007_199_254_740_993n; + const requests: URL[] = []; + const fetcher = vi.fn(async (input) => { + const url = new URL(input.toString()); + requests.push(url); + if (requests.length === 1) { + const events = Array.from({ length: 500 }, (_, index) => eventJson(firstId + BigInt(index), 1)); + return rawJson(`{"events":[${events.join(",")}]}`); + } + return rawJson(`{"events":[${eventJson(firstId + 500n, 1)}]}`); + }); + const rpc = new HypersnapHttpRpc({ baseUrl: "https://node.example", fetcher }); + const first = await rpc.getEvents(1, firstId.toString()); + + expect(first.events).toHaveLength(500); + expect(first.events[0]?.id).toBe(firstId.toString()); + expect(first.events.at(-1)?.id).toBe((firstId + 499n).toString()); + expect(first.nextPageToken).toBeInstanceOf(Uint8Array); + const second = await rpc.getEvents(1, firstId.toString(), first.nextPageToken); + expect(second.events.map((event) => event?.id)).toEqual([(firstId + 500n).toString()]); + expect(second.nextPageToken).toBeUndefined(); + expect(requests[0]?.searchParams.get("pageSize")).toBe("500"); + expect(requests[0]?.searchParams.get("reverse")).toBe("false"); + expect(requests[1]?.searchParams.get("from_event_id")).toBe((firstId + 500n).toString()); + + await expect(rpc.getEvents(2, firstId.toString(), first.nextPageToken)).rejects.toThrow(/invalid.*page token/i); + }); + + it("rejects invalid response media types, oversized bodies, and out-of-order pages", async () => { + const wrongType = new HypersnapHttpRpc({ + baseUrl: "https://node.example", + fetcher: async () => new Response("{}", { headers: { "content-type": "text/plain" } }) + }); + await expect(wrongType.getInfo()).rejects.toThrow(/content-type/); + + const oversized = new HypersnapHttpRpc({ + baseUrl: "https://node.example", + maxResponseBytes: 4, + fetcher: async () => rawJson("{\"tooLong\":true}") + }); + await expect(oversized.getInfo()).rejects.toThrow(/body-size limit/); + + const unordered = new HypersnapHttpRpc({ + baseUrl: "https://node.example", + fetcher: async () => rawJson(`{"events":[${eventJson(12n, 1)},${eventJson(11n, 1)}]}`) + }); + await expect(unordered.getEvents(1, "10")).rejects.toThrow(/not strictly ordered/); + }); + + it("propagates caller aborts and per-request timeouts", async () => { + const pendingFetch: HypersnapHttpFetch = async (_input, init) => new Promise((_resolve, reject) => { + const signal = init?.signal; + const abort = (): void => reject(signal?.reason ?? new Error("aborted")); + if (signal?.aborted) abort(); + else signal?.addEventListener("abort", abort, { once: true }); + }); + const rpc = new HypersnapHttpRpc({ baseUrl: "https://node.example", timeoutMs: 1_000, fetcher: pendingFetch }); + const controller = new AbortController(); + const aborted = rpc.getInfo(controller.signal); + controller.abort(); + await expect(aborted).rejects.toMatchObject({ name: "AbortError" }); + + const timed = new HypersnapHttpRpc({ baseUrl: "https://node.example", timeoutMs: 5, fetcher: pendingFetch }); + await expect(timed.getInfo()).rejects.toMatchObject({ name: "TimeoutError" }); + }); + + it("simulates Subscribe with a validated head probe and bounded forward polling", async () => { + const requests: URL[] = []; + let reverseCalls = 0; + const fetcher = vi.fn(async (input) => { + const url = new URL(input.toString()); + requests.push(url); + if (url.searchParams.get("reverse") === "true") { + reverseCalls += 1; + const id = reverseCalls === 1 ? 10n : 12n; + return rawJson(`{"events":[${eventJson(id, 1)}]}`); + } + expect(url.searchParams.get("from_event_id")).toBe("11"); + expect(url.searchParams.get("stop_id")).toBe("13"); + return rawJson(`{"events":[${eventJson(11n, 1)},${eventJson(12n, 1)}]}`); + }); + const rpc = new HypersnapHttpRpc({ baseUrl: "https://node.example", pollIntervalMs: 1, fetcher }); + const received: string[] = []; + const onError = vi.fn(); + const active: { subscription?: RpcSubscription } = {}; + const subscription = rpc.subscribe(1, undefined, (event: RawHubEvent) => { + received.push(String(event.id)); + if (event.id === "12") active.subscription?.cancel(); + }, onError); + active.subscription = subscription; + + await subscription.ready; + expect(received[0]).toBe("10"); + await subscription.done; + expect(received).toEqual(["10", "11", "12"]); + expect(onError).not.toHaveBeenCalled(); + expect(requests.some((url) => url.searchParams.get("reverse") === "true" && url.searchParams.get("pageSize") === "1")).toBe(true); + expect(requests.some((url) => url.searchParams.get("stop_id") === "13")).toBe(true); + }); + + it("closes pending subscriptions without leaking an error callback", async () => { + const fetcher: HypersnapHttpFetch = async (_input, init) => new Promise((_resolve, reject) => { + const signal = init?.signal; + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + const rpc = new HypersnapHttpRpc({ baseUrl: "https://node.example", fetcher }); + const onError = vi.fn(); + const subscription = rpc.subscribe(1, undefined, () => undefined, onError); + rpc.close(); + + await expect(subscription.ready).rejects.toMatchObject({ name: "AbortError" }); + await expect(subscription.done).resolves.toBeUndefined(); + expect(onError).not.toHaveBeenCalled(); + await expect(rpc.getInfo()).rejects.toMatchObject({ name: "AbortError" }); + }); +}); + +function json(value: unknown): Response { + return rawJson(JSON.stringify(value)); +} + +function rawJson(value: string): Response { + return new Response(value, { headers: { "content-type": "application/json; charset=utf-8" } }); +} + +function eventJson(id: bigint, shard: number, timestamp = 142732800): string { + const block = id >> 14n; + return `{"type":"HUB_EVENT_TYPE_BLOCK_CONFIRMED","id":${id},"blockConfirmedBody":{"blockNumber":${block},"shardIndex":${shard},"timestamp":${timestamp},"totalEvents":1},"blockNumber":${block},"shardIndex":${shard}}`; +} + +function mergeEventJson(id: bigint, shard: number): string { + const block = id >> 14n; + return `{"type":"HUB_EVENT_TYPE_MERGE_MESSAGE","id":${id},"mergeMessageBody":{"message":{"hash":"0xaabbccddeeff00112233445566778899aabbccdd","data":{"type":"MESSAGE_TYPE_CAST_ADD","fid":42,"timestamp":142732800,"network":"FARCASTER_NETWORK_MAINNET"}}},"blockNumber":${block},"shardIndex":${shard}}`; +} diff --git a/apps/collector/src/hypersnap-http-rpc.ts b/apps/collector/src/hypersnap-http-rpc.ts new file mode 100644 index 0000000..c81f81b --- /dev/null +++ b/apps/collector/src/hypersnap-http-rpc.ts @@ -0,0 +1,782 @@ +import { MinimumIntervalGate, type NodeInfo, type RawHubEvent } from "@snapmeter/protocol"; +import type { CollectorRpc, RpcSubscription } from "./rpc.js"; + +const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_POLL_INTERVAL_MS = 1_000; +const DEFAULT_MAX_RESPONSE_BYTES = 20 * 1024 * 1024; +const EVENT_PAGE_SIZE = 500; +const MAX_UINT64 = 18_446_744_073_709_551_615n; +const MAX_UINT32 = 4_294_967_295; +const PAGE_TOKEN_PREFIX = "snapmeter:hypersnap-http:v1"; +const FARCASTER_EPOCH_SECONDS = 1_609_459_200; +const MAX_FUTURE_SKEW_SECONDS = 300; +const MAX_BLOCK_TIMESTAMP_CACHE_ENTRIES = 4_096; + +export type HypersnapHttpFetch = ( + input: string | URL | Request, + init?: RequestInit +) => Promise; + +export interface HypersnapHttpRpcConfig { + baseUrl: string | URL; + expectedPeerId?: string; + timeoutMs?: number; + pollIntervalMs?: number; + minimumIntervalMs?: number; + maxResponseBytes?: number; + fetcher?: HypersnapHttpFetch; +} + +interface JsonRecord { + [key: string]: unknown; +} + +/** + * A read-only CollectorRpc adapter for Hypersnap's HTTPS JSON API. + * + * The API has no streaming route or event continuation token. Subscribe is + * therefore simulated with an initial reverse head probe followed by bounded + * forward polls, while continuation tokens encode the next exact uint64 ID. + */ +export class HypersnapHttpRpc implements CollectorRpc { + readonly #baseUrl: URL; + readonly #expectedPeerId?: string; + readonly #timeoutMs: number; + readonly #pollIntervalMs: number; + readonly #maxResponseBytes: number; + readonly #fetcher: HypersnapHttpFetch; + readonly #requestStartGate: MinimumIntervalGate; + readonly #blockTimestamps = new Map(); + readonly #closed = new AbortController(); + readonly #subscriptionCancels = new Set<() => void>(); + + constructor(config: HypersnapHttpRpcConfig) { + this.#baseUrl = validateBaseUrl(config.baseUrl); + this.#expectedPeerId = optionalPeerId(config.expectedPeerId); + this.#timeoutMs = boundedInteger(config.timeoutMs, DEFAULT_TIMEOUT_MS, 1, 120_000, "timeoutMs"); + this.#pollIntervalMs = boundedInteger(config.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS, 1, 3_600_000, "pollIntervalMs"); + this.#maxResponseBytes = boundedInteger( + config.maxResponseBytes, + DEFAULT_MAX_RESPONSE_BYTES, + 1, + 100 * 1024 * 1024, + "maxResponseBytes" + ); + this.#fetcher = config.fetcher ?? fetch; + this.#requestStartGate = new MinimumIntervalGate(boundedInteger( + config.minimumIntervalMs, + 0, + 0, + 3_600_000, + "minimumIntervalMs" + )); + } + + async getInfo(signal?: AbortSignal): Promise { + const body = record(await this.#requestJson("v1/info", {}, signal), "GetInfo response"); + const peerId = requiredString(body.peer_id, "GetInfo peer_id", 256); + if (this.#expectedPeerId !== undefined && peerId !== this.#expectedPeerId) { + throw new Error("Hypersnap HTTP endpoint peer_id did not match the pinned identity"); + } + const shardValues = array(body.shardInfos, "GetInfo shardInfos", 1_024); + return { + version: requiredString(body.version, "GetInfo version", 256), + peerId, + numShards: safeInteger(body.numShards, "GetInfo numShards", 0, MAX_UINT32), + shardInfos: shardValues.map((value, index) => { + const shard = record(value, `GetInfo shardInfos[${index}]`); + return { + shardId: safeInteger(shard.shardId, `GetInfo shardInfos[${index}].shardId`, 0, MAX_UINT32), + maxHeight: safeInteger(shard.maxHeight, `GetInfo shardInfos[${index}].maxHeight`, 0, Number.MAX_SAFE_INTEGER), + blockDelay: safeInteger(shard.blockDelay, `GetInfo shardInfos[${index}].blockDelay`, 0, Number.MAX_SAFE_INTEGER), + mempoolSize: safeInteger(shard.mempoolSize, `GetInfo shardInfos[${index}].mempoolSize`, 0, Number.MAX_SAFE_INTEGER) + }; + }) + }; + } + + async getEvent(shardIndex: number, id: string, signal?: AbortSignal): Promise { + const shard = positiveShard(shardIndex); + const eventId = uint64String(id, "event id", false); + const value = await this.#requestJson("v1/eventById", { + event_id: eventId, + shard_index: String(shard) + }, signal); + const [event] = await this.#canonicalizeEventTimes([normalizeEvent(value, shard)], shard, signal); + if (!event) throw new Error("eventById returned an empty response"); + if (String(event.id) !== eventId) throw new Error("eventById returned a different event id"); + return event; + } + + async getEvents( + shardIndex: number, + startId: string, + pageToken?: Uint8Array, + stopId?: string, + signal?: AbortSignal + ): Promise<{ events: Array; nextPageToken?: Uint8Array }> { + const shard = positiveShard(shardIndex); + const requestedStart = uint64String(startId, "start event id", true); + const effectiveStart = pageToken === undefined + ? requestedStart + : decodePageToken(pageToken, shard); + if (compareUint64(effectiveStart, requestedStart) < 0) { + throw new Error("Hypersnap HTTP page token precedes the requested start id"); + } + const exclusiveStop = stopId === undefined ? undefined : uint64String(stopId, "stop event id", true); + if (exclusiveStop !== undefined && compareUint64(effectiveStart, exclusiveStop) >= 0) { + return { events: [] }; + } + + const query: Record = { + from_event_id: effectiveStart, + shard_index: String(shard), + pageSize: String(EVENT_PAGE_SIZE), + reverse: "false" + }; + if (exclusiveStop !== undefined) query.stop_id = exclusiveStop; + const events = await this.#eventsRequest(query, shard, EVENT_PAGE_SIZE, signal); + validateForwardPage(events, effectiveStart, exclusiveStop); + + if (events.length < EVENT_PAGE_SIZE) return { events }; + const lastId = String(events.at(-1)?.id); + if (lastId === String(MAX_UINT64)) return { events }; + const nextId = incrementUint64(lastId); + if (exclusiveStop !== undefined && compareUint64(nextId, exclusiveStop) >= 0) return { events }; + return { events, nextPageToken: encodePageToken(shard, nextId) }; + } + + subscribe( + shardIndex: number, + fromId: string | undefined, + onEvent: (event: RawHubEvent) => void, + onError: (error: Error) => void + ): RpcSubscription { + this.#assertOpen(); + const shard = positiveShard(shardIndex); + const requestedStart = fromId === undefined ? undefined : uint64String(fromId, "subscription from id", true); + const controller = new AbortController(); + let cancelled = false; + let finished = false; + let readySettled = false; + let resolveReady: (() => void) | undefined; + let rejectReady: ((error: Error) => void) | undefined; + let resolveDone: (() => void) | undefined; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + const done = new Promise((resolve) => { resolveDone = resolve; }); + + const cancel = (): void => { + if (finished) return; + cancelled = true; + controller.abort(abortError("Hypersnap HTTP subscription cancelled")); + finish(); + }; + const finish = (error?: Error): void => { + if (finished) return; + finished = true; + this.#subscriptionCancels.delete(cancel); + if (!readySettled) { + readySettled = true; + if (error !== undefined) rejectReady?.(error); + else rejectReady?.(abortError("Hypersnap HTTP subscription ended before readiness")); + } + resolveDone?.(); + if (error !== undefined && !cancelled) { + try { + onError(error); + } catch { + // Observer failures must not leave the subscription lifecycle open. + } + } + }; + this.#subscriptionCancels.add(cancel); + + void this.#pollSubscription(shard, requestedStart, controller.signal, (event) => { + if (controller.signal.aborted) return; + onEvent(event); + if (!readySettled) { + readySettled = true; + resolveReady?.(); + } + }).then( + () => finish(), + (error: unknown) => finish(asError(error)) + ); + + return { cancel, ready, done }; + } + + close(): void { + if (this.#closed.signal.aborted) return; + this.#closed.abort(abortError("Hypersnap HTTP RPC client closed")); + for (const cancel of [...this.#subscriptionCancels]) cancel(); + this.#subscriptionCancels.clear(); + } + + async #pollSubscription( + shard: number, + requestedStart: string | undefined, + signal: AbortSignal, + emit: (event: RawHubEvent) => void + ): Promise { + let nextId: string | undefined; + while (!signal.aborted) { + const headEvents = await this.#eventsRequest({ + from_event_id: "0", + shard_index: String(shard), + pageSize: "1", + reverse: "true" + }, shard, 1, signal); + if (headEvents.length === 0) { + await abortableDelay(this.#pollIntervalMs, signal); + continue; + } + + const head = headEvents[0] as RawHubEvent; + const headId = String(head.id); + if (nextId === undefined) { + emit(head); + nextId = incrementUint64(headId); + if (requestedStart !== undefined && compareUint64(requestedStart, nextId) > 0) nextId = requestedStart; + } else if (compareUint64(headId, nextId) >= 0) { + const stopId = incrementUint64(headId); + let token: Uint8Array | undefined; + let pages = 0; + do { + const page = await this.getEvents(shard, nextId, token, stopId, signal); + pages += 1; + if (pages > 100_000) throw new Error("Hypersnap HTTP subscription poll exceeded its page safety bound"); + for (const event of page.events) { + if (event === null) continue; + emit(event); + nextId = incrementUint64(String(event.id)); + } + token = page.nextPageToken; + } while (token !== undefined && !signal.aborted); + } + await abortableDelay(this.#pollIntervalMs, signal); + } + } + + async #eventsRequest( + query: Record, + shard: number, + maximumEvents: number, + signal?: AbortSignal + ): Promise { + const body = record(await this.#requestJson("v1/events", query, signal), "events response"); + const events = array(body.events, "events response events", maximumEvents).map((value) => normalizeEvent(value, shard)); + return this.#canonicalizeEventTimes(events, shard, signal); + } + + async #canonicalizeEventTimes( + events: RawHubEvent[], + shard: number, + signal?: AbortSignal + ): Promise { + for (const event of events) this.#rememberBlockTimestamp(event); + for (const event of events) { + if (isBlockConfirmation(event)) continue; + const blockNumber = uint64String(event.blockNumber, "HubEvent blockNumber", true); + const cacheKey = blockTimestampKey(shard, blockNumber); + let timestamp = this.#blockTimestamps.get(cacheKey); + if (timestamp === undefined) { + const blockEventId = (BigInt(blockNumber) << 14n).toString(); + const value = await this.#requestJson("v1/eventById", { + event_id: blockEventId, + shard_index: String(shard) + }, signal); + const confirmation = normalizeEvent(value, shard); + if (!isBlockConfirmation(confirmation) + || String(confirmation.id) !== blockEventId + || String(confirmation.blockNumber) !== blockNumber + || confirmation.timestamp === undefined) { + throw new Error("Hypersnap HTTP endpoint did not return the canonical block confirmation timestamp"); + } + this.#rememberBlockTimestamp(confirmation); + timestamp = this.#blockTimestamps.get(cacheKey); + } + if (timestamp === undefined) throw new Error("canonical block timestamp was unavailable"); + if (event.timestamp !== undefined + && uint64String(event.timestamp, "HubEvent timestamp", true) !== timestamp) { + throw new Error("HubEvent timestamp conflicted with its canonical block confirmation"); + } + event.timestamp = timestamp; + } + return events; + } + + #rememberBlockTimestamp(event: RawHubEvent): void { + if (!isBlockConfirmation(event) || event.timestamp === undefined) return; + const blockNumber = uint64String(event.blockNumber, "block confirmation number", true); + const shard = positiveShard(Number(event.shardIndex)); + const cacheKey = blockTimestampKey(shard, blockNumber); + const timestamp = uint64String(event.timestamp, "block confirmation timestamp", true); + const known = this.#blockTimestamps.get(cacheKey); + if (known !== undefined && known !== timestamp) { + throw new Error("block confirmation timestamp conflicted with a previously observed value"); + } + if (known === undefined) { + this.#blockTimestamps.set(cacheKey, timestamp); + if (this.#blockTimestamps.size > MAX_BLOCK_TIMESTAMP_CACHE_ENTRIES) { + const oldest = this.#blockTimestamps.keys().next().value as string | undefined; + if (oldest !== undefined) this.#blockTimestamps.delete(oldest); + } + } + } + + async #requestJson(path: string, query: Record, signal?: AbortSignal): Promise { + this.#assertOpen(); + if (signal?.aborted) throw abortError("Hypersnap HTTP request aborted"); + await this.#requestStartGate.waitForStart(signal); + const url = new URL(path, this.#baseUrl); + for (const [name, value] of Object.entries(query)) url.searchParams.set(name, value); + const requestScope = requestAbortScope(signal, this.#closed.signal, this.#timeoutMs); + try { + const response = await this.#fetcher(url, { + method: "GET", + headers: { accept: "application/json" }, + redirect: "error", + credentials: "omit", + cache: "no-store", + signal: requestScope.signal + }); + const contentType = response.headers.get("content-type") ?? ""; + if (!/^application\/json(?:\s*;|$)/i.test(contentType)) { + throw new Error("Hypersnap HTTP response content-type was not application/json"); + } + if (!response.ok) throw new Error(`Hypersnap HTTP request failed with status ${response.status}`); + const text = await readBoundedUtf8(response, this.#maxResponseBytes, requestScope.signal); + try { + return JSON.parse(quoteUnsafeJsonIntegers(text)) as unknown; + } catch (error) { + throw new Error("Hypersnap HTTP response contained invalid JSON", { cause: error }); + } + } catch (error) { + if (requestScope.signal.aborted) throw abortReason(requestScope.signal.reason); + throw error; + } finally { + requestScope.cleanup(); + } + } + + #assertOpen(): void { + if (this.#closed.signal.aborted) throw abortError("Hypersnap HTTP RPC client is closed"); + } +} + +function validateBaseUrl(value: string | URL): URL { + let url: URL; + try { + url = new URL(value.toString()); + } catch { + throw new Error("Hypersnap HTTP base URL must be an absolute HTTPS URL"); + } + if (url.protocol !== "https:") throw new Error("Hypersnap HTTP base URL must use HTTPS"); + if (url.username || url.password) throw new Error("Hypersnap HTTP base URL must not contain credentials"); + if (url.search || url.hash) throw new Error("Hypersnap HTTP base URL must not contain a query or fragment"); + if (!url.hostname) throw new Error("Hypersnap HTTP base URL must contain a hostname"); + url.pathname = `${url.pathname.replace(/\/+$/, "")}/`; + return url; +} + +function optionalPeerId(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const result = value.trim(); + if (!result || result.length > 256 || /\s/.test(result) || hasControlCharacters(result)) { + throw new Error("expectedPeerId must be a nonempty peer identifier without whitespace"); + } + return result; +} + +function boundedInteger(value: number | undefined, fallback: number, minimum: number, maximum: number, name: string): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new Error(`${name} must be an integer from ${minimum} through ${maximum}`); + } + return result; +} + +function positiveShard(value: number): number { + if (!Number.isSafeInteger(value) || value < 1 || value > MAX_UINT32) { + throw new Error("shard index must be a positive uint32"); + } + return value; +} + +function uint64String(value: unknown, name: string, allowZero: boolean): string { + const text = typeof value === "string" + ? value + : typeof value === "number" && Number.isSafeInteger(value) ? String(value) : ""; + if (!/^\d+$/.test(text)) throw new Error(`${name} must be an exact decimal uint64 string`); + let parsed: bigint; + try { + parsed = BigInt(text); + } catch { + throw new Error(`${name} must be an exact decimal uint64 string`); + } + if (parsed > MAX_UINT64 || (!allowZero && parsed === 0n)) throw new Error(`${name} is outside the supported uint64 range`); + return parsed.toString(); +} + +function safeInteger(value: unknown, name: string, minimum: number, maximum: number): number { + const text = typeof value === "string" && /^\d+$/.test(value) ? value : undefined; + const result = text === undefined ? value : Number(text); + if (typeof result !== "number" || !Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new Error(`${name} must be a safe integer from ${minimum} through ${maximum}`); + } + return result; +} + +function requiredString(value: unknown, name: string, maximumLength: number): string { + if (typeof value !== "string" || !value || value.length > maximumLength || hasControlCharacters(value)) { + throw new Error(`${name} must be a nonempty bounded string`); + } + return value; +} + +function record(value: unknown, name: string): JsonRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${name} must be an object`); + return value as JsonRecord; +} + +function array(value: unknown, name: string, maximumLength: number): unknown[] { + if (!Array.isArray(value) || value.length > maximumLength) throw new Error(`${name} must be a bounded array`); + return value; +} + +function normalizeEvent(value: unknown, expectedShard: number): RawHubEvent { + const source = record(value, "HubEvent"); + const id = uint64String(source.id, "HubEvent id", false); + const shardIndex = safeInteger(source.shardIndex, "HubEvent shardIndex", 1, MAX_UINT32); + if (shardIndex !== expectedShard) throw new Error("HubEvent shardIndex did not match the requested shard"); + const eventType = knownHubEventType(source.type); + const normalized: JsonRecord = { ...source, type: eventType, id, shardIndex }; + const blockNumber = uint64String(source.blockNumber, "HubEvent blockNumber", true); + if ((BigInt(id) >> 14n).toString() !== blockNumber) { + throw new Error("HubEvent id did not encode its declared block number"); + } + normalized.blockNumber = blockNumber; + + const mergeBody = optionalRecord(source.mergeMessageBody); + const message = optionalRecord(mergeBody?.message); + const data = optionalRecord(message?.data); + const isMergeMessage = eventType === 1 || eventType === "HUB_EVENT_TYPE_MERGE_MESSAGE"; + if (isMergeMessage && data === undefined) throw new Error("merge HubEvent did not contain message data"); + if (data !== undefined) { + const normalizedData: JsonRecord = { ...data }; + const messageType = knownMessageType(data.type); + normalizedData.type = messageType; + normalizedData.fid = uint64String(data.fid, "message fid", false); + if (data.network !== 1 && data.network !== "1" && data.network !== "FARCASTER_NETWORK_MAINNET") { + throw new Error("message network was not Farcaster mainnet"); + } + normalizedData.network = "FARCASTER_NETWORK_MAINNET"; + const messageTimestamp = uint64String(data.timestamp, "message timestamp", true); + validateFarcasterTimestamp(messageTimestamp, "message timestamp"); + normalizedData.timestamp = messageTimestamp; + const hash = requiredString(message?.hash, "message hash", 42); + if (!/^0x[0-9a-f]{40}$/i.test(hash)) throw new Error("message hash was not a 20-byte hex value"); + normalized.mergeMessageBody = { ...mergeBody, message: { ...message, hash: hash.toLowerCase(), data: normalizedData } }; + } + + const blockBody = optionalRecord(source.blockConfirmedBody); + const hasBlockConfirmationType = eventType === 11 || eventType === "HUB_EVENT_TYPE_BLOCK_CONFIRMED"; + if (hasBlockConfirmationType && blockBody === undefined) throw new Error("block-confirmed HubEvent did not contain its body"); + if (blockBody !== undefined) { + const normalizedBlock: JsonRecord = { ...blockBody }; + if (blockBody.blockNumber !== undefined) { + normalizedBlock.blockNumber = uint64String(blockBody.blockNumber, "block number", true); + if (normalizedBlock.blockNumber !== blockNumber) throw new Error("block confirmation number did not match its HubEvent"); + } + if (blockBody.shardIndex !== undefined && safeInteger(blockBody.shardIndex, "block shardIndex", 1, MAX_UINT32) !== expectedShard) { + throw new Error("block confirmation shard did not match its HubEvent"); + } + if (blockBody.timestamp !== undefined) { + const blockTimestamp = uint64String(blockBody.timestamp, "block timestamp", true); + validateFarcasterTimestamp(blockTimestamp, "block timestamp"); + normalizedBlock.timestamp = blockTimestamp; + normalized.timestamp = blockTimestamp; + } + if (blockBody.totalEvents !== undefined) normalizedBlock.totalEvents = uint64String(blockBody.totalEvents, "block totalEvents", true); + normalized.blockConfirmedBody = normalizedBlock; + } + + if (source.timestamp !== undefined) { + const eventTimestamp = uint64String(source.timestamp, "HubEvent timestamp", true); + validateFarcasterTimestamp(eventTimestamp, "HubEvent timestamp"); + if (isBlockConfirmation(normalized as RawHubEvent) + && normalized.timestamp !== undefined + && String(normalized.timestamp) !== eventTimestamp) { + throw new Error("block-confirmed HubEvent timestamp conflicted with its body timestamp"); + } + normalized.timestamp = eventTimestamp; + } + return normalized as RawHubEvent; +} + +function isBlockConfirmation(event: RawHubEvent): boolean { + return event.type === 11 || event.type === "11" || event.type === "HUB_EVENT_TYPE_BLOCK_CONFIRMED"; +} + +function blockTimestampKey(shard: number, blockNumber: string): string { + return `${shard}:${blockNumber}`; +} + +function knownHubEventType(value: unknown): number | string { + const knownNames = new Set([ + "HUB_EVENT_TYPE_NONE", + "HUB_EVENT_TYPE_MERGE_MESSAGE", + "HUB_EVENT_TYPE_PRUNE_MESSAGE", + "HUB_EVENT_TYPE_REVOKE_MESSAGE", + "HUB_EVENT_TYPE_MERGE_USERNAME_PROOF", + "HUB_EVENT_TYPE_MERGE_ON_CHAIN_EVENT", + "HUB_EVENT_TYPE_MERGE_FAILURE", + "HUB_EVENT_TYPE_BLOCK_CONFIRMED", + "HUB_EVENT_TYPE_CHANNEL_OWNER_CHANGE_HINT" + ]); + if (typeof value === "string" && knownNames.has(value)) return value; + const numeric = typeof value === "number" && Number.isInteger(value) + ? value + : typeof value === "string" && /^\d+$/.test(value) ? Number(value) : -1; + if ([0, 1, 2, 3, 6, 9, 10, 11, 12].includes(numeric)) return numeric; + throw new Error("HubEvent type was not in the reviewed enum set"); +} + +function knownMessageType(value: unknown): number | string { + const knownNames = new Set([ + "MESSAGE_TYPE_NONE", + "MESSAGE_TYPE_CAST_ADD", + "MESSAGE_TYPE_CAST_REMOVE", + "MESSAGE_TYPE_REACTION_ADD", + "MESSAGE_TYPE_REACTION_REMOVE", + "MESSAGE_TYPE_LINK_ADD", + "MESSAGE_TYPE_LINK_REMOVE", + "MESSAGE_TYPE_VERIFICATION_ADD_ETH_ADDRESS", + "MESSAGE_TYPE_VERIFICATION_REMOVE", + "MESSAGE_TYPE_USER_DATA_ADD", + "MESSAGE_TYPE_USERNAME_PROOF", + "MESSAGE_TYPE_FRAME_ACTION", + "MESSAGE_TYPE_LINK_COMPACT_STATE", + "MESSAGE_TYPE_LEND_STORAGE", + "MESSAGE_TYPE_KEY_ADD", + "MESSAGE_TYPE_KEY_REMOVE", + "MESSAGE_TYPE_CHANNEL_UPDATE", + "MESSAGE_TYPE_CHANNEL_MEMBER", + "MESSAGE_TYPE_CHANNEL_PIN", + "MESSAGE_TYPE_CHANNEL_MODERATE" + ]); + if (typeof value === "string" && knownNames.has(value)) return value; + const numeric = typeof value === "number" && Number.isInteger(value) + ? value + : typeof value === "string" && /^\d+$/.test(value) ? Number(value) : -1; + if ((numeric >= 0 && numeric <= 8) || (numeric >= 11 && numeric <= 21)) return numeric; + throw new Error("message type was not in the reviewed enum set"); +} + +function validateFarcasterTimestamp(value: string, name: string): void { + const seconds = BigInt(value); + const maximum = BigInt(Math.floor(Date.now() / 1_000) - FARCASTER_EPOCH_SECONDS + MAX_FUTURE_SKEW_SECONDS); + if (seconds <= 0n || seconds > maximum) throw new Error(`${name} was outside the accepted canonical time range`); +} + +function optionalRecord(value: unknown): JsonRecord | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonRecord : undefined; +} + +function hasControlCharacters(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code <= 31 || code === 127) return true; + } + return false; +} + +function validateForwardPage(events: readonly RawHubEvent[], startId: string, stopId: string | undefined): void { + let previous: string | undefined; + for (const event of events) { + const id = String(event.id); + if (compareUint64(id, startId) < 0) throw new Error("Hypersnap HTTP forward page preceded its requested start id"); + if (stopId !== undefined && compareUint64(id, stopId) >= 0) throw new Error("Hypersnap HTTP forward page crossed its exclusive stop id"); + if (previous !== undefined && compareUint64(id, previous) <= 0) throw new Error("Hypersnap HTTP forward page was not strictly ordered"); + previous = id; + } +} + +function compareUint64(left: string, right: string): number { + const a = BigInt(left); + const b = BigInt(right); + return a < b ? -1 : a > b ? 1 : 0; +} + +function incrementUint64(value: string): string { + const next = BigInt(uint64String(value, "event id", true)) + 1n; + if (next > MAX_UINT64) throw new Error("event id reached the uint64 maximum"); + return next.toString(); +} + +function encodePageToken(shard: number, nextId: string): Uint8Array { + return new TextEncoder().encode(`${PAGE_TOKEN_PREFIX}:${shard}:${nextId}`); +} + +function decodePageToken(token: Uint8Array, expectedShard: number): string { + if (token.length === 0 || token.length > 128) throw new Error("invalid Hypersnap HTTP page token"); + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(token); + } catch { + throw new Error("invalid Hypersnap HTTP page token"); + } + const match = /^snapmeter:hypersnap-http:v1:(\d+):(\d+)$/.exec(text); + if (match === null || Number(match[1]) !== expectedShard) throw new Error("invalid Hypersnap HTTP page token"); + return uint64String(match[2], "page token event id", true); +} + +async function readBoundedUtf8(response: Response, maximumBytes: number, signal: AbortSignal): Promise { + const declared = response.headers.get("content-length"); + if (declared !== null) { + if (!/^\d+$/.test(declared) || BigInt(declared) > BigInt(maximumBytes)) { + throw new Error("Hypersnap HTTP response exceeded the body-size limit"); + } + } + if (response.body === null) throw new Error("Hypersnap HTTP response body was empty"); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + while (true) { + if (signal.aborted) throw abortReason(signal.reason); + const result = await reader.read(); + if (result.done) break; + total += result.value.byteLength; + if (total > maximumBytes) throw new Error("Hypersnap HTTP response exceeded the body-size limit"); + chunks.push(result.value); + } + } catch (error) { + await reader.cancel().catch(() => undefined); + throw error; + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (error) { + throw new Error("Hypersnap HTTP response was not valid UTF-8", { cause: error }); + } +} + +/** Quote only unsafe integer tokens before JSON.parse so uint64 values remain exact. */ +function quoteUnsafeJsonIntegers(json: string): string { + let output = ""; + let index = 0; + let inString = false; + let escaped = false; + while (index < json.length) { + const character = json[index] as string; + if (inString) { + output += character; + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === "\"") inString = false; + index += 1; + continue; + } + if (character === "\"") { + inString = true; + output += character; + index += 1; + continue; + } + if (character === "-" || /\d/.test(character)) { + const match = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/.exec(json.slice(index)); + if (match !== null) { + const token = match[0]; + if (!/[.eE]/.test(token) && unsafeIntegerToken(token)) output += JSON.stringify(token); + else output += token; + index += token.length; + continue; + } + } + output += character; + index += 1; + } + return output; +} + +function unsafeIntegerToken(token: string): boolean { + const digits = token.startsWith("-") ? token.slice(1) : token; + if (digits.length > 16) return true; + return BigInt(token) > BigInt(Number.MAX_SAFE_INTEGER) || BigInt(token) < BigInt(Number.MIN_SAFE_INTEGER); +} + +interface AbortScope { + signal: AbortSignal; + cleanup(): void; +} + +function requestAbortScope(caller: AbortSignal | undefined, closed: AbortSignal, timeoutMs: number): AbortScope { + const controller = new AbortController(); + const signals = caller === undefined ? [closed] : [caller, closed]; + const listeners: Array<{ signal: AbortSignal; listener: () => void }> = []; + const relay = (source: AbortSignal): void => { + if (!controller.signal.aborted) controller.abort(source.reason ?? abortError("Hypersnap HTTP request aborted")); + }; + for (const source of signals) { + if (source.aborted) relay(source); + else { + const listener = (): void => relay(source); + source.addEventListener("abort", listener, { once: true }); + listeners.push({ signal: source, listener }); + } + } + const timeout = setTimeout(() => { + if (controller.signal.aborted) return; + const error = new Error(`Hypersnap HTTP request timed out after ${timeoutMs}ms`); + error.name = "TimeoutError"; + controller.abort(error); + }, timeoutMs); + return { + signal: controller.signal, + cleanup: () => { + clearTimeout(timeout); + for (const entry of listeners) entry.signal.removeEventListener("abort", entry.listener); + } + }; +} + +function abortableDelay(milliseconds: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const timeout = setTimeout(finish, milliseconds); + const onAbort = (): void => finish(); + function finish(): void { + clearTimeout(timeout); + signal.removeEventListener("abort", onAbort); + resolve(); + } + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +function abortError(message: string): Error { + const error = new Error(message); + error.name = "AbortError"; + return error; +} + +function abortReason(reason: unknown): Error { + return reason instanceof Error ? reason : abortError("Hypersnap HTTP request aborted"); +} + +function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/apps/collector/src/rpc.ts b/apps/collector/src/rpc.ts index c3300a9..799e956 100644 --- a/apps/collector/src/rpc.ts +++ b/apps/collector/src/rpc.ts @@ -1,4 +1,6 @@ import { SnapchainRpcClient, type NodeInfo, type RawHubEvent, type RpcConfig } from "@snapmeter/protocol"; +import type { RpcTransportConfig } from "./config.js"; +import { HypersnapHttpRpc } from "./hypersnap-http-rpc.js"; export interface RpcSubscription { cancel(): void; @@ -25,9 +27,21 @@ export interface CollectorRpc { close(): void; } -export type RpcFactory = (config: RpcConfig) => CollectorRpc; +export type RpcFactory = (config: RpcTransportConfig) => CollectorRpc; -export const defaultRpcFactory: RpcFactory = (config) => new ProtocolRpcAdapter(new SnapchainRpcClient(config)); +export const defaultRpcFactory: RpcFactory = (config) => { + if (config.transport === "https-json") { + if (config.authorization || config.apiKey) throw new Error("Hypersnap HTTP public reads do not accept collector credentials"); + return new HypersnapHttpRpc({ + baseUrl: config.url, + expectedPeerId: config.expectedPeerId, + timeoutMs: config.timeoutMs, + pollIntervalMs: config.pollIntervalMs, + minimumIntervalMs: config.getEventsMinIntervalMs + }); + } + return new ProtocolRpcAdapter(new SnapchainRpcClient(config as RpcConfig)); +}; class ProtocolRpcAdapter implements CollectorRpc { constructor(readonly client: SnapchainRpcClient) {} diff --git a/apps/dashboard/worker/index.worker.test.ts b/apps/dashboard/worker/index.worker.test.ts index 0f04556..df8323f 100644 --- a/apps/dashboard/worker/index.worker.test.ts +++ b/apps/dashboard/worker/index.worker.test.ts @@ -409,14 +409,23 @@ describe("Worker API", () => { const fresh = batch(); fresh.cursors = [{ source: "snapchain", shard: 1, eventId: "11", verifiedAtMs: Date.now() }]; + // The Cloudflare harness can take long enough for this test to cross a + // UTC minute boundary. Saturate both the current and immediately next + // window so the assertion tests replay accounting, not wall-clock luck. + const requestWindowStartMs = Math.floor(Date.now() / 60_000) * 60_000; + for (const saturatedWindow of [requestWindowStartMs, requestWindowStartMs + 60_000]) { + await env.DB.prepare( + "INSERT INTO rate_windows(source, collector_id, window_start_ms, batch_count) VALUES ('snapchain', ?, ?, 300) ON CONFLICT(source, collector_id, window_start_ms) DO UPDATE SET batch_count=300" + ).bind(SOURCE_RATE_SCOPE, saturatedWindow).run(); + } const limited = await SELF.fetch(await signedRequest(fresh)); expect(limited.status).toBe(429); expect(await limited.json()).toMatchObject({ error: "rate_limited" }); } finally { await env.DB.prepare("DELETE FROM ingest_batches WHERE batch_id=?").bind(payload.batchId).run(); await env.DB.prepare( - "DELETE FROM rate_windows WHERE source='snapchain' AND collector_id=? AND window_start_ms=?" - ).bind(SOURCE_RATE_SCOPE, windowStartMs).run(); + "DELETE FROM rate_windows WHERE source='snapchain' AND collector_id=? AND window_start_ms BETWEEN ? AND ?" + ).bind(SOURCE_RATE_SCOPE, windowStartMs, Math.floor(Date.now() / 60_000) * 60_000 + 60_000).run(); } }); diff --git a/docs/architecture.md b/docs/architecture.md index 6947b17..7e61c12 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -5,16 +5,16 @@ SnapMeter separates private node access from public reads: ```text -Snapchain HubService ----+ signed, replay-safe batches - +--> collector ------------------------+ -Hypersnap HubService ----+ SQLite + outbox + per-shard state | - v +Snapchain HubService --------------------+ signed, replay-safe batches + +--> collector ------------------------+ +Hypersnap local gRPC (preferred) --------+ SQLite + outbox + per-shard state | +Hypersnap HTTPS events (fallback only) --+ v mobile/desktop browser <---- Worker read API <---- D1 + LiveRoom Durable Object ^ | +--------------- hibernating WebSocket -----------+ ``` -The collector is the only component that talks to node gRPC endpoints. The Worker is the only public ingress. Browsers use same-origin HTTP reads and a public read-only WebSocket; they never see RPC or ingest credentials. +The collector is the only component that talks to node RPC endpoints. Snapchain and the preferred local Hypersnap source use gRPC; the optional Hypersnap fallback uses a read-only HTTPS JSON API. The Worker is the only public write ingress. Browsers use same-origin HTTP reads and a public read-only WebSocket; they never see RPC or ingest credentials. ## Collector @@ -35,6 +35,16 @@ Each `(source instance, shard)` owns: The local database retains only analytics metadata: event identity/type, FID, action/receipt times, replay classification, actor-window/day membership, compact time buckets, source health, cursors, and outbox state. It does not retain or upload cast text, signatures, or private RPC metadata. Retention is bounded but must preserve at least 31 days needed for exact metric reconstruction. +### Hypersnap endpoint state machine + +The Hypersnap source has one preferred role and at most one fallback role. They are replicas for one source identity, not two streams: only one client/session and one set of shard workers is active at a time. + +Before activation, either candidate must pass exact version/peer pins where configured, full positive-shard coverage, the maximum block-delay threshold, durable endpoint enrollment, and cursor continuity. For each nonzero durable cursor, `GetEvent`/`eventById` must return the same source/shard/event ID and normalized SHA-256 event fingerprint already stored locally. This prevents silent history replacement when changing transport or operator. It does not prove events not yet observed by the collector. + +Startup tries preferred local gRPC first and then the HTTPS fallback. An active endpoint is abandoned after the configured count of repeated discovery failures or sustained incomplete coverage. While fallback is active, successful preferred probes accumulate at the configured recovery interval; a failed probe resets the count. Reaching the recovery-success threshold closes the fallback session, clears transient connection state, opens the preferred endpoint, and reconciles from the same durable cursors. Any incompatibility leaves the source unavailable/partial rather than resetting history. + +The HTTP fallback maps `/v1/info`, `/v1/eventById`, and `/v1/events` to the collector RPC boundary. There is no public streaming method, so its `Subscribe` role is a bounded poller; `GetEvents` remains the durable authority. Default five-second head polling and one-second global request-start pacing keep two idle shard pollers near 0.4 requests per second while preserving request capacity for replay, trading a few seconds of pulse latency for bounded pressure on a public service. + ## Replay and completeness `Subscribe.from_id` and `GetEvents.start_id` are inclusive. `GetEvents.stop_id` is exclusive. Reconciliation freezes a highest already-handled live event ID as its exclusive bound, pages one shard at a time, overlaps the last durable ID, and deduplicates. Because the bound event entered through the same idempotent handler before it became the bound, the scan fills lower gaps without applying that event twice. @@ -79,4 +89,7 @@ D1 holds compact minute buckets, daily metrics/membership, snapshots, source sta | Cloud outage | Keep aggregates in the durable outbox and do not claim cloud acknowledgement. | | Duplicate/out-of-order batch | Reject replay or idempotently return the existing result; do not double-count. | | Stale collector | Public source status becomes stale/disconnected even if the site itself remains available. | -| Hypersnap RPC unavailable | Hypersnap becomes `unavailable`; derived metrics are not promoted to verified. | +| Preferred Hypersnap RPC unavailable | Activate the enrolled HTTPS replica only after its identity, shards, delay, cursor, and fingerprints validate; otherwise Hypersnap becomes `unavailable`. | +| Fallback active and preferred recovers | Require consecutive healthy preferred probes, then switch one session at a time and reconcile before trusting live pulses. | +| Endpoint identity/version/shards drift | Reject activation. An environment change alone cannot override durable enrollment. | +| Cursor predates fallback retention | Reject fallback continuity and remain unavailable/partial; never reset the cursor. | diff --git a/docs/data-sources.md b/docs/data-sources.md index 10dd58c..a52b0f8 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -39,6 +39,48 @@ It does not mean the Hyper write succeeded. `[hyper].enabled=true`, the `hyper:v Hypersnap also uses upstream internal gRPC port `3383`. The SnapMeter default `127.0.0.1:4383` is solely a loopback host remap to avoid colliding with a co-located Snapchain node. Upstream configuration still uses the `SNAPCHAIN_` prefix; there is no upstream `HYPERSNAP_` namespace at the inspected commit. +### Preferred local node and HTTPS fallback + +SnapMeter can treat two Hypersnap endpoints as ordered replicas of the same canonical event source; it never sums or concurrently presents them as independent networks: + +1. `HYPERSNAP_GRPC_URL` is the preferred local/private native gRPC endpoint. +2. `HYPERSNAP_FALLBACK_HTTP_URL` is an optional read-only HTTPS JSON endpoint used only while the preferred endpoint cannot pass activation/health checks. +3. While fallback is active, the collector periodically probes the preferred endpoint and switches back only after `HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES` consecutive successful probes. + +The HTTPS adapter calls `/v1/info`, `/v1/eventById`, and ordered `/v1/events`. Because the public API has no streaming route, it simulates the latency channel with bounded head/forward polling; durable fixed-range event reconciliation remains authoritative. `HYPERSNAP_FALLBACK_POLL_INTERVAL_MS` controls polling and `HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS` paces all fallback request starts across shards. + +| Environment variable | Public-template value | Purpose | +|---|---:|---| +| `HYPERSNAP_EXPECTED_PEER_ID` | blank | Optional exact preferred/local peer pin. | +| `HYPERSNAP_EXPECTED_VERSION` | blank | Optional exact preferred/local version pin. | +| `HYPERSNAP_RPC_TIMEOUT_MS` | `5000` | Per-call timeout shared by both Hypersnap roles, independent of any longer hosted Snapchain timeout; allowed range 250-120,000 ms. | +| `HYPERSNAP_FALLBACK_HTTP_URL` | `https://haatz.quilibrium.com` | Optional fallback base URL; clearing it disables the fallback when both identity pins are also cleared. | +| `HYPERSNAP_FALLBACK_EXPECTED_PEER_ID` | `12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc` | Mandatory exact public-role peer pin. | +| `HYPERSNAP_FALLBACK_EXPECTED_VERSION` | `0.13.3` | Mandatory exact public-role version pin. | +| `HYPERSNAP_FALLBACK_POLL_INTERVAL_MS` | `5000` | Delay between HTTP head polls; across two idle shards this is about 0.4 requests/second, leaving request capacity for replay; allowed range 250-60,000 ms. | +| `HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS` | `1000` | Minimum delay between all fallback request starts; allowed range 0-3,600,000 ms. | +| `HYPERSNAP_FAILOVER_AFTER_FAILURES` | `3` | Repeated active-endpoint discovery failures before switching; allowed range 1-100. | +| `HYPERSNAP_PREFERRED_RECOVERY_INTERVAL_MS` | `60000` | Preferred probe interval while fallback is active; allowed range 5,000-3,600,000 ms. | +| `HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES` | `3` | Consecutive preferred probes required before returning; allowed range 1-100. | +| `HYPERSNAP_MAX_BLOCK_DELAY_SECONDS` | `30` | Maximum per-data-shard delay allowed for either role; allowed range 0-86,400 seconds. | + +The public default is `https://haatz.quilibrium.com`. On 2026-08-13 it was the only currently healthy public endpoint displayed by the [official Hypersnap portal](https://hypersnap.org/), and its advertised data shards reported zero block delay during verification. The portal says it checks nodes every minute, but publishes no historical uptime series or node-creation evidence. SnapMeter therefore makes **no claim** that this endpoint is the oldest node or has the highest historical uptime; it is a reviewed current fallback, not an availability SLA. Upstream's [node guide](https://hypersnap.org/run-a-node) documents HTTP `3381` and gRPC `3383`, while the public hostname exposes the reviewed HTTP API through HTTPS. + +The observed Haatz `0.13.3` version corresponds to upstream Hypersnap commit `ce408646fd09d886f275b74757341a1d328728ab`. A source diff from that commit to SnapMeter's inspected `2eee4c9f2a7833ce7971dfef028480abbe9c4720` pin changes only the root `Cargo.toml` package version from `0.13.3` to `0.13.4`; no protocol or runtime source differs. That is compatibility evidence for this adapter review, **not binary attestation**: the remote endpoint self-reports its version and peer ID, and SnapMeter cannot prove what commit produced the running binary. The exact observed values are change-detection pins only. + +The fallback is fail-closed: + +- HTTPS is mandatory and URLs containing credentials, query strings, or fragments are rejected. +- `HYPERSNAP_FALLBACK_EXPECTED_PEER_ID` and `HYPERSNAP_FALLBACK_EXPECTED_VERSION` are mandatory exact pins. The checked-in public values are public identity metadata, not credentials. +- `HYPERSNAP_MAX_BLOCK_DELAY_SECONDS` applies to both roles, and every expected positive data shard must be present before activation. +- The first accepted `(role, transport, canonical URL, peer ID, version, shard set)` is durably enrolled. Later drift is rejected even if the environment pin was edited. +- A candidate must return the durable cursor event for every shard, and its normalized event fingerprint must match local history. A fallback cannot adopt an older cursor that lacks a stored fingerprint. +- Only one role is active. Switching resets transient source state, resumes inclusively from the shared durable per-shard cursor, reconciles, and deduplicates before any new live pulse can be trusted. + +The peer ID is reported by `/v1/info`; it is not a cryptographic attestation of every event or of the endpoint operator. TLS authenticates the configured hostname, and fingerprint/cursor checks detect conflicting history already observed by SnapMeter, but a public replica remains a third-party trust dependency. It cannot change Hypersnap from `derived` to `verified`. + +A retention probe of the public endpoint during implementation exposed only roughly three days of canonical events. Retention is operator-controlled and not guaranteed by the portal. The fallback therefore cannot reconstruct a 30-day cold start and may be unable to bridge an outage older than its retained cursor. In either case SnapMeter fails closed or remains visibly partial; it never resets the cursor or fabricates older coverage. + ## Discovery and health For each endpoint, the collector: diff --git a/docs/local-reconstruction.md b/docs/local-reconstruction.md index 9321c7f..cf09432 100644 --- a/docs/local-reconstruction.md +++ b/docs/local-reconstruction.md @@ -44,7 +44,7 @@ Optional by task: - PowerShell 5.1 or PowerShell 7 on Windows for the bootstrap and Scheduled Task scripts. - A local Docker engine; Docker Compose 2.24.4 or later is required only for the documented `!override` node-port fragment. - A Cloudflare account for remote deployment. The workspace already pins Wrangler, so no global Wrangler installation is needed. -- Private or authenticated Snapchain-compatible gRPC endpoints for live collection. +- A private/authenticated Snapchain-compatible gRPC endpoint for live Snapchain collection; Hypersnap may use a local gRPC node and the optional reviewed read-only HTTPS fallback. Confirm the required toolchain before installing: @@ -132,17 +132,25 @@ SNAPCHAIN_GRPC_URL=127.0.0.1:3383 HYPERSNAP_GRPC_URL=127.0.0.1:4383 SNAPCHAIN_GRPC_TLS=false HYPERSNAP_GRPC_TLS=false +HYPERSNAP_RPC_TIMEOUT_MS=5000 +HYPERSNAP_FALLBACK_HTTP_URL=https://haatz.quilibrium.com +HYPERSNAP_FALLBACK_EXPECTED_PEER_ID=12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc +HYPERSNAP_FALLBACK_EXPECTED_VERSION=0.13.3 ``` Both upstream node types use internal gRPC port `3383`. Port `4383` is only the loopback host remap for a co-located Hypersnap node. Never publish either native plaintext RPC port to the Internet. +The public template configures local Hypersnap gRPC as preferred and an HTTPS canonical-event API as fallback. The [official Hypersnap portal](https://hypersnap.org/) listed that endpoint as healthy when the configuration was reviewed, but it does not provide historical uptime or node-age proof. Peer ID and version are exact public identity pins, not secrets; a mismatch fails closed. The HTTP API has no event stream, so the collector polls its canonical `/v1/events` route while retaining ordered reconciliation and the same durable per-shard cursors. + +This fallback is a convenience/trust dependency, not private reconstruction material and not an independent Hyper-write source. It keeps `HYPERSNAP_SOURCE_MODE=derived`. A live probe found only roughly three days of event retention, so a new database remains partial for the 30-day window until enough prospective history accumulates. To reconstruct without a third-party public source, leave all `HYPERSNAP_FALLBACK_*` values blank together and run only the reviewed local node; if neither is available, set `HYPERSNAP_SOURCE_MODE=unavailable`. + Choose a data directory outside the Git checkout. On Windows the default is `%LOCALAPPDATA%\SnapMeter`; an explicit secondary-drive layout is also valid: ```dotenv SNAPMETER_DATA_DIR=D:\SnapMeter\collector ``` -The collector creates and migrates `snapmeter.sqlite3` itself, including its WAL, cursors, local identity, schema-v3 actor pseudonym key, bounded analytics state, and durable delivery outbox. These files must remain untracked. A clean reconstruction creates a new collector ID and key inside its new database; neither value comes from production or belongs in configuration. Never inspect, print, extract, or export the key separately. +The collector creates and migrates `snapmeter.sqlite3` itself, including its WAL, cursors, local identity, schema-v4 actor pseudonym key and endpoint enrollments, bounded analytics state, and durable delivery outbox. These files must remain untracked. A clean reconstruction creates a new collector ID and key inside its new database; neither value comes from production or belongs in configuration. Never inspect, print, extract, or export the key separately. Pair that new collector only with the clean local or remote D1 dataset created for the reconstruction. Migration `0005_collector_binding.sql` makes the first non-doctor delivery claim that dataset's global collector slot; an empty doctor probe only validates access and does not claim it. An existing production D1 dataset rejects a newly reconstructed database with HTTP 409 `collector_identity_conflict`. Production failover therefore restores the entire stopped, WAL-consistent collector state rather than reconstructing a new database; follow the [Windows runbook](windows-runbook.md). @@ -193,7 +201,7 @@ The last two commands must print the exact full SHAs above. Build and configure `docker-compose.nodes.override.yml` is an illustrative Compose 2.24.4+ fragment for a separately reviewed upstream-node project. It is not a standalone node launcher. It binds Snapchain host port `3383` and Hypersnap host port `4383` to each container's internal port `3383`, both on loopback. -An exact 30-day cold start requires at least 31 days of authoritative event retention or a separately trusted history source. The pinned default Snapchain HubEvent retention is only three days. Missing older history must remain visibly partial until sufficient prospective coverage accumulates. +An exact 30-day cold start requires at least 31 days of authoritative event retention or a separately trusted history source. The pinned default Snapchain HubEvent retention is only three days, and the reviewed public Hypersnap fallback exposed a similarly short observed range. Missing older history must remain visibly partial until sufficient prospective coverage accumulates. ## Reconstruct Cloudflare resources diff --git a/docs/security.md b/docs/security.md index 01ccf28..1fcde24 100644 --- a/docs/security.md +++ b/docs/security.md @@ -2,9 +2,9 @@ ## Assets and trust boundaries -Sensitive assets are the ingest HMAC secret, optional RPC authorization/API-key metadata, Cloudflare API token/account configuration, local actor/cursor database, and retry outbox. The schema-v3 collector database also contains the authoritative collector ID and actor pseudonym key. The public dashboard, summary APIs, and WebSocket clients are untrusted readers. Node RPC endpoints and collector state stay private. +Sensitive assets are the ingest HMAC secret, optional RPC authorization/API-key metadata, Cloudflare API token/account configuration, local actor/cursor database, and retry outbox. The schema-v4 collector database also contains the authoritative collector ID, actor pseudonym key, endpoint enrollments, and event fingerprints. The public dashboard, summary APIs, WebSocket clients, and optional third-party Hypersnap HTTPS replica are untrusted. Native node RPC endpoints and collector state stay private. -Primary threats are forged/replayed ingest, duplicate or out-of-order delivery, secret leakage through logs/bundles, public node exposure, oversized/schema-confusing requests, SQL injection, source-quality spoofing, WebSocket abuse, and unbounded local/cloud storage. +Primary threats are forged/replayed ingest, duplicate or out-of-order delivery, secret leakage through logs/bundles, public node exposure, malicious or drifting fallback data, oversized/schema-confusing requests, SQL injection, source-quality spoofing, WebSocket abuse, and unbounded local/cloud storage. ## Ingest controls @@ -35,7 +35,7 @@ Migration `0005_collector_binding.sql` gives each D1 dataset one global `collect For failover, stop the collector and take a SQLite-consistent backup of the entire collector state, including `snapmeter.sqlite3` and any associated WAL/SHM state. Restore or clone that stopped state as one unit so the collector ID, pseudonym key, cursors, deduplication records, and outbox stay aligned. Never inspect, print, extract, export, copy, or persist the pseudonym key separately. A fresh database is a new collector identity and is not a replacement for this backup. -Accepting an intentionally new database requires an explicit operator reset of the D1 collector binding: `DELETE FROM collector_binding WHERE slot=1`. Perform that statement only after backing up D1 and completing the metric-continuity plan. Treat the reset as a metric-data boundary: preferably perform it at a UTC-day boundary when no reconciliation will add actors to earlier retained days, or clear and deliberately rebuild the affected cloud actor-day membership before accepting the new ID. Changing the ingest secret does not reset this binding. Schema v3 is forward-only; if a release fails after migration, restore the stopped pre-upgrade database with its compatible binary or fix forward, rather than running a pre-v3 binary against a v3 database. +Accepting an intentionally new database requires an explicit operator reset of the D1 collector binding: `DELETE FROM collector_binding WHERE slot=1`. Perform that statement only after backing up D1 and completing the metric-continuity plan. Treat the reset as a metric-data boundary: preferably perform it at a UTC-day boundary when no reconciliation will add actors to earlier retained days, or clear and deliberately rebuild the affected cloud actor-day membership before accepting the new ID. Changing the ingest secret does not reset this binding. Schema v4 is forward-only; if a release fails after migration, restore the stopped pre-upgrade database with its compatible binary or fix forward, rather than running a pre-v4 binary against a v4 database. ## Secrets @@ -49,9 +49,27 @@ Accepting an intentionally new database requires an explicit operator reset of t If a secret is exposed, stop affected ingest, rotate the Worker secret/token, update collectors securely, review replay/idempotency logs, and invalidate the old value. Git history rewriting is a separate release-owner decision; deleting a working-tree file alone is insufficient. +## Public Hypersnap fallback + +The optional fallback is a read-only public HTTPS dependency and receives no SnapMeter ingest secret, API key, authorization metadata, cookie, raw browser request, or private collector state. Requests use `GET`, `Accept: application/json`, no credentials/cache, no redirects, bounded response bodies, strict JSON/type/uint64 validation, timeouts, and global request-start pacing. + +Controls before either Hypersnap role can become active include: + +- exact configured version and peer-ID pins for the public role; +- HTTPS hostname authentication with embedded URL credentials/query/fragment rejected; +- full positive-shard coverage and a bounded block-delay threshold; +- durable enrollment of role, transport, canonical URL, peer ID, version, and shard set; +- `GetEvent`/`eventById` cursor continuity and a SHA-256 fingerprint match against locally observed normalized event content; +- one active role at a time, followed by inclusive reconciliation and deduplication on every switch; +- hysteresis before returning to the preferred local node. + +These controls limit accidental drift, replay discontinuity, and equivocation against already observed history. They do not turn a public operator into a trusted authority: `/v1/info` self-reports the peer ID/version, TLS authenticates only the hostname, and an endpoint can still lie consistently about previously unseen canonical events. The fallback therefore remains `derived`, its availability is not an uptime claim, and operators who cannot accept the dependency must disable all fallback identity/URL values together or mark Hypersnap unavailable. + +The checked-in peer ID, version, and public URL are not secrets. An exact pin or durable enrollment mismatch is an intentional stop condition. Do not bypass it by deleting the collector database/enrollment; review the endpoint/upstream change, back up state, and ship an explicit re-enrollment migration. + ## Network and browser -Bind local native gRPC to loopback/private interfaces; it is plaintext at the inspected upstream commits. Use TLS termination and optional authorization for remote access. Never open inbound public firewall rules for collector convenience. +Bind local native gRPC to loopback/private interfaces; it is plaintext at the inspected upstream commits. Use TLS termination and optional authorization for remote gRPC access. Never open inbound public firewall rules for collector convenience. The Hypersnap fallback is outbound HTTPS only; the upstream [operator guide](https://hypersnap.org/run-a-node) does not justify exposing local `3381`/`3383` ports. Read APIs are same-origin by default. Apply restrictive security headers, no permissive credentialed CORS, and a CSP appropriate for bundled assets/WebSockets. Public WebSockets are read-only, schema-versioned, rate/connection-limited, and receive only aggregate packets. Durable Object messages must not contain secrets or raw social content. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ce5cdd7..a13ee67 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -24,6 +24,8 @@ Test-NetConnection 127.0.0.1 -Port 4383 - Check loopback binding and firewall before changing software. - Do not automatically start a large node; verify storage/bandwidth/retention plans first. +For the Hypersnap HTTPS fallback, do not use `Test-NetConnection 127.0.0.1`. Run `check-health.ps1` or `doctor`; they probe the configured `/v1/info` endpoint without printing credentials. The fallback URL must be absolute HTTPS with no embedded credentials, query, or fragment, and both expected peer ID and expected version are required. + ## Doctor reports protocol or shard problems - A `shard_infos` entry for shard 0 is normal; it is block health, not an event subscription shard. @@ -58,6 +60,20 @@ This can be correct. Idle sources have no fake beat, and replay/catch-up/reconci `DERIVED` is the expected honest mode at the pinned Hypersnap commit. Public RPC exposes canonical merges, not per-message Hyper outcomes. It cannot be changed by enabling `[hyper]` or observing `hyper:v1`. `unavailable` means the configured canonical source is disconnected or too stale/partial to support the inference. +## Hypersnap fallback does not activate or return local + +Look for redacted `source.endpoint_rejected`, `source.endpoint_switching`, and `source.preferred_probe_failed` records, then run `doctor`. Common fail-closed causes are: + +- `/v1/info` peer ID or version no longer matches the exact environment pin; +- peer/version/URL/shard set differs from the role's durable enrollment; +- one or more positive data shards is absent, or block delay exceeds `HYPERSNAP_MAX_BLOCK_DELAY_SECONDS`; +- `eventById` cannot return the current durable cursor because the public endpoint's roughly three-day retention has passed; +- the cursor event's normalized fingerprint conflicts with the event already stored locally; +- an upgraded/fresh database has a legacy cursor without a bound fingerprint, which a fallback is not allowed to adopt; +- the preferred node has not yet passed `HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES` consecutive probes at the configured interval. + +Do not fix these failures by clearing cursors, editing `source_endpoint_enrollment`, widening block delay without investigation, or changing the expected peer/version until the endpoint and upstream change have been reviewed. Stop the collector, back up the whole data directory, confirm the [official public-node listing](https://hypersnap.org/), inspect the current `/v1/info` response through `doctor`, and use a reviewed release/migration for intentional re-enrollment. If both roles are unusable, keep Hypersnap visibly unavailable. + ## Cloud ingest rejects a batch - Confirm collector and Worker secrets match without printing either. diff --git a/docs/windows-runbook.md b/docs/windows-runbook.md index 0bb517e..9e36e03 100644 --- a/docs/windows-runbook.md +++ b/docs/windows-runbook.md @@ -4,7 +4,7 @@ - Windows 11 with PowerShell 5.1 or PowerShell 7. - Node.js 24 or later and pnpm 11.19 available to the account that will run the task. -- Two reachable, private Snapchain-compatible gRPC endpoints, or an explicit decision to run only the available source. +- A reachable Snapchain source and either a preferred local Hypersnap gRPC node, the reviewed HTTPS fallback, or an explicit decision to mark the missing source unavailable. - A Cloudflare ingest URL and matching Wrangler-managed HMAC secret for production delivery. - NTFS storage with room for at least 31 days of bounded analytics state plus retry headroom. @@ -28,6 +28,18 @@ SNAPCHAIN_GRPC_URL=127.0.0.1:3383 HYPERSNAP_GRPC_URL=127.0.0.1:4383 SNAPCHAIN_GRPC_TLS=false HYPERSNAP_GRPC_TLS=false +HYPERSNAP_EXPECTED_PEER_ID= +HYPERSNAP_EXPECTED_VERSION= +HYPERSNAP_RPC_TIMEOUT_MS=5000 +HYPERSNAP_FALLBACK_HTTP_URL=https://haatz.quilibrium.com +HYPERSNAP_FALLBACK_EXPECTED_PEER_ID=12D3KooWMYfkXiNcn9LifPkLYiHtGmXYnknYG1yFBD53rUseUMUc +HYPERSNAP_FALLBACK_EXPECTED_VERSION=0.13.3 +HYPERSNAP_FALLBACK_POLL_INTERVAL_MS=5000 +HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS=1000 +HYPERSNAP_FAILOVER_AFTER_FAILURES=3 +HYPERSNAP_PREFERRED_RECOVERY_INTERVAL_MS=60000 +HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES=3 +HYPERSNAP_MAX_BLOCK_DELAY_SECONDS=30 SNAPCHAIN_GRPC_API_KEY= SNAPCHAIN_RPC_MIN_INTERVAL_MS=0 SNAPMETER_INGEST_URL= @@ -39,6 +51,8 @@ After a verified deployment, fill the URL with the exact origin plus `/api/v1/in For Neynar-hosted Snapchain, set `SNAPCHAIN_GRPC_URL=snapchain-grpc-api.neynar.com:443`, `SNAPCHAIN_GRPC_TLS=true`, and `SNAPCHAIN_GRPC_API_KEY` to the key from the Neynar developer portal. Set `SNAPCHAIN_RPC_MIN_INTERVAL_MS=250` to serialize GetEvents request starts across both shard workers at no more than four starts per second. If no independent Hypersnap source is available, set `HYPERSNAP_SOURCE_MODE=unavailable`. The collector never prints the API-key value. +The checked-in Hypersnap fallback is the public node that the [official portal](https://hypersnap.org/) showed healthy during implementation. The portal does not publish node age or historical uptime, so do not describe it as the oldest or highest-uptime node. Its exact peer/version pins are public identity metadata. Leave the optional preferred `HYPERSNAP_EXPECTED_*` values blank until you have recorded the local node's actual identity; the collector still durably enrolls the first accepted identity. + ## Interactive operation ```powershell @@ -51,6 +65,14 @@ For Neynar-hosted Snapchain, set `SNAPCHAIN_GRPC_URL=snapchain-grpc-api.neynar.c Stop an interactive run with `Ctrl+C`. Graceful shutdown preserves transaction boundaries and outbox state. Backfill is bounded reconciliation and must update historical buckets without visual pulses. +## Hypersnap failover operation + +`HYPERSNAP_GRPC_URL` remains preferred even when it is still syncing. At startup the collector uses the HTTPS fallback if the local endpoint is unreachable, incomplete, over `HYPERSNAP_MAX_BLOCK_DELAY_SECONDS`, or incompatible. During a fallback session it probes local every `HYPERSNAP_PREFERRED_RECOVERY_INTERVAL_MS`; the defaults require three consecutive successes before switching back. A failed preferred probe resets that recovery count. `HYPERSNAP_RPC_TIMEOUT_MS=5000` bounds both Hypersnap transports independently, so a longer Snapchain/Neynar timeout does not make fallback probes hang for the same duration. + +Activation is stricter than a successful TCP/HTTP probe. The collector requires all expected positive data shards, exact configured peer/version pins, a stable durable enrollment, and the same event fingerprint at each existing cursor. The HTTPS endpoint exposes canonical events only, so Hypersnap remains `derived`. A live probe found only about three days of retained events; expect the 30-day metric to remain partial after cold start, and expect failover to be rejected if the durable cursor has already fallen outside public retention. + +Use `doctor` before the first continuous run and after any endpoint, peer, or version change. If a reviewed upstream upgrade changes the version, endpoint identity, or shard set, an environment-pin edit alone is intentionally insufficient: stop the collector, back up its complete data directory, inspect the upstream/API change, and use a release-provided enrollment migration. Do not edit SQLite, delete cursors, or erase enrollment to force acceptance. + ## Scheduled Task Default, per-user startup after logon: @@ -81,7 +103,7 @@ Uninstalling the task does not delete databases, logs, `.env`, or outbox data. ## Data, logs, and permissions -If `SNAPMETER_DATA_DIR` is empty, scripts resolve `%LOCALAPPDATA%\SnapMeter`. Production boot tasks should use an explicit absolute path, commonly `C:\ProgramData\SnapMeter`. Logs default to a `logs` child directory. The runner keeps `collector-*.log` files for 14 days by default; set `SNAPMETER_LOG_RETENTION_DAYS` from 1 through 365. Cleanup resolves and verifies every target under the configured log directory before deletion. The SQLite database, lock, health snapshot, and outbox remain local. Schema v3 keeps the authoritative collector ID and actor pseudonym key inside the SQLite database; never print, extract, or export that key separately. +If `SNAPMETER_DATA_DIR` is empty, scripts resolve `%LOCALAPPDATA%\SnapMeter`. Production boot tasks should use an explicit absolute path, commonly `C:\ProgramData\SnapMeter`. Logs default to a `logs` child directory. The runner keeps `collector-*.log` files for 14 days by default; set `SNAPMETER_LOG_RETENTION_DAYS` from 1 through 365. Cleanup resolves and verifies every target under the configured log directory before deletion. The SQLite database, lock, health snapshot, and outbox remain local. Schema v4 keeps the authoritative collector ID, actor pseudonym key, endpoint enrollments, and event fingerprints inside the SQLite database; never print, extract, or export that key separately. Check free space regularly: @@ -99,14 +121,14 @@ The production D1 dataset accepts only its registered collector ID. A separately For backup or failover, stop the Scheduled Task, confirm the collector process has exited, and copy the entire resolved data directory as one SQLite-consistent unit. Preserve `snapmeter.sqlite3` together with any `snapmeter.sqlite3-wal` and `snapmeter.sqlite3-shm` files, the outbox, and their access controls. Restore that stopped state as a unit before starting the replacement host. Do not copy only a live main database, and do not move the pseudonym key into `.env` or another key store. -An intentional move to a fresh database requires the guarded manual reset of `collector_binding` documented in [Cloudflare deployment](deployment.md#collector-binding). Prefer a UTC-day boundary when no older-day reconciliation is pending; otherwise clear and deliberately rebuild affected cloud actor-day membership so two key domains cannot inflate DAU. This is an operator recovery action, not automatic failover. A schema-v3 database must not be opened by a pre-v3 collector binary; restore a compatible pre-upgrade database or fix forward. +An intentional move to a fresh database requires the guarded manual reset of `collector_binding` documented in [Cloudflare deployment](deployment.md#collector-binding). Prefer a UTC-day boundary when no older-day reconciliation is pending; otherwise clear and deliberately rebuild affected cloud actor-day membership so two key domains cannot inflate DAU. This is an operator recovery action, not automatic failover. A schema-v4 database must not be opened by a pre-v4 collector binary; restore a compatible pre-upgrade database or fix forward. ## Firewall and port binding - Bind local gRPC only to `127.0.0.1` when the collector is on the same host. - Do not create public inbound firewall rules for `3383` or `4383`. - If a private remote endpoint is required, restrict inbound source addresses, terminate TLS, and use optional authorization metadata. -- Permit outbound HTTPS to the deployed Worker and any authenticated TLS proxy. +- Permit outbound HTTPS to the deployed Worker, any authenticated TLS proxy, and the configured Hypersnap HTTPS fallback. - Snapchain gossip (`3382/UDP`) and HTTP compatibility (`3381/TCP`) are node concerns, not collector requirements. The node override example requires Docker Compose 2.24.4 or later for the `!override` tag. It replaces inherited port lists, then binds both RPC mappings to loopback; this avoids silently retaining a colliding or public base mapping. Add any required HTTP/gossip ports back with distinct reviewed publications. @@ -118,7 +140,7 @@ docker compose --profile collector up -d docker compose --profile collector logs -f collector ``` -The Compose service exposes no inbound port, stores SQLite state in a named volume, and caps container JSON logs at three 10 MiB files. It uses `host.docker.internal:3383` and `host.docker.internal:4383` by default; set `SNAPCHAIN_GRPC_URL_DOCKER`/`HYPERSNAP_GRPC_URL_DOCKER` in the shell or Compose environment for another topology. `docker-compose.nodes.override.yml` is a reference fragment for Compose 2.24.4+ to merge into a reviewed upstream node configuration, not a standalone node launcher. +The Compose service exposes no inbound port, stores SQLite state in a named volume, and caps container JSON logs at three 10 MiB files. It uses `host.docker.internal:3383` and `host.docker.internal:4383` by default; set `SNAPCHAIN_GRPC_URL_DOCKER`/`HYPERSNAP_GRPC_URL_DOCKER` in the shell or Compose environment for another topology. The fallback URL and identity/policy variables pass through the existing `.env` file, and the container needs outbound HTTPS access. `docker-compose.nodes.override.yml` is a reference fragment for Compose 2.24.4+ to merge into a reviewed upstream node configuration, not a standalone node launcher. ## WSL2 and Docker Desktop diff --git a/packages/protocol/src/classifier.test.ts b/packages/protocol/src/classifier.test.ts index d41f440..1d03a5f 100644 --- a/packages/protocol/src/classifier.test.ts +++ b/packages/protocol/src/classifier.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from "vitest"; -import { HYPERSNAP_CLASSIFIER_VERSION, MESSAGE_TYPES, actionFamilyForMessage, isHyperEligible, isSnapchainQualifying } from "./classifier"; +import { HYPERSNAP_CLASSIFIER_VERSION, HYPERSNAP_PUBLIC_COMPAT_SHA, MESSAGE_TYPES, actionFamilyForMessage, isHyperEligible, isSnapchainQualifying } from "./classifier"; import { FARCASTER_EPOCH_MS, normalizeMergeEvent } from "./rpc"; describe("Hyper eligibility classifier", () => { it("is versioned to the inspected Hypersnap source", () => { expect(HYPERSNAP_CLASSIFIER_VERSION).toMatch(/^2eee4c9f2a78/); + expect(HYPERSNAP_PUBLIC_COMPAT_SHA).toBe("ce408646fd09d886f275b74757341a1d328728ab"); }); it.each([ @@ -41,7 +42,7 @@ describe("canonical event normalization", () => { id: "99", shardIndex: 2, timestamp: "100", - mergeMessageBody: { message: { data: { type: "MESSAGE_TYPE_CAST_ADD", fid: "42", timestamp: "90" } } } + mergeMessageBody: { message: { data: { type: "MESSAGE_TYPE_CAST_ADD", fid: "42", timestamp: "90", network: "FARCASTER_NETWORK_MAINNET" } } } }; it("uses confirmed HubEvent time ahead of message time", () => { @@ -53,7 +54,9 @@ describe("canonical event normalization", () => { it("derives only eligible Hypersnap observations", () => { expect(normalizeMergeEvent(base, "hypersnap", "derived", FARCASTER_EPOCH_MS + 101_000, false)).not.toBeNull(); - expect(normalizeMergeEvent({ ...base, mergeMessageBody: { message: { data: { type: MESSAGE_TYPES.KEY_ADD, fid: "42", timestamp: "90" } } } }, "hypersnap", "derived", FARCASTER_EPOCH_MS + 101_000, false)).toBeNull(); + expect(normalizeMergeEvent({ ...base, mergeMessageBody: { message: { data: { type: MESSAGE_TYPES.KEY_ADD, fid: "42", timestamp: "90", network: 1 } } } }, "hypersnap", "derived", FARCASTER_EPOCH_MS + 101_000, false)).toBeNull(); + expect(normalizeMergeEvent({ ...base, mergeMessageBody: { message: { data: { type: 1, fid: "42", timestamp: "90", network: 2 } } } }, "hypersnap", "derived", FARCASTER_EPOCH_MS + 101_000, false)).toBeNull(); + expect(normalizeMergeEvent({ ...base, mergeMessageBody: { message: { data: { type: 1, fid: "42", timestamp: "90" } } } }, "hypersnap", "derived", FARCASTER_EPOCH_MS + 101_000, false)).toBeNull(); }); it("rejects failures, maintenance, and missing FIDs", () => { diff --git a/packages/protocol/src/classifier.ts b/packages/protocol/src/classifier.ts index 728ad84..b6a8fee 100644 --- a/packages/protocol/src/classifier.ts +++ b/packages/protocol/src/classifier.ts @@ -2,6 +2,7 @@ import type { ActionFamily } from "@snapmeter/contracts"; export const SNAPCHAIN_UPSTREAM_SHA = "6152402aea2dbe732fb73076f674b038bfd4aee5"; export const HYPERSNAP_UPSTREAM_SHA = "2eee4c9f2a7833ce7971dfef028480abbe9c4720"; +export const HYPERSNAP_PUBLIC_COMPAT_SHA = "ce408646fd09d886f275b74757341a1d328728ab"; export const HYPERSNAP_CLASSIFIER_VERSION = `${HYPERSNAP_UPSTREAM_SHA.slice(0, 12)}.1`; export const MESSAGE_TYPES = { diff --git a/packages/protocol/src/rpc.test.ts b/packages/protocol/src/rpc.test.ts index 4a0815f..ce7efd4 100644 --- a/packages/protocol/src/rpc.test.ts +++ b/packages/protocol/src/rpc.test.ts @@ -6,9 +6,50 @@ import { invokeUnaryWithAbort, MinimumIntervalGate, observeSubscriptionStream, + rawHubEventFingerprint, type RawHubEvent } from "./rpc"; +describe("canonical HubEvent fingerprints", () => { + it("matches equivalent gRPC byte fields and HTTP hex fields", () => { + const common = { + type: "HUB_EVENT_TYPE_MERGE_MESSAGE", + id: "123", + shardIndex: 1, + blockNumber: "9", + timestamp: "101" + }; + const grpc = { + ...common, + mergeMessageBody: { message: { + hash: Buffer.from("aabb", "hex"), + data: { type: "MESSAGE_TYPE_CAST_ADD", fid: "42", timestamp: "100", network: "FARCASTER_NETWORK_MAINNET" } + } } + } satisfies RawHubEvent; + const http = { + ...common, + mergeMessageBody: { message: { + hash: "0xaabb", + data: { type: 1, fid: "42", timestamp: 100, network: 1 } + } } + } satisfies RawHubEvent; + expect(rawHubEventFingerprint(grpc, 1)).toBe(rawHubEventFingerprint(http, 1)); + expect(rawHubEventFingerprint({ ...http, mergeMessageBody: { message: { + ...http.mergeMessageBody.message, + data: { ...http.mergeMessageBody.message.data, fid: "43" } + } } }, 1)).not.toBe(rawHubEventFingerprint(http, 1)); + expect(rawHubEventFingerprint({ ...http, timestamp: "102" }, 1)) + .not.toBe(rawHubEventFingerprint(http, 1)); + }); + + it("normalizes the channel-owner hint enum consistently across transports", () => { + const common = { id: "147456", shardIndex: 1, blockNumber: "9", timestamp: "101" }; + expect(rawHubEventFingerprint({ ...common, type: 12 }, 1)).toBe( + rawHubEventFingerprint({ ...common, type: "HUB_EVENT_TYPE_CHANNEL_OWNER_CHANGE_HINT" }, 1) + ); + }); +}); + describe("Snapchain RPC metadata", () => { it("sends proxy authorization and hosted-provider API keys under distinct headers", () => { const metadata = createRpcMetadata({ authorization: "Bearer proxy-secret", apiKey: "neynar-secret" }); @@ -85,6 +126,47 @@ describe("Snapchain live subscription request", () => { await subscription.done; expect(cancel).toHaveBeenCalledTimes(1); }); + + it("contains an observer exception before readiness and terminates the stream", async () => { + const emitter = new EventEmitter(); + const cancel = vi.fn(); + const stream = Object.assign(emitter, { cancel }) as unknown as Parameters[0]; + const failure = new Error("event fingerprint conflict"); + const onError = vi.fn(); + const subscription = observeSubscriptionStream(stream, () => { throw failure; }, onError); + const rejection = expect(subscription.ready).rejects.toBe(failure); + + expect(() => emitter.emit("data", { id: "123" } satisfies RawHubEvent)).not.toThrow(); + + await rejection; + await subscription.done; + expect(onError).toHaveBeenCalledOnce(); + expect(onError).toHaveBeenCalledWith(failure); + expect(cancel).toHaveBeenCalledOnce(); + subscription.cancel(); + expect(cancel).toHaveBeenCalledOnce(); + }); + + it("contains an observer exception after readiness and reports it once", async () => { + const emitter = new EventEmitter(); + const cancel = vi.fn(); + const stream = Object.assign(emitter, { cancel }) as unknown as Parameters[0]; + const failure = new Error("late event fingerprint conflict"); + const onError = vi.fn(() => { throw new Error("observer error handler failed"); }); + const onEvent = vi.fn((event: RawHubEvent) => { + if (event.id === "124") throw failure; + }); + const subscription = observeSubscriptionStream(stream, onEvent, onError); + + emitter.emit("data", { id: "123" } satisfies RawHubEvent); + await subscription.ready; + expect(() => emitter.emit("data", { id: "124" } satisfies RawHubEvent)).not.toThrow(); + + await subscription.done; + expect(onError).toHaveBeenCalledOnce(); + expect(onError).toHaveBeenCalledWith(failure); + expect(cancel).toHaveBeenCalledOnce(); + }); }); describe("shared GetEvents start throttling", () => { diff --git a/packages/protocol/src/rpc.ts b/packages/protocol/src/rpc.ts index 40b1030..67b2ddf 100644 --- a/packages/protocol/src/rpc.ts +++ b/packages/protocol/src/rpc.ts @@ -1,7 +1,8 @@ import { credentials, loadPackageDefinition, Metadata, type ClientReadableStream, type ClientUnaryCall, type ServiceError } from "@grpc/grpc-js"; import { loadSync } from "@grpc/proto-loader"; import { fileURLToPath } from "node:url"; -import { actionFamilyForMessage, isHyperEligible } from "./classifier"; +import { createHash } from "node:crypto"; +import { actionFamilyForMessage, isHyperEligible, messageTypeNumber } from "./classifier"; import type { Source, SourceMode } from "@snapmeter/contracts"; import type { ActivityRecord } from "@snapmeter/metrics"; @@ -25,6 +26,7 @@ export interface ShardInfo { export interface NodeInfo { version: string; + peerId?: string; numShards: number; shardInfos: ShardInfo[]; } @@ -35,10 +37,43 @@ export interface RawHubEvent { shardIndex?: number; timestamp?: number | string; blockNumber?: number | string; - mergeMessageBody?: { message?: { data?: { type?: number | string; fid?: number | string; timestamp?: number | string } } }; + mergeMessageBody?: { + message?: { + data?: { type?: number | string; fid?: number | string; timestamp?: number | string; network?: number | string }; + hash?: unknown; + }; + }; blockConfirmedBody?: Record; } +export function rawHubEventFingerprint(event: RawHubEvent, fallbackShard: number): string { + const merge = event.mergeMessageBody?.message; + const data = merge?.data; + const block = event.blockConfirmedBody; + const canonical = JSON.stringify({ + type: hubEventTypeNumber(event.type), + id: exactIntegerText(event.id), + shard: Number(event.shardIndex) > 0 ? Number(event.shardIndex) : fallbackShard, + blockNumber: exactIntegerText(event.blockNumber), + timestamp: exactIntegerText(event.timestamp), + message: data ? { + type: messageTypeNumber(data.type), + fid: exactIntegerText(data.fid), + timestamp: exactIntegerText(data.timestamp), + network: networkNumber(data.network), + hash: byteLikeText(merge?.hash) + } : null, + confirmed: block ? { + blockNumber: exactIntegerText(block.blockNumber), + shard: Number(block.shardIndex ?? fallbackShard), + timestamp: exactIntegerText(block.timestamp), + blockHash: byteLikeText(block.blockHash), + totalEvents: exactIntegerText(block.totalEvents) + } : null + }); + return createHash("sha256").update("snapmeter-hub-event-v1\0").update(canonical).digest("hex"); +} + export interface ProtocolRpcSubscription { cancel(): void; ready: Promise; @@ -105,7 +140,13 @@ export class SnapchainRpcClient { mempoolSize: Number(shard.mempoolSize ?? 0) })) : []; const declared = Number(response.numShards ?? 0); - return { version: String(response.version ?? "unknown"), numShards: declared || shardInfos.filter((shard) => shard.shardId > 0).length, shardInfos }; + const peerId = String(response.peerId ?? response.peer_id ?? "").trim(); + return { + version: String(response.version ?? "unknown"), + ...(peerId ? { peerId } : {}), + numShards: declared || shardInfos.filter((shard) => shard.shardId > 0).length, + shardInfos + }; } async getEvent(shardIndex: number, id: string, signal?: AbortSignal): Promise { @@ -212,6 +253,7 @@ export function observeSubscriptionStream( let rejectReady: ((error: Error) => void) | undefined; let readySettled = false; let doneSettled = false; + let cancelIssued = false; const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; @@ -232,17 +274,40 @@ export function observeSubscriptionStream( doneSettled = true; settleDone?.(); }; + const reportError = (error: Error): void => { + try { + onError(error); + } catch { + // Observer error handlers must never become uncaught EventEmitter errors. + } + }; + const cancelUnderlying = (): void => { + if (cancelIssued) return; + cancelIssued = true; + try { stream.cancel(); } catch { /* cancellation is best-effort */ } + }; + const terminateWithError = (error: Error, cancelStream: boolean): void => { + if (doneSettled) return; + failBeforeReady(error); + reportError(error); + finish(); + if (cancelStream) cancelUnderlying(); + }; stream.on("data", (event: RawHubEvent) => { + if (doneSettled) return; // Snapchain sends response metadata before its spawned subscription task // installs the broadcast receiver. Incorporate the first data event // before declaring readiness so the subsequent fixed-bound replay sees it. - onEvent(event); + try { + onEvent(event); + } catch (error) { + terminateWithError(error instanceof Error ? error : new Error(String(error)), true); + return; + } markReady(); }); stream.on("error", (error: Error) => { - failBeforeReady(error); - onError(error); - finish(); + terminateWithError(error, false); }); stream.on("end", () => { failBeforeReady(new Error("subscription ended before becoming ready")); @@ -256,7 +321,7 @@ export function observeSubscriptionStream( cancel: () => { failBeforeReady(rpcAbortError("subscription cancelled")); finish(); - stream.cancel(); + cancelUnderlying(); }, ready, done @@ -366,6 +431,7 @@ export function normalizeMergeEvent( if (event.type !== 1 && event.type !== "1" && event.type !== "HUB_EVENT_TYPE_MERGE_MESSAGE") return null; const data = event.mergeMessageBody?.message?.data; if (!data) return null; + if (source === "hypersnap" && networkNumber(data.network) !== 1) return null; const family = actionFamilyForMessage(data.type); if (!family) return null; if (source === "hypersnap" && sourceMode === "derived" && !isHyperEligible(data.type)) return null; @@ -389,3 +455,60 @@ export function normalizeMergeEvent( isReplay }; } + +function hubEventTypeNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isInteger(value)) return value; + if (typeof value === "string") { + if (/^\d+$/.test(value)) return Number(value); + const known: Record = { + HUB_EVENT_TYPE_NONE: 0, + HUB_EVENT_TYPE_MERGE_MESSAGE: 1, + HUB_EVENT_TYPE_PRUNE_MESSAGE: 2, + HUB_EVENT_TYPE_REVOKE_MESSAGE: 3, + HUB_EVENT_TYPE_MERGE_USERNAME_PROOF: 6, + HUB_EVENT_TYPE_MERGE_ON_CHAIN_EVENT: 9, + HUB_EVENT_TYPE_MERGE_FAILURE: 10, + HUB_EVENT_TYPE_BLOCK_CONFIRMED: 11, + HUB_EVENT_TYPE_CHANNEL_OWNER_CHANGE_HINT: 12 + }; + return known[value] ?? null; + } + return null; +} + +function networkNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isInteger(value)) return value; + if (typeof value === "string") { + if (/^\d+$/.test(value)) return Number(value); + const known: Record = { + FARCASTER_NETWORK_NONE: 0, + FARCASTER_NETWORK_MAINNET: 1, + FARCASTER_NETWORK_TESTNET: 2, + FARCASTER_NETWORK_DEVNET: 3 + }; + return known[value] ?? null; + } + return null; +} + +function exactIntegerText(value: unknown): string | null { + if (typeof value === "bigint") return value.toString(); + if (typeof value === "number" && Number.isSafeInteger(value)) return String(value); + if (typeof value === "string" && /^\d+$/.test(value)) return BigInt(value).toString(); + return null; +} + +function byteLikeText(value: unknown): string | null { + if (value instanceof Uint8Array) return Buffer.from(value).toString("hex"); + if (typeof value !== "string" || value.length > 4_096) return null; + if (/^0x[0-9a-f]+$/i.test(value)) return value.slice(2).toLowerCase(); + try { + const bytes = Buffer.from(value, "base64"); + if (bytes.length > 0 && bytes.toString("base64").replace(/=+$/, "") === value.replace(/=+$/, "")) { + return bytes.toString("hex"); + } + } catch { + // Non-binary strings are still normalized below. + } + return value; +} diff --git a/scripts/SnapMeter.Common.psm1 b/scripts/SnapMeter.Common.psm1 index b22a329..118a875 100644 --- a/scripts/SnapMeter.Common.psm1 +++ b/scripts/SnapMeter.Common.psm1 @@ -136,6 +136,56 @@ function Test-SnapMeterIntegerValue { } } +function Get-SnapMeterHttpsUri { + param([Parameter(Mandatory = $true)][string]$Value, [string]$Name = 'HTTPS URL') + + [Uri]$uri = $null + if (-not [Uri]::TryCreate($Value.Trim(), [UriKind]::Absolute, [ref]$uri) -or + $uri.Scheme -ne 'https' -or + [string]::IsNullOrWhiteSpace($uri.Host) -or + -not [string]::IsNullOrEmpty($uri.UserInfo) -or + -not [string]::IsNullOrEmpty($uri.Query) -or + -not [string]::IsNullOrEmpty($uri.Fragment)) { + throw "$Name must be an absolute HTTPS URL without credentials, a query, or a fragment." + } + return $uri +} + +function Test-SnapMeterPeerIdValue { + param([AllowEmptyString()][string]$Value, [Parameter(Mandatory = $true)][string]$Name) + + if ([string]::IsNullOrWhiteSpace($Value)) { + return + } + $normalized = $Value.Trim() + if ($normalized.Length -gt 128 -or $normalized -notmatch '^[1-9A-HJ-NP-Za-km-z]+$') { + throw "$Name must be a base58 peer identifier no longer than 128 characters." + } +} + +function Test-SnapMeterVersionValue { + param([AllowEmptyString()][string]$Value, [Parameter(Mandatory = $true)][string]$Name) + + if ([string]::IsNullOrWhiteSpace($Value)) { + return + } + $normalized = $Value.Trim() + if ($normalized.Length -gt 64 -or $normalized -notmatch '^[0-9A-Za-z][0-9A-Za-z._/+:-]*$') { + throw "$Name contains an invalid version identifier." + } +} + +function Get-SnapMeterHypersnapInfoUri { + param([Parameter(Mandatory = $true)][string]$Value) + + $baseUri = Get-SnapMeterHttpsUri -Value $Value -Name 'HYPERSNAP_FALLBACK_HTTP_URL' + $builder = [UriBuilder]::new($baseUri) + if (-not $builder.Path.EndsWith('/')) { + $builder.Path += '/' + } + return [Uri]::new($builder.Uri, 'v1/info') +} + function Test-SnapMeterConfiguration { param([switch]$RequireCloud) @@ -151,6 +201,16 @@ function Test-SnapMeterConfiguration { Test-SnapMeterBooleanValue -Value $env:HYPERSNAP_GRPC_TLS -Name 'HYPERSNAP_GRPC_TLS' foreach ($rule in @( @{ Name = 'SNAPMETER_RPC_TIMEOUT_MS'; Minimum = 250L; Maximum = 120000L }, + @{ Name = 'SNAPCHAIN_RPC_TIMEOUT_MS'; Minimum = 250L; Maximum = 120000L }, + @{ Name = 'HYPERSNAP_RPC_TIMEOUT_MS'; Minimum = 250L; Maximum = 120000L }, + @{ Name = 'SNAPCHAIN_RPC_MIN_INTERVAL_MS'; Minimum = 0L; Maximum = 3600000L }, + @{ Name = 'HYPERSNAP_RPC_MIN_INTERVAL_MS'; Minimum = 0L; Maximum = 3600000L }, + @{ Name = 'HYPERSNAP_FALLBACK_RPC_MIN_INTERVAL_MS'; Minimum = 0L; Maximum = 3600000L }, + @{ Name = 'HYPERSNAP_FALLBACK_POLL_INTERVAL_MS'; Minimum = 250L; Maximum = 60000L }, + @{ Name = 'HYPERSNAP_FAILOVER_AFTER_FAILURES'; Minimum = 1L; Maximum = 100L }, + @{ Name = 'HYPERSNAP_PREFERRED_RECOVERY_INTERVAL_MS'; Minimum = 5000L; Maximum = 3600000L }, + @{ Name = 'HYPERSNAP_PREFERRED_RECOVERY_SUCCESSES'; Minimum = 1L; Maximum = 100L }, + @{ Name = 'HYPERSNAP_MAX_BLOCK_DELAY_SECONDS'; Minimum = 0L; Maximum = 86400L }, @{ Name = 'SNAPMETER_RECONCILE_INTERVAL_MS'; Minimum = 1000L; Maximum = 3600000L }, @{ Name = 'SNAPMETER_DISCOVERY_INTERVAL_MS'; Minimum = 5000L; Maximum = 3600000L }, @{ Name = 'SNAPMETER_SNAPSHOT_INTERVAL_MS'; Minimum = 1000L; Maximum = 300000L }, @@ -167,6 +227,26 @@ function Test-SnapMeterConfiguration { Test-SnapMeterIntegerValue -Value $configuredValue -Name $rule.Name -Minimum $rule.Minimum -Maximum $rule.Maximum } + Test-SnapMeterPeerIdValue -Value $env:SNAPCHAIN_EXPECTED_PEER_ID -Name 'SNAPCHAIN_EXPECTED_PEER_ID' + Test-SnapMeterPeerIdValue -Value $env:HYPERSNAP_EXPECTED_PEER_ID -Name 'HYPERSNAP_EXPECTED_PEER_ID' + Test-SnapMeterVersionValue -Value $env:SNAPCHAIN_EXPECTED_VERSION -Name 'SNAPCHAIN_EXPECTED_VERSION' + Test-SnapMeterVersionValue -Value $env:HYPERSNAP_EXPECTED_VERSION -Name 'HYPERSNAP_EXPECTED_VERSION' + + $fallbackUrlConfigured = -not [string]::IsNullOrWhiteSpace($env:HYPERSNAP_FALLBACK_HTTP_URL) + $fallbackPeerConfigured = -not [string]::IsNullOrWhiteSpace($env:HYPERSNAP_FALLBACK_EXPECTED_PEER_ID) + $fallbackVersionConfigured = -not [string]::IsNullOrWhiteSpace($env:HYPERSNAP_FALLBACK_EXPECTED_VERSION) + if (-not $fallbackUrlConfigured -and ($fallbackPeerConfigured -or $fallbackVersionConfigured)) { + throw 'Hypersnap fallback identity pins require HYPERSNAP_FALLBACK_HTTP_URL.' + } + if ($fallbackUrlConfigured) { + [void](Get-SnapMeterHttpsUri -Value $env:HYPERSNAP_FALLBACK_HTTP_URL -Name 'HYPERSNAP_FALLBACK_HTTP_URL') + if (-not $fallbackPeerConfigured -or -not $fallbackVersionConfigured) { + throw 'A Hypersnap HTTPS fallback requires both HYPERSNAP_FALLBACK_EXPECTED_PEER_ID and HYPERSNAP_FALLBACK_EXPECTED_VERSION.' + } + } + Test-SnapMeterPeerIdValue -Value $env:HYPERSNAP_FALLBACK_EXPECTED_PEER_ID -Name 'HYPERSNAP_FALLBACK_EXPECTED_PEER_ID' + Test-SnapMeterVersionValue -Value $env:HYPERSNAP_FALLBACK_EXPECTED_VERSION -Name 'HYPERSNAP_FALLBACK_EXPECTED_VERSION' + $snapchainMode = if ([string]::IsNullOrWhiteSpace($env:SNAPCHAIN_SOURCE_MODE)) { 'verified' } else { $env:SNAPCHAIN_SOURCE_MODE.ToLowerInvariant() } $hypersnapMode = if ([string]::IsNullOrWhiteSpace($env:HYPERSNAP_SOURCE_MODE)) { 'derived' } else { $env:HYPERSNAP_SOURCE_MODE.ToLowerInvariant() } if ($snapchainMode -notin @('verified', 'unavailable')) { @@ -278,6 +358,7 @@ Export-ModuleMember -Function @( 'Resolve-SnapMeterPath', 'Import-SnapMeterEnvironment', 'Get-SnapMeterEndpoint', + 'Get-SnapMeterHypersnapInfoUri', 'Test-SnapMeterBooleanValue', 'Test-SnapMeterIntegerValue', 'Test-SnapMeterConfiguration', diff --git a/scripts/bootstrap.ps1 b/scripts/bootstrap.ps1 index f57da2b..5e1ad33 100644 --- a/scripts/bootstrap.ps1 +++ b/scripts/bootstrap.ps1 @@ -52,7 +52,23 @@ foreach ($source in @( )) { $endpoint = Get-SnapMeterEndpoint -Value $source.Value $reachable = Test-NetConnection -ComputerName $endpoint.Host -Port $endpoint.Port -InformationLevel Quiet -WarningAction SilentlyContinue - Write-Host ("{0} TCP probe: {1}" -f $source.Name, $(if ($reachable) { 'reachable' } else { 'not reachable; doctor will report source unavailable' })) + $unreachableMessage = if ($source.Name -eq 'Hypersnap' -and -not [string]::IsNullOrWhiteSpace($env:HYPERSNAP_FALLBACK_HTTP_URL)) { + 'not reachable; doctor will try the configured HTTPS fallback' + } else { + 'not reachable; doctor will report source unavailable' + } + Write-Host ("{0} TCP probe: {1}" -f $source.Name, $(if ($reachable) { 'reachable' } else { $unreachableMessage })) +} + +if (-not [string]::IsNullOrWhiteSpace($env:HYPERSNAP_FALLBACK_HTTP_URL)) { + try { + $fallbackInfoUri = Get-SnapMeterHypersnapInfoUri -Value $env:HYPERSNAP_FALLBACK_HTTP_URL + $fallbackResponse = Invoke-WebRequest -Uri $fallbackInfoUri -Method Get -Headers @{ Accept = 'application/json' } -MaximumRedirection 0 -TimeoutSec 15 -UseBasicParsing + $fallbackReachable = $fallbackResponse.StatusCode -ge 200 -and $fallbackResponse.StatusCode -lt 300 + Write-Host ("Hypersnap HTTPS fallback probe: {0}" -f $(if ($fallbackReachable) { 'reachable' } else { 'not reachable; doctor will report compatibility details' })) + } catch { + Write-Host 'Hypersnap HTTPS fallback probe: not reachable; doctor will report compatibility details' + } } $lockFile = Join-Path $repositoryRoot 'pnpm-lock.yaml' diff --git a/scripts/check-health.ps1 b/scripts/check-health.ps1 index 239a8f1..2ea2d13 100644 --- a/scripts/check-health.ps1 +++ b/scripts/check-health.ps1 @@ -15,16 +15,36 @@ Test-SnapMeterConfiguration $pnpmPath = Get-SnapMeterCommandPath -Names @('pnpm.cmd', 'pnpm') $failures = [System.Collections.Generic.List[string]]::new() -foreach ($source in @( - @{ Name = 'Snapchain'; Value = $env:SNAPCHAIN_GRPC_URL }, - @{ Name = 'Hypersnap'; Value = $env:HYPERSNAP_GRPC_URL } -)) { - $endpoint = Get-SnapMeterEndpoint -Value $source.Value - $reachable = Test-NetConnection -ComputerName $endpoint.Host -Port $endpoint.Port -InformationLevel Quiet -WarningAction SilentlyContinue - Write-Host ("{0} TCP: {1}" -f $source.Name, $(if ($reachable) { 'reachable' } else { 'unreachable' })) - if (-not $reachable) { - $failures.Add("$($source.Name) TCP endpoint is unreachable") + +$snapchainEndpoint = Get-SnapMeterEndpoint -Value $env:SNAPCHAIN_GRPC_URL +$snapchainReachable = Test-NetConnection -ComputerName $snapchainEndpoint.Host -Port $snapchainEndpoint.Port -InformationLevel Quiet -WarningAction SilentlyContinue +Write-Host ("Snapchain TCP: {0}" -f $(if ($snapchainReachable) { 'reachable' } else { 'unreachable' })) +if (-not $snapchainReachable -and $env:SNAPCHAIN_SOURCE_MODE -ne 'unavailable') { + $failures.Add('Snapchain TCP endpoint is unreachable') +} + +$hypersnapEndpoint = Get-SnapMeterEndpoint -Value $env:HYPERSNAP_GRPC_URL +$hypersnapPrimaryReachable = Test-NetConnection -ComputerName $hypersnapEndpoint.Host -Port $hypersnapEndpoint.Port -InformationLevel Quiet -WarningAction SilentlyContinue +Write-Host ("Hypersnap primary TCP: {0}" -f $(if ($hypersnapPrimaryReachable) { 'reachable' } else { 'unreachable' })) + +$hypersnapFallbackConfigured = -not [string]::IsNullOrWhiteSpace($env:HYPERSNAP_FALLBACK_HTTP_URL) +$hypersnapFallbackReachable = $false +if ($hypersnapFallbackConfigured) { + try { + $fallbackInfoUri = Get-SnapMeterHypersnapInfoUri -Value $env:HYPERSNAP_FALLBACK_HTTP_URL + $fallbackResponse = Invoke-WebRequest -Uri $fallbackInfoUri -Method Get -Headers @{ Accept = 'application/json' } -MaximumRedirection 0 -TimeoutSec 15 -UseBasicParsing + $hypersnapFallbackReachable = $fallbackResponse.StatusCode -ge 200 -and $fallbackResponse.StatusCode -lt 300 + } catch { + $hypersnapFallbackReachable = $false } + Write-Host ("Hypersnap HTTPS fallback: {0}" -f $(if ($hypersnapFallbackReachable) { 'reachable' } else { 'unreachable' })) +} + +$hypersnapMode = if ([string]::IsNullOrWhiteSpace($env:HYPERSNAP_SOURCE_MODE)) { 'derived' } else { $env:HYPERSNAP_SOURCE_MODE.ToLowerInvariant() } +if ($hypersnapMode -ne 'unavailable' -and -not $hypersnapPrimaryReachable -and -not $hypersnapFallbackReachable) { + $failures.Add('No reachable Hypersnap primary or HTTPS fallback was found') +} elseif ($hypersnapFallbackConfigured -and -not $hypersnapFallbackReachable) { + Write-Warning 'Hypersnap fallback is unreachable; collection can continue only while the preferred endpoint remains healthy.' } Push-Location $repositoryRoot diff --git a/vitest.config.ts b/vitest.config.ts index e6a44ac..1caf6dd 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,9 @@ export default defineConfig({ test: { include: ["packages/**/*.test.ts", "apps/collector/src/**/*.test.ts"], maxWorkers: 4, - testTimeout: 15_000, + // Real SQLite/WAL migration and network-free collector integration tests + // can contend on Windows when four workers run concurrently. + testTimeout: 30_000, coverage: { provider: "v8", reporter: ["text", "json-summary"],