diff --git a/.claude/skills/review-pr/SKILL.md b/.claude/skills/review-pr/SKILL.md
new file mode 100644
index 0000000..7f5bb93
--- /dev/null
+++ b/.claude/skills/review-pr/SKILL.md
@@ -0,0 +1,857 @@
+---
+name: review-pr
+description: Review a GitHub pull request or local Git range against @questdb/nodejs-client TypeScript ILP/QWP client coding standards
+argument-hint: "[PR number or URL | --range=..] [--level=0..3]"
+allowed-tools: Bash, Read, Grep, Glob, Agent
+---
+
+# Review a Node.js client pull request
+
+**Usage:** `/review-pr [PR number or URL | --range=..] [--level=0..3]`
+
+Review the PR or local range identified by the invocation arguments. When this skill
+is run as `/skill:review-pr `, the `` are appended as a `User:` message;
+treat that text as `$ARGUMENTS`. Parse exactly one review target: a PR number/URL,
+or `--range=..`. The range head may be omitted (`--range=..`) to
+review the working tree, including uncommitted changes. If both targets are supplied,
+stop and ask which was intended. If neither is supplied, ask for one.
+
+Use `Bash` only for read-only `gh` and Git queries, plus repository validation
+commands when evidence requires them. Use `Read`, `Grep`, `Glob`, and fresh-context
+agents through the Agent tool. Do not edit the primary working tree, push, post
+comments, or mutate the PR. Step 3b may create an isolated temporary worktree solely
+to verify a regression test against reverted production hunks; remove it afterward.
+
+## Review mindset
+
+You are a senior QuestDB engineer performing a blocking code review.
+`@questdb/nodejs-client` is mission-critical software: it serializes rows into the
+QuestDB InfluxDB Line Protocol (ILP) over HTTP/HTTPS or TCP/TCPS, and into the QuestDB
+Wire Protocol (QWP) over WebSocket or fire-and-forget UDP, with a browser build, an
+egress query path, and a crash-safe Node store-and-forward journal. A bug can silently
+corrupt bytes, drop or duplicate rows, abandon persisted data, leak credentials,
+exhaust resources, or break supported Node.js and browser consumers.
+
+**A review that blocks on everything blocks on nothing.** Every finding costs an
+author and CI round-trip. Reserve blocking severity for defects with a real user
+consequence, report other issues at the severity their evidence earns, and approve
+when the gates pass. Zero findings is a successful outcome.
+
+- **Assume nothing is correct until verified.** Read surrounding source and tests;
+ do not review the diff in isolation.
+- **Treat the diff as the entry point, not the boundary.** Contract changes often
+ break unchanged callers, overrides, transports, protocol versions, or generated
+ type consumers.
+- **Discovery is not a finding.** Every concern, including agent output, is an
+ untrusted hypothesis until it passes Step 3b. Omit anything unproved.
+- **Falsify before explaining.** Search for guards, validation, retries, alternate
+ callers, unsupported configurations, and identical base behavior before building
+ a failure narrative. Failure to disprove is not proof.
+- **Keep the PR blast radius small.** The PR owns defects it introduces or exposes.
+ Pre-existing behavior that is unchanged from base does not block it; a fully proved
+ pre-existing bug may leave as an adjacent issue draft.
+- **Do not praise the code.** Focus on defects, risks, and missing evidence.
+- **Think adversarially.** Exercise `null`/`undefined`, empty strings and arrays,
+ `NaN`/`Infinity`, imprecise `number` integers, `bigint`, multi-byte UTF-8, all ILP
+ delimiters, maximum buffer sizes, retries after uncertain sends, connection drops,
+ TLS/auth failures, and every negotiated protocol version. For QWP also exercise
+ mid-frame socket loss, replay after a restart, a NACK of an already replayed frame,
+ a full or externally locked journal directory, a role-rejected or capability-gapped
+ endpoint, and a truncated or hostile server frame.
+- **Store-and-forward promises no data loss.** Once rows enter the journal, only a
+ rejection that is deterministic under byte-identical replay may abandon them, and a
+ transient outage must never end the replay loop or surface to the producer. Treat a
+ breach of the store-and-forward checklist as Critical.
+- **Demand efficient hot paths.** Per-row and per-cell work scales to millions of
+ rows. Avoid allocations, repeated scans, redundant conversions, extra buffer copies,
+ and suboptimal algorithms there. Bounded setup/configuration work is less severe.
+- **Check what is missing.** Look for absent error handling, cleanup, tests, public
+ exports, TSDoc, README changes, deprecation wiring, and cross-transport parity.
+- **Untested behavior is a coverage risk, not proof of a defect.** A missing test is
+ Critical only when a supported, reachable regression could cause material user harm
+ and existing safeguards do not contain it.
+- **Verify every PR claim.** Reproduce fixes where practical, check performance claims
+ against the actual multiplier, and treat the PR description as a hypothesis.
+- **Assess reachability before reporting.** Drop theoretical paths that callers,
+ validation, configuration, or buffer bounds make impossible.
+- **Never review generated artifacts as source.** `dist/cjs/**`, `dist/es/**`, and
+ `docs/**` are generated. Review their `src/**/*.ts` or documentation source instead.
+
+## Review level
+
+Parse `$ARGUMENTS` for `--level=N`, `-lN`, or a bare digit `0`-`3`. Default to
+level 0. Strip the level token and any `--range=` token before passing a PR target
+to `gh`.
+
+| Level | What runs |
+|-------|-----------|
+| **0 (default)** | Steps 1, 2, 2.4, 2.5f, 2.6, and 4. Review inline without agent fanout. Build a compact coverage map and apply the Step 3b admission gate inline from a blank evidence form. |
+| **1** | Add Steps 2.5a and 2.5e when tests change. Run Agent 1 plus at most two applicable roles from Agents 2-7, 9-13, and 14-15. Independently falsify each surviving atomic candidate. |
+| **2** | Run all of Step 2.5, restricting 2.5b to exported/public/protected symbols, transport interfaces, shared helpers, and configuration options. Run Agent 1 plus at most four change-relevant roles. Independently falsify each surviving candidate. |
+| **3** | Run the full workflow. Select at most six applicable discovery roles: Agent 1 always; Agent 8 when changed symbols have out-of-diff callers; Agents 2-7 and 14-15 when their domains are touched; Agents 9-13 for changed tests or a fix claim; Agent 10 only when a distinct adversarial pass is warranted. Depth comes from evidence, not agent count. |
+
+State the selected level at the start of the review. If defaulted, mention that level
+3 exists for a full mission-critical pass. Changes to `src/buffer/**`, `src/_qwp/**`,
+`src/qwp-node/**`, transport/auth/TLS, protocol negotiation, flush semantics, or any
+public entry point (`src/index.ts`, `src/qwp/index.ts`, `src/qwp/node.ts`,
+`src/qwp/browser.ts`) are high risk; recommend level 3, but honor an explicit lower
+level and state the limitation. Replay-journal, ack-watermark, drainer, and failover
+changes stay high risk regardless of how small the diff is.
+
+## Spawning review agents
+
+Steps 3 and 3b use fresh-context, read-only Agent tasks. Discovery tasks receive the
+diff, Step 2.4 gitlink verdicts, the Step 2.5 surface map, the Step 2.6 coverage map,
+the chosen role, and the candidate contract. Agents 10 and Step 3b falsifiers are
+deliberate reduced-context exceptions.
+
+Use a shared temporary artifact for large maps instead of pasting them into every
+prompt. Never pass a discovery narrative, proposed severity/fix, votes, or verification
+claims to a falsifier. The parent owns role selection, the private candidate ledger,
+admission, severity, deduplication, and the final report.
+
+## Step 1: Gather review context
+
+Every mode must end with `$BASE` and `$HEAD` identified. Behavioral findings require
+the same trigger at both revisions unless the surface is genuinely new.
+
+### GitHub PR
+
+```bash
+PR=''
+gh pr view "$PR" --json number,title,body,labels,state,baseRefOid,headRefOid
+gh pr diff "$PR"
+gh pr view "$PR" --comments
+BASE=$(gh pr view "$PR" --json baseRefOid --jq .baseRefOid)
+HEAD=$(gh pr view "$PR" --json headRefOid --jq .headRefOid)
+```
+
+Also inspect the commit subjects with a read-only query when available. Do not check
+out the PR into the primary working tree merely to review it.
+
+### Local range (`--range`)
+
+```bash
+BASE=''
+HEAD=''
+git diff "$BASE"${HEAD:+"...$HEAD"} --stat
+git diff "$BASE"${HEAD:+"...$HEAD"}
+git diff "$BASE"${HEAD:+"...$HEAD"} --name-only
+git status --porcelain
+```
+
+With an empty head, include staged and unstaged tracked changes. `git diff` omits
+untracked files, so read any untracked source/test files that belong to the change.
+In range mode skip Step 2 because there is no PR metadata, state that fact, and run
+all other selected steps normally.
+
+## Step 2: PR title and description
+
+Skip this step in range mode.
+
+Check the repository conventions in `CONTRIBUTING.md` and recent accepted PRs:
+
+- Title follows Conventional Commits: `type(scope): description`.
+- Description explains end-user impact, not only implementation details.
+- A bug fix links or closes its issue.
+- Tone is analytical and avoids superlatives.
+- Public API, option/default, export, or compatibility changes are explicit.
+- README/TSDoc updates accompany user-visible behavior where needed.
+- New or renamed options document their defaults and deprecation path through
+ `SenderOptions.resolveDeprecated`.
+- New or renamed QWP keys are wired through `src/qwp-node/client-config.ts`, validated
+ against the transports that support them, and documented in `QWP.md`.
+- A changed public QWP surface updates `test/qwp/public-api-contract.ts`.
+
+## Step 2.4: Submodule boundaries (mandatory at every level)
+
+Treat submodule gitlink changes as opaque. Detect mode `160000` pointer moves, record
+the path and old/new hashes, and classify each as exactly:
+
+```bash
+git diff --raw "$BASE"${HEAD:+"...$HEAD"} | awk '$1 ~ /^:160000/ || $2 == "160000"'
+```
+
+- **OPAQUE** — the superproject changes only the gitlink. Do not enter the submodule,
+ fetch its branches, expand the commit range, inspect its files, attribute upstream
+ behavior changes to this PR, or report findings from its contents. Assume the
+ referenced changes were already merged and reviewed upstream.
+
+Review submodule contents only when the user explicitly requests that as an independent
+task. A genuine integration defect remains in scope only when code in the superproject
+diff calls or configures the bumped submodule incorrectly; file the finding at that
+superproject callsite and do not use an expanded submodule range as evidence.
+
+Repeat each `OPAQUE` verdict in Step 4 so the scope decision is auditable. If no
+gitlinks changed, state `Submodules: none` in the summary.
+
+## Step 2.5: Map the change surface
+
+Use `rg` and `rg --files` (or Grep/Glob equivalents) rather than reasoning about
+callers from memory. The resulting map is input to every normal Step 3 agent.
+
+### 2.5a Semantic delta per changed symbol
+
+For every modified or added function, method, class, abstract/protected member,
+exported type/constant, interface, and configuration option, record:
+
+- **Symbol:** fully qualified name.
+- **Before:** signature, sync/async return shape, thrown errors and inputs, state
+ mutation (`hasTable`, `hasSymbols`, `hasColumns`, `position`, `endOfLastRow`),
+ allocation behavior, protocol versions, and exact wire bytes where applicable.
+- **After:** the same fields.
+- **Delta:** the concrete behavioral difference. Use `no behavioral change` only
+ after checking; words such as “refactored” or “simplified” are insufficient.
+
+### 2.5b Callsite inventory
+
+For every changed exported/public/protected symbol, base-class member, shared helper,
+transport-interface method, or option name, search all source, tests, README/examples,
+and exports. Group results by file and include overrides and implementations.
+
+At minimum check:
+
+- All four public entry points — `src/index.ts`, `src/qwp/index.ts`, `src/qwp/node.ts`,
+ `src/qwp/browser.ts` — and emitted public type implications.
+- `SenderBufferBase` plus `SenderBufferV1`/`V2`/`V3` overrides and `createBuffer`.
+- `SenderTransport` plus Undici, stdlib HTTP, and TCP implementations.
+- `SenderOptions.resolveAuto`, `resolveDeprecated`, config parsing, `fromConfig`, and
+ `fromEnv` for option changes, plus `src/qwp-node/client-config.ts` for QWP keys.
+- Changed `src/_qwp/_core/**` constants and codecs against both the ingress encoder and
+ the egress decoder; one cap or type byte is normally read by both sides.
+- `QwpSender` and the writer helpers, `QwpIngressSession`, `QwpEgressSession`,
+ `QwpClient`, the reconnecting connections in `src/_qwp/_internal/**`, and the UDP
+ sender.
+- `QwpNodeFileReplayStore`, `QwpNodeOrphanDrainer`, the advisory lock, and the segment
+ maintenance worker for any store-and-forward change.
+- Unit/integration tests and test helpers, including `test/qwp/**` and its fixtures.
+- `test/qwp/public-api-contract.ts` for any exported QWP symbol, type, or option.
+- `README.md`, `QWP.md`, and examples for public symbols/options.
+
+A changed shared symbol with no recorded `rg` command is a skill violation. Never
+assert “only used here” without the search trace.
+
+### 2.5c Implicit contract list
+
+For each changed symbol, record before versus after for every applicable contract:
+
+- Inputs that throw synchronously and which callers catch or propagate.
+- `null`/`undefined`: accept, reject, or omit. `strictNullChecks` is off, so validate
+ runtime behavior rather than trusting the signature.
+- Buffer capacity: bytes reserved by `checkCapacity(data, base)` versus bytes emitted.
+- Row state: reads/transitions of `hasTable`, `hasSymbols`, `hasColumns`, `position`,
+ and `endOfLastRow`, including empty-row closure.
+- Sync/async shape and whether every caller awaits it.
+- ILP bytes: separators, escaping, marker bytes, byte order, arrays, timestamp units,
+ decimal payloads, and protocol-version applicability.
+- Transport lifecycle, retry/idempotency, auto-flush behavior, auth, TLS, and cleanup.
+- Number precision: `number` versus `bigint`, especially LONG and nanosecond values.
+- Configuration name/default/validation/deprecation behavior.
+- Allocation and complexity on setup, per-row, and per-cell paths.
+
+### 2.5d Cross-context exposure list
+
+List places where the change is visible but the diff does not touch, grouped by:
+
+- Per-row/per-cell buffer-build hot path.
+- Protocol-version fanout (ILP v1/v2/v3, and the QWP frame version with its negotiated
+ caps and capabilities).
+- Transport fanout (Undici, stdlib HTTP, TCP/TCPS, QWP WebSocket ingress, QWP egress,
+ Node UDP).
+- Runtime fanout (Node entry points versus the browser build, which must stay free of
+ Node built-ins, `ws`, and `qwp-node` imports).
+- Flush, commit, retry, replay, and lazy auto-flush paths.
+- Reconnect, failover, role/capability rejection, and poison-frame escalation.
+- Store-and-forward journal, orphan drainer, advisory locking, and the maintenance
+ worker thread.
+- Protocol negotiation, durable-ACK capability negotiation, and configuration parsing.
+- Auth/TLS and resource lifecycle.
+- Worker-thread use (one mutable `Sender` per worker) and multi-process use of a single
+ store-and-forward directory.
+- Public ESM/CJS/type surface across all four entry points.
+- Tests, helpers, README, `QWP.md`, and examples.
+
+Every listed context must be checked in Step 3.
+
+### 2.5e Test surface and helper inventory
+
+Run when tests are added or changed. Use repository searches to record:
+
+- Existing setup/teardown, fixtures, mock HTTP/proxy helpers, buffer hex helpers,
+ custom matchers, and parameterized-test patterns the change could reuse.
+- Callers of any changed shared test helper or fixture.
+- The production symbols each changed test actually exercises.
+- Whether the assertion observes public behavior, exact wire bytes, transport calls,
+ resource cleanup, or only implementation details.
+
+### 2.5f Build and runtime profile (mandatory at every level)
+
+Record current facts with file/line citations; do not rely on this list becoming stale:
+
+- TypeScript flags from `tsconfig.json`, especially `strictNullChecks`,
+ `noImplicitAny`, and `noUncheckedIndexedAccess`.
+- Node.js version floor and `@types/node` version.
+- Runtime dependencies and what each covers: `undici` (HTTP) and `ws` (Node QWP
+ WebSocket). There is no native dependency: store-and-forward locking is pure
+ JavaScript in `advisory-lock.ts`, using a `.lock.owner` directory as the mutex with
+ an mtime heartbeat for stale recovery. Reintroducing a native addon would break
+ every consumer on a platform or Node major it has no binary for, so treat a new
+ `optionalDependencies` entry or a compiled binary in the bundle as a finding.
+ `fzstd` is a devDependency that the bundler inlines; making it an external import
+ would break installs.
+- Dual ESM/CJS build and every `package.json` exports subpath (`.`, `./qwp`,
+ `./qwp/browser`, `./qwp/node`), plus which sources each subpath is allowed to import.
+- ILP protocol default/negotiation and TCP's explicit-version requirement.
+- QWP `QWP_VERSION`, the `/write/v4` ingress and `/read/v1` egress routes, the caps in
+ `src/_qwp/_core/constants.ts`, and the capabilities negotiated per connection.
+- `worker_threads` use by the segment maintenance worker, and the `Date.now()` /
+ `Math.random()` dependencies in backoff, episode, and timeout accounting that
+ deterministic tests must be able to control.
+- `Buffer.write` versus `writeInt*` boundary semantics. A short `Buffer.write` can
+ silently truncate, while numeric writes throw out of bounds; `writeInt8` requires
+ `-128..127` and marker bytes above 127 must be sign-folded.
+
+Put the relevant facts at the top of normal Step 3 prompts. Agent 10 receives only
+the reduced context defined below.
+
+## Step 2.6: Test coverage map (mandatory at every level)
+
+For every production behavioral change, including each new branch/error/NULL/boundary
+path, build an internal row containing:
+
+- **Change:** symbol and exact behavior/path.
+- **Test:** exact test file and name found through recorded `rg`/`rg --files` searches.
+- **Failure link:** assertion and why it fails if the behavior regresses.
+- **Reachability/population:** supported API/configuration/event and affected users.
+- **Credible consequence:** concrete recurrence and observed harm.
+- **Change risk:** complexity, caller breadth, state/resource sensitivity, safeguards.
+- **Stable test design:** least invasive meaningful unit/integration/fault-injection
+ assertion and observation seam.
+- **Effort/fragility evidence:** concrete setup, nondeterminism, platform, or production
+ seam costs; “hard to test” alone is not evidence.
+- **Dimensions:** applicable protocol, transport, runtime (Node/browser), happy/error,
+ NULL, boundary, concurrency, retry, reconnect/replay, crash-recovery, and
+ resource-cleanup dimensions.
+- **Disposition:** `COVERED`, `CRITICAL GAP`, `MODERATE GAP`, `ACCEPTED GAP`, or `EXEMPT`.
+
+Mark rows with no effective assertion `UNTESTED` before classification. Missing tests
+alone never establish Critical severity:
+
+- **Critical gap:** a supported reachable regression can cause data loss/corruption,
+ a security failure, outage/hang, compatibility break, unbounded resource loss, or
+ similarly material harm; safeguards do not contain it; Step 3b admits it.
+- **Moderate gap:** meaningful but bounded exposure, including most bug fixes without
+ an effective regression test.
+- **Accepted gap:** localized low-risk behavior where a stable test is demonstrably
+ disproportionate or more fragile than the code and existing safeguards are strong.
+- **Exempt:** verified non-behavioral source, documentation, generated-output, or CI
+ changes.
+
+Publish only admitted gaps. Keep covered, accepted, exempt, and omitted rows private
+unless the user asks for the complete map.
+
+## Step 3: Change-specific candidate discovery
+
+Use fresh-context, read-only Agent tasks. Select only roles materially touched by the
+change and obey the review-level cap. Agent count is never evidence.
+
+Every normal discovery task receives the diff, change-surface map, coverage map, and
+these candidate rules:
+
+- Generate atomic, falsifiable hypotheses; do not assign severity, propose fixes,
+ write persuasive titles, or claim verification.
+- Cite the exact changed hunk or unchanged callsite contract allegedly broken.
+- Name the supported-state producer: exact public API call, option, protocol, server
+ response, runtime, or event that creates every trigger. Use `producer: unknown`
+ rather than inventing one.
+- Give reachability, head observation, same-trigger base observation, user symptom,
+ evidence commands/artifacts, and strongest counterevidence. Mark unchecked fields
+ `unknown`.
+- Universal claims such as “never”, “only”, “no retry”, or “all transports” require
+ an exhaustive caller/event-source inventory.
+- Do not split supporting mechanisms into findings without independent consequences.
+- Pre-existing unchanged behavior is not a PR finding. Fully proved pre-existing bugs
+ may be proposed as adjacent issues only after Step 3b.
+- Returning no candidate is valid and preferred to speculation.
+
+### Agent roles
+
+**Agent 1 — Correctness and ILP semantics:** Check nullish omission, separators,
+escaping, input validation, integer precision, timestamp conversion, float edge cases,
+array shape/type/emptiness, decimal encoding, error paths, and exact v1/v2/v3 wire
+behavior. Check every changed symbol against its callers and overrides.
+
+**Agent 2 — Buffer and byte-encoding safety:** Reconstruct bytes and capacity math.
+Check every write against `checkCapacity`, UTF-8/escaping expansion, signed marker
+bytes, little-endian numeric/dimension encoding, `position`, overlapping compaction,
+resize/max-size behavior, `toBufferView` aliasing, `toBufferNew` mutation, and decimal
+two's-complement bounds.
+
+**Agent 3 — Transport, negotiation, auth, and TLS:** Check serializer negotiation,
+TCP explicit versions, retry classification/idempotency, Undici/stdlib parity, Basic/
+Bearer/JWK credentials, secret exposure, TLS verification/custom roots, timeouts, and
+connect/send/close behavior.
+
+**Agent 4 — Async, concurrency, and flush semantics:** Check every Promise/`await`,
+ordering across `at`/`atNow`/`tryFlush`/`flush`, row loss after `toBufferNew` compaction,
+uncertain-send duplication, lazy interval/row-count auto-flush, and unsafe sharing or
+interleaving of mutable Sender state.
+
+**Agent 5 — Resource management and lifecycle:** Trace sockets, Undici pools/agents,
+user-supplied versus owned agents, timers, abort controllers, listeners, and buffer
+views on success, failure, and early return. Verify failed connect/send/TLS paths close
+or preserve ownership correctly.
+
+**Agent 6 — Performance and algorithmic optimality:** For each loop, scan, allocation,
+copy, conversion, and data structure, state complexity and the best feasible approach.
+Focus on per-row/per-cell `toString`, string concatenation, repeated `Buffer.byteLength`,
+per-character writes, resize copying, large arrays, and avoidable buffer copies. Every
+candidate must state its multiplier or fixed bound and whether users wait on the path.
+
+**Agent 7 — Public API, compatibility, and code quality:** Check `src/index.ts`, ESM/
+CJS exports, `.d.ts` implications, TSDoc, option defaults/deprecations, supported Node
+APIs, README/examples, unsound casts, dead code/imports, ESLint, Prettier, naming, and
+member ordering. Separate compatibility defects from cosmetics.
+
+**Agent 8 — Cross-context caller impact:** Walk every 2.5b callsite with callers up to
+two levels. For each, return `SAFE`, `CANDIDATE`, or `INSUFFICIENT_EVIDENCE` and state
+whether the new contract breaks valid inputs, row state, bytes, sync/async shape,
+protocol subclasses, transports, config readers, error/retry paths, or worker contexts.
+
+**Agent 9 — Test coverage:** Recheck every Step 2.6 test and failure link, add missed
+behavior rows, and mutation-spot-check the most dangerous changed conditions. Check
+the matrix of protocols, transports, auth/TLS, auto-flush, resize, escaping, nullish
+values, arrays, precision, timestamps, retry/error, and resource cleanup.
+
+**Agent 10 — Fresh-context adversarial:** Receive only the diff and changed filenames.
+Instruction: “Generate a small set of falsifiable ways this code could be wrong and
+try to disprove each before returning it.” It may inspect the repository but receives
+no surface map, checklists, prior candidates, severities, or fixes.
+
+**Agent 11 — Test efficacy and correctness:** Trace each changed test from production
+symbol to assertion. Find vacuous assertions, tests that do not reach the changed path,
+wrong/stale expected wire bytes, happy-path-only coverage, swallowed asynchronous
+assertion failures, timing-dependent synchronization, and cleanup failures.
+
+**Agent 12 — Test-code quality:** Search the 2.5e inventory before flagging duplicated
+setup or helpers. Check parameterization opportunities, misleading names, copy/paste
+residue, debug output, commented code, unjustified skipped tests, brittle implementation
+assertions, and unnecessary casts. Name a real reusable alternative for each complaint.
+
+**Agent 13 — Regression-test efficacy:** For a bug-fix claim, identify which production
+hunk each test depends on. A candidate survives only if the test passes at head and
+fails when the production fix is reverted in an isolated scratch worktree.
+
+**Agent 14 — QWP wire format and protocol sessions:** Reconstruct frame headers,
+LEB128 varints, column encodings, Gorilla bit packing, zstd framing, symbol-dictionary
+IDs with their delta/reset flags, decimal scale, geohash bits, array shape, and NULL
+bitmaps against the caps in `src/_qwp/_core/constants.ts`. Check the ingress encoder and
+the egress decoder together because both read the same constants. Check status-byte to
+category to policy mapping, per-table transaction grouping, durable-ACK negotiation,
+ingress cap splitting, and that a truncated, oversized, or hostile server frame is
+rejected before it is allocated, copied, or trusted.
+
+**Agent 15 — Store-and-forward, replay, and failover:** Verify the durability contract
+in the checklist below. Trace the cumulative ack watermark, replay from
+`ackedFsn + 1`, segment format and checkpoints, append backpressure and deadlines,
+cross-process advisory locking, orphan-slot quarantine, poison-frame strike accounting,
+capability-gap episodes, reconnect budgets, and endpoint health/zone ranking. Any path
+that abandons accepted rows, advances the watermark past an unacknowledged frame, or
+ends the steady-state replay loop on a transient failure is a data-loss candidate.
+
+Combine outputs into a private candidate ledger. Split compound narratives into atomic
+propositions, deduplicate by proposition plus evidence, and record dependencies. Do not
+draft severity, fixes, or report prose yet.
+
+## Step 3b: Independently falsify, prove, and admit candidates
+
+Use this state machine without shortcuts:
+
+`HYPOTHESIS → FALSIFYING → PROVEN → ADMITTED`
+
+Missing proof, unresolved contradiction, failed reproduction, unsupported producer,
+or dependency on an omitted premise ends at `OMITTED`. “Could not disprove” is not
+`PROVEN`, and there is no public downgraded/false-positive section.
+
+At levels 1-3, launch one fresh-context falsifier per atomic candidate. Give it only:
+
+1. The neutral proposition.
+2. Repository plus base/head identities (or captured working-tree diff hash) and
+ relevant filenames.
+3. Raw evidence/artifact paths.
+
+Do not send the discovery narrative, severity, fix, author identity, votes, or claims
+that anyone verified it. At level 0, apply the same protocol inline from a blank form.
+
+The falsifier first constructs the strongest disproof: missing producer, unsupported
+configuration, impossible version pairing, omitted caller, retry, guard, validation,
+cleanup, downstream containment, or identical/better base behavior. Only a surviving
+candidate receives affirmative proof.
+
+Admit a behavioral candidate only when every applicable field has cited evidence:
+
+- **Attribution:** changed hunk, or unchanged callsite plus changed contract.
+- **Supported-state producer:** exact supported API/config/protocol/runtime/event.
+- **Reachability:** complete producer-to-symptom path, including guards, retries,
+ dispatch, ownership, and cleanup.
+- **Head observation:** executed trigger and observed result at the reviewed revision.
+- **Base observation:** identical trigger/result at `$BASE`, or `N/A — genuinely new
+ surface` with proof.
+- **User symptom:** independently observable consequence.
+- **Counterevidence search:** strongest disproof and why it does not apply.
+- **Artifact:** command/test, output, environment/configuration, and revision identity.
+
+Runtime-shape, race, ordering, retry, restart, resource-lifetime, compatibility, and
+wire-format claims require executed artifacts; static reading alone cannot admit them.
+For fully static compile errors or standards violations, mark runtime-only fields
+`N/A — static` and cite the complete source proof. Coverage searches prove absence of
+a test, not the reachability or impact needed for a Critical gap.
+
+Apply these special burdens:
+
+- Universal negatives require an exhaustive inventory and executed probe.
+- Concurrency/order candidates must force or observe the interleaving.
+- Regression-test candidates must run green at head and red with the production fix
+ reverted in a scratch worktree, never the primary working tree.
+- If execution is impossible, record the limitation privately and omit the behavioral
+ candidate rather than replacing evidence with confident prose.
+- If a parent premise is omitted, omit every dependent candidate.
+
+Then independently verify Node-client specifics:
+
+1. Read exact source lines in `src/**/*.ts`, not generated output, and trace callers,
+ interfaces, factories, and v1/v2/v3 overrides.
+2. Count every emitted byte against capacity, including escaped multi-byte UTF-8,
+ separators, suffixes, marker bytes, dimension headers, and decimal payloads.
+3. Reconstruct expected wire bytes and compare them with both production output and
+ byte-level test expectations.
+4. Validate nullish behavior at runtime because TypeScript nullability may be disabled.
+5. Trace `toBufferNew`/compaction relative to awaited sends for loss/duplication claims.
+6. Trace retry classes and whether the server could have durably accepted an uncertain
+ send before replay.
+7. Trace every socket, agent, timer, abort controller, listener, and buffer view through
+ success/error/early return; never destroy a user-supplied agent.
+8. For performance, prove complexity, hot/cold placement, call frequency, multiplier
+ or fixed bound, and a materially better feasible implementation.
+9. For public API/config claims, check every export, parser, default, deprecation path,
+ README example, ESM/CJS output implication, and supported Node version.
+10. For test efficacy, prove the assertion reaches the change and would fail under the
+ claimed regression. Recompute expected hex/bytes rather than trusting fixtures.
+11. For QWP wire claims, reconstruct the frame bytes for encode and decode, and check
+ every length, cap, and flag against `src/_qwp/_core/constants.ts` rather than against
+ an assumed peer behavior.
+12. For replay, ack, reconnect, or failover claims, trace the cumulative ack watermark
+ and prove which frames a restart, NACK, or non-orderly close resends or drops.
+ Classify the failure through `qwpDefaultSenderErrorPolicy` before calling anything
+ terminal.
+13. For store-and-forward claims, execute against a real directory: fill it, hold its
+ lock from a second process, truncate or corrupt a segment, and kill the process
+ between append and checkpoint. Durability and crash-recovery claims need journal
+ artifacts, never source reading alone.
+14. Derive a fix only after admission, then verify it compiles and closes all admitted
+ paths without creating a compatibility, ownership, or retry defect.
+
+### Net user impact and ledger classification
+
+Before assigning severity, answer in order:
+
+- **Population:** named supported API/config/protocol/runtime population.
+- **Delta vs base:** observed difference for the identical trigger.
+- **Magnitude/frequency:** per cell, row, flush, request, Sender lifetime, or once.
+- **Offsets:** validation, retry, server rejection, type/build gate, operational process,
+ or other containment before the user sees harm.
+- **Net:** `net-negative`, `net-neutral`, or `net-positive`. Only net-negative behavioral
+ candidates may be findings.
+
+Classify ledger entries as:
+
+- **ADMITTED in-diff** — proved defect inside the diff.
+- **ADMITTED out-of-diff-breakage** — proved unchanged caller broken by this PR's
+ changed contract.
+- **OMITTED pre-existing/not-attributed** — same or worse behavior exists at base and
+ this PR does not expose a new path.
+- **OMITTED false** — counterevidence disproves it.
+- **OMITTED unverified** — required producer, path, observation, artifact, or dependency
+ is missing.
+
+Keep omitted candidates and disproofs private. A fully proved pre-existing bug may
+become an adjacent issue draft; false or unverified candidates never do. Verify every
+enumerated instance independently rather than sampling and generalizing.
+
+## Review checklists
+
+### Correctness and wire format
+
+- Nullish omission must not emit a separator or leave invalid row state.
+- `number` LONG values beyond `2^53` lose precision; nanosecond timestamps require
+ `bigint`; v1 timestamps use microseconds while v2+ preserve nanoseconds.
+- Reject or intentionally encode `NaN`, `Infinity`, invalid units, invalid types, and
+ unsupported protocol features.
+- Verify table/symbol/column escaping for space, comma, equals, newline, carriage
+ return, quote, backslash, and multi-byte UTF-8.
+- Validate irregular/non-homogeneous/empty arrays and v2 dimension/type bytes.
+- Verify v3 decimal sign, scale, length, two's complement, and big-endian payload.
+
+### Buffer and byte safety
+
+- Every write has capacity for actual escaped UTF-8 bytes and suffix/marker bytes.
+- `writeInt8` values stay in `-128..127`; sign-fold unsigned marker bytes.
+- Doubles, int32 values, and dimensions use correct little-endian width/order.
+- Do not retain `toBufferView` across mutation; account for `toBufferNew` compaction.
+- Verify overlapping compact copies, growth termination, `max_buf_size`, and exact
+ `position` advancement.
+
+### Transport, protocol, auth, and TLS
+
+- Negotiated serializer matches the server; TCP requires an explicit version.
+- Retriable classification, backoff, and time budgets are correct; uncertain replay
+ cannot silently duplicate accepted rows.
+- Undici and stdlib HTTP agree on auth, TLS, timeout, retry, and response handling.
+- Basic/Bearer/JWK credentials are correct and never logged or included in errors.
+- Verification is disabled only explicitly; custom CA/roots are applied.
+- QWP endpoint selection honors the health and zone ranking; a background drainer
+ publishes health observations but never resets foreground classifications.
+- Upgrade failures are classified into a `QwpUpgradeError` kind, and a browser's opaque
+ upgrade error is never reported as a specific cause.
+- WebSocket close codes carry no policy meaning; classify by status byte and upgrade
+ kind instead.
+
+### QWP wire format and sessions
+
+- Frame header magic, version, flags, table count, and payload length agree between
+ encoder and decoder, and every cap in `src/_qwp/_core/constants.ts` is enforced on both
+ sides.
+- Varints stay inside uint64; row, column, name-length, array-element, and dictionary
+ limits are checked on encode and on decode.
+- Symbol dictionary IDs stay dense and connection-scoped; delta and reset flags match
+ what the peer reconstructs, and a `DICTIONARY_GAP` rejection triggers catch-up rather
+ than a terminal failure.
+- Gorilla, zstd, and raw encodings round-trip; decompression respects
+ `QWP_MAX_ZSTD_DECOMPRESSED_SIZE`, and every server-supplied length is validated before
+ it is allocated or copied.
+- Decimal scale, geohash bits, long256 words, UUID, IPv4, binary, and array shape
+ validation match the documented bounds for each column and bind type.
+- Server-supplied text decodes as fatal UTF-8 into a `QwpProtocolError`, never into a
+ silently mangled value.
+- Transactions are atomic per table, not across a flush; closing publishes staged rows
+ without committing them.
+- Durable ACK is requested through the Node upgrade header or the browser subprotocol,
+ and an unconfirmed capability fails with `QwpDurableAckUnavailableError`.
+- Ingress splitting respects the negotiated cap, and a single row above the cap fails
+ with `QwpBatchTooLargeError` instead of being dropped.
+- The browser entry point stays free of Node built-ins, `ws`, and `qwp-node` symbols.
+
+### Store-and-forward and durability
+
+A breach here is Critical: the contract is that a running producer neither loses data
+nor hard-fails on a transient outage.
+
+- The steady-state replay loop does not surface transport or server errors to the
+ producer. Journal exhaustion and its append deadline are the errors a caller may see.
+- Node foreground replay is unbounded after startup. Attempt and duration budgets apply
+ to `"sync"` startup and to the browser/memory policy only; a budget that latches a
+ running sender terminal during a long outage is a data-loss defect.
+- Backoff is exponential with full jitter and a capped per-attempt delay, while the
+ store-and-forward retry loop itself stays uncapped.
+- NACK policy follows `qwpDefaultSenderErrorPolicy`: `WRITE_ERROR`, `INTERNAL_ERROR`,
+ `DICTIONARY_GAP`, and an unknown status retry from `ackedFsn + 1`; `NOT_WRITABLE`
+ retries elsewhere; only rejections that are deterministic under byte-identical replay
+ go terminal. An unrecognized status byte fails open to retry, never closed.
+- The ack watermark never advances past a NACKed or unacknowledged frame, and abandoned
+ bytes are quarantined and reported through `QwpSenderError` rather than dropped.
+- Repeated rejection escalates through the poison-frame detector, honoring
+ `maxFrameRejections` and `poisonMinEscalationWindowMs`. Normal and going-away closes,
+ `NOT_WRITABLE`, and dictionary catch-up must not consume strikes, and a transient
+ class must not consume a capability-gap episode budget.
+- Orphan-drainer terminals are the ones that are terminal by design — authentication,
+ protocol, poison frame, and an exhausted capability-gap episode — and they quarantine
+ the slot behind its `.failed` sentinel for an operator. Any other terminal is a
+ finding.
+- Segment magic, format version, and checkpoint invariants hold; a torn, truncated, or
+ foreign-version segment is quarantined instead of replayed.
+- Advisory locking is fail-closed: a directory owned by another process yields
+ `QwpReplayStoreLockedError`, a release that cannot be proved stays on the retry list,
+ and the maintenance worker is stopped on every exit path.
+- UDP ingress is fire-and-forget by contract — no acknowledgement, no replay, no
+ durability claim. Review it for datagram sizing and socket cleanup, not against the
+ guarantees above.
+
+### Async, concurrency, and resources
+
+- Await every Promise; preserve send order and error propagation.
+- Understand that compaction precedes the awaited send and auto-flush is lazy.
+- Do not invite concurrent mutation or share a Sender across workers.
+- Close owned sockets/pools/agents/timers/listeners on every path; preserve user-owned
+ agents; do not retain stale buffer views.
+- Close QWP sockets, keepalive and ACK timers, reconnect timers, the maintenance worker
+ thread, and advisory locks on every path, including a failed upgrade, an aborted
+ replay, and a quarantined slot.
+
+### Performance
+
+- Avoid per-row/per-cell allocations, repeated `toString`/`Buffer.byteLength`, string
+ concatenation, and avoidable conversions/scans.
+- Avoid per-character writes where safe bulk copying exists, resize thrashing, needless
+ buffer copies, and O(n²) work over rows/cells/array elements.
+- State the data multiplier for hot-path findings; bounded setup costs are Moderate at
+ most unless they create an outage or compatibility failure.
+
+### Public API and code quality
+
+- Export new public symbols; treat removals/renames/signature/default changes as
+ compatibility changes.
+- Only the four documented entry points are public. Paths containing `internal`,
+ `qwp-node`, or `src` are implementation details even when a bundler resolves them.
+- A changed exported QWP symbol, option, constant, or error updates
+ `test/qwp/public-api-contract.ts` and `QWP.md`.
+- Keep TSDoc/types accurate and avoid casts that hide runtime null/type problems.
+- Wire renamed options through parsing, validation, `resolveDeprecated`, `resolveAuto`,
+ `fromConfig`, and `fromEnv` as applicable, and through the QWP config parser for QWP
+ keys.
+- Update README/examples for user-visible behavior.
+- Keep ESLint/Prettier clean; remove dead code/imports; follow local naming/order.
+
+### Tests
+
+- Cover each changed protocol/transport/auth/TLS/configuration path that behaves
+ differently, plus error, nullish, boundary, resize, retry, and cleanup paths.
+- Use byte-level assertions for serializer changes and transport-level assertions for
+ network/auth changes.
+- Recompute expected hex/bytes and ensure assertions can fail and reach production code.
+- QWP changes need frame-level assertions, and behavior that depends on it needs a
+ reconnect, replay, restart, or lock-contention test. Reuse the fake sockets, fixtures,
+ and interop helpers already in `test/qwp/`.
+- A bug fix needs a regression test that fails without the fix unless the Step 2.6
+ proportionality analysis admits a non-Critical gap.
+- Prefer existing helpers and deterministic synchronization; avoid brittle timing,
+ debug residue, misleading names, and implementation-only assertions.
+
+### TODOs and commit messages
+
+- Scan added/changed lines for `TODO`, `FIXME`, `HACK`, `XXX`, and `WORKAROUND`.
+ Distinguish moved comments from newly deferred work and verify referenced issues.
+- Check Conventional Commit subjects against `CONTRIBUTING.md`; descriptions should
+ state user impact where relevant.
+
+## Step 4: Output
+
+Present only **ADMITTED** findings. Omitted hypotheses, disproofs, retractions, agent
+counts, candidate counts, and the private ledger never appear. Do not publish a concern
+and retract it later. Keep the report actionable; if a normal PR produces more than
+about seven findings, rerun admission and remove dependent, duplicate, not-attributed,
+or low-value items.
+
+Every Critical and Moderate finding begins with three lines written from the completed
+admission form:
+
+- **Problem:** what is wrong, at most 12 words.
+- **Net impact:** supported population and magnitude, at most 12 words.
+- **Evidence:** decisive artifact/static proof and reviewed revision identity.
+
+Then provide only the minimal producer → path → symptom trace, base comparison, exact
+file/line, in-diff versus out-of-diff-breakage classification, and suggested fix.
+
+### Severity classification
+
+Severity is determined by reachable user consequence, not checklist category.
+
+**Critical** requires a supported trigger and one of:
+
+- Wrong/missing/duplicated/corrupted data or ILP/QWP wire bytes.
+- Abandoned, silently dropped, or unreplayable store-and-forward data, or an ack
+ watermark advanced past an unacknowledged frame.
+- Crash, hang, outage, unbounded loop, OOM, or unbounded socket/timer/listener leak.
+- A steady-state replay loop that ends, or surfaces a transient transport failure to the
+ producer, instead of retrying.
+- Credential exposure, auth/TLS bypass, or another security failure.
+- Silent/misleading failure that makes ingestion appear successful or undiagnosable.
+- Public API, config, runtime, module-system, protocol, or rolling-version compatibility
+ break affecting existing supported consumers.
+- User-observable throughput/latency/network regression multiplied per row/cell/request.
+- An admitted Critical coverage gap meeting Step 2.6's full reachability/impact burden.
+
+Every behavioral Critical must complete: “user does X → sees Y,” with an executed
+same-trigger base comparison. A performance Critical states the multiplier. A theory
+without a supported trigger is omitted, not preserved as Moderate.
+
+**Moderate** covers admitted attributable issues with bounded/developer-facing impact:
+proved weak tests, missing internal-path coverage, documentation defects, concrete
+standards violations, or bounded setup/configuration costs. Dynamic speculation and
+unchanged hardening opportunities are omitted.
+
+**Minor** covers concrete cosmetics on changed lines: naming, ordering, formatting,
+or comment wording.
+
+Exclude merge mechanics, tautologies true of every similar PR, deliberate project
+decisions without evidence they are wrong, generated artifacts as source, and all
+contents behind `OPAQUE` submodule gitlink bumps.
+
+### Critical
+
+List blocking admitted issues in descending user impact. Include the three summary
+lines, population/base delta/magnitude/offsets/net-negative determination, exact file
+and lines, supported trigger and symptom, executed artifacts, classification, contract
+and caller for out-of-diff breakage, and a fix scoped to this PR.
+
+### Moderate
+
+List non-blocking admitted issues with the three summary lines and decisive evidence.
+
+### Minor
+
+List optional, concrete cosmetics. Omit the section when empty.
+
+### Adjacent findings (not blocking — file as GitHub issues)
+
+Include only fully proved pre-existing bugs encountered in changed files or mapped
+callers that this PR does not introduce, expose, or worsen. They never affect the
+verdict and are never proposed as changes to this PR. For each provide:
+
+- **Problem:** issue-title-length summary.
+- **Net impact:** population and magnitude.
+- **Location:** exact file and lines.
+- **Symptom/reachability:** observed path, or named guard if latent.
+- **Suggested fix:** one or two lines.
+- **Standalone severity:** Critical, Moderate, or Minor.
+
+Offer to file them; never file without permission.
+
+### Coverage map
+
+State the test-gate result and number of admitted coverage gaps. Render admitted gap
+rows with their recorded search and failure link. Do not expose covered/accepted/exempt
+rows or omitted-candidate counts unless asked.
+
+### Summary
+
+Choose exactly one verdict:
+
+- **approve** — no open Critical findings and the test gate passes.
+- **approve with comments** — both gates pass, but named Moderate items remain.
+- **request changes** — at least one Critical finding is open or the test gate fails.
+- **needs discussion** — product, architecture, or compatibility decision is required.
+
+Apply these hard gates:
+
+- **Correctness gate:** any admitted Critical requires `request changes`. Omitted
+ hypotheses never affect the verdict.
+- **Test gate:** fails only for admitted Critical coverage gaps. Zero test changes or
+ missing regression coverage alone does not fail it.
+- Before finalizing, re-audit each rendered behavioral finding for strongest disproof,
+ supported producer, independent falsifier context, dynamic head/base evidence,
+ dependency survival, net-negative user impact, and post-admission severity.
+- If both gates pass, approve plainly; Moderate/Minor items do not justify withholding
+ approval.
+
+Also state:
+
+- Test-gate result and admitted gap count.
+- Regressions or tradeoffs.
+- Submodule verdicts (`path: OPAQUE — contents excluded`) or `Submodules: none`.
+- Admitted split: in-diff / out-of-diff-breakage.
+- Severity distribution.
+- At levels 0-1, the callsite-analysis limitation rather than implying exhaustive
+ out-of-diff coverage.
+
+Do not state agent counts, candidate counts, rejected/false-positive counts, or
+retraction history.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c87581..4d67c3c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -6,7 +6,11 @@ on:
- main
pull_request:
schedule:
- - cron: '15 2,10,18 * * *'
+ - cron: "15 2,10,18 * * *"
+ # GitHub disables a scheduled workflow after 60 days of repository
+ # inactivity, which stops it running on pull requests too. Dispatch is
+ # how to re-run it after that without pushing to a branch.
+ workflow_dispatch:
jobs:
test:
@@ -37,5 +41,125 @@ jobs:
- name: Type-checking
run: pnpm typecheck
+ # tsconfig.qwp-browser.json is where the browser contract lives: strict,
+ # no @types/node, DOM lib, over src/qwp/** minus node.ts. `pnpm typecheck`
+ # does not cover it, so without this step nothing stops a Node built-in or
+ # a strict-null violation reaching the browser entry point.
+ - name: Type-checking (browser)
+ run: pnpm typecheck:qwp-browser
+
+ # `pnpm typecheck` covers src plus a single contract file, so nothing
+ # type-checked test/** at all: a test could reference a deleted export
+ # and still pass, because vitest strips types without checking them.
+ - name: Type-checking (tests)
+ run: pnpm typecheck:test
+
+ - name: Type-checking (benchmarks)
+ run: pnpm typecheck:bench
+
+ - name: Linting (benchmarks)
+ run: pnpm lint:bench
+
- name: Tests
run: pnpm test
+
+ # Loads the built bundles through package.json `exports`, the way a
+ # consumer does. Every other suite imports from `src/`, where all four
+ # entry points share one module instance and cross-bundle defects are
+ # invisible.
+ - name: Built package tests
+ run: pnpm test:dist
+
+ # test:dist asserts runtime behaviour only. The compiled writers promise
+ # per-column row typing, which lives entirely in the emitted .d.ts files
+ # and is therefore invisible to every suite that imports from `src/`,
+ # where all four entry points share one module instance.
+ - name: Type-checking (built package consumer)
+ run: pnpm typecheck:dist
+
+ # The same gate publish.yml runs. Keeping it here too means a chunk that
+ # escapes `files`, or a regression in the check itself, fails a pull
+ # request instead of first being discovered during a release.
+ - name: Check for build artifacts
+ run: node scripts/check-build-artifacts.mjs
+
+ # Drives the built browser bundle in real Chromium against a local mock
+ # server. Tests that need a live QuestDB belong in the server repositories,
+ # where the topology and authentication fixtures already exist.
+ qwp-browser:
+ name: QWP browser bundle
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Setup node
+ uses: actions/setup-node@v4
+ with:
+ node-version: 20
+
+ - uses: pnpm/action-setup@v4
+ with:
+ version: 9
+ run_install: true
+
+ - name: Install Chromium
+ run: pnpm exec playwright install --with-deps chromium
+
+ - name: Browser bundle tests
+ run: pnpm test:qwp-browser
+
+ enterprise-qwp-e2e:
+ name: Dispatch Enterprise QWP E2E
+ runs-on: ubuntu-latest
+ # Azure credentials are not exposed to fork PRs. The repository variable
+ # keeps this dormant until the Enterprise pipeline and PAT are configured.
+ if: >-
+ vars.ENTERPRISE_E2E_ENABLED == 'true' &&
+ (github.event_name != 'pull_request' ||
+ github.event.pull_request.head.repo.full_name == github.repository)
+ steps:
+ - name: Queue Enterprise TypeScript-client E2E
+ env:
+ ENT_DISPATCH_PAT: ${{ secrets.ENT_DISPATCH_PAT }}
+ CLIENT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+ CLIENT_PR_NUMBER: ${{ github.event.pull_request.number || '' }}
+ CLIENT_BRANCH: ${{ github.head_ref || github.ref_name }}
+ run: |
+ set -euo pipefail
+
+ if [ -z "${ENT_DISPATCH_PAT:-}" ]; then
+ echo "ENT_DISPATCH_PAT is not configured" >&2
+ exit 1
+ fi
+
+ ORG_URL="https://dev.azure.com/questdb/"
+ PROJECT="questdb-enterprise"
+ PIPELINE_NAME="build-and-test-e2e-javascript-client"
+ PIPELINES=$(curl -fsS -u ":${ENT_DISPATCH_PAT}" \
+ "${ORG_URL}${PROJECT}/_apis/pipelines?api-version=7.0")
+ PIPELINE_ID=$(echo "$PIPELINES" | jq -r --arg name "$PIPELINE_NAME" \
+ '.value[] | select(.name == $name) | .id' | head -1)
+ if [ -z "$PIPELINE_ID" ] || [ "$PIPELINE_ID" = "null" ]; then
+ echo "Enterprise pipeline '$PIPELINE_NAME' is not registered" >&2
+ exit 1
+ fi
+
+ BODY=$(jq -n \
+ --arg commit "$CLIENT_COMMIT" \
+ --arg pr "$CLIENT_PR_NUMBER" \
+ --arg branch "$CLIENT_BRANCH" \
+ '{
+ templateParameters: {
+ javascriptClientCommit: $commit,
+ javascriptClientPrNumber: $pr,
+ clientBranch: $branch
+ }
+ }')
+ RESPONSE=$(curl -fsS -u ":${ENT_DISPATCH_PAT}" \
+ -H "Content-Type: application/json" \
+ -X POST \
+ -d "$BODY" \
+ "${ORG_URL}${PROJECT}/_apis/pipelines/${PIPELINE_ID}/runs?api-version=7.0")
+ echo "Enterprise E2E queued: $(echo "$RESPONSE" | jq -r '._links.web.href')"
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 2e30f98..1c91ae3 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -29,16 +29,32 @@ jobs:
- name: Type-checking
run: pnpm typecheck
+ # tsconfig.qwp-browser.json is where the browser contract lives: strict,
+ # no @types/node, DOM lib, over src/qwp/** minus node.ts. `pnpm typecheck`
+ # does not cover it, so without this step nothing stops a Node built-in or
+ # a strict-null violation reaching the browser entry point.
+ - name: Type-checking (browser)
+ run: pnpm typecheck:qwp-browser
+
- name: Tests
run: pnpm test
- name: Build
run: pnpm build
- - name: Check for build artifacts
+ # Guards the emitted .d.ts row typing, which no runtime suite can see.
+ - name: Type-checking (built package consumer)
run: |
- [ -f dist/cjs/index.js ] || (echo "CJS build missing" && exit 1)
- [ -f dist/es/index.mjs ] || (echo "ESM build missing" && exit 1)
+ pnpm exec tsc --noEmit -p tsconfig.dist-types.json
+ pnpm exec tsc --noEmit -p tsconfig.dist-types.cjs.json
+
+ # Every subpath in `exports`, not just the root: a publish that omits
+ # ./qwp, ./qwp/browser or ./qwp/node resolves to nothing for consumers.
+ # Entry bundles also import shared chunks that no `exports` entry names,
+ # so the walk follows relative imports: a chunk left out of `files`
+ # publishes a package whose every entry resolves to a missing file.
+ - name: Check for build artifacts
+ run: node scripts/check-build-artifacts.mjs
- name: Publish
uses: JS-DevTools/npm-publish@v3
diff --git a/CLAUDE.md b/CLAUDE.md
index 8260426..c18ca40 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
## Project Overview
-This is the QuestDB Node.js client library (@questdb/nodejs-client) that provides data ingestion capabilities to QuestDB databases. The client supports multiple transport protocols (HTTP/HTTPS, TCP/TCPS) and authentication methods.
+This is the QuestDB JavaScript client library (@questdb/nodejs-client) that provides data ingestion capabilities to QuestDB databases. The client supports multiple transport protocols (HTTP/HTTPS, TCP/TCPS) and authentication methods.
## Development Commands
diff --git a/QWP.md b/QWP.md
new file mode 100644
index 0000000..08af0a5
--- /dev/null
+++ b/QWP.md
@@ -0,0 +1,1510 @@
+# QuestDB Wire Protocol (QWP)
+
+This guide covers QWP ingress and egress from Node.js and browser applications.
+It describes the supported public entry points, delivery semantics, authentication,
+failure handling, and migration from the existing Node.js sender and the low-level
+QWP API.
+
+QWP support is currently a preview. The documented exports are the compatibility
+baseline for the first QWP release, but may still change before that release. Once
+released, changes to this documented surface follow the package's semantic-versioning
+policy. Imports from internal source paths are never supported.
+
+## Choose an entry point
+
+| Entry point | Runtime | Use it for |
+| ------------------------------------ | ------------------ | ----------------------------------------------------------------------------------------- |
+| `@questdb/nodejs-client` | Node.js | Existing `Sender`, including QWP ingress selected with `ws::`, `wss::`, or `udp::` |
+| `@questdb/nodejs-client/qwp/browser` | Browser | Browser-safe QWP ingress, egress, authentication bootstrap, sessions, and codecs |
+| `@questdb/nodejs-client/qwp/node` | Node.js | QWP ingress and egress with upgrade headers, TLS agents, and persistent store-and-forward |
+| `@questdb/nodejs-client/qwp` | Browser or Node.js | Shared protocol codecs and low-level session abstractions for advanced integrations |
+
+The three QWP subpaths are declared both in `exports` and in `typesVersions`, so
+they resolve under every TypeScript `moduleResolution` setting, including the
+legacy `node10` that `module: "commonjs"` still implies by default. Keep the two
+declarations in step: `exports` alone leaves a `node10` consumer with
+`TS2307: Cannot find module '@questdb/nodejs-client/qwp/node'` at compile time
+while the same import works perfectly at runtime.
+
+Do not import the package root from browser code. It retains the existing Node.js
+transports and dependencies for backward compatibility. The browser entry point has
+no Node.js imports. Node-only features remain in `qwp/node`, so supporting browsers
+does not require redesigning or breaking the existing client.
+
+QWP uses `/write/v4` for ingress and `/read/v1` for egress. A server must expose
+these WebSocket routes; optional features are enabled only when negotiation confirms
+that the server supports them.
+
+## Ingress
+
+### Node.js through the existing `Sender`
+
+Changing `http::` or `tcp::` to `ws::` selects QWP while preserving the familiar
+fluent row API:
+
+```typescript
+import { Sender } from "@questdb/nodejs-client";
+
+const sender = await Sender.fromConfig(
+ "wss::addr=questdb.example:9000;token=REST_OR_OIDC_TOKEN;auto_flush=off",
+);
+await sender.connect();
+
+try {
+ await sender
+ .table("trades")
+ .symbol("symbol", "ETH-USD")
+ .floatColumn("price", 2_615.54)
+ .timestampColumn("received_at", Date.now(), "ms")
+ .at(Date.now(), "ms");
+ await sender.flush();
+} finally {
+ await sender.close();
+}
+```
+
+`username` plus `password` selects HTTP Basic authentication for the WebSocket
+upgrade. `token` selects Bearer authentication. Use `wss::` in production.
+
+`Sender.fromConfig()` uses the same Java-compatible `ws::`/`wss::` vocabulary
+as `connectQwpNodeClient()`. Comma-separated or repeated `addr` values configure
+ordered failover endpoints, and ingress, egress, pool, and reserved policy keys
+are validated from one schema. The standalone sender applies ingress-owned keys;
+keys owned only by egress or the pooled facade are accepted as intentional no-ops.
+
+## Configuration-string keys
+
+Every `ws::`/`wss::` connect string is parsed by one schema, shared with the
+other QuestDB clients, whichever entry point builds the client —
+`Sender.fromConfig()`, `SenderOptions.fromConfig()`, `connectQwpNodeClient()`,
+or `connectQwpNodeQuery()`. An unrecognised key is rejected with
+`unknown configuration key: `; a legacy ILP key adds a hint pointing at
+where it applies instead.
+
+Keys are grouped by the component that applies them. A client applies the keys
+its own side owns and accepts the rest as intentional no-ops, so one connect
+string can configure a sender, a query client, or the pooled facade. Every key
+also has a programmatic equivalent on the corresponding options object; the
+connect string is the portable spelling.
+
+The complete connect string is parsed and validated before typed overrides are
+applied. When both forms set the same option, the typed value wins. In
+particular, `Sender.fromConfig()` applies `qwp.webSocket.failoverUrls`,
+`target`, `zone`, and `senderId` after URL parsing. The primary ingress URL
+continues to come from `addr`, because the typed object intentionally omits
+`url`.
+
+### Connection
+
+| Key | Value | Default | Meaning |
+| -------------------- | ------------------ | --------- | ---------------------------------------------------------------------------------------- |
+| `addr` | `host[:port]` | port 9000 | Endpoint. Repeat the key, or comma-separate, for ordered failover. |
+| `username`, `user` | string | — | HTTP Basic user for the WebSocket upgrade. |
+| `password`, `pass` | string | — | HTTP Basic password. |
+| `token` | string | — | Bearer token; alternative to Basic. |
+| `tls_verify` | `on`, `unsafe_off` | on | Certificate verification. `unsafe_off` disables it. |
+| `tls_roots` | path | — | PEM file containing trusted private-CA certificates. PKCS#12 is not supported. |
+| `tls_roots_password` | string | — | Unsupported by Node; convert PKCS#12 roots to PEM and omit this key. |
+| `auth_timeout_ms` | integer ms | `15000` | Deadline for the upgrade and authentication exchange. |
+| `connect_timeout` | integer ms | `15000` | Deadline for the TCP/TLS transport, and for the upgrade unless `auth_timeout_ms` is set. |
+
+### Ingress
+
+| Key | Value | Default | Meaning |
+| ----------------------------------------------- | ---------------- | --------- | ------------------------------------------------------------------------ |
+| `auto_flush` | `on`, `off` | on | Master switch for all auto-flush triggers. |
+| `auto_flush_rows` | integer | `1000` | Flush after this many staged rows. |
+| `auto_flush_bytes` | integer or `off` | off | Flush once staged rows reach this estimated size. |
+| `auto_flush_interval` | integer ms | `100` | Flush when this long has passed. Checked as rows are added. |
+| `close_flush_timeout_millis` | integer ms | `5000` | Bound on `close()`'s ACK drain. `0` or negative is a fast close. |
+| `transaction` | `on`, `off` | off | Group each flush into a per-table transaction. |
+| `request_durable_ack` | `on`, `off` | off | Require durable ACKs; fails if the server cannot confirm them. |
+| `durable_ack_keepalive_interval_millis` | integer ms | — | Poll interval for durable-ACK progress. |
+| `max_name_len` | integer | `127` | Maximum table and column name length, in UTF-8 bytes. |
+| `sender_id` | string | `default` | Identifies this producer to the server and in the journal. |
+| `max_frame_rejections` | integer | — | Rejections of one frame before the poison-frame detector escalates. |
+| `poison_min_escalation_window_millis` | integer ms | — | Minimum dwell before a poison frame may escalate. |
+| `catch_up_cap_gap_min_escalation_window_millis` | integer ms | `300000` | Minimum dwell before an orphan symbol-dictionary cap gap is quarantined. |
+| `connection_listener_inbox_capacity` | integer | — | Bound on the connection-event inbox before events are dropped. |
+| `error_inbox_capacity` | integer | — | Bound on the `onSenderError` inbox before events are dropped. |
+
+### Reconnect and failover
+
+| Key | Value | Default | Meaning |
+| ---------------------------------- | --------------------------- | ------- | -------------------------------------------------------------------------------------- |
+| `reconnect_initial_backoff_millis` | integer ms | — | First reconnect delay; grows exponentially with jitter. |
+| `reconnect_max_backoff_millis` | integer ms | — | Ceiling for one reconnect delay. |
+| `reconnect_max_duration_millis` | integer ms | — | Budget for a reconnect episode. This is the QWP replacement for ILP's `retry_timeout`. |
+| `failover` | `on`, `off` | — | Enables endpoint failover for egress. |
+| `failover_max_attempts` | integer ≥ 1 | — | Failover attempts before giving up. |
+| `failover_backoff_initial_ms` | integer ms | — | First failover delay. |
+| `failover_backoff_max_ms` | integer ms | — | Ceiling for one failover delay. |
+| `failover_max_duration_ms` | integer ms | — | Budget for a failover episode. |
+| `target` | `any`, `primary`, `replica` | — | Server role this client will accept, on both ingress and egress. |
+| `zone` | string | — | Preferred topology zone when ranking endpoints, on both ingress and egress. |
+
+### Store-and-forward (Node only)
+
+Setting `sf_dir` turns on the persistent journal; the rest tune it. A default
+shown as a dash is applied downstream of the connect string, by the sender or
+session that consumes it.
+
+| Key | Value | Default | Meaning |
+| --------------------------- | ------------------------------ | ------------- | ----------------------------------------------------------------- |
+| `sf_dir` | path | — | Journal directory. Enables store-and-forward. |
+| `sf_durability` | `memory`, `periodic`, `append` | `memory` | Local durability barrier after each vectored append. |
+| `sf_max_total_bytes` | integer bytes | `10737418240` | Journal ceiling. Reaching it is the one error a producer sees. |
+| `sf_max_segment_bytes` | integer bytes | `4194304` | Size of one segment file. |
+| `sf_sync_interval_millis` | integer ms | — | Checkpoint interval when `sf_durability=periodic`. |
+| `sf_append_deadline_millis` | integer ms | `30000` | How long an append waits for space before failing. |
+| `initial_connect_retry` | `off`, `sync`, `async` | `off` | Startup policy when the server is unreachable. Requires `sf_dir`. |
+| `drain_orphans` | `on`, `off` | off | Adopt and drain journals left by crashed producers. |
+| `max_background_drainers` | integer | — | Concurrent orphan drainers. |
+
+### Egress
+
+| Key | Value | Default | Meaning |
+| ------------------- | --------------------- | ---------- | -------------------------------------------------- |
+| `max_batch_rows` | integer, 1..1048576 | — | Rows the server puts in one result batch. |
+| `initial_credit` | integer ≥ 0 | — | Starting flow-control credit for a query. |
+| `buffer_pool_size` | integer ≥ 1 | — | Reusable result buffers held per session. |
+| `compression` | `raw`, `zstd`, `auto` | negotiated | Result compression to negotiate. |
+| `compression_level` | integer, 1..22 | — | zstd level requested from the server. |
+| `client_id` | string | — | Identifies this client in server-side diagnostics. |
+
+### Pool
+
+Applied by the pooled facade; a standalone sender or query client ignores them.
+
+| Key | Value | Default | Meaning |
+| ------------------------- | ----------- | ------- | --------------------------------------------- |
+| `sender_pool_min` | integer | — | Senders kept warm. |
+| `sender_pool_max` | integer | — | Sender ceiling. |
+| `query_pool_min` | integer | — | Query sessions kept warm. |
+| `query_pool_max` | integer | — | Query-session ceiling. |
+| `acquire_timeout_ms` | integer ms | — | How long `acquire()` waits for a free entry. |
+| `query_close_timeout_ms` | integer ms | — | Bound on closing a borrowed query session. |
+| `idle_timeout_ms` | integer ms | — | Idle time before a pooled entry is reaped. |
+| `max_lifetime_ms` | integer ms | — | Absolute lifetime of a pooled entry. |
+| `housekeeper_interval_ms` | integer ms | — | How often the pool reaps aged entries. |
+| `lazy_connect` | `on`, `off` | off | Start without blocking on a first connection. |
+
+### Reserved
+
+`on_write_error`, `on_server_error`, `on_internal_error`, `on_parse_error`,
+`on_schema_error` and `on_security_error` are part of the shared vocabulary and
+are accepted, but this client does not yet apply them: server-error policy comes
+from `qwpDefaultSenderErrorPolicy` and the `onSenderError` stream. They are
+listed so a connect string written for another QuestDB client is not rejected.
+
+### Node.js fire-and-forget UDP
+
+`udp::` selects Node-only QWP v1 over IPv4 UDP while retaining the fluent row API:
+
+```typescript
+import { Sender } from "@questdb/nodejs-client";
+
+const sender = await Sender.fromConfig(
+ "udp::addr=239.1.2.3:9007;max_datagram_size=1400;multicast_ttl=1",
+);
+await sender.connect();
+await sender
+ .table("trades")
+ .symbol("symbol", "ETH-USD")
+ .floatColumn("price", 2615.54)
+ .atNow();
+await sender.close();
+```
+
+The default port is 9007, the maximum datagram size (`max_datagram_size`) is 1400
+bytes, and the multicast TTL (`multicast_ttl`) is zero. Each datagram is
+self-contained, contains exactly one table, and uses an inline schema plus
+table-local symbol dictionaries. Batches are split at row boundaries;
+`QwpUdpDatagramTooLargeError` is raised before transmission when one row cannot
+fit. `connectQwpNodeUdpSender()` and `connectQwpNodeUdp()` expose the same
+transport from `qwp/node`.
+
+UDP provides no authentication, TLS, server or durable ACK, transactions,
+reconnection, compression, or store-and-forward. Local socket errors are delivered
+to `QwpNodeUdpOptions.onError`; like the Java sender, they are observational and do
+not retry rows that may already have been handed to the network. UDP is unavailable
+from the browser entry point.
+
+Advanced QWP options are accepted in the second argument:
+
+```typescript
+const sender = await Sender.fromConfig(
+ "wss::addr=questdb.example:9000;token=REST_OR_OIDC_TOKEN;initial_connect_retry=async",
+ {
+ qwp: {
+ webSocket: {
+ requestDurableAck: true,
+ connectTimeoutMs: 5_000,
+ authTimeoutMs: 15_000,
+ failoverUrls: ["wss://questdb-dr.example:9000/write/v4"],
+ target: "any",
+ zone: "eu-west-1a",
+ senderId: "producer-a",
+ storeAndForward: {
+ directory: "/var/lib/my-service/qwp-replay/producer-a",
+ maxBytes: 512 * 1024 * 1024,
+ durability: "periodic",
+ checkpointIntervalMs: 5_000,
+ backpressurePolicy: "wait",
+ appendDeadlineMs: 30_000,
+ catchUpCapGapMinEscalationWindowMs: 300_000,
+ drainOrphans: true,
+ maxBackgroundDrainers: 4,
+ },
+ },
+ sender: {
+ awaitDurableAck: true,
+ autoFlushRows: 10_000,
+ autoFlushBytes: 4 * 1024 * 1024,
+ },
+ session: {
+ reconnect: {
+ maxAttempts: 0,
+ maxDurationMs: 0,
+ },
+ },
+ },
+ },
+);
+```
+
+Node bounds connection establishment in two phases. `connectTimeoutMs` covers
+DNS plus the TCP/TLS connection; after that succeeds, `authTimeoutMs` independently
+covers the authenticated HTTP request and WebSocket upgrade. Both default to 15
+seconds, so one endpoint attempt can take up to their sum. A timeout is reported as
+`QwpUpgradeError` with `timeoutPhase` set to `"connect"` or `"authentication"`.
+Browsers cannot observe the transport boundary, so their `connectTimeoutMs` continues
+to cover the complete WebSocket opening lifecycle and they do not expose
+`authTimeoutMs`.
+
+Give each active sender its own store-and-forward directory. The Node.js journal
+persists frames and their symbol dictionary before sending. Set
+`initialConnectMode: "async"` when a persistent sender must start while every
+endpoint is offline. Unless
+`awaitServerAck: true` or `awaitDurableAck: true` is selected, `flush()` resolves once
+the complete logical flush reaches the configured local journal boundary; a background
+drainer then sends it in order. The default `"append"` boundary is locally durable,
+while `"periodic"` and `"memory"` trade that immediate guarantee for throughput.
+Applications can therefore keep publishing during an outage until the configured
+`maxBytes` applies backpressure. A failed journal publication leaves the high-level
+rows staged so the caller can retry.
+
+`initialConnectMode` selects persistent startup behavior: `"off"` (the default)
+makes one
+fail-fast attempt, `"sync"` retries on the caller within the configured reconnect
+budget, and `"async"` returns immediately while
+the background replay loop connects. `Sender.fromConfig()` also accepts
+`initial_connect_retry=off|sync|async` when `qwp.webSocket.storeAndForward` is
+supplied. Initial authentication, upgrade, and capability failures remain terminal.
+When no mode is explicit, configuring any reconnect duration/backoff key promotes
+the initial connection to `"sync"`, so that budget also governs startup.
+After a foreground persistent sender has connected successfully at least once, the
+same failures are retried indefinitely so credential rotation and rolling capability
+changes cannot strand its journal. The configured reconnect attempt/duration budget
+therefore bounds `"sync"` startup and non-persistent reconnects, not steady-state
+foreground store-and-forward recovery.
+
+The connect-string key
+`catch_up_cap_gap_min_escalation_window_millis` is the equivalent of
+`catchUpCapGapMinEscalationWindowMs`.
+
+`durability` controls the local persistence barrier:
+
+- `"append"` (the default) issues a data-only durability barrier after every vectored
+ positional frame write; manifest and directory metadata retain full barriers;
+ hot-spare creation and activation are durable before publication resolves.
+- `"periodic"` checkpoints segment files, symbol metadata, and directory changes in the
+ background. The default interval is 5 seconds, and `close()` performs a final
+ checkpoint. A power failure can lose the most recent checkpoint window.
+- `"memory"` relies on operating-system writeback. It survives an orderly close and
+ normally a process failure, but it makes no power-loss durability promise.
+
+`backpressurePolicy: "error"` preserves the existing immediate
+`QwpReplayStoreFullError` behavior. Set it to `"wait"` to pause publication until an
+ACK advances the checksummed cursor, then a bounded background trimmer deletes fully
+drained segments.
+`appendDeadlineMs` bounds each such pause (30 seconds by
+default) and expiry raises `QwpReplayStoreAppendTimeoutError`. Waiting appenders do
+not hold the journal mutation queue, so ACK cleanup can continue. Direct users of
+`QwpNodeFileReplayStore` can inspect `metrics` for pending records and segments,
+checkpoint work, checkpoint failures, active waiters, stalls, and timeouts.
+
+The persisted symbol dictionary is monotonic for one open journal generation and
+cannot be reclaimed by an ACK alone. It counts toward the `maxBytes` target together
+with each complete fixed-segment reservation, including the hot spare. The journal
+preserves up to 32 MiB (or the configured target when smaller) for live frame segments
+if dictionary growth uses all remaining headroom. Dictionary persistence itself is
+never rejected by the target, so actual disk usage can exceed it by the current
+dictionary overshoot and at most one liveness segment. Frame growth beyond that
+allowance remains backpressured until background ACK trimming frees complete segments.
+A partly acknowledged segment remains charged to the disk budget until its last live
+record is acknowledged.
+Once every frame is acknowledged,
+`close()` removes the dictionary under the journal lock; the next clean start uses a
+fresh symbol-ID space. A partially drained close retains the dictionary required by
+the surviving frames.
+
+The journal takes an exclusive lock when it is loaded and holds it until the sender
+or session closes. A second live Node.js process using the same directory fails with
+`QwpReplayStoreLockedError` before recovery or cleanup can mutate journal contents,
+unless the first has stopped heartbeating long enough to be reclaimed — in which case
+it is the first that stops writing, as described under the heartbeat below.
+Ownership is held by a `.lock.owner` directory created next to the slot: `mkdir` is
+the only exclusive-by-construction filesystem operation available on every supported
+platform without a native addon, so exactly one process can create it. The holder PID
+is recorded in `.lock.pid` for diagnostics, and the stable `.lock` file is created and
+left in place so a slot keeps the on-disk shape a Java client expects. Short-lived
+locks under the shared parent directory's `.slot-locks` child serialize orphan
+adoption with close/rename/recreate quarantine transitions.
+
+**A Node.js client and a Java client must not use one persistence directory at the
+same time.** The Java client locks `.lock` with `flock` on Unix and `LockFileEx` on
+Windows. The Node.js client does not participate in those kernel locks, so the two
+runtimes will not see each other's lock and can both open the same slot, corrupting
+the journal. The persistence format itself remains cross-client: a directory written
+by one runtime can be handed to the other once the first has closed it. Only
+concurrent access is unsupported, and only between runtimes — two Node.js processes
+still exclude each other correctly.
+
+A kernel lock disappears the instant its holder dies; a directory does not. The holder
+therefore refreshes the owner directory's mtime every 5 seconds, and a contender
+reclaims a slot whose mtime has not advanced for 15 seconds. A contender also reclaims
+immediately when the owner record names a process that no longer exists on the same
+host, which is the common case after a crash. A stale owner directory is renamed aside
+before removal, so two contenders racing to reclaim one slot cannot both win it. Each
+acquisition also writes a token into the owner record and checks it before removing
+anything, so a release can never take away a directory that has since been handed to
+somebody else.
+
+If a holder is paused long enough for its heartbeat to lapse — `SIGSTOP`, a suspended
+VM, a stalled filesystem, or any synchronous section that blocks the event loop for
+more than 15 seconds — its lock can be reclaimed while it still believes it holds it.
+Such a holder stops writing: once it can no longer vouch for its own lock, every
+append, checkpoint and acknowledgement on that journal fails with
+`QwpReplayStoreLockLostError`, and the sender falls back to whatever its durability
+policy does when the journal is unavailable. This is deliberately conservative — the
+holder fails as soon as a contender _could_ have taken the slot, not only once one
+demonstrably has — because the alternative is writing at offsets the new owner now
+owns. A frame's sequence is derived from its position in the segment, so a same-width
+overwrite would otherwise reopen as a complete journal with the new owner's
+acknowledged frames missing and nothing reported.
+
+New journals use the cross-client SFA persistence layout. Fixed-size
+`sf-.sfa` files have the Java/Rust 24-byte `SF01` header and
+`[crc32c, payloadLength, payload]` frame envelope. `sf-manifest.bin` and
+`.ack-watermark` use the shared dual-slot checksummed metadata layout, while
+`.symbol-dict` uses the shared chunked `SYD1` representation. TypeScript tests load
+Java-produced segment and dictionary fixtures and compare TypeScript output with the
+same normalized bytes.
+
+Each segment reserves `maxSegmentBytes` of target payload data (4 MiB by default)
+plus one frame header so a maximum-sized frame fits. The active segment and one
+pre-sized temporary hot spare keep open file handles; rotation activates the spare.
+A process-wide, unreferenced worker provisions replacements, checkpoints dirty paths,
+and performs ACK-driven unlink and directory barriers. ACK trimming advances the
+durable manifest head before handing removal to that worker and runs in bounded
+background batches. Frame append uses a vectored header-plus-payload write, avoiding
+an additional payload-sized journal buffer.
+
+A background provisioning, checkpoint or trim failure is parked on the store and
+raised from the next journal call, then cleared by the next successful batch. Because
+such a fault is transient — a briefly full, read-only or descriptor-starved volume —
+reaching one while applying a server acknowledgement reconnects and replays rather
+than ending the sender: a filesystem hiccup must not cost a running producer. Failures
+that are verdicts on the journal itself carry `retryable: false` and stay terminal;
+today those are `QwpReplayStoreCorruptionError` and `QwpReplayStoreLockLostError`. The
+store persists its acknowledgement cursor before it mutates anything, so a fault at
+that moment leaves exactly the state a crash at that moment would leave, and replay
+resumes from the persisted watermark.
+
+Recovery validates segment CRCs with a reusable 64 KiB scanner and indexes only frame
+sequence, file offset, and payload length. The reconnect loop reads one payload from
+its retained segment handle when it is ready to send it; it does not materialize the
+complete persisted backlog. Fresh background store-and-forward frames likewise drop
+their resident payload after journal publication and are read back on demand. Memory
+therefore scales with the active encoding/send window rather than total disk backlog.
+
+Recovery also handles the canonical creation crash window in which a valid SFA
+segment becomes durable before its manifest.
+
+On startup, a dictionary sidecar truncated at a complete-block boundary is rebuilt
+from the ordered symbol deltas embedded in surviving committed frames and healed
+before replay. A corrupt or stale dictionary sidecar is replaced when those committed
+frames independently reconstruct a complete dense dictionary from ID zero. If the
+frame journal is structurally corrupt, or the surviving deltas contain a dictionary
+gap or conflict that cannot be reconstructed, the foreground slot is renamed to
+`.unreplayable-N`, marked with `.failed`, and preserved for inspection. The
+sender then starts once with a clean slot at the configured path.
+`onRecoveryQuarantine` receives the original and quarantine paths plus the terminal
+cause and a typed `senderError`. The shared `onSenderError` callback receives the same
+`data-loss` / `abandoned` verdict and its `quarantinedPath`. This build-time recovery
+notification is synchronous because no connected sender dispatcher exists yet;
+callback failures cannot interrupt recovery. Quarantined paths are never adopted by
+the orphan scanner. Operational filesystem errors are not quarantined and still fail
+startup, so a temporary permissions or disk problem cannot be mistaken for data
+corruption.
+
+For a standalone sender, `drainOrphans: true` scans sibling directories beneath the
+configured journal directory's parent, excludes the sender's own directory, and
+adopts record-bearing slots left by failed producers. Adoption is lock-protected and
+uses an independent QWP connection per slot, bounded by `maxBackgroundDrainers` (4 by
+default). The scanner runs immediately and then every 30 seconds; set
+`orphanScanIntervalMs: 0` for a startup-only scan. Terminal recovery failures create
+`.failed` in the slot so a corrupt or permanently rejected head cannot cause a hot
+retry loop. After inspection or repair, call `retryQwpNodeOrphanSlot(slotDirectory)`
+to make it eligible again. `onOrphanDrainEvent` reports discovery, drain, lock
+contention, quarantine, scanner failures, durable-ACK capability gaps, and transient
+all-replica windows through a bounded asynchronous inbox. An abandoned slot also
+reports a typed `data-loss` sender error. Callback exceptions cannot interrupt
+recovery.
+
+Blocking (`off` or `sync`) foreground startup fails immediately if every usable
+endpoint lacks durable-ACK support. Asynchronous foreground startup and steady-state
+store-and-forward reconnects retain their records and retry through rolling upgrades.
+An orphan slot retries a consecutive durable-ACK capability-gap episode until either
+16 connection sweeps or the configured reconnect `maxDurationMs` is reached, then it
+is quarantined behind `.failed` (`maxDurationMs: 0` disables only the time half of
+the budget). A transport outage or an all-replica window resets both halves of this
+orphan budget; neither transient condition can itself quarantine persisted data. The
+`durable-ack-unavailable`,
+`durable-ack-persistent-failure`, and `primary-unavailable` orphan events expose the
+distinction to operators.
+
+A foreground sender retries a symbol-dictionary catch-up entry that is too large for
+the current target forever because a larger-cap node may return. An orphan drainer
+quarantines that slot only after 16 consecutive incompatible-cap observations and a
+minimum five-minute dwell. Tune the dwell with
+`catchUpCapGapMinEscalationWindowMs`; an unrelated transport or upgrade failure resets
+the episode so outage time cannot accidentally satisfy it.
+
+Keep sibling adoption off unless the parent is a dedicated store-and-forward group:
+every record-bearing child directory that is not the foreground slot is considered
+eligible. Browser senders never scan or persist local slots.
+
+An offline sender cannot inspect the server-advertised batch cap before its first
+publication. Set `qwp.session.maxBatchSizeBytes` to a value no greater than the
+smallest target node's cap when offline startup is required.
+
+Set `awaitServerAck: true` when a particular flush must observe QuestDB's protocol ACK
+before returning. `awaitDurableAck: true` implies server-ACK waiting and additionally
+waits for replicated/durable progress. Browser senders use the in-memory replay
+publication boundary by default and do not offer persistent disk publication.
+
+A crash after the server accepts a frame but before local acknowledgement cleanup can
+replay that frame, so delivery is at least once. Applications that require exactly-once
+effects should use their own stable event key or another idempotency strategy. Closing
+a persistent sender stops its drainer but preserves published, unacknowledged frames for
+the next sender using that directory.
+
+### Direct high-level API
+
+Use `QwpSender` directly when QWP-only column types or detailed session controls are
+needed:
+
+```typescript
+import * as qwp from "@questdb/nodejs-client/qwp";
+import { connectQwpNodeSender } from "@questdb/nodejs-client/qwp/node";
+
+const sender = await connectQwpNodeSender(
+ {
+ url: "wss://questdb.example:9000/write/v4",
+ authorization: `Bearer ${token}`,
+ },
+ {
+ autoFlushRows: 5_000,
+ autoFlushBytes: 4 * 1024 * 1024,
+ autoFlushIntervalMs: 1_000,
+ encode: { symbolDictionary: "delta", gorilla: true },
+ },
+);
+
+try {
+ await sender
+ .table("telemetry")
+ .symbol("device", "sensor-7")
+ .longColumn("sequence", 42n)
+ .uuidColumn("event_id", "9f1c96b2-54b8-4d85-bb24-e82c6f1ac120")
+ .at(1_775_000_000_000, "ms");
+ await sender.flush();
+} finally {
+ await sender.close();
+}
+```
+
+A row in progress is the columns staged so far plus the table selected by
+`table()`. When a setter or `at()` rejects a value, the sender discards both, so a
+half-built row can never reach QuestDB and the next row starts from `table()` again:
+
+```typescript
+for (const reading of readings) {
+ try {
+ await sender
+ .table("telemetry")
+ .symbol("device", reading.device)
+ .floatColumn("value", reading.value)
+ .at(reading.timestamp, "ms");
+ } catch (error) {
+ // Only this row is gone. Rows staged earlier stay pending.
+ log.warn(error);
+ }
+}
+await sender.flush();
+```
+
+Setters called after a failure raise `table name must be set before adding columns`
+rather than quietly joining a fresh row. `cancelRow()` discards a row in progress the
+same way without an error, and `reset()` remains the heavier option that also drops
+every row staged since the last flush.
+
+### Compiled object-row writers
+
+For repeated rows with one table schema, compile a table-bound writer instead of
+sharing the fluent row-builder state:
+
+```typescript
+const trades = sender.writer("trades", {
+ symbol: qwp.symbol(),
+ side: qwp.symbol(),
+ price: qwp.double(),
+ quantity: qwp.long(),
+ timestamp: qwp.designatedTimestamp("ns"),
+});
+
+await trades.row({
+ symbol: "ETH-USD",
+ side: "sell",
+ price: 2615.54,
+ quantity: 42n,
+ timestamp: 1_723_000_000_000_000_000n,
+});
+
+await trades.rows([
+ {
+ symbol: "BTC-USD",
+ side: "buy",
+ price: 39_269.98,
+ quantity: 7n,
+ timestamp: 1_723_000_001_000_000_000n,
+ },
+]);
+```
+
+`rows()` accepts `Iterable` and `AsyncIterable` sources and applies the sender's
+normal auto-flush, batch-cap, backpressure, transaction, symbol-dictionary, and ACK
+settings. The schema is validated once.
+
+The schema vocabulary covers every column type the fluent row API can write:
+
+| Field | QuestDB type | Accepted row values |
+| --------------------------- | -------------------- | -------------------------------------------------------------------------------------- |
+| `symbol()` | SYMBOL | `string` |
+| `varchar()` | VARCHAR | `string` |
+| `char()` | CHAR | `string` of one UTF-16 code unit |
+| `bool()` | BOOLEAN | `boolean` |
+| `byte()` | BYTE | `number` |
+| `short()` | SHORT | `number` |
+| `int32()` | INT | `number` |
+| `int64()`, `long()` | LONG | `bigint` |
+| `float32()` | FLOAT | `number` |
+| `float64()`, `double()` | DOUBLE | `number` |
+| `timestamp(unit)` | TIMESTAMP | `number` or `bigint`; `"ns"` requires `bigint` |
+| `designatedTimestamp(unit)` | designated TIMESTAMP | as above, required in every row |
+| `date()` | DATE | `number` or `bigint` milliseconds since the epoch |
+| `binary()` | BINARY | `Uint8Array`, copied on append |
+| `uuid()` | UUID | canonical UUID text, 16 canonical big-endian bytes, or `{ low, high }` |
+| `long256()` | LONG256 | unsigned 256-bit `bigint`, `0x` hex text, four little-endian words, or `{ words }` |
+| `ipv4()` | IPV4 | dotted-quad text or signed/unsigned packed address; `0.0.0.0` is the NULL sentinel |
+| `geohash(precisionBits)` | GEOHASH | raw bits, base-32 text of `precisionBits / 5` characters, or `{ bits, precisionBits }` |
+| `decimal64(scale)` | DECIMAL64 | unscaled `bigint`, decimal text, `number`, or `{ unscaled, scale }` |
+| `decimal128(scale)` | DECIMAL128 | as above, scale up to 38 |
+| `decimal256(scale)` | DECIMAL256 | as above, scale up to 76 |
+| `doubleArray()` | DOUBLE[] | nested `number` arrays of uniform shape, or `{ dimensions, values }` |
+| `longArray()` | LONG[] | nested `bigint`/`number` arrays of uniform shape, or `{ dimensions, values }` |
+
+LONG, LONG256, and nanosecond timestamp inputs are `bigint` so they cannot silently
+lose precision. The record forms are exactly what the egress result views hand back,
+so a query result value can be written straight into a row without conversion.
+
+Widths are spelled out deliberately. The fluent row API predates these names and its
+`floatColumn()` and `intColumn()` are 64-bit despite reading as 32-bit, with
+`float32Column()` and `int32Column()` as the narrow forms. Compiled writers avoid the
+ambiguity: `float32()`/`float64()` and `int32()`/`int64()` mean exactly what they say.
+
+Geohash precision and decimal scale belong to the column, not the value, so they are
+fixed when the schema is compiled and validated against the sender's staged schema on
+every append. Decimal text and `{ unscaled, scale }` values are rescaled to the
+column's scale when that is exact, and rejected when it would round: at
+`decimal64(2)`, `"1.50"` stages as `150n` and `"1.005"` raises `QwpWriterRowError`.
+Base-32 geohash text carries five bits per character, so `geohash(20)` accepts
+`"u33d"` and rejects `"u33"`.
+
+Regular fields may be absent, `null`, or `undefined`, which writes a NULL. A schema
+may contain at most one designated timestamp and, when present, that field is required
+in every row. Unknown object keys and type mismatches raise `QwpWriterRowError`; bulk
+errors include the zero-based row index. A failing row is never partly staged. Rows
+successfully completed before a later iterable row fails remain available to flush.
+
+Compiled writers are also available through the regular Node `Sender` when it uses a
+QWP transport. Calling `writer()` for an HTTP or TCP ILP sender raises an error. A
+writer obtained from a pooled sender lease cannot be used after the lease is closed.
+
+Like the Java QWP sender, `flush()` and `commit()` resolve after the complete
+logical flush reaches the local ingress/replay publication boundary. They do
+not wait for a server ACK by default. Set `awaitServerAck: true` for an
+implicit ACK barrier, or use the explicit sequence API below.
+
+For producer-controlled acknowledgement barriers, publish first and wait for the
+cumulative ACK watermark separately:
+
+```typescript
+await sender
+ .table("telemetry")
+ .symbol("device", "sensor-7")
+ .longColumn("sequence", 43n)
+ .atNow();
+
+const sequence = await sender.flushAndGetSequence();
+await sender.waitForAcknowledged(sequence, 5_000);
+```
+
+`flushAndGetSequence()` always resolves at the publication boundary, independently
+of `awaitServerAck`, and returns the highest stable frame sequence published by that
+call. It returns `-1n` when there was nothing to publish. `publishedSequence` and
+`acknowledgedSequence` expose the current immutable watermarks. ACK waits are
+cumulative, so one later acknowledgement resolves all covered waits and callers may
+wait for different sequences concurrently. When durable ACK was negotiated, the
+acknowledged watermark advances only after QuestDB reports durable progress;
+otherwise it follows ordinary protocol OK responses. A deadline failure raises
+`QwpIngressAckTimeoutError` without closing an otherwise healthy session.
+
+Rows are staged until an auto-flush boundary or an explicit `flush()`. A `null` or
+`undefined` column value omits that column from the row. `atNow()` asks QuestDB to
+assign the designated timestamp; `at(value, unit)` sends an explicit `ns`, `us`, or
+`ms` timestamp. `close()` publishes completed rows and waits for the committed-frame
+ACK watermark for up to `closeFlushTimeoutMs` (5 seconds by default, matching the
+Java client). Set it to `0` or a negative value for a fast close, which publishes
+without the ACK drain; publication itself stays bounded, so `close()` always
+returns. An unfinished row is still discarded with a warning.
+The configuration-string equivalent is `close_flush_timeout_millis`.
+
+`autoFlushBytes` is a soft threshold over estimated raw column-buffer storage and is
+disabled by default (`0`). It combines with `autoFlushRows` and
+`autoFlushIntervalMs`: reaching any enabled threshold flushes after the completed row,
+so one row of overshoot is possible. Once connected, an enabled byte threshold is
+clamped to 90% of the server-advertised batch cap. Schema and symbol-dictionary
+overhead make this an estimate; exact encoded-size enforcement and automatic frame
+splitting remain the ingress session's responsibility. `sender.metrics.pendingBytes`
+and `sender.metrics.effectiveAutoFlushBytes` expose the live estimate and applied
+threshold. Configuration strings use `auto_flush_bytes=N`; `off` is equivalent to
+zero.
+
+The sender automatically maintains connection-scoped symbol IDs, emits dictionary
+deltas, tracks acknowledgements, and splits multi-row batches at the smaller of the
+client cap and the server-advertised cap. One row that cannot fit is rejected with
+`QwpBatchTooLargeError` before it is sent.
+
+Low-level Node sessions expose `publishFrame()`, `publishTables()`, and
+`publishTablesDelta()` for local-publication semantics. Their `send*()` counterparts
+continue to return the server ACK. Use the publication methods only with persistent
+store-and-forward when local durability is the intended completion boundary.
+`sendFrameWithPublication()`, `sendTablesWithPublication()`, and
+`sendTablesDeltaWithPublication()` expose both boundaries from one operation: await
+`publication` before releasing retryable source rows, then await `acknowledgement`
+when server acceptance is also required. If a split logical batch cannot be fully
+journaled, its unattempted suffix is suppressed and the operation's publication
+promise rejects.
+
+### Browser ingress
+
+Browser applications must use the browser entry point and a same-origin WebSocket
+route (directly or through a reverse proxy):
+
+```typescript
+import { connectQwpBrowserSender } from "@questdb/nodejs-client/qwp/browser";
+
+const url = new URL("/write/v4", location.href);
+url.protocol = location.protocol === "https:" ? "wss:" : "ws:";
+
+const sender = await connectQwpBrowserSender({ url }, { autoFlush: false });
+
+try {
+ await sender.table("page_events").symbol("kind", "view").atNow();
+ await sender.flush();
+} finally {
+ await sender.close();
+}
+```
+
+The browser WebSocket API cannot set `Authorization` or arbitrary `X-QWP-*`
+upgrade headers. When authentication is enabled, create QuestDB's HttpOnly session
+cookies over REST before opening the WebSocket:
+
+```typescript
+import {
+ bootstrapQwpBrowserSession,
+ connectQwpBrowserSender,
+} from "@questdb/nodejs-client/qwp/browser";
+
+await bootstrapQwpBrowserSession({
+ url: new URL("/exec", location.href),
+ authentication: { type: "bearer", token: oidcOrRestAccessToken },
+ // QuestDB Enterprise only; omit this to use the logged-in principal.
+ serviceAccount: "market_data_writer",
+});
+
+const sender = await connectQwpBrowserSender({ url });
+```
+
+Basic authentication is also accepted as `{ type: "basic", username, password }`.
+The application obtains OIDC tokens from its identity provider; this package does
+not run an interactive OIDC flow. The bootstrap request uses
+`credentials: "include"`. REST and WebSocket endpoints therefore need the same
+browser origin, or correctly configured credentialed CORS and cookie attributes.
+JavaScript never reads `qdb_session` or the Enterprise `qdbServiceAccount` cookie.
+
+Set `sessionBootstrap` on the WebSocket options to repeat bootstrap before every
+initial, reconnect, and failover attempt:
+
+```typescript
+const sender = await connectQwpBrowserSender({
+ url,
+ sessionBootstrap: {
+ authentication: { type: "bearer", token: oidcOrRestAccessToken },
+ serviceAccount: "market_data_writer",
+ },
+});
+```
+
+### Transactions and durable acknowledgement
+
+Transactional auto-flush keeps automatically emitted frames in an open server-side
+transaction. `commit()` (an alias for `flush()`) publishes the group-closing frame.
+The example also waits for its cumulative durable acknowledgement because it enables
+`awaitDurableAck`:
+
+```typescript
+const sender = await connectQwpBrowserSender(
+ { url, requestDurableAck: true },
+ {
+ transactional: true,
+ autoFlushRows: 10_000,
+ awaitDurableAck: true,
+ durableAckTimeoutMs: 30_000,
+ },
+);
+
+for (const event of events) {
+ await sender
+ .table("events")
+ .symbol("source", event.source)
+ .longColumn("value", event.value)
+ .at(event.timestamp, "ms");
+}
+await sender.commit();
+```
+
+Transactions are atomic per table, not across all tables in one flush. Closing a
+sender publishes locally staged transactional rows but does not implicitly commit;
+QuestDB rolls the open server transaction back. The sender logs a warning in this case.
+
+In browsers, durable ACK capability is negotiated with a WebSocket subprotocol;
+Node.js uses upgrade headers. Setting `awaitDurableAck` automatically requests the
+capability unless `requestDurableAck` was set explicitly. The connection fails with
+`QwpDurableAckUnavailableError` when the server does not confirm it. Browser durable
+tracking is in memory only. Persistent store-and-forward is intentionally Node-only.
+
+Browser ingress adds `qwp_browser_handshake=v1` to the WebSocket URL. Compatible
+servers send a small `SERVER_INFO` message immediately after the upgrade, and the
+sender uses its exact ingress payload cap for automatic splitting. Older servers
+ignore the query parameter; after a bounded 250 ms negotiation window the client
+continues in unknown-cap mode. Set `ingressNegotiationTimeoutMs` to tune that window,
+or keep using `maxBatchSizeBytes` as a local compatibility limit.
+
+### Reconnect, failover, and roles
+
+The preferred URL and `failoverUrls` form one endpoint set. Endpoints are ranked by
+observed health (`healthy`, unknown, transient rejection, transport error, topology
+rejection) and then by zone affinity; configuration order breaks ties. Health outranks
+zone, so a known healthy cross-zone node is preferred to an untried local node. Every
+connection sweep can still try every endpoint, allowing role and health changes to
+recover. A non-orderly close demotes the selected endpoint before the next sweep.
+Each standalone Node sender/drainer family, and each pooled orphan scanner, shares one
+live health ledger among its walkers while keeping independent sweep cursors, so
+concurrent drainers cannot consume one another's endpoint attempts. After a foreground
+round is exhausted, stale classifications are reset while learned zone tiers persist;
+the most recent successful same-zone endpoint remains sticky. Background orphan
+drainers publish health observations but never reset foreground classifications.
+Ingress reconnect is enabled by default for factory-created browser and Node sessions.
+Unacknowledged frames are retained in memory and replayed at least once after a
+transport failure. The built-in memory replay queue is capped at 128 MiB. When the
+cap is full, publication waits for ACK-driven trimming for at most 30 seconds, then
+rejects with `QwpMemoryReplayAppendTimeoutError`; a single frame that can never fit
+is rejected immediately with `QwpMemoryReplayFrameTooLargeError`. Set
+`memoryReplayMaxBytes` and `memoryReplayAppendDeadlineMs` on ingress session options
+to tune these bounds. The accounting includes a fixed per-frame allowance so many
+small frames cannot bypass the byte cap.
+
+The default memory policy uses full-jitter backoff from 100 ms to 5 seconds and a
+five-minute per-outage deadline; the initial connection remains fail-fast. Set
+`reconnect: false` for one fixed connection. Supplying a `reconnect` object tunes the
+bounds, emits lifecycle events through `onEvent`, and retains the earlier opt-in
+behavior of retrying initial connection establishment.
+
+Each retry delay is selected between zero and the current exponential ceiling,
+preventing clients disconnected together from retrying in lockstep. Configured attempt
+and duration bounds apply to browser/memory reconnect and Node `"sync"` startup. A
+Node foreground store-and-forward replay loop remains unbounded after startup. Without
+`storeAndForward`, both Node and browser ingress replay only for the lifetime of the
+process or page; configuring a Node directory makes the same replay crash-safe.
+
+Ingress also detects a replay head that is repeatedly NACKed or followed by a
+non-orderly WebSocket close. `maxFrameRejections` controls the strike threshold and
+`poisonMinEscalationWindowMs` (5 seconds by default) prevents a brief outage from
+being mistaken for a deterministic poison frame. Normal and going-away closes,
+`NOT_WRITABLE`, and retriable symbol-dictionary catch-up rejections are retried with
+pacing but do not count as poison strikes.
+
+Node.js sees the rejected upgrade status and `X-QuestDB-Role`, so a read-only replica
+or catching-up primary can be classified and skipped. Browsers deliberately expose
+an opaque upgrade error because their WebSocket API hides the HTTP response. Avoid
+placing ingress replica endpoints in a browser endpoint list unless the proxy routes
+writers to a primary.
+
+### Observability
+
+Use immutable metrics snapshots for polling and callbacks for event-driven telemetry:
+
+```typescript
+import {
+ QWP_INGRESS_PROGRESS_KIND,
+ createQwpNodeSender,
+} from "@questdb/nodejs-client/qwp/node";
+
+const sender = createQwpNodeSender(
+ { url: "ws://localhost:9000/write/v4" },
+ {},
+ {
+ reconnect: {
+ onEvent: (event) => console.info("QWP connection", event),
+ },
+ onProgress: (event) => {
+ if (event.kind === QWP_INGRESS_PROGRESS_KIND.ACKNOWLEDGED) {
+ console.info("accepted through", event.sequence);
+ }
+ },
+ onError: (event) => console.error("QWP ingress", event.error),
+ onSenderError: (error) => {
+ console.error(
+ "QWP rejection",
+ error.category,
+ error.appliedPolicy,
+ error.fromFsn,
+ error.toFsn,
+ );
+ },
+ },
+);
+
+await sender.connect();
+console.info(sender.metrics);
+```
+
+Callbacks are placed on bounded asynchronous inboxes and never invoked inside ACK,
+reconnect, or orphan-recovery protocol stacks. Connection events default to 64 retained
+entries and errors to 256; `connectionListenerInboxCapacity` and
+`errorInboxCapacity` (or their snake-case unified-string keys) tune those bounds.
+Overflow drops the oldest pending entry and retains the newest state. Inspect
+`droppedProgressNotifications`, `droppedConnectionNotifications`, and
+`droppedErrorNotifications` in the immutable ingress metrics; non-zero values mean an
+observer is not keeping up. Callback failures are contained. Callbacks still execute on
+the JavaScript event loop, so CPU-bound synchronous work should be moved to an
+application worker.
+
+`onSenderError` is the Java-parity rejection stream. Its immutable payload includes
+`category`, applied policy, raw server status/message, wire message sequence, inclusive
+stable `[fromFsn, toFsn]` correlation range, optional single-table attribution, and
+`quarantinedPath` for abandoned persistent data. The legacy `onError` callback remains
+available for timeouts and general session failures; classified NACK events also expose
+the same payload as `event.senderError`. When `onSenderError` is omitted, QWP logs
+retriable rejections at `warn` and terminal rejections or abandoned data at `error`.
+General asynchronous session failures are likewise logged when `onError` is omitted,
+so a background store-and-forward failure is never silent by default. Reconnect and
+orphan-drain fallbacks use the same bounded asynchronous error inbox; direct session
+fallback logging adds no callback or close-time dependency. Both paths work in browsers
+and Node.js.
+
+## Egress
+
+QWP egress streams typed result batches. One connection executes one active query at
+a time.
+
+```typescript
+import { connectQwpNodeEgress } from "@questdb/nodejs-client/qwp/node";
+
+const session = await connectQwpNodeEgress(
+ {
+ url: "wss://questdb.example:9000/read/v1",
+ failoverUrls: [
+ "wss://questdb-replica-2.example:9000/read/v1",
+ "wss://questdb-primary.example:9000/read/v1",
+ ],
+ target: "replica",
+ zone: "eu-west-1a",
+ authorization: `Bearer ${token}`,
+ compression: "zstd",
+ compressionLevel: 3,
+ maxBatchRows: 4096,
+ },
+ { queryTimeoutMs: 30_000, bufferPoolSize: 4 },
+);
+
+try {
+ const query = await session.query(
+ "select timestamp, symbol, price from trades where symbol = $1",
+ {
+ binds: (binds) => binds.setVarchar(0, "ETH-USD"),
+ initialCredit: 1024 * 1024,
+ },
+ );
+
+ for await (const batch of query) {
+ console.info(batch.columns);
+ for (const row of batch.rows()) console.info(row);
+ }
+
+ const completion = await query.completion;
+ console.info(completion);
+} finally {
+ await session.close();
+}
+```
+
+### Bounded reusable result views
+
+`query()` keeps its convenient materialized batches. For hot paths, `queryViews()`
+avoids allocating a JavaScript value array for every column and delivers one
+reusable batch view through an awaited callback:
+
+```typescript
+const query = await session.queryViews(
+ "select timestamp, symbol, price from trades",
+ async (batch) => {
+ const timestamp = batch.column(0);
+ const symbol = batch.column(1);
+ const price = batch.column(2);
+
+ // Fixed-width values are read directly from the QWP little-endian bytes.
+ for (let row = 0; row < batch.rowCount; row++) {
+ if (!price.isNull(row)) {
+ consume(
+ timestamp.getLong(row),
+ symbol.getSymbol(row),
+ price.getDouble(row),
+ );
+ }
+ }
+
+ // Raw views are available for vectorized consumers.
+ consumePackedDoubles(price.valuesBytes()!);
+ },
+ { initialCredit: 256 * 1024 },
+);
+await query.completion;
+```
+
+For conventional row-major processing, the same batch also owns one reusable
+`QwpResultRowView`:
+
+```typescript
+batch.forEachRow((row) => {
+ if (!row.isNull(2)) {
+ consume(row.getLong(0), row.getSymbol(1), row.getDouble(2));
+ }
+});
+
+// Direct indexed access uses the same flyweight.
+const first = batch.row(0);
+consume(first.rowIndex, first.getString(1));
+```
+
+`forEachRow()` is synchronous, visits rows in index order, propagates callback
+exceptions, and re-points the same row object on every iteration. Do not retain
+the row object or any zero-copy value returned from it; copy the value inside the
+current invocation when it must survive. Calling `batch.row(index)` also returns
+that shared object, re-pointed to the requested row.
+
+The batch, its column objects, and every `Uint8Array`/`Int32Array` returned by a
+column or row are valid only until the callback settles. The decoder reuses those
+objects and its NULL-index, symbol-ID, array-offset, and Gorilla-timestamp scratch
+storage for later batches. Copy an individual byte view with `.slice()`, or call
+`batch.materialize()` inside the callback, when data must be retained.
+
+Raw fixed-width, NULL, VARCHAR/BINARY, and array data views point into the current
+decoded frame; Zstd results point into that batch's decompressed buffer. Accessors
+such as `getString()` and `get()` decode or construct only the requested cell. The
+callback is awaited before automatic credit is replenished, so the configured
+credit window bounds server read-ahead while application work is in progress.
+
+`target` accepts `any` (the default), `primary`, or `replica`. Primary routing also
+accepts standalone servers and a primary completing catch-up, matching the Java
+client. Both keys apply to ingress and egress alike. `zone` is an opaque, case-insensitive preference for `any` and `replica`;
+cross-zone endpoints remain eligible. It is ignored for `primary`, which must be
+followed across zones. The client validates the authoritative role and zone from the
+first QWP `SERVER_INFO` frame before accepting an endpoint, so the same guarantees
+work in browsers even though browser WebSocket APIs hide upgrade response headers.
+
+Bind indexes are zero-based in the client: index `0` is SQL placeholder `$1`.
+`QwpBindValues` supports booleans, integer and floating-point values, dates,
+microsecond and nanosecond timestamps, strings, UUIDs, LONG256, geohashes,
+decimals, and typed nulls. Set values in ascending index order. `bindPayload` and
+`bindCount` remain advanced escape hatches for pre-encoded data.
+
+Set per-query `resetDictionary: true` to ask the server to reset its
+connection-scoped egress symbol dictionary before execution. The client sends the
+flag only when `SERVER_INFO` advertises `QUERY_FLAGS`; older servers receive the
+same flag-free request as the default path, so this option remains safe during a
+rolling upgrade.
+
+Matching Java, the high-level client defaults `initialCredit` to zero, allowing
+unbounded server send-ahead. Set a positive session-level or per-query value to bound
+wire buffering, particularly in browsers. With positive credit, the exact wire size
+of each consumed batch is replenished automatically. Set `autoCredit: false` and call
+`query.grantCredit()` for manual control.
+
+Both materialized `query()` results and zero-copy `queryViews()` use a client-side
+decoded-batch pool with four slots by default. Set the session-level
+`bufferPoolSize` to tune this bound. Materialized decoding pauses when all slots are
+queued until iteration requests another batch. For `queryViews()`, callbacks remain
+serial and callback-scoped, while the receive loop continues decoding into the other
+reusable slots; a slow callback stalls decoding only after the pool fills. This bound
+is independent of QWP credit, so `initialCredit: 0` no longer permits an unbounded
+queue of decoded batches. Protocol credit remains the stronger end-to-end bound,
+particularly in browsers where the WebSocket implementation may buffer raw frames
+before JavaScript reads them.
+
+A session `queryTimeoutMs` supplies the default deadline; per-query `timeoutMs`
+overrides it, and zero disables it. Expiry rejects iteration and `completion` with
+`QwpEgressQueryTimeoutError`, sends QWP `CANCEL`, and drains the terminal response
+before the connection accepts another query. Breaking out of `for await` early also
+discards buffered batches, restores their flow-control credit, sends `CANCEL`, and
+rejects `completion` with `QwpEgressQueryAbandonedError`. Call `query.cancel()` for
+explicit cancellation.
+
+`await query.awaitCompletion(timeoutMs)` bounds only the caller's wait and returns
+`false` without cancelling when the timeout expires, matching Java
+`Completion.await(timeout, unit)`. `query.isDone()` reports terminal state. Use the
+query deadline options only when timeout should actively cancel the server query.
+The initial and reconnect `SERVER_INFO` timeout defaults to five seconds, matching
+Java, and remains configurable through `serverInfoTimeoutMs`.
+
+Cancellation draining is bounded by `cancelDrainTimeoutMs` (5 seconds by default).
+Late batches are decoded and credited while the terminal response is pending. If the
+server does not terminate the query within the bound, the client fails with
+`QwpEgressQueryCancelTimeoutError` and closes the unusable connection instead of
+leaving the session permanently occupied.
+
+Node.js and browsers can request Zstd with `compression: "zstd"` or `"auto"` and a
+level from 1 through 22. Raw remains the compatibility default. Node uses
+`X-QWP-Accept-Encoding`; browsers send the same preference in the URL's
+`qwp_accept_encoding` parameter. Compatible servers report the effective codec and
+operator-forced level in the existing egress `SERVER_INFO` message. Check
+`session.negotiatedCompression` after the handshake. Older servers ignore the query
+parameter and safely remain raw. The decoder handles raw and Zstd batches in both
+runtimes.
+
+Set transport-level `maxBatchRows` from 1 through 1,048,576 to ask QuestDB for
+smaller `RESULT_BATCH` messages. The server clamps the request to its hard cap. Node
+sends `X-QWP-Max-Batch-Rows`; browsers use the `qwp_max_batch_rows` URL parameter,
+which requires a server that supports browser QWP negotiation. Older servers ignore
+the browser parameter and keep their configured batch size.
+
+A single `RESULT_BATCH` may declare at most `QWP_MAX_CELLS_PER_BATCH` cells --
+32Mi, its rows multiplied by its columns. The row and column caps bound each
+dimension on its own, and a compressed body detaches the grid they describe from
+the bytes on the wire: an all-NULL column is one bit per cell before Zstd, so
+without this bound a few kilobytes of RLE-compressed bitmap declares a result no
+heap can hold. The bound is checked before any column is read, and 32Mi cells sits
+far above any plausible result -- the widest supported table at 16k rows, or a full
+1,048,576-row batch at 32 columns. Lower `maxBatchRows` for genuinely wide tables.
+
+Opening a connection runs under two deadlines. `connect_timeout` covers the TCP/TLS
+transport, and `auth_timeout_ms` takes over for the WebSocket upgrade and the
+authentication exchange as soon as the transport connects; both default to 15
+seconds. Setting only `connect_timeout` bounds both phases with that value, so an
+endpoint that accepts TCP and never answers the upgrade -- a stalled proxy or load
+balancer -- fails inside the budget you asked for rather than 15 seconds later. Set
+`auth_timeout_ms` as well when the upgrade legitimately needs longer than the
+transport.
+
+Egress failover is enabled by default in Node.js and browsers. A transport failure or
+invalid protocol response closes and deprioritizes that endpoint, reconnects, resets
+connection-scoped decoding state, and re-executes the active query. The default policy
+uses eight connection sweeps, full-jitter backoff starting at 50 ms and capped at one
+second, and a 30-second outage deadline. `QUERY_ERROR` remains a query result and does
+not trigger failover.
+
+`session.ready` resolves once with the initial `SERVER_INFO`. Read
+`session.serverInfo` for the immutable snapshot from the currently bound endpoint:
+role, zone, cluster and node IDs, epoch, capabilities, server clock, and negotiated
+compression. Reading the property is non-perturbing and never initiates a failover
+walk. If an endpoint dies, it continues to report the previous snapshot until the
+transport successfully rebinds, then refreshes to the new endpoint.
+
+Re-execution is at least once: a statement may have completed before its response was
+lost, and a consumer may already have observed a prefix of SELECT rows. Queued but
+unconsumed batches are discarded automatically. Configure `onReplayReset` when the
+application must clear an accumulated prefix before batches restart at sequence zero;
+the callback is an optional notification, not an opt-in. Set `reconnect: false` to use
+one fixed connection and surface failures without replay. Supplying a `reconnect`
+object tunes the failover bounds and also retains the earlier opt-in behavior of
+retrying initial connection establishment.
+
+Browser egress uses the same session API:
+
+```typescript
+import { connectQwpBrowserEgress } from "@questdb/nodejs-client/qwp/browser";
+
+const readUrl = new URL("/read/v1", location.href);
+readUrl.protocol = location.protocol === "https:" ? "wss:" : "ws:";
+
+const session = await connectQwpBrowserEgress({
+ url: readUrl,
+ failoverUrls: ["wss://replica-2.example/read/v1"],
+ target: "replica",
+ zone: "eu-west-1a",
+ compression: "zstd",
+ compressionLevel: 3,
+ sessionBootstrap: {
+ authentication: { type: "bearer", token: oidcOrRestAccessToken },
+ },
+});
+```
+
+## Combined pooled client
+
+Use `QwpClient` when one long-lived application component needs both ingestion
+and concurrent queries. The Node and browser entry points provide configured
+factories; each borrowed handle exclusively owns one pooled WebSocket until its
+`close()` returns it:
+
+For Node, the recommended common-case API accepts one Java-style
+`ws::`/`wss::` cluster string. Every `addr` entry is shared by ingress and
+egress; the facade derives `/write/v4` and `/read/v1`, applies the same
+authentication and TLS configuration to both sides, and validates ingress,
+egress, and pool settings before opening a socket:
+
+```typescript
+import { connectQwpNodeClient } from "@questdb/nodejs-client/qwp/node";
+
+const db = await connectQwpNodeClient(
+ "wss::" +
+ "addr=node-a.example:9000,node-b.example:9000;" +
+ `token=${token};` +
+ "target=replica;zone=eu-west-1a;" +
+ "sender_pool_max=2;query_pool_max=8;",
+);
+```
+
+Repeated `addr=` keys also accumulate endpoints. Programmatic overrides for
+callbacks, custom agents, store-and-forward, sender/session settings, and pool
+sizes may be passed as the second argument. The whole string is still validated
+before overrides are applied, matching the Java builder's fail-fast behavior.
+
+A custom `wss://` agent is the WebSocket upgrade's sole TLS channel, so it
+carries its own certificate verification and cannot be combined with
+`tls_verify`, `tls_roots`, or `tls_roots_password` — that combination is
+rejected rather than silently dropping either. Configure verification on the
+agent instead, and pass an `https.Agent` for `wss` (a plain `http.Agent` is for
+`ws`).
+
+The Node client accepts `tls_roots` only as valid PEM-encoded CA certificates.
+Password-protected PKCS#12 trust stores and `tls_roots_password` are rejected:
+Node's `pfx` option represents client private-key/certificate identity, not
+additional trusted roots. Export the CA certificates to PEM and omit the
+password key.
+
+Set `lazy_connect=on` to tolerate an unavailable cluster during startup. In the
+JavaScript client, ingress uses memory replay by default, or persistent replay when
+`sf_dir` is present, with `initial_connect_retry=async`; egress uses
+`query_pool_min=0` and connects on the first query. Explicit
+`initial_connect_retry=off|sync` or a positive `query_pool_min` conflicts with
+`lazy_connect` and is rejected before the client is created:
+
+```typescript
+const db = await connectQwpNodeClient(
+ "wss::addr=node-a.example,node-b.example;" + "lazy_connect=on;",
+);
+```
+
+For unified strings with `sf_dir`, Java-compatible defaults apply: memory
+durability, a 10 GiB total journal cap, 4 MiB frame/segment batches, a 30-second
+capacity wait, a 60-second close drain, and fail-fast initial connection. Set
+`sender_id` to name the disk slot base; pooled senders use `-`.
+Without `sf_dir`, `sf_max_total_bytes` and `sf_append_deadline_millis` tune the
+built-in memory replay queue instead.
+The parser also supports `max_name_len` and the Java listener/error inbox
+capacity keys. Those capacities actively bound asynchronous connection and
+typed-error delivery and are reflected in ingress drop counters.
+
+The object form remains available for cases where constructing the two sides
+separately is useful:
+
+```typescript
+import { connectQwpNodeClient } from "@questdb/nodejs-client/qwp/node";
+
+const db = await connectQwpNodeClient({
+ ingress: {
+ url: "wss://questdb.example:9000/write/v4",
+ authorization: `Bearer ${token}`,
+ },
+ egress: {
+ url: "wss://questdb.example:9000/read/v1",
+ authorization: `Bearer ${token}`,
+ target: "replica",
+ zone: "eu-west-1a",
+ },
+ pool: {
+ senderPoolMin: 1,
+ senderPoolMax: 2,
+ queryPoolMin: 1,
+ queryPoolMax: 8,
+ acquireTimeoutMs: 5_000,
+ idleTimeoutMs: 60_000,
+ maxLifetimeMs: 30 * 60_000,
+ housekeepingIntervalMs: 5_000,
+ },
+});
+
+try {
+ const sender = await db.borrowSender();
+ try {
+ await sender.table("trades").symbol("symbol", "ETH-USD").atNow();
+ } finally {
+ // Flushes completed rows and returns the sender; the socket stays pooled.
+ await sender.close();
+ }
+
+ const [prices, volumes] = await Promise.all([
+ db.borrowQuery(),
+ db.borrowQuery(),
+ ]);
+ try {
+ // These use independent egress WebSockets and may execute concurrently.
+ const drain = async (lease, sql) => {
+ const query = await lease.query(sql);
+ for await (const batch of query) consume(batch);
+ await query.completion;
+ };
+ await Promise.all([
+ drain(prices, "select * from latest_prices"),
+ drain(volumes, "select * from hourly_volumes"),
+ ]);
+ } finally {
+ await Promise.all([prices.close(), volumes.close()]);
+ }
+} finally {
+ await db.close();
+}
+```
+
+Browser applications can likewise describe the cluster, REST/OIDC
+authentication bootstrap, and failover order once. A cluster URL may be an
+origin, a reverse-proxy base path, or an existing `/write/v4` or `/read/v1`
+endpoint; the facade derives both protocol routes while preserving query
+parameters. Omit `sessionBootstrap.url` to derive the matching `/exec` route
+for every failover endpoint:
+
+```typescript
+import { connectQwpBrowserClient } from "@questdb/nodejs-client/qwp/browser";
+
+const db = await connectQwpBrowserClient({
+ cluster: {
+ url: "wss://node-a.example/qdb",
+ failoverUrls: ["wss://node-b.example/qdb"],
+ sessionBootstrap: {
+ authentication: { type: "bearer", token: oidcOrRestAccessToken },
+ serviceAccount: "analytics",
+ },
+ },
+ ingress: { requestDurableAck: true },
+ egress: {
+ target: "replica",
+ zone: "eu-west-1a",
+ compression: "zstd",
+ },
+ pool: { senderPoolMax: 2, queryPoolMax: 8 },
+});
+```
+
+`url`, `failoverUrls`, and `sessionBootstrap` belong to `cluster` in this
+unified form and are rejected if repeated under `ingress` or `egress`.
+Side-specific timeouts, WebSocket factories, durable-ACK settings, routing, and
+compression remain available as explicit overrides. The original split object
+form with complete `ingress` and `egress` trees remains supported for advanced
+cases that intentionally connect the two sides differently.
+
+`connectQwpNodeClient()` and `connectQwpBrowserClient()` prewarm each configured
+pool minimum. Their `createQwp*Client()` counterparts are lazy. Pools grow to
+their maximum under concurrent borrows and apply one FIFO acquisition deadline;
+exhaustion raises `QwpPoolAcquireTimeoutError`. Query handles are single-flight,
+but separate borrowed handles run concurrently. Returning a handle with an active
+query sends `CANCEL` and waits for the session's bounded cancellation drain; a
+connection that cannot drain is closed instead of being handed to another borrower.
+Each query lease exposes the same refreshed snapshot as `lease.serverInfo`; accessing
+it after returning the lease raises `QwpClientClosedError` rather than exposing a
+pooled connection now owned by another borrower.
+The shared housekeeper closes excess connections after `idleTimeoutMs` and recycles
+connections older than `maxLifetimeMs` once they are idle, while always retaining
+each configured pool minimum. Set either timeout to zero to disable that policy;
+`housekeepingIntervalMs` controls how quickly an expired idle connection is noticed.
+Prefer returning application-owned leases before calling `QwpClient.close()`.
+If shutdown races a borrower, it rejects queued borrowers, closes idle connections,
+and cancels active queries before closing every borrowed query connection. A query
+lease that is never returned therefore cannot retain a WebSocket after client
+shutdown; subsequent operations on it fail as closed. Borrowed senders remain under
+their producer's ownership: shutdown waits up to `acquireTimeoutMs` (capped at five
+seconds) for them to return and never closes a sender underneath its borrower. A
+sender returned during or after shutdown is closed instead of re-entering the pool,
+while a sender that outlives the bounded wait owns its eventual teardown.
+
+Pooled sender `close()` flushes completed rows, discards an unfinished row with a
+warning, and resets staging before reuse. With Node store-and-forward enabled, the
+configured directory is treated as a pool root and each stable sender slot owns a
+`sender-N` child directory, avoiding journal lock conflicts. The configured
+`senderPoolMin` remains authoritative. A client-level recovery scanner reserves and
+drains inactive canonical slots independently of foreground pool connections, both
+inside the current range and outside it after `senderPoolMax` is reduced. Foreground
+creation and recovery share an atomic slot coordinator, so neither can acquire a
+managed journal while the other owns it. This managed-slot recovery is automatic;
+`drainOrphans: true` additionally adopts noncanonical sibling slots beneath the pool
+root.
+
+## Error handling and cleanup
+
+The public error classes preserve enough context for policy decisions:
+
+| Error | Meaning |
+| ---------------------------------- | ----------------------------------------------------------------------------------------------------------- |
+| `QwpUpgradeError` | Classified authentication, role, version, capability, timeout, transport, or browser-opaque upgrade failure |
+| `QwpRoleMismatchError` | A connected endpoint's advertised role does not satisfy the requested egress target |
+| `QwpPoolAcquireTimeoutError` | Every pooled connection is leased beyond the configured acquisition deadline |
+| `QwpPoolResourceError` | Creating a new pooled sender or query connection failed |
+| `QwpClientClosedError` | The pooled client or an individual returned lease is already closed |
+| `QwpDurableAckUnavailableError` | Durable acknowledgement was required but not negotiated |
+| `QwpSendTimeoutError` | A send did not drain before its deadline; delivery is unknown |
+| `QwpSenderCloseTimeoutError` | Sender shutdown could not publish and ACK-drain all committed ingress frames within its deadline |
+| `QwpIngressNackError` | QuestDB rejected an ingress frame |
+| `QwpIngressAckTimeoutError` | The cumulative ingress ACK watermark did not reach the requested sequence before its deadline |
+| `QwpBatchTooLargeError` | One encoded row cannot fit the effective ingress cap |
+| `QwpReconnectExhaustedError` | The configured reconnect boundary was reached |
+| `QwpReplayRejectedError` | A replayed frame was rejected and retained for inspection |
+| `QwpReplayStoreFullError` | The Node.js replay journal reached its configured size |
+| `QwpReplayStoreAppendTimeoutError` | The Node.js replay journal did not regain capacity before the configured append deadline |
+| `QwpReplayStoreCheckpointError` | A periodic Node.js replay-journal checkpoint failed; operations fail closed until a retry succeeds |
+| `QwpReplayStoreLockedError` | Another process owns the configured Node.js replay directory |
+| `QwpEgressQueryError` | QuestDB returned a terminal query error |
+| `QwpEgressQueryAbandonedError` | Result iteration ended before the server completed the query |
+| `QwpEgressQueryTimeoutError` | The client deadline expired and cancellation began |
+| `QwpEgressQueryCancelTimeoutError` | A cancelled query did not produce a terminal server response before the drain deadline |
+| `QwpEgressReplayRequiredError` | Deprecated compatibility type from the former explicit replay opt-in |
+
+Always close senders and sessions in `finally`. Sender publication plus ACK draining is
+bounded by `closeFlushTimeoutMs`; the subsequent WebSocket closing handshake is bounded
+by `closeTimeoutMs`. In Node, `connectTimeoutMs` and `authTimeoutMs` independently
+bound transport connection and authenticated upgrade. `sendTimeoutMs`, acknowledgement
+timeouts, and query deadlines cover later lifecycle phases; configure each according
+to the deployment rather than using one very large catch-all value.
+
+## Migration guide
+
+### Existing Node.js `Sender`
+
+For the common fluent API, migration is primarily a transport change:
+
+```diff
+- const sender = await Sender.fromConfig("http::addr=localhost:9000");
++ const sender = await Sender.fromConfig("ws::addr=localhost:9000");
+```
+
+Review these behavioral differences before rollout:
+
+- QWP `flush()` uses the Java-compatible local-publication boundary by default in
+ browsers and Node.js. Set `awaitServerAck` for a protocol ACK barrier, or
+ `awaitDurableAck` to wait through durable upload. With Node persistent
+ store-and-forward, local publication means durable journal append.
+- QWP symbol dictionaries are connection-scoped and automatic.
+- Table and column identifiers are rejected locally using the Java client's rules;
+ column identity is case-insensitive and preserves the spelling first declared.
+- Large batches are split to the negotiated WebSocket payload cap.
+- QWP transactional auto-flush is per table and must be explicitly committed.
+- Browser and Node QWP ingress reconnect by default with in-memory, at-least-once
+ replay. That queue has a 128 MiB cap and a bounded 30-second capacity wait by
+ default. Configure Node store-and-forward when replay must survive process failure.
+- HTTP/TCP-only keys do not carry over to `ws::`; use the unified QWP connect-string
+ vocabulary. Programmatic callbacks, custom agents, and other non-string hooks remain
+ available under `extraOptions.qwp`.
+- Auto-flush defaults differ from the ILP transports, matching the Java client's
+ separate WebSocket defaults: `auto_flush_rows` is `1000` where `http::` uses
+ `75000` and `tcp::` uses `600`, and `auto_flush_interval` is `100` ms where both
+ use `1000` ms. A workload migrated on the one-line change above therefore sends
+ smaller batches far more often; set both keys explicitly to keep its previous
+ batching.
+
+Roll out `ws::` per sender instance so the existing protocols can remain in service
+during migration.
+
+### Low-level QWP ingress
+
+Code that manually creates `QwpTableBuffer` and calls
+`QwpIngressSession.sendTables()` can normally move to `connectQwpNodeSender()` or
+`connectQwpBrowserSender()`. Keep low-level sessions only when an application needs
+to produce encoded table buffers itself. The high-level sender owns batching, symbol
+deltas, ACK tracking, auto-flush, transactions, and durable waits.
+
+### Java client concepts
+
+The TypeScript high-level sender follows the Java client's core model—fluent rows,
+automatic batching, connection-scoped symbol dictionaries, negotiated caps, durable
+acknowledgement, and persistent replay—but uses runtime-specific connection factories:
+
+| Java client concept | TypeScript API |
+| ---------------------------- | ------------------------------------------------------------- |
+| Sender/builder configuration | `Sender.fromConfig()` in Node.js, or `connectQwp*Sender()` |
+| Fluent table row | `table()`, typed column methods, `at()` / `atNow()` |
+| Local publish/commit | `flush()` / `commit()` |
+| Explicit ACK barrier | `flushAndGetSequence()` plus `waitForAcknowledged()` |
+| Durable delivery | `requestDurableAck` plus `awaitDurableAck` |
+| Store-and-forward | Node `storeAndForward`; intentionally unavailable in browsers |
+| Fire-and-forget UDP ingress | Node `udp::` or `connectQwpNodeUdpSender()` |
+| Query parameters | `session.query(sql, { binds })` |
+| Materialized result batches | `for await (const batch of query)` |
+| Reusable result views | `queryViews()` with column views or `forEachRow()` row views |
+| Egress row/buffer bounds | `maxBatchRows` and session `bufferPoolSize` |
+
+Unlike Java's dedicated dispatcher threads, TypeScript callback inboxes schedule work on
+later JavaScript event-loop turns. This keeps user callbacks out of protocol call stacks,
+but CPU-bound callback code still blocks the runtime and belongs in a Worker or
+`worker_threads` task.
+
+## Development benchmarks
+
+The repository includes diagnostic QWP benchmarks for ingress encoding, fluent sender
+construction, symbol dictionaries, egress materialization and reusable views, Zstd,
+store-and-forward persistence/recovery, and live completion-boundary latency. See
+[`benchmarks/README.md`](benchmarks/README.md) for commands and result interpretation.
+They are intentionally not CI performance gates.
+
+## Public API policy
+
+Only the four package entry points listed at the top are public. In particular,
+paths containing `internal`, `qwp-node`, or `src` are implementation details even if
+a bundler can resolve them. The compatibility contract checks the documented
+high-level constructors, session classes, errors, constants, and option signatures
+from the shared, browser, and Node entry points. Additional low-level codec exports
+from `qwp` are intended for advanced integrations; prefer high-level APIs when no
+custom encoder or transport is required.
diff --git a/README.md b/README.md
index 0ce8d45..bf43535 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ pnpm add @questdb/nodejs-client
## Compatibility table
| QuestDB client version | Supported Node.js versions | Default HTTP Agent |
-|------------------------|----------------------------|---------------------|
+| ---------------------- | -------------------------- | ------------------- |
| ^4.0.0 | v20 and above | Undici Http Agent |
| ^3.0.0 | v16 and above | Standard Http Agent |
@@ -28,12 +28,12 @@ Use the stdlib_http option to switch to the standard HTTP/HTTPS modules.
## Configuration options
Detailed description of the client's configuration options can be found in
-the {@link SenderOptions} documentation.
+the {@link index.SenderOptions | SenderOptions} documentation.
## Examples
The examples below demonstrate how to use the client.
-For more details, please, check the {@link Sender}'s documentation.
+For more details, please, check the {@link index.Sender | Sender}'s documentation.
### Basic API usage
@@ -65,6 +65,421 @@ async function run() {
run().then(console.log).catch(console.error);
```
+### Null and undefined values
+
+Passing `null` or `undefined` as a column or symbol value omits that column from
+the row, and QuestDB records the omission as NULL. This is the model the QuestDB
+clients share — the Java client puts it as "to mark the value NULL, omit the
+column from the row" — with the JavaScript client doing the omission for you, so a
+record with optional fields needs no branching:
+
+```typescript
+const trade: { side?: string; amount?: number } = { amount: 0.011 };
+
+await sender
+ .table("trades")
+ .symbol("symbol", "BTC-USD")
+ .symbol("side", trade.side) // undefined -> column omitted -> NULL
+ .floatColumn("price", 39269.98)
+ .floatColumn("amount", trade.amount)
+ .at(Date.now(), "ms");
+// wire: trades,symbol=BTC-USD price=39269.98,amount=0.011
+```
+
+This applies to every column method on both the ILP (`http`/`https`/`tcp`/`tcps`)
+and QWP (`ws`/`wss`/`udp`) senders, and to the compiled QWP writers. The one
+method that spreads a single value over several arguments, `long256Column`,
+omits its column when _all four_ words are nullish; a partial set is rejected
+rather than treated as NULL.
+
+Two consequences are worth knowing:
+
+- An omitted column is not created on a table that does not already have it. The
+ omission carries no type, so schema-on-write has nothing to infer from.
+- A row in which _every_ value is nullish behaves differently per protocol. ILP
+ has no way to encode a row with no fields, so `at()`/`atNow()` rejects it with
+ "The row must have a symbol or column set before it is closed". QWP is
+ columnar and can express it, so the row is sent with no columns — carrying
+ only its designated timestamp.
+- A rejected `at()`/`atNow()` on ILP discards the row it could not close,
+ including its table name, and leaves rows already in the buffer alone. Catch
+ the error and start the next row from `table()`; there is no need to `reset()`
+ and nothing already buffered is lost.
+
+**Changed in this release.** Earlier versions threw a type error for most nullish
+values, and protocol v2 encoded `arrayColumn(name, null)` as an explicit NULL
+array marker. Both now omit the column instead. If your code relied on the throw
+as a data-quality guard, validate before calling the sender.
+
+### QWP ingress from Node.js or a browser
+
+See the [complete QWP guide](./QWP.md) for ingress and egress APIs, the combined
+pooled client, browser authentication, delivery semantics, migration guidance, and
+the public API policy.
+
+Node.js applications can select QWP through the regular `Sender` API:
+
+```typescript
+import { Sender } from "@questdb/nodejs-client";
+
+const sender = await Sender.fromConfig("ws::addr=127.0.0.1:9000");
+await sender.connect();
+await sender
+ .table("trades")
+ .symbol("symbol", "ETH-USD")
+ .floatColumn("price", 2615.54)
+ .at(Date.now(), "ms");
+await sender.flush();
+await sender.close();
+```
+
+For repeated object rows, compile the table schema once. The resulting writer
+validates each complete row before changing sender state and accepts both individual
+rows and synchronous or asynchronous iterables:
+
+```typescript
+import * as qwp from "@questdb/nodejs-client/qwp";
+
+const trades = sender.writer("trades", {
+ symbol: qwp.symbol(),
+ side: qwp.symbol(),
+ price: qwp.double(),
+ quantity: qwp.long(),
+ timestamp: qwp.designatedTimestamp("ns"),
+});
+
+await trades.row({
+ symbol: "ETH-USD",
+ side: "sell",
+ price: 2615.54,
+ quantity: 42n,
+ timestamp: 1_723_000_000_000_000_000n,
+});
+await trades.rows(moreTrades);
+```
+
+The schema vocabulary covers every QuestDB column type the fluent row API can write,
+including `date()`, `char()`, `binary()`, `uuid()`, `long256()`, `ipv4()`,
+`geohash(precisionBits)`, `decimal64/128/256(scale)`, `doubleArray()`, and
+`longArray()`. See [QWP.md](QWP.md#compiled-object-row-writers) for the accepted value
+forms of each field.
+
+The regular `Sender` accepts the same unified QWP configuration vocabulary as
+the pooled Node client. Use comma-separated or repeated `addr` values for
+failover; standalone ingress validates but otherwise ignores egress- and
+pool-only keys.
+
+Node.js also supports fire-and-forget QWP-over-UDP through the same API:
+
+```typescript
+const sender = await Sender.fromConfig(
+ "udp::addr=239.1.2.3:9007;max_datagram_size=1400;multicast_ttl=1",
+);
+await sender.connect();
+await sender
+ .table("trades")
+ .symbol("symbol", "ETH-USD")
+ .floatColumn("price", 2615.54)
+ .atNow();
+await sender.close();
+```
+
+UDP datagrams are self-contained and split at row boundaries. UDP has no
+authentication, acknowledgements, transactions, retry, or store-and-forward and is
+not available in browsers. See the QWP guide for the lower-level Node UDP API.
+
+QWP `flush()` resolves at the local publication boundary by default in both
+Node.js and browsers, matching the Java QWP sender. Set
+`qwp.sender.awaitServerAck: true` to wait for QuestDB's protocol ACK instead,
+or `awaitDurableAck: true` to wait through durable upload. When Node QWP is
+configured with `qwp.webSocket.storeAndForward`, the publication boundary is
+the local durable journal, so the sender can accept flushes while QuestDB is
+offline and a background drainer reconnects and sends them in order.
+Set `initialConnectMode` to `"off"` (the default), `"sync"`, or `"async"` to
+choose fail-fast, bounded blocking, or background startup. Supplying reconnect
+budget settings without an explicit mode promotes initial startup to `"sync"`,
+matching the Java client. The configuration-string
+equivalent is `initial_connect_retry`, used together with the store-and-forward
+options in `extraOptions.qwp`.
+Persistent frames are coalesced into fixed-size 4 MiB `.sfa` segments by default,
+using the shared Java/Rust/Python SFA envelope, manifest, ACK watermark, and symbol
+dictionary formats. The active segment and a pre-sized temporary hot spare keep open
+handles. A shared worker provisions spares, checkpoints files, and trims acknowledged
+segments. Recovery keeps only frame offsets in memory and reads payloads from disk as
+they are sent, so a large persisted backlog is not duplicated on the JavaScript heap.
+Set `drainOrphans: true` when sibling journal directories share a dedicated parent:
+the Node client scans and drains slots left by failed producer processes with bounded
+concurrency. Pooled QWP clients recover idle in-range and out-of-range `sender-N`
+slots automatically without raising `senderPoolMin`, including leftovers after
+`senderPoolMax` is reduced. Terminally bad slots are marked `.failed` for inspection
+and can be re-enabled with
+`retryQwpNodeOrphanSlot()`. This persistent mode is Node-only; browser senders
+use the in-memory replay boundary.
+
+Browser applications use the browser entry point, which has no Node.js
+dependencies. Cookies are supplied by the browser during a same-origin
+WebSocket upgrade. Browser and non-persistent Node ingress reconnect by default and
+retain unacknowledged frames in memory; set `reconnect: false` in the session options
+for a fixed connection. Only Node store-and-forward survives process failure.
+
+```typescript
+import { connectQwpBrowserSender } from "@questdb/nodejs-client/qwp/browser";
+
+const url = new URL("/write/v4", location.href);
+url.protocol = location.protocol === "https:" ? "wss:" : "ws:";
+const sender = await connectQwpBrowserSender({ url }, { autoFlush: false });
+await sender.table("events").longColumn("value", 42n).atNow();
+await sender.flush();
+await sender.close();
+```
+
+For batches larger than the automatic flush threshold, transactional mode
+keeps each auto-flushed frame in an open server-side transaction. An explicit
+`flush()` (or its `commit()` alias) publishes the group-closing frame. Set
+`awaitServerAck: true`, or wait on the sequence returned by
+`flushAndGetSequence()`, when the call must also observe the cumulative ACK.
+QuestDB guarantees this atomicity per table; a flush that contains multiple
+tables is not one cross-table transaction.
+
+```typescript
+const sender = await connectQwpBrowserSender(
+ { url },
+ {
+ autoFlushRows: 10_000,
+ autoFlushBytes: 4 * 1024 * 1024,
+ transactional: true,
+ },
+);
+
+for (const event of events) {
+ await sender
+ .table("events")
+ .symbol("source", event.source)
+ .longColumn("value", event.value)
+ .at(event.timestamp, "ms");
+}
+await sender.commit();
+await sender.close();
+```
+
+QWP `close()` publishes completed rows and waits up to 5 seconds for their
+committed-frame ACK watermark. Configure `closeFlushTimeoutMs` (or
+`close_flush_timeout_millis` in a `ws::` string); `0` publishes without waiting.
+An unfinished row is not completed implicitly.
+
+The server intentionally withholds ACKs for deferred frames until commit. The
+sender pipelines transactional auto-flushes without waiting for those ACKs,
+then publishes the group-closing frame at `flush()`/`commit()`. With
+`awaitServerAck` or `awaitDurableAck`, that call also waits for all covered
+ACKs; durable waiting starts only after the transaction commits. Closing
+without an explicit commit abandons the open transaction and logs a warning;
+QuestDB rolls it back when the WebSocket disconnects.
+
+Ingress sessions expose browser-safe progress/error callbacks and immutable
+metrics snapshots. Reconnect events remain on `reconnect.onEvent`, keeping
+connection topology separate from batch acceptance and durable progress.
+
+```typescript
+import {
+ QWP_INGRESS_PROGRESS_KIND,
+ createQwpBrowserSender,
+} from "@questdb/nodejs-client/qwp/browser";
+
+const sender = createQwpBrowserSender(
+ { url },
+ { autoFlush: false },
+ {
+ reconnect: {
+ onEvent: (event) => console.info("QWP connection", event),
+ },
+ onProgress: (event) => {
+ if (event.kind === QWP_INGRESS_PROGRESS_KIND.ACKNOWLEDGED) {
+ console.info("accepted through", event.sequence);
+ }
+ },
+ onError: (event) => console.error("QWP ingress", event.error),
+ onSenderError: (error) =>
+ console.error(
+ "QWP rejection",
+ error.category,
+ error.appliedPolicy,
+ error.fromFsn,
+ error.toFsn,
+ ),
+ },
+);
+
+await sender.connect();
+const snapshot = sender.metrics;
+console.info(
+ snapshot.totalRowsPublished,
+ snapshot.ingress?.totalFramesReplayed,
+);
+```
+
+Snapshots distinguish the client-session acceptance sequence from persistent
+replay watermarks. With durable ACKs, `replayAcknowledgedFrameSequence`
+advances only after the durable watermark covers a frame. Observer callbacks are
+dispatched asynchronously through bounded, drop-oldest inboxes, so they do not run
+inside ACK or reconnect protocol stacks. The metrics snapshot exposes delivered and
+dropped progress, connection, and error notification counters.
+`connectionListenerInboxCapacity` and `errorInboxCapacity` tune the Java-compatible
+64/256 defaults. `onSenderError` receives typed category/policy, wire status, message
+sequence, stable frame-sequence range, and quarantine context. If it is omitted,
+retriable rejections are logged at `warn` and terminal rejections or abandoned data at
+`error`; general asynchronous ingress failures are also logged when `onError` is
+omitted. Observer exceptions are contained, but CPU-bound callbacks should still move
+work to a Worker because browser and Node JavaScript share the event loop.
+
+When QuestDB authentication is enabled, establish the browser's HttpOnly
+`qdb_session` cookie over REST before opening a QWP WebSocket. A QuestDB REST
+token and an OIDC access token both use the `bearer` form. The application is
+responsible for obtaining an OIDC token from its identity provider; the client
+does not run an interactive OIDC authorization flow.
+
+```typescript
+import {
+ bootstrapQwpBrowserSession,
+ connectQwpBrowserSender,
+} from "@questdb/nodejs-client/qwp/browser";
+
+await bootstrapQwpBrowserSession({
+ url: new URL("/exec", location.href),
+ authentication: { type: "bearer", token: oidcOrRestAccessToken },
+ // QuestDB Enterprise only; omit to use the authenticated principal.
+ serviceAccount: "market_data_writer",
+});
+
+const sender = await connectQwpBrowserSender({ url }, { autoFlush: false });
+```
+
+The bootstrap can also be attached to the connection options. It then runs
+before each initial, reconnect, or failover WebSocket attempt:
+
+```typescript
+const sender = await connectQwpBrowserSender(
+ {
+ url,
+ sessionBootstrap: {
+ authentication: {
+ type: "basic",
+ username: "admin",
+ password: "quest",
+ },
+ },
+ },
+ { autoFlush: false },
+);
+```
+
+The REST request uses `credentials: "include"`. The default bootstrap URL is
+`/exec` beside `/write/v4` or `/read/v1`; set `sessionBootstrap.url` explicitly
+when a reverse proxy exposes a different REST path. The REST and WebSocket
+routes should be served from the same browser origin (or configured with
+credentialed CORS), otherwise the browser may decline to store or send the
+HttpOnly cookies. JavaScript deliberately never reads `qdb_session` or the
+Enterprise `qdbServiceAccount` cookie.
+
+Browsers can request durable ingress acknowledgements without custom HTTP
+headers. The client offers a QWP WebSocket subprotocol and verifies that the
+server selected it before sending data. Browser keepalives use side-effect-free,
+table-less QWP poll frames because the WebSocket API does not expose
+protocol-level PING frames. A poll completes once published: durable progress
+arrives independently, and an open deferred transaction may intentionally
+prevent the server from sending a cumulative OK for that poll. Supplying
+`durableAckKeepaliveMs` requires durable negotiation (`requestDurableAck: true`,
+either explicit or implied by `awaitDurableAck`); manual polls and durable waits
+reject locally when the capability was not negotiated.
+
+```typescript
+const sender = await connectQwpBrowserSender(
+ { url, requestDurableAck: true },
+ { autoFlush: false, awaitDurableAck: true },
+);
+```
+
+Browser durable ACKs are an in-memory delivery confirmation only. Persistent
+store-and-forward remains available exclusively through the Node.js entry
+point. In-memory ingress replay is capped at 128 MiB and waits at most 30 seconds
+for ACK-driven trimming by default; tune `memoryReplayMaxBytes` and
+`memoryReplayAppendDeadlineMs` in the ingress session options when needed.
+
+### Zstd-compressed QWP egress
+
+Node.js egress clients can opt into compressed result batches during the
+WebSocket upgrade. Raw batches remain the default for compatibility.
+
+```typescript
+import { connectQwpNodeEgress } from "@questdb/nodejs-client/qwp/node";
+
+const session = await connectQwpNodeEgress(
+ {
+ url: "ws://127.0.0.1:9000/read/v1",
+ compression: "zstd",
+ compressionLevel: 3,
+ },
+ {
+ queryTimeoutMs: 30_000,
+ },
+);
+try {
+ const query = await session.query("select * from trades", {
+ initialCredit: 1024 * 1024,
+ });
+ console.log("effective Zstd level", session.negotiatedZstdLevel);
+ for await (const batch of query) {
+ for (const row of batch.rows()) console.log(row);
+ }
+ await query.completion;
+} finally {
+ await session.close();
+}
+```
+
+Zstd decoding and negotiation are also included in the browser entry point.
+Because browsers cannot set the `X-QWP-Accept-Encoding` upgrade header, the
+client sends the same preference through the WebSocket URL's
+`qwp_accept_encoding` parameter. No proxy-injected compression header is
+required. Older servers ignore the parameter and safely continue with raw
+batches.
+
+Level `1` is the lowest-CPU default and is usually the right starting point.
+Higher values trade server CPU for wire size; the client accepts levels 1–22,
+while the server may clamp the request or apply an operator-configured level.
+`session.negotiatedCompression` and `session.negotiatedZstdLevel` report what
+the active server actually selected and refresh after reconnection or failover.
+Both `"zstd"` and `"auto"` advertise Zstd followed by raw fallback, and the
+server still sends an individual batch raw when compression would make it
+larger.
+
+Matching the Java client, egress queries default `initialCredit` to zero, meaning
+unbounded server send-ahead. Set a positive session or per-query value to bound wire
+buffering—particularly in browsers. With positive credit, the client automatically
+replenishes the exact wire size of each result batch after consumption. Set
+`autoCredit: false` to manage credit explicitly through `query.grantCredit()`.
+
+For allocation-sensitive consumers, `session.queryViews(sql, onBatch)` supplies
+bounded, reusable column views instead of materializing every value into JavaScript
+arrays. Typed accessors read fixed-width values directly from QWP bytes, and raw
+byte views are available for vectorized processing. The callback is awaited before
+credit is replenished, while the receive loop decodes ahead through the bounded
+reusable buffer pool. Views are invalid when their callback returns; copy a byte
+view with `.slice()` or call `batch.materialize()` inside the callback to retain
+data. Tune the default four-slot pool with the session's `bufferPoolSize`.
+
+`queryTimeoutMs` sets the session's default query deadline; a per-query
+`timeoutMs` overrides it, and zero disables the deadline. When a deadline
+expires, the client rejects iteration and `query.completion` with
+`QwpEgressQueryTimeoutError`, sends QWP `CANCEL`, and waits for the terminal
+server response before accepting another query on that connection. Breaking out
+of `for await` early cancels the query too. `cancelDrainTimeoutMs` bounds that
+wait (5 seconds by default); an unresponsive cancellation closes the connection
+with `QwpEgressQueryCancelTimeoutError` instead of wedging the session.
+To bound only the caller's wait without cancelling, use
+`await query.awaitCompletion(timeoutMs)`. It returns `false` on timeout and leaves
+the query active, matching Java `Completion.await(timeout, unit)`. The SERVER_INFO
+handshake timeout defaults to five seconds on both clients.
+
### Authentication and secure connection
#### Username and password authentication with HTTP transport
@@ -80,7 +495,7 @@ async function run() {
// pass the authentication details to the sender
// for secure connection use 'https' protocol instead of 'http'
const sender = await Sender.fromConfig(
- `http::addr=127.0.0.1:9000;username=${USER};password=${PWD}`
+ `http::addr=127.0.0.1:9000;username=${USER};password=${PWD}`,
);
// add rows to the buffer of the sender
@@ -114,7 +529,7 @@ async function run() {
// pass the authentication details to the sender
// for secure connection use 'https' protocol instead of 'http'
const sender = await Sender.fromConfig(
- `http::addr=127.0.0.1:9000;token=${TOKEN}`
+ `http::addr=127.0.0.1:9000;token=${TOKEN}`,
);
// add rows to the buffer of the sender
@@ -148,7 +563,7 @@ async function run() {
// pass the authentication details to the sender
const sender = await Sender.fromConfig(
- `tcp::addr=127.0.0.1:9009;username=${CLIENT_ID};token=${PRIVATE_KEY}`
+ `tcp::addr=127.0.0.1:9009;username=${CLIENT_ID};token=${PRIVATE_KEY}`,
);
await sender.connect();
@@ -178,42 +593,42 @@ import { Sender } from "@questdb/nodejs-client";
async function run() {
// create a sender
- const sender = await Sender.fromConfig('http::addr=localhost:9000');
+ const sender = await Sender.fromConfig("http::addr=localhost:9000");
// order book snapshots to ingest
const orderBooks = [
{
- symbol: 'BTC-USD',
- exchange: 'Coinbase',
+ symbol: "BTC-USD",
+ exchange: "Coinbase",
timestamp: Date.now(),
- bidPrices: [50100.25, 50100.20, 50100.15, 50100.10, 50100.05],
+ bidPrices: [50100.25, 50100.2, 50100.15, 50100.1, 50100.05],
bidSizes: [0.5, 1.2, 2.1, 0.8, 3.5],
- askPrices: [50100.30, 50100.35, 50100.40, 50100.45, 50100.50],
- askSizes: [0.6, 1.5, 1.8, 2.2, 4.0]
+ askPrices: [50100.3, 50100.35, 50100.4, 50100.45, 50100.5],
+ askSizes: [0.6, 1.5, 1.8, 2.2, 4.0],
},
{
- symbol: 'ETH-USD',
- exchange: 'Coinbase',
+ symbol: "ETH-USD",
+ exchange: "Coinbase",
timestamp: Date.now(),
- bidPrices: [2850.50, 2850.45, 2850.40, 2850.35, 2850.30],
+ bidPrices: [2850.5, 2850.45, 2850.4, 2850.35, 2850.3],
bidSizes: [5.0, 8.2, 12.5, 6.8, 15.0],
- askPrices: [2850.55, 2850.60, 2850.65, 2850.70, 2850.75],
- askSizes: [4.5, 7.8, 10.2, 8.5, 20.0]
- }
+ askPrices: [2850.55, 2850.6, 2850.65, 2850.7, 2850.75],
+ askSizes: [4.5, 7.8, 10.2, 8.5, 20.0],
+ },
];
try {
// add rows to the buffer of the sender
for (const orderBook of orderBooks) {
await sender
- .table('order_book_l2')
- .symbol('symbol', orderBook.symbol)
- .symbol('exchange', orderBook.exchange)
- .arrayColumn('bid_prices', orderBook.bidPrices)
- .arrayColumn('bid_sizes', orderBook.bidSizes)
- .arrayColumn('ask_prices', orderBook.askPrices)
- .arrayColumn('ask_sizes', orderBook.askSizes)
- .at(orderBook.timestamp, 'ms');
+ .table("order_book_l2")
+ .symbol("symbol", orderBook.symbol)
+ .symbol("exchange", orderBook.exchange)
+ .arrayColumn("bid_prices", orderBook.bidPrices)
+ .arrayColumn("bid_sizes", orderBook.bidSizes)
+ .arrayColumn("ask_prices", orderBook.askPrices)
+ .arrayColumn("ask_sizes", orderBook.askSizes)
+ .at(orderBook.timestamp, "ms");
}
// flush the buffer of the sender, sending the data to QuestDB
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
new file mode 100644
index 0000000..9d10e92
--- /dev/null
+++ b/THIRD_PARTY_NOTICES.md
@@ -0,0 +1,23 @@
+# Third-party notices
+
+This product bundles `fzstd` 0.1.1, which is available under the MIT License:
+
+Copyright (c) 2020 Arjun Barrett
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/benchmarks/README.md b/benchmarks/README.md
new file mode 100644
index 0000000..865e90c
--- /dev/null
+++ b/benchmarks/README.md
@@ -0,0 +1,86 @@
+# QWP benchmarks
+
+These benchmarks are diagnostic tools, not CI performance gates. Run them on a
+quiet, pinned machine and compare commits on the same host.
+
+```bash
+# Encoder, high-level sender, egress, and store-and-forward benchmarks.
+pnpm bench
+
+# Live QuestDB on localhost:9000.
+pnpm bench:e2e
+
+# Override the live server and sample sizes.
+QDB_ADDR=host:9000 BENCH_ROWS=10000 pnpm bench:e2e
+
+# Use real storage for persistence measurements.
+QWP_BENCH_DIR=/var/tmp/qwp-bench pnpm bench
+
+pnpm typecheck:bench
+pnpm lint:bench
+pnpm format:bench
+pnpm vitest run benchmarks/*.test.ts
+```
+
+Set `QWP_BENCH_DURABLE_ACK=1` for an additional live durable-ACK arm. The
+server must advertise durable acknowledgements. The default E2E run measures
+local WebSocket publication, protocol ACK, and local store-and-forward append
+as separate completion contracts.
+
+## Workloads
+
+- `trades`: one low-cardinality symbol, two doubles, and a designated timestamp.
+- `wide`: 50 data columns across symbol, long, double, and varchar families.
+- `sparse`: eight potential long columns with deterministic 30% nulls.
+- `highCardinalitySymbols`: one distinct symbol per row up to 100,000 values.
+
+All workloads use a deterministic xorshift generator. Encoder floors perform
+only the minimum int64, UTF-8, or `Map` work, without QWP schema, null, or frame
+overhead. They are comparison baselines, not performance targets.
+
+## Reading Vitest output
+
+Vitest reports benchmark callbacks per second (`hz`):
+
+- encoder and sender callbacks process 10,000 rows;
+- materialized/view egress callbacks process 10,000 rows;
+- the compressed egress callback processes 100 rows;
+- persistence append callbacks write 100 4 KiB frames.
+
+Multiply `hz` by the corresponding unit count before reporting rows or appends
+per second. Check `rme` before treating small differences as meaningful.
+
+The persistence suite prints a 256 MiB write-and-fsync baseline. It benchmarks
+all three file-store policies:
+
+- `memory`: file writes relying on operating-system page-cache writeback;
+- `periodic`: file writes plus background checkpoints;
+- `append`: a persistence barrier after every frame.
+
+It also measures full recovery of 1,000 frame references and lazy 4 KiB payload
+reads. Segment rolls, hot-spare provisioning, checkpoint timers, and background
+trimming can make the distribution bimodal. Report percentiles or the complete
+distribution rather than only its mean.
+
+Each append callback also acknowledges its preceding prefix, retaining one live
+record so the journal remains in steady state without exhausting its configured
+capacity. The reported number therefore includes normal ACK bookkeeping and trim
+scheduling.
+
+Confirm that `QWP_BENCH_DIR` is not tmpfs before describing any result as disk
+performance. The directory and the baseline must live on the same filesystem.
+
+## E2E completion boundaries
+
+The live benchmark flushes every measured row so each sample contains a real
+completion boundary. Its arms are deliberately not interchangeable:
+
+- local publication means the WebSocket accepted the frame;
+- protocol ACK means QuestDB accepted the frame;
+- local SF append means the frame crossed the configured local persistence
+ boundary;
+- optional durable ACK means the server reported durable upload.
+
+Each repetition uses a disjoint timestamp range. Store-and-forward repetitions
+also use distinct sender IDs so recovered dictionaries and replay slots do not
+turn later repetitions into warm-recovery measurements.
diff --git a/benchmarks/e2e.ts b/benchmarks/e2e.ts
new file mode 100644
index 0000000..82dcb76
--- /dev/null
+++ b/benchmarks/e2e.ts
@@ -0,0 +1,179 @@
+import { it } from "vitest";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Sender } from "../src";
+import { BENCHMARK_WORKLOADS } from "./workloads";
+
+const ADDRESS = process.env.QDB_ADDR ?? "localhost:9000";
+const ROWS = Number(process.env.BENCH_ROWS ?? 5000);
+const WARMUP_ROWS = Number(process.env.BENCH_WARMUP_ROWS ?? 500);
+const REPEATS = Number(process.env.BENCH_REPEATS ?? 3);
+const TIMESTAMP_STRIDE = 1_000_000_000n;
+const SF_ROOT = process.env.QWP_BENCH_DIR ?? tmpdir();
+
+type SenderExtraOptions = NonNullable[1]>;
+
+interface ArmOptions {
+ label: string;
+ table: string;
+ configuration: (repeat: number, sfDirectory: string) => string;
+ extraOptions?: SenderExtraOptions;
+}
+
+function positiveInteger(value: number, name: string): number {
+ if (!Number.isSafeInteger(value) || value < 1) {
+ throw new RangeError(`${name} must be a positive safe integer`);
+ }
+ return value;
+}
+
+function nonNegativeInteger(value: number, name: string): number {
+ if (!Number.isSafeInteger(value) || value < 0) {
+ throw new RangeError(`${name} must be a non-negative safe integer`);
+ }
+ return value;
+}
+
+function percentile(
+ sorted: readonly number[],
+ percentileValue: number,
+): number {
+ if (sorted.length === 0) return Number.NaN;
+ const rank = Math.ceil((percentileValue / 100) * sorted.length) - 1;
+ return sorted[Math.max(0, Math.min(sorted.length - 1, rank))];
+}
+
+async function measureArm(
+ options: ArmOptions,
+ repeat: number,
+ sfDirectory: string,
+): Promise {
+ const sender = await Sender.fromConfig(
+ options.configuration(repeat, sfDirectory),
+ options.extraOptions,
+ );
+ const samples: number[] = [];
+ const timestampOffset = BigInt(repeat) * TIMESTAMP_STRIDE;
+ try {
+ await sender.connect();
+ const allRows = BENCHMARK_WORKLOADS.trades.rows(WARMUP_ROWS + ROWS);
+ for (const row of allRows.slice(0, WARMUP_ROWS)) {
+ sender
+ .table(options.table)
+ .symbol("symbol", row.symbols[0][1])
+ .floatColumn("price", row.doubles[0][1])
+ .floatColumn("amount", row.doubles[1][1]);
+ await sender.at(row.timestamp + timestampOffset);
+ }
+ await sender.flush();
+
+ for (const row of allRows.slice(WARMUP_ROWS)) {
+ sender
+ .table(options.table)
+ .symbol("symbol", row.symbols[0][1])
+ .floatColumn("price", row.doubles[0][1])
+ .floatColumn("amount", row.doubles[1][1]);
+ await sender.at(row.timestamp + timestampOffset);
+ const started = process.hrtime.bigint();
+ await sender.flush();
+ samples.push(Number(process.hrtime.bigint() - started) / 1000);
+ }
+ } finally {
+ await sender.close();
+ }
+ return samples.sort((left, right) => left - right);
+}
+
+function report(label: string, runs: readonly number[][]): void {
+ console.log(`\n${label}`);
+ for (const value of [50, 90, 99, 99.9]) {
+ const samples = runs.map((run) => percentile(run, value));
+ const minimum = Math.min(...samples).toFixed(1);
+ const maximum = Math.max(...samples).toFixed(1);
+ console.log(
+ ` p${value}\t${minimum} - ${maximum} us (${runs.length} repeats)`,
+ );
+ }
+}
+
+it("measures QWP ingress completion boundaries", async () => {
+ positiveInteger(ROWS, "BENCH_ROWS");
+ nonNegativeInteger(WARMUP_ROWS, "BENCH_WARMUP_ROWS");
+ positiveInteger(REPEATS, "BENCH_REPEATS");
+
+ await mkdir(SF_ROOT, { recursive: true });
+ const sfDirectory = await mkdtemp(join(SF_ROOT, "qwp-bench-e2e-"));
+ console.log(
+ `QWP E2E latency: ${ADDRESS}, ${ROWS} rows, ${REPEATS} repeats per arm`,
+ );
+ console.log(`SF directory: ${sfDirectory}`);
+ console.log(
+ `Verify real storage before quoting SF results: df -T ${SF_ROOT}`,
+ );
+
+ const arms: ArmOptions[] = [
+ {
+ label: "flush() = local WebSocket publication",
+ table: "bench_e2e_publication",
+ configuration: () => `ws::addr=${ADDRESS};auto_flush=off`,
+ extraOptions: {
+ qwp: { sender: { autoFlush: false, closeFlushTimeoutMs: 0 } },
+ },
+ },
+ {
+ label: "flush() = server protocol ACK",
+ table: "bench_e2e_ack",
+ configuration: () => `ws::addr=${ADDRESS};auto_flush=off`,
+ extraOptions: {
+ qwp: {
+ sender: {
+ autoFlush: false,
+ awaitServerAck: true,
+ closeFlushTimeoutMs: 0,
+ },
+ },
+ },
+ },
+ {
+ label: "flush() = local SF append durability",
+ table: "bench_e2e_sf",
+ configuration: (repeat, directory) =>
+ `ws::addr=${ADDRESS};auto_flush=off;sf_dir=${directory};` +
+ `sender_id=bench-${repeat};sf_durability=append`,
+ extraOptions: {
+ qwp: { sender: { autoFlush: false, closeFlushTimeoutMs: 0 } },
+ },
+ },
+ ];
+
+ if (process.env.QWP_BENCH_DURABLE_ACK === "1") {
+ arms.push({
+ label: "flush() = server durable ACK",
+ table: "bench_e2e_durable_ack",
+ configuration: () =>
+ `ws::addr=${ADDRESS};auto_flush=off;request_durable_ack=on`,
+ extraOptions: {
+ qwp: {
+ sender: {
+ autoFlush: false,
+ awaitDurableAck: true,
+ closeFlushTimeoutMs: 0,
+ },
+ },
+ },
+ });
+ }
+
+ try {
+ for (const arm of arms) {
+ const runs: number[][] = [];
+ for (let repeat = 0; repeat < REPEATS; repeat++) {
+ runs.push(await measureArm(arm, repeat, sfDirectory));
+ }
+ report(arm.label, runs);
+ }
+ } finally {
+ await rm(sfDirectory, { recursive: true, force: true });
+ }
+});
diff --git a/benchmarks/egress.bench.ts b/benchmarks/egress.bench.ts
new file mode 100644
index 0000000..0e43148
--- /dev/null
+++ b/benchmarks/egress.bench.ts
@@ -0,0 +1,144 @@
+import { bench, describe } from "vitest";
+import {
+ decodeQwpEgressMessage,
+ encodeQwpFrame,
+ encodeQwpGorilla,
+ QWP_COLUMN_TYPE,
+ QWP_EGRESS_MESSAGE,
+ QWP_FLAG_DELTA_SYMBOL_DICTIONARY,
+ QWP_FLAG_GORILLA,
+ QWP_FLAG_ZSTD,
+ QwpByteWriter,
+ QwpResultBatchDecoder,
+ writeQwpVarint,
+} from "../src/_qwp/_core";
+
+const ROWS = 10_000;
+let sink = 0;
+
+function writeString(writer: QwpByteWriter, value: string): void {
+ const bytes = new TextEncoder().encode(value);
+ writeQwpVarint(writer, bytes.byteLength);
+ writer.writeBytes(bytes);
+}
+
+function resultFrame(rowCount: number): Uint8Array {
+ const payload = new QwpByteWriter();
+ payload.writeUint8(QWP_EGRESS_MESSAGE.RESULT_BATCH).writeBigUint64(1n);
+ writeQwpVarint(payload, 0); // batch sequence
+ writeString(payload, "bench_result");
+ writeQwpVarint(payload, rowCount);
+ writeQwpVarint(payload, 4);
+ for (const [name, type] of [
+ ["id", QWP_COLUMN_TYPE.INT],
+ ["price", QWP_COLUMN_TYPE.DOUBLE],
+ ["name", QWP_COLUMN_TYPE.VARCHAR],
+ ["timestamp", QWP_COLUMN_TYPE.TIMESTAMP],
+ ] as const) {
+ writeString(payload, name);
+ payload.writeUint8(type);
+ }
+
+ payload.writeUint8(0); // no INT nulls
+ for (let row = 0; row < rowCount; row++) payload.writeInt32(row);
+
+ payload.writeUint8(0); // no DOUBLE nulls
+ for (let row = 0; row < rowCount; row++) {
+ payload.writeFloat64(1000 + (row % 1000) / 10);
+ }
+
+ payload.writeUint8(0); // no VARCHAR nulls
+ const text = Array.from(
+ { length: rowCount },
+ (_, row) => `value-${row % 100}`,
+ );
+ let textOffset = 0;
+ payload.writeUint32(0);
+ for (const value of text) {
+ textOffset += new TextEncoder().encode(value).byteLength;
+ payload.writeUint32(textOffset);
+ }
+ for (const value of text) payload.writeUtf8(value);
+
+ payload.writeUint8(0).writeUint8(1); // no nulls, Gorilla encoded
+ payload.writeBytes(
+ encodeQwpGorilla(
+ Array.from(
+ { length: rowCount },
+ (_, row) => 1_700_000_000_000_000n + BigInt(row) * 1000n,
+ ),
+ ),
+ );
+ return encodeQwpFrame(payload.toUint8Array(), QWP_FLAG_GORILLA, 1);
+}
+
+// A standard Zstd frame containing a 100-row QWP INT result body.
+const COMPRESSED_INT_RESULT_BODY = Uint8Array.from([
+ 40, 181, 47, 253, 96, 153, 0, 157, 0, 0, 96, 0, 0, 0, 100, 1, 1, 120, 4, 0,
+ 42, 0, 0, 1, 0, 138, 171, 46, 9,
+]);
+
+function compressedResultFrame(): Uint8Array {
+ const payload = new QwpByteWriter();
+ payload.writeUint8(QWP_EGRESS_MESSAGE.RESULT_BATCH).writeBigUint64(1n);
+ writeQwpVarint(payload, 0);
+ payload.writeBytes(COMPRESSED_INT_RESULT_BODY);
+ return encodeQwpFrame(
+ payload.toUint8Array(),
+ QWP_FLAG_DELTA_SYMBOL_DICTIONARY | QWP_FLAG_ZSTD,
+ 1,
+ );
+}
+
+const decoded = decodeQwpEgressMessage(resultFrame(ROWS));
+if (decoded.kind !== "result-batch") {
+ throw new Error("benchmark frame is not a result batch");
+}
+const compressed = decodeQwpEgressMessage(compressedResultFrame());
+if (compressed.kind !== "result-batch") {
+ throw new Error("compressed benchmark frame is not a result batch");
+}
+
+describe("QWP egress batch decoding", () => {
+ bench(`materialized / ${ROWS} rows`, () => {
+ const batch = new QwpResultBatchDecoder().decode(decoded);
+ sink += batch.rowCount + batch.columns.length;
+ });
+
+ const viewDecoder = new QwpResultBatchDecoder();
+ bench(`reusable column views / ${ROWS} rows`, () => {
+ viewDecoder.resetQuerySchema();
+ const batch = viewDecoder.decodeView(decoded);
+ sink += batch.rowCount + batch.columnCount;
+ });
+
+ const columnDecoder = new QwpResultBatchDecoder();
+ bench(`column-view traversal / ${ROWS} rows`, () => {
+ columnDecoder.resetQuerySchema();
+ const batch = columnDecoder.decodeView(decoded);
+ const ids = batch.column(0);
+ const prices = batch.column(1);
+ const names = batch.column(2);
+ for (let row = 0; row < batch.rowCount; row++) {
+ sink += ids.getInt(row) + prices.getDouble(row);
+ sink += names.getString(row)?.length ?? 0;
+ }
+ });
+
+ const rowDecoder = new QwpResultBatchDecoder();
+ bench(`row-view traversal / ${ROWS} rows`, () => {
+ rowDecoder.resetQuerySchema();
+ const batch = rowDecoder.decodeView(decoded);
+ batch.forEachRow((row) => {
+ sink += row.getInt(0) + row.getDouble(1);
+ sink += row.getString(2)?.length ?? 0;
+ });
+ });
+
+ bench("Zstd decompress + materialize / 100 INT rows", () => {
+ const batch = new QwpResultBatchDecoder().decode(compressed);
+ sink += batch.rowCount + Number(batch.columns[0].values[0]);
+ });
+});
+
+export const egressBenchmarkSink = (): number => sink;
diff --git a/benchmarks/encoder.bench.ts b/benchmarks/encoder.bench.ts
new file mode 100644
index 0000000..5267855
--- /dev/null
+++ b/benchmarks/encoder.bench.ts
@@ -0,0 +1,90 @@
+import { bench, describe } from "vitest";
+import {
+ encodeQwpIngressFrame,
+ QWP_COLUMN_TYPE,
+ QwpSymbolDictionary,
+ QwpTableBuffer,
+} from "../src/_qwp/_core";
+import {
+ floorInternSymbols,
+ floorWriteLongs,
+ floorWriteStrings,
+} from "./floors";
+import { buildBenchmarkTable } from "./tables";
+import { BENCHMARK_WORKLOADS } from "./workloads";
+
+const ROWS = 10_000;
+let sink = 0;
+
+describe("QWP ingress frame encoder", () => {
+ for (const name of ["trades", "wide", "sparse"] as const) {
+ const table = buildBenchmarkTable(BENCHMARK_WORKLOADS[name].rows(ROWS));
+ bench(`${name} / Gorilla off`, () => {
+ sink += encodeQwpIngressFrame([table], { gorilla: false }).byteLength;
+ });
+ bench(`${name} / Gorilla on`, () => {
+ sink += encodeQwpIngressFrame([table], { gorilla: true }).byteLength;
+ });
+ }
+});
+
+describe("encoder floors", () => {
+ const longs = BENCHMARK_WORKLOADS.sparse
+ .rows(ROWS)
+ .flatMap((row) => row.longs.map(([, value]) => value))
+ .slice(0, ROWS);
+ const longTable = new QwpTableBuffer("floor_long");
+ for (const value of longs) {
+ longTable
+ .getOrCreateColumn("value", QWP_COLUMN_TYPE.LONG)
+ ?.values.push(value);
+ longTable.nextRow();
+ }
+
+ bench("floor / writeBigInt64LE", () => {
+ sink += floorWriteLongs(longs).byteLength;
+ });
+ bench("QWP / single long column", () => {
+ sink += encodeQwpIngressFrame([longTable], {
+ gorilla: false,
+ }).byteLength;
+ });
+
+ const strings = BENCHMARK_WORKLOADS.wide
+ .rows(ROWS)
+ .flatMap((row) => row.strings.map(([, value]) => value))
+ .slice(0, ROWS);
+ const stringTable = new QwpTableBuffer("floor_varchar");
+ for (const value of strings) {
+ stringTable
+ .getOrCreateColumn("value", QWP_COLUMN_TYPE.VARCHAR)
+ ?.values.push(value);
+ stringTable.nextRow();
+ }
+
+ bench("floor / UTF-8 write", () => {
+ sink += floorWriteStrings(strings).byteLength;
+ });
+ bench("QWP / single varchar column", () => {
+ sink += encodeQwpIngressFrame([stringTable], {
+ gorilla: false,
+ }).byteLength;
+ });
+});
+
+describe("symbol interning", () => {
+ const symbols = BENCHMARK_WORKLOADS.highCardinalitySymbols
+ .rows(ROWS)
+ .map((row) => row.symbols[0][1]);
+
+ bench("floor / Map", () => {
+ sink += floorInternSymbols(symbols).length;
+ });
+ bench("QwpSymbolDictionary.getOrAdd", () => {
+ const dictionary = new QwpSymbolDictionary();
+ for (const symbol of symbols) dictionary.getOrAdd(symbol);
+ sink += dictionary.size;
+ });
+});
+
+export const benchmarkSink = (): number => sink;
diff --git a/benchmarks/floors.test.ts b/benchmarks/floors.test.ts
new file mode 100644
index 0000000..6c39bf0
--- /dev/null
+++ b/benchmarks/floors.test.ts
@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+import {
+ floorInternSymbols,
+ floorWriteLongs,
+ floorWriteStrings,
+} from "./floors";
+
+describe("benchmark floors", () => {
+ it("writes eight bytes per long", () => {
+ const bytes = floorWriteLongs([1n, 2n, 3n]);
+ expect(bytes).toHaveLength(24);
+ expect(new DataView(bytes.buffer).getBigInt64(8, true)).toBe(2n);
+ });
+
+ it("writes UTF-8 values back to back", () => {
+ expect(floorWriteStrings(["ab", "cd"]).toString("utf8")).toBe("abcd");
+ });
+
+ it("interns symbols to dense IDs", () => {
+ expect(floorInternSymbols(["a", "b", "a"])).toEqual([0, 1, 0]);
+ });
+});
diff --git a/benchmarks/floors.ts b/benchmarks/floors.ts
new file mode 100644
index 0000000..39c3f50
--- /dev/null
+++ b/benchmarks/floors.ts
@@ -0,0 +1,38 @@
+import { Buffer } from "node:buffer";
+
+/** Minimum byte movement for a flat int64 column, without QWP framing. */
+export function floorWriteLongs(values: readonly bigint[]): Uint8Array {
+ const bytes = new Uint8Array(values.length * 8);
+ const view = new DataView(bytes.buffer);
+ let offset = 0;
+ for (const value of values) {
+ view.setBigInt64(offset, BigInt.asIntN(64, value), true);
+ offset += 8;
+ }
+ return bytes;
+}
+
+/** Minimum UTF-8 copying work, without offsets, nulls, or QWP framing. */
+export function floorWriteStrings(values: readonly string[]): Buffer {
+ let length = 0;
+ for (const value of values) length += Buffer.byteLength(value, "utf8");
+ const bytes = Buffer.allocUnsafe(length);
+ let offset = 0;
+ for (const value of values) offset += bytes.write(value, offset, "utf8");
+ return bytes;
+}
+
+/** Naive per-row symbol interning baseline. */
+export function floorInternSymbols(values: readonly string[]): number[] {
+ const ids = new Map();
+ const result: number[] = [];
+ for (const value of values) {
+ let id = ids.get(value);
+ if (id === undefined) {
+ id = ids.size;
+ ids.set(value, id);
+ }
+ result.push(id);
+ }
+ return result;
+}
diff --git a/benchmarks/persistence.bench.ts b/benchmarks/persistence.bench.ts
new file mode 100644
index 0000000..23b13f1
--- /dev/null
+++ b/benchmarks/persistence.bench.ts
@@ -0,0 +1,169 @@
+import { beforeAll, bench, describe } from "vitest";
+import { mkdtemp, mkdir, open, rm, unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+ QWP_SF_DURABILITY,
+ QwpNodeFileReplayStore,
+ type QwpSfDurability,
+} from "../src/qwp-node/file-replay-store";
+
+const FRAME = new Uint8Array(4096).fill(0x41);
+const APPENDS = 100;
+const RECOVERY_FRAMES = 1000;
+const MAX_BYTES = 64 * 1024 * 1024;
+const MAX_SEGMENT_BYTES = 4 * 1024 * 1024;
+
+interface StoreState {
+ store: QwpNodeFileReplayStore;
+ nextSequence: bigint;
+}
+
+let root: string;
+let pageCache: StoreState;
+let periodic: StoreState;
+let durable: StoreState;
+let recoveryDirectory: string;
+let lazyReader: QwpNodeFileReplayStore;
+let lazySequences: bigint[];
+let lazyCursor = 0;
+let sink = 0;
+
+async function diskBaseline(directory: string): Promise {
+ const path = join(directory, "disk-baseline.tmp");
+ const file = await open(path, "wx", 0o600);
+ const block = new Uint8Array(4 * 1024 * 1024);
+ const blocks = 64;
+ const started = process.hrtime.bigint();
+ try {
+ for (let index = 0; index < blocks; index++) await file.write(block);
+ await file.sync();
+ } finally {
+ await file.close();
+ await unlink(path).catch(() => undefined);
+ }
+ const seconds = Number(process.hrtime.bigint() - started) / 1e9;
+ const mebibytes = (block.byteLength * blocks) / (1024 * 1024);
+ console.log(
+ `[disk baseline] ${mebibytes} MiB write + fsync: ${(mebibytes / seconds).toFixed(1)} MiB/s`,
+ );
+}
+
+async function createStore(
+ name: string,
+ durability: QwpSfDurability,
+): Promise {
+ const directory = join(root, name);
+ const store = new QwpNodeFileReplayStore({
+ directory,
+ durability,
+ checkpointIntervalMs:
+ durability === QWP_SF_DURABILITY.PERIODIC ? 1000 : undefined,
+ maxBytes: MAX_BYTES,
+ maxSegmentBytes: MAX_SEGMENT_BYTES,
+ });
+ await store.loadReferences();
+ return { store, nextSequence: 0n };
+}
+
+async function appendBatch(state: StoreState): Promise {
+ for (let index = 0; index < APPENDS; index++) {
+ await state.store.append({
+ frameSequence: state.nextSequence++,
+ payload: FRAME,
+ });
+ }
+ // Keep one record live so repeated iterations exercise steady-state segment
+ // use instead of retiring the active segment after every benchmark body.
+ await state.store.acknowledgeThrough(state.nextSequence - 2n);
+}
+
+async function seedBacklog(directory: string): Promise {
+ const store = new QwpNodeFileReplayStore({
+ directory,
+ durability: QWP_SF_DURABILITY.MEMORY,
+ maxBytes: MAX_BYTES,
+ maxSegmentBytes: MAX_SEGMENT_BYTES,
+ });
+ await store.loadReferences();
+ for (let index = 0; index < RECOVERY_FRAMES; index++) {
+ await store.append({ frameSequence: BigInt(index), payload: FRAME });
+ }
+ await store.close();
+}
+
+beforeAll(async () => {
+ const configuredRoot = process.env.QWP_BENCH_DIR ?? tmpdir();
+ await mkdir(configuredRoot, { recursive: true });
+ root = await mkdtemp(join(configuredRoot, "qwp-bench-"));
+ console.log(`[store-and-forward] benchmark directory: ${root}`);
+ console.log(
+ `[store-and-forward] verify real storage when quoting results: df -T ${configuredRoot}`,
+ );
+ await diskBaseline(root);
+
+ [pageCache, periodic, durable] = await Promise.all([
+ createStore("page-cache", QWP_SF_DURABILITY.MEMORY),
+ createStore("periodic", QWP_SF_DURABILITY.PERIODIC),
+ createStore("append", QWP_SF_DURABILITY.APPEND),
+ ]);
+
+ recoveryDirectory = join(root, "recovery");
+ const lazyDirectory = join(root, "lazy-read");
+ await seedBacklog(recoveryDirectory);
+ await seedBacklog(lazyDirectory);
+ lazyReader = new QwpNodeFileReplayStore({
+ directory: lazyDirectory,
+ durability: QWP_SF_DURABILITY.MEMORY,
+ maxBytes: MAX_BYTES,
+ maxSegmentBytes: MAX_SEGMENT_BYTES,
+ });
+ lazySequences = (await lazyReader.loadReferences()).map(
+ (reference) => reference.frameSequence,
+ );
+
+ return async () => {
+ await Promise.all([
+ pageCache.store.close(),
+ periodic.store.close(),
+ durable.store.close(),
+ lazyReader.close(),
+ ]);
+ await rm(root, { recursive: true, force: true });
+ };
+});
+
+describe(`QwpNodeFileReplayStore / ${APPENDS} appends`, () => {
+ bench("durability=memory (page-cache write)", async () => {
+ await appendBatch(pageCache);
+ });
+
+ bench("durability=periodic", async () => {
+ await appendBatch(periodic);
+ });
+
+ bench("durability=append (fsync per frame)", async () => {
+ await appendBatch(durable);
+ });
+});
+
+describe("store-and-forward recovery", () => {
+ bench(`recover ${RECOVERY_FRAMES} frame references`, async () => {
+ const store = new QwpNodeFileReplayStore({
+ directory: recoveryDirectory,
+ durability: QWP_SF_DURABILITY.MEMORY,
+ maxBytes: MAX_BYTES,
+ maxSegmentBytes: MAX_SEGMENT_BYTES,
+ });
+ const references = await store.loadReferences();
+ sink += references.length;
+ await store.close();
+ });
+
+ bench("lazy read / 4 KiB payload", async () => {
+ const sequence = lazySequences[lazyCursor++ % lazySequences.length];
+ sink += (await lazyReader.readPayload(sequence)).byteLength;
+ });
+});
+
+export const persistenceBenchmarkSink = (): number => sink;
diff --git a/benchmarks/sender.bench.ts b/benchmarks/sender.bench.ts
new file mode 100644
index 0000000..36a89d5
--- /dev/null
+++ b/benchmarks/sender.bench.ts
@@ -0,0 +1,162 @@
+import { beforeAll, bench, describe } from "vitest";
+import {
+ encodeQwpIngressFrame,
+ QWP_STATUS,
+ QwpSymbolDictionary,
+ type QwpIngressEncodeOptions,
+ type QwpIngressResponse,
+ type QwpTableBuffer,
+} from "../src/_qwp/_core";
+import { QwpSender, type QwpSenderSession } from "../src/_qwp/sender";
+import { BENCHMARK_WORKLOADS, type BenchmarkRow } from "./workloads";
+
+const ROWS = 10_000;
+let sink = 0;
+
+class EncodingSession implements QwpSenderSession {
+ private readonly dictionary = new QwpSymbolDictionary();
+ private confirmedMaxSymbolId = -1;
+ private publishedSequence = -1n;
+
+ get publishedFrameSequence(): bigint {
+ return this.publishedSequence;
+ }
+
+ get acknowledgedFrameSequence(): bigint {
+ return this.publishedSequence;
+ }
+
+ async sendTables(
+ tables: readonly QwpTableBuffer[],
+ options: QwpIngressEncodeOptions = {},
+ ): Promise {
+ this.encode(tables, options);
+ return this.response();
+ }
+
+ async sendTablesDelta(
+ tables: readonly QwpTableBuffer[],
+ options: Pick = {},
+ ): Promise {
+ this.encodeDelta(tables, options);
+ return this.response();
+ }
+
+ async publishTables(
+ tables: readonly QwpTableBuffer[],
+ options: QwpIngressEncodeOptions = {},
+ ): Promise {
+ this.encode(tables, options);
+ }
+
+ async publishTablesDelta(
+ tables: readonly QwpTableBuffer[],
+ options: Pick = {},
+ ): Promise {
+ this.encodeDelta(tables, options);
+ }
+
+ async waitForDurable(): Promise {}
+
+ async close(): Promise {}
+
+ private encode(
+ tables: readonly QwpTableBuffer[],
+ options: QwpIngressEncodeOptions,
+ ): void {
+ sink += encodeQwpIngressFrame(tables, options).byteLength;
+ this.publishedSequence++;
+ }
+
+ private encodeDelta(
+ tables: readonly QwpTableBuffer[],
+ options: Pick,
+ ): void {
+ sink += encodeQwpIngressFrame(tables, {
+ ...options,
+ dictionary: this.dictionary,
+ confirmedMaxSymbolId: this.confirmedMaxSymbolId,
+ }).byteLength;
+ this.confirmedMaxSymbolId = this.dictionary.size - 1;
+ this.publishedSequence++;
+ }
+
+ private response(): QwpIngressResponse {
+ return {
+ status: QWP_STATUS.OK,
+ sequence: this.publishedSequence,
+ tables: [],
+ };
+ }
+}
+
+async function fillSender(
+ sender: QwpSender,
+ rows: readonly BenchmarkRow[],
+): Promise {
+ for (const row of rows) {
+ sender.table(row.table);
+ for (const [name, value] of row.symbols) sender.symbol(name, value);
+ for (const [name, value] of row.longs) sender.longColumn(name, value);
+ for (const [name, value] of row.doubles) {
+ sender.doubleColumn(name, value);
+ }
+ for (const [name, value] of row.strings) {
+ sender.stringColumn(name, value);
+ }
+ await sender.at(row.timestamp);
+ }
+}
+
+function senderFor(
+ session: EncodingSession,
+ symbolDictionary: "delta" | "full",
+): QwpSender {
+ return new QwpSender(async () => session, {
+ autoFlush: false,
+ closeFlushTimeoutMs: 0,
+ encode: { symbolDictionary },
+ });
+}
+
+describe("high-level QwpSender build and encode", () => {
+ for (const name of ["trades", "wide", "sparse"] as const) {
+ const rows = BENCHMARK_WORKLOADS[name].rows(ROWS);
+ bench(name, async () => {
+ const sender = senderFor(new EncodingSession(), "full");
+ await fillSender(sender, rows);
+ await sender.flush();
+ });
+ }
+});
+
+describe("high-level symbol dictionary modes", () => {
+ const rows = BENCHMARK_WORKLOADS.highCardinalitySymbols.rows(ROWS);
+ const steadySession = new EncodingSession();
+
+ beforeAll(async () => {
+ const sender = senderFor(steadySession, "delta");
+ await fillSender(sender, rows);
+ await sender.flush();
+ });
+
+ bench("full dictionary", async () => {
+ const sender = senderFor(new EncodingSession(), "full");
+ await fillSender(sender, rows);
+ await sender.flush();
+ });
+
+ bench("delta dictionary / cold", async () => {
+ const sender = senderFor(new EncodingSession(), "delta");
+ await fillSender(sender, rows);
+ await sender.flush();
+ });
+
+ bench("delta dictionary / confirmed steady state", async () => {
+ const sender = senderFor(steadySession, "delta");
+ await fillSender(sender, rows);
+ await sender.flush();
+ });
+});
+
+export const senderBenchmarkSink = (): number => sink;
diff --git a/benchmarks/tables.ts b/benchmarks/tables.ts
new file mode 100644
index 0000000..e71a08c
--- /dev/null
+++ b/benchmarks/tables.ts
@@ -0,0 +1,29 @@
+import { QWP_COLUMN_TYPE, QwpTableBuffer } from "../src/_qwp/_core";
+import type { BenchmarkRow } from "./workloads";
+
+export function buildBenchmarkTable(
+ rows: readonly BenchmarkRow[],
+): QwpTableBuffer {
+ const table = new QwpTableBuffer(rows[0].table);
+ for (const row of rows) {
+ for (const [name, value] of row.symbols) {
+ table.getOrCreateColumn(name, QWP_COLUMN_TYPE.SYMBOL)?.values.push(value);
+ }
+ for (const [name, value] of row.longs) {
+ table.getOrCreateColumn(name, QWP_COLUMN_TYPE.LONG)?.values.push(value);
+ }
+ for (const [name, value] of row.doubles) {
+ table.getOrCreateColumn(name, QWP_COLUMN_TYPE.DOUBLE)?.values.push(value);
+ }
+ for (const [name, value] of row.strings) {
+ table
+ .getOrCreateColumn(name, QWP_COLUMN_TYPE.VARCHAR)
+ ?.values.push(value);
+ }
+ table
+ .getOrCreateColumn("", QWP_COLUMN_TYPE.TIMESTAMP)
+ ?.values.push(row.timestamp);
+ table.nextRow();
+ }
+ return table;
+}
diff --git a/benchmarks/validate.test.ts b/benchmarks/validate.test.ts
new file mode 100644
index 0000000..335034c
--- /dev/null
+++ b/benchmarks/validate.test.ts
@@ -0,0 +1,92 @@
+import { describe, expect, it } from "vitest";
+import {
+ encodeQwpIngressFrame,
+ QWP_COLUMN_TYPE,
+ QwpSymbolDictionary,
+ QwpTableBuffer,
+} from "../src/_qwp/_core";
+import { buildBenchmarkTable } from "./tables";
+import { BENCHMARK_WORKLOADS, type BenchmarkRow } from "./workloads";
+
+const BASE_TIMESTAMP = 1_700_000_000_000_000n;
+
+function encode(
+ rows: readonly BenchmarkRow[],
+ dictionary?: QwpSymbolDictionary,
+ confirmedMaxSymbolId?: number,
+): Uint8Array {
+ return encodeQwpIngressFrame([buildBenchmarkTable(rows)], {
+ dictionary,
+ confirmedMaxSymbolId,
+ });
+}
+
+describe("benchmark wire-format invariants", () => {
+ it("encodes trades to a plausible number of bytes per row", () => {
+ const rows = BENCHMARK_WORKLOADS.trades.rows(10_000);
+ const bytesPerRow = encode(rows).byteLength / rows.length;
+ expect(bytesPerRow).toBeGreaterThan(14);
+ expect(bytesPerRow).toBeLessThan(24);
+ });
+
+ it("compacts null values instead of writing placeholders", () => {
+ const sparse = BENCHMARK_WORKLOADS.sparse.rows(2000);
+ const sparseBytes = encode(sparse).byteLength;
+ const dense = sparse.map((row) => ({
+ ...row,
+ nulls: [],
+ longs: ["a", "b", "c", "d", "e", "f", "g", "h"].map(
+ (name) => [name, 1n] as [string, bigint],
+ ),
+ }));
+ expect(sparseBytes).toBeLessThan(encode(dense).byteLength * 0.9);
+ });
+
+ it("emits fewer bytes after a symbol-dictionary baseline is confirmed", () => {
+ const rows = BENCHMARK_WORKLOADS.highCardinalitySymbols.rows(5000);
+ const fullBytes = encode(rows).byteLength;
+ const dictionary = new QwpSymbolDictionary();
+ encode(rows, dictionary, -1);
+ const deltaBytes = encode(rows, dictionary, dictionary.size - 1).byteLength;
+ expect(deltaBytes).toBeLessThan(fullBytes);
+ });
+
+ it("does not treat a populated but unconfirmed dictionary as steady state", () => {
+ const rows = BENCHMARK_WORKLOADS.highCardinalitySymbols.rows(5000);
+ const fullBytes = encode(rows).byteLength;
+ const dictionary = new QwpSymbolDictionary();
+ encode(rows, dictionary, -1);
+ const coldBytes = encode(rows, dictionary, -1).byteLength;
+ expect(coldBytes).toBeGreaterThan(fullBytes * 0.9);
+ });
+
+ it("does not apply Gorilla encoding to long columns", () => {
+ const table = new QwpTableBuffer("gorilla_long");
+ for (let index = 0; index < 5000; index++) {
+ table
+ .getOrCreateColumn("value", QWP_COLUMN_TYPE.LONG)
+ ?.values.push(BigInt(index));
+ table.nextRow();
+ }
+ expect(encodeQwpIngressFrame([table], { gorilla: true }).byteLength).toBe(
+ encodeQwpIngressFrame([table], { gorilla: false }).byteLength,
+ );
+ });
+
+ it("compresses regularly spaced timestamps with Gorilla encoding", () => {
+ const table = new QwpTableBuffer("gorilla_timestamp");
+ for (let index = 0; index < 5000; index++) {
+ table
+ .getOrCreateColumn("timestamp", QWP_COLUMN_TYPE.TIMESTAMP)
+ ?.values.push(BASE_TIMESTAMP + BigInt(index) * 1000n);
+ table.nextRow();
+ }
+ const uncompressed = encodeQwpIngressFrame([table], {
+ gorilla: false,
+ }).byteLength;
+ const compressed = encodeQwpIngressFrame([table], {
+ gorilla: true,
+ }).byteLength;
+ expect(compressed).toBeLessThan(uncompressed / 2);
+ });
+});
diff --git a/benchmarks/workloads.test.ts b/benchmarks/workloads.test.ts
new file mode 100644
index 0000000..cfee948
--- /dev/null
+++ b/benchmarks/workloads.test.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { BENCHMARK_WORKLOADS } from "./workloads";
+
+function stringify(value: unknown): string {
+ return JSON.stringify(value, (_key, item) =>
+ typeof item === "bigint" ? item.toString() : item,
+ );
+}
+
+describe("benchmark workloads", () => {
+ it("are deterministic across calls", () => {
+ expect(stringify(BENCHMARK_WORKLOADS.trades.rows(100))).toBe(
+ stringify(BENCHMARK_WORKLOADS.trades.rows(100)),
+ );
+ });
+
+ it("builds the advertised trade shape", () => {
+ const row = BENCHMARK_WORKLOADS.trades.rows(1)[0];
+ expect(row.symbols).toHaveLength(1);
+ expect(row.doubles).toHaveLength(2);
+ });
+
+ it("builds 50 wide data columns", () => {
+ const row = BENCHMARK_WORKLOADS.wide.rows(1)[0];
+ expect(
+ row.symbols.length +
+ row.longs.length +
+ row.doubles.length +
+ row.strings.length,
+ ).toBe(50);
+ });
+
+ it("builds high-cardinality symbols", () => {
+ const rows = BENCHMARK_WORKLOADS.highCardinalitySymbols.rows(5000);
+ expect(new Set(rows.map((row) => row.symbols[0][1])).size).toBeGreaterThan(
+ 4000,
+ );
+ });
+
+ it("makes roughly 30 percent of sparse values null", () => {
+ const rows = BENCHMARK_WORKLOADS.sparse.rows(1000);
+ const nulls = rows.reduce((total, row) => total + row.nulls.length, 0);
+ const ratio = nulls / (rows.length * 8);
+ expect(ratio).toBeGreaterThan(0.2);
+ expect(ratio).toBeLessThan(0.4);
+ });
+});
diff --git a/benchmarks/workloads.ts b/benchmarks/workloads.ts
new file mode 100644
index 0000000..63586a8
--- /dev/null
+++ b/benchmarks/workloads.ts
@@ -0,0 +1,132 @@
+export interface BenchmarkRow {
+ table: string;
+ symbols: [string, string][];
+ longs: [string, bigint][];
+ doubles: [string, number][];
+ strings: [string, string][];
+ /** Column names deliberately left unset for this row. */
+ nulls: string[];
+ timestamp: bigint;
+}
+
+export interface BenchmarkWorkload {
+ name: string;
+ columns: number;
+ rows(count: number): BenchmarkRow[];
+}
+
+/** Deterministic, dependency-free xorshift32 generator. */
+function random(seed: number): () => number {
+ let state = seed || 0x9e3779b9;
+ const next = (): number => {
+ state ^= state << 13;
+ state ^= state >>> 17;
+ state ^= state << 5;
+ return (state >>> 0) / 0x100000000;
+ };
+ // Small xorshift seeds start near zero. Discard the poorly diffused prefix.
+ for (let index = 0; index < 16; index++) next();
+ return next;
+}
+
+const BASE_TIMESTAMP = 1_700_000_000_000_000n;
+
+function trades(count: number): BenchmarkRow[] {
+ const next = random(1);
+ const symbols = ["ETH-USD", "BTC-USD", "SOL-USD", "ADA-USD"];
+ return Array.from({ length: count }, (_, index) => ({
+ table: "bench_trades",
+ symbols: [["symbol", symbols[index % symbols.length]]],
+ longs: [],
+ doubles: [
+ ["price", 1000 + next() * 5000],
+ ["amount", next()],
+ ],
+ strings: [],
+ nulls: [],
+ timestamp: BASE_TIMESTAMP + BigInt(index) * 1000n,
+ }));
+}
+
+function wide(count: number): BenchmarkRow[] {
+ const next = random(2);
+ const rows: BenchmarkRow[] = [];
+ for (let index = 0; index < count; index++) {
+ const longs: [string, bigint][] = [];
+ const doubles: [string, number][] = [];
+ const strings: [string, string][] = [];
+ for (let column = 0; column < 20; column++) {
+ longs.push([`l${column}`, BigInt(Math.floor(next() * 1e6))]);
+ doubles.push([`d${column}`, next() * 1000]);
+ }
+ for (let column = 0; column < 9; column++) {
+ strings.push([`s${column}`, `v${Math.floor(next() * 100)}`]);
+ }
+ rows.push({
+ table: "bench_wide",
+ symbols: [["sym", `s${index % 16}`]],
+ longs,
+ doubles,
+ strings,
+ nulls: [],
+ timestamp: BASE_TIMESTAMP + BigInt(index) * 1000n,
+ });
+ }
+ return rows;
+}
+
+function highCardinalitySymbols(count: number): BenchmarkRow[] {
+ return Array.from({ length: count }, (_, index) => ({
+ table: "bench_highcard",
+ symbols: [["sym", `sym-${index % 100_000}`]],
+ longs: [["v", BigInt(index)]],
+ doubles: [],
+ strings: [],
+ nulls: [],
+ timestamp: BASE_TIMESTAMP + BigInt(index) * 1000n,
+ }));
+}
+
+function sparse(count: number): BenchmarkRow[] {
+ const next = random(4);
+ const names = ["a", "b", "c", "d", "e", "f", "g", "h"];
+ const rows: BenchmarkRow[] = [];
+ for (let index = 0; index < count; index++) {
+ const longs: [string, bigint][] = [];
+ const nulls: string[] = [];
+ for (const name of names) {
+ if (next() < 0.3) nulls.push(name);
+ else longs.push([name, BigInt(Math.floor(next() * 1e6))]);
+ }
+ rows.push({
+ table: "bench_sparse",
+ symbols: [],
+ longs,
+ doubles: [],
+ strings: [],
+ nulls,
+ timestamp: BASE_TIMESTAMP + BigInt(index) * 1000n,
+ });
+ }
+ return rows;
+}
+
+export type BenchmarkWorkloadName =
+ | "trades"
+ | "wide"
+ | "highCardinalitySymbols"
+ | "sparse";
+
+export const BENCHMARK_WORKLOADS: Record<
+ BenchmarkWorkloadName,
+ BenchmarkWorkload
+> = {
+ trades: { name: "trades", columns: 4, rows: trades },
+ wide: { name: "wide", columns: 50, rows: wide },
+ highCardinalitySymbols: {
+ name: "highCardinalitySymbols",
+ columns: 3,
+ rows: highCardinalitySymbols,
+ },
+ sparse: { name: "sparse", columns: 8, rows: sparse },
+};
diff --git a/examples.manifest.yaml b/examples.manifest.yaml
index 428a4dd..bfdcaa9 100644
--- a/examples.manifest.yaml
+++ b/examples.manifest.yaml
@@ -2,12 +2,12 @@
lang: javascript
path: examples/basic.js
header: |-
- NodeJS client library [repo](https://github.com/questdb/nodejs-questdb-client).
+ JavaScript client library [repo](https://github.com/questdb/nodejs-questdb-client).
- name: ilp-auth
lang: javascript
path: examples/auth.js
header: |-
- NodeJS client library [repo](https://github.com/questdb/nodejs-questdb-client).
+ JavaScript client library [repo](https://github.com/questdb/nodejs-questdb-client).
auth:
kid: testapp
d: 9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8
@@ -20,7 +20,7 @@
lang: javascript
path: examples/auth_tls.js
header: |-
- NodeJS client library [repo](https://github.com/questdb/nodejs-questdb-client).
+ JavaScript client library [repo](https://github.com/questdb/nodejs-questdb-client).
auth:
kid: testapp
d: 9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8
@@ -33,5 +33,5 @@
lang: javascript
path: examples/basic.js
header: |-
- NodeJS client library [repo](https://github.com/questdb/nodejs-questdb-client).
+ JavaScript client library [repo](https://github.com/questdb/nodejs-questdb-client).
conf: http::addr=localhost:9000
diff --git a/examples/qwp-basic.ts b/examples/qwp-basic.ts
new file mode 100644
index 0000000..c0ba737
--- /dev/null
+++ b/examples/qwp-basic.ts
@@ -0,0 +1,19 @@
+import { Sender } from "@questdb/nodejs-client";
+
+async function main(): Promise {
+ const sender = await Sender.fromConfig("ws::addr=localhost:9000");
+ await sender.connect();
+ try {
+ await sender
+ .table("trades")
+ .symbol("symbol", "ETH-USD")
+ .floatColumn("price", 2_615.54)
+ .floatColumn("amount", 0.00044)
+ .at(Date.now(), "ms");
+ await sender.flush();
+ } finally {
+ await sender.close();
+ }
+}
+
+void main();
diff --git a/examples/qwp-browser.ts b/examples/qwp-browser.ts
new file mode 100644
index 0000000..7203fee
--- /dev/null
+++ b/examples/qwp-browser.ts
@@ -0,0 +1,15 @@
+import { connectQwpBrowserSender } from "@questdb/nodejs-client/qwp/browser";
+
+async function main(): Promise {
+ const url = new URL("/write/v4", location.href);
+ url.protocol = location.protocol === "https:" ? "wss:" : "ws:";
+ const sender = await connectQwpBrowserSender({ url }, { autoFlush: false });
+ try {
+ await sender.table("events").longColumn("value", 42n).atNow();
+ await sender.flush();
+ } finally {
+ await sender.close();
+ }
+}
+
+void main();
diff --git a/package.json b/package.json
index 2290156..0cc7620 100644
--- a/package.json
+++ b/package.json
@@ -1,31 +1,89 @@
{
"name": "@questdb/nodejs-client",
"version": "4.2.0",
- "description": "QuestDB Node.js Client",
+ "description": "QuestDB JavaScript Client",
"scripts": {
"test": "vitest",
+ "test:qwp-browser": "pnpm build && vitest run --config vitest.qwp-browser.config.ts",
+ "test:dist": "pnpm build && vitest run --config vitest.dist.config.ts",
+ "typecheck:dist": "pnpm build && tsc --noEmit -p tsconfig.dist-types.json && tsc --noEmit -p tsconfig.dist-types.cjs.json",
"build": "bunchee",
"eslint": "eslint src/**",
"typecheck": "tsc --noEmit",
+ "typecheck:qwp-browser": "tsc --noEmit -p tsconfig.qwp-browser.json",
+ "typecheck:test": "tsc --noEmit -p tsconfig.test.json",
+ "bench": "vitest bench --run benchmarks",
+ "bench:e2e": "vitest run --config vitest.bench-e2e.config.ts",
+ "typecheck:bench": "tsc --noEmit -p tsconfig.bench.json",
+ "lint:bench": "eslint 'benchmarks/**/*.ts' vitest.bench-e2e.config.ts",
+ "format:bench": "prettier --write 'benchmarks/**/*.{ts,md}' tsconfig.bench.json vitest.bench-e2e.config.ts",
"format": "prettier --write '{src,test}/**/*.{ts,js,json}'",
- "docs": "typedoc --out docs src/index.ts",
+ "docs": "typedoc",
"preview:docs": "serve docs"
},
"files": [
+ "QWP.md",
+ "THIRD_PARTY_NOTICES.md",
+ "dist/_qwp",
"dist/cjs",
"dist/es"
],
"main": "dist/cjs/index.js",
"module": "dist/es/index.mjs",
"types": "dist/cjs/index.d.ts",
+ "typesVersions": {
+ "*": {
+ "qwp": [
+ "./dist/cjs/qwp/index.d.ts"
+ ],
+ "qwp/browser": [
+ "./dist/cjs/qwp/browser.d.ts"
+ ],
+ "qwp/node": [
+ "./dist/cjs/qwp/node.d.ts"
+ ]
+ }
+ },
"exports": {
- "import": {
- "types": "./dist/es/index.d.mts",
- "default": "./dist/es/index.mjs"
+ ".": {
+ "import": {
+ "types": "./dist/es/index.d.mts",
+ "default": "./dist/es/index.mjs"
+ },
+ "require": {
+ "types": "./dist/cjs/index.d.ts",
+ "default": "./dist/cjs/index.js"
+ }
+ },
+ "./qwp": {
+ "import": {
+ "types": "./dist/es/qwp/index.d.mts",
+ "default": "./dist/es/qwp/index.mjs"
+ },
+ "require": {
+ "types": "./dist/cjs/qwp/index.d.ts",
+ "default": "./dist/cjs/qwp/index.js"
+ }
+ },
+ "./qwp/browser": {
+ "import": {
+ "types": "./dist/es/qwp/browser.d.mts",
+ "default": "./dist/es/qwp/browser.mjs"
+ },
+ "require": {
+ "types": "./dist/cjs/qwp/browser.d.ts",
+ "default": "./dist/cjs/qwp/browser.js"
+ }
},
- "require": {
- "types": "./dist/cjs/index.d.ts",
- "default": "./dist/cjs/index.js"
+ "./qwp/node": {
+ "import": {
+ "types": "./dist/es/qwp/node.d.mts",
+ "default": "./dist/es/qwp/node.mjs"
+ },
+ "require": {
+ "types": "./dist/cjs/qwp/node.d.ts",
+ "default": "./dist/cjs/qwp/node.js"
+ }
}
},
"repository": {
@@ -42,8 +100,11 @@
"@eslint/js": "^9.16.0",
"@microsoft/tsdoc": "^0.15.1",
"@types/node": "^22.15.17",
+ "@types/ws": "^8.18.1",
"bunchee": "^6.5.1",
"eslint": "^9.26.0",
+ "fzstd": "0.1.1",
+ "playwright": "^1.62.1",
"prettier": "^3.5.3",
"serve": "^14.2.4",
"testcontainers": "^10.25.0",
@@ -53,6 +114,7 @@
"vitest": "^3.1.3"
},
"dependencies": {
- "undici": "^7.8.0"
+ "undici": "^7.8.0",
+ "ws": "^8.21.3"
}
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d824b63..9040014 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -11,6 +11,9 @@ importers:
undici:
specifier: ^7.8.0
version: 7.8.0
+ ws:
+ specifier: ^8.21.3
+ version: 8.21.3
devDependencies:
'@eslint/js':
specifier: ^9.16.0
@@ -21,12 +24,21 @@ importers:
'@types/node':
specifier: ^22.15.17
version: 22.15.17
+ '@types/ws':
+ specifier: ^8.18.1
+ version: 8.18.1
bunchee:
specifier: ^6.5.1
version: 6.5.1(typescript@5.7.2)
eslint:
specifier: ^9.26.0
version: 9.26.0
+ fzstd:
+ specifier: 0.1.1
+ version: 0.1.1
+ playwright:
+ specifier: ^1.62.1
+ version: 1.62.1
prettier:
specifier: ^3.5.3
version: 3.5.3
@@ -726,6 +738,9 @@ packages:
'@types/unist@3.0.3':
resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==}
+ '@types/ws@8.18.1':
+ resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
+
'@typescript-eslint/eslint-plugin@8.32.0':
resolution: {integrity: sha512-/jU9ettcntkBFmWUzzGgsClEi2ZFiikMX5eEQsmxIAWMOn4H3D4rvHssstmAHGVvrYnaMqdWWWg0b5M6IN/MTQ==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
@@ -1397,6 +1412,11 @@ packages:
fs-constants@1.0.0:
resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
+ fsevents@2.3.2:
+ resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
+ engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
+ os: [darwin]
+
fsevents@2.3.3:
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@@ -1405,6 +1425,9 @@ packages:
function-bind@1.1.2:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
+ fzstd@0.1.1:
+ resolution: {integrity: sha512-dkuVSOKKwh3eas5VkJy1AW1vFpet8TA/fGmVA5krThl8YcOVE/8ZIoEA1+U1vEn5ckxxhLirSdY837azmbaNHA==}
+
get-caller-file@2.0.5:
resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
engines: {node: 6.* || 8.* || >= 10.*}
@@ -1442,6 +1465,7 @@ packages:
glob@10.4.5:
resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==}
+ deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
hasBin: true
globals@14.0.0:
@@ -1868,6 +1892,16 @@ packages:
resolution: {integrity: sha512-ueGLflrrnvwB3xuo/uGob5pd5FN7l0MsLf0Z87o/UQmRtwjvfylfc9MurIxRAWywCYTgrvpXBcqjV4OfCYGCIQ==}
engines: {node: '>=16.20.0'}
+ playwright-core@1.62.1:
+ resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==}
+ engines: {node: '>=20'}
+ hasBin: true
+
+ playwright@1.62.1:
+ resolution: {integrity: sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==}
+ engines: {node: '>=20'}
+ hasBin: true
+
postcss@8.4.49:
resolution: {integrity: sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==}
engines: {node: ^10 || ^12 || >=14}
@@ -2308,6 +2342,7 @@ packages:
uuid@10.0.0:
resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}
+ deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
hasBin: true
vary@1.1.2:
@@ -2407,6 +2442,18 @@ packages:
wrappy@1.0.2:
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
+ ws@8.21.3:
+ resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==}
+ engines: {node: '>=10.0.0'}
+ peerDependencies:
+ bufferutil: ^4.0.1
+ utf-8-validate: '>=5.0.2'
+ peerDependenciesMeta:
+ bufferutil:
+ optional: true
+ utf-8-validate:
+ optional: true
+
y18n@5.0.8:
resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
engines: {node: '>=10'}
@@ -2970,6 +3017,10 @@ snapshots:
'@types/unist@3.0.3': {}
+ '@types/ws@8.18.1':
+ dependencies:
+ '@types/node': 22.15.17
+
'@typescript-eslint/eslint-plugin@8.32.0(@typescript-eslint/parser@8.32.0(eslint@9.26.0)(typescript@5.7.2))(eslint@9.26.0)(typescript@5.7.2)':
dependencies:
'@eslint-community/regexpp': 4.12.1
@@ -3762,11 +3813,16 @@ snapshots:
fs-constants@1.0.0: {}
+ fsevents@2.3.2:
+ optional: true
+
fsevents@2.3.3:
optional: true
function-bind@1.1.2: {}
+ fzstd@0.1.1: {}
+
get-caller-file@2.0.5: {}
get-east-asian-width@1.3.0: {}
@@ -4149,6 +4205,14 @@ snapshots:
pkce-challenge@5.0.0: {}
+ playwright-core@1.62.1: {}
+
+ playwright@1.62.1:
+ dependencies:
+ playwright-core: 1.62.1
+ optionalDependencies:
+ fsevents: 2.3.2
+
postcss@8.4.49:
dependencies:
nanoid: 3.3.8
@@ -4797,6 +4861,8 @@ snapshots:
wrappy@1.0.2: {}
+ ws@8.21.3: {}
+
y18n@5.0.8: {}
yaml@2.6.1: {}
diff --git a/scripts/check-build-artifacts.mjs b/scripts/check-build-artifacts.mjs
new file mode 100644
index 0000000..eaa2808
--- /dev/null
+++ b/scripts/check-build-artifacts.mjs
@@ -0,0 +1,101 @@
+// Verifies that every package `exports` target exists and is included by
+// `npm pack`, along with every shared chunk those entries import. Entry bundles
+// import chunks that no `exports` entry names, so checking only the untarred
+// tree would miss a chunk left out of `files` and publish broken entry points.
+//
+// This lives in a file rather than inline in the workflow because the pattern
+// below needs both quote characters, which cannot survive a single-quoted
+// `node -e` argument in a YAML block scalar.
+import { existsSync, readFileSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { dirname, join, relative, resolve, sep } from "node:path";
+
+// Only specifiers that name an emitted file. Matching every `from "./x"` in the
+// raw text would also match prose inside a comment the bundler preserved -- a
+// sentence such as "the factories from './qwp'" is not an import, and treating
+// it as one reports a build artifact that was never meant to exist.
+const SPECIFIER =
+ /(?:\bfrom|\brequire\(|\bimport\()\s*["'](\.[^"']*\.(?:d\.)?[mc]?[jt]s)["']/g;
+
+const { exports: map, typesVersions } = JSON.parse(
+ readFileSync("package.json", "utf8"),
+);
+
+const missing = [];
+const seen = new Set();
+
+const walk = (file, from) => {
+ if (!existsSync(file)) {
+ missing.push(`${from} -> ${file}`);
+ return;
+ }
+ const key = resolve(file);
+ if (seen.has(key)) return;
+ seen.add(key);
+ const source = readFileSync(file, "utf8");
+ for (const [, specifier] of source.matchAll(SPECIFIER)) {
+ walk(join(dirname(file), specifier), file);
+ }
+};
+
+for (const [subpath, conditions] of Object.entries(map)) {
+ for (const target of Object.values(conditions)) {
+ for (const file of Object.values(target)) {
+ walk(file, subpath);
+ }
+ }
+}
+
+// typesVersions is what TypeScript's legacy node10 resolution reads instead of
+// `exports`, so a target missing here breaks those consumers with a TS2307 that
+// no runtime test can see.
+for (const [subpath, targets] of Object.entries(typesVersions?.["*"] ?? {})) {
+ for (const target of targets) {
+ walk(target, `typesVersions ${subpath}`);
+ }
+}
+
+if (missing.length > 0) {
+ console.error(`missing build artifacts:\n ${missing.join("\n ")}`);
+ process.exit(1);
+}
+
+let pack;
+try {
+ [pack] = JSON.parse(
+ execFileSync(
+ process.platform === "win32" ? "npm.cmd" : "npm",
+ ["pack", "--dry-run", "--json"],
+ {
+ encoding: "utf8",
+ stdio: ["ignore", "pipe", "pipe"],
+ },
+ ),
+ );
+} catch (error) {
+ const stderr = error?.stderr?.toString().trim();
+ console.error(`npm pack --dry-run failed${stderr ? `:\n${stderr}` : ""}`);
+ process.exit(1);
+}
+
+if (!Array.isArray(pack?.files)) {
+ console.error("npm pack --dry-run returned no package file manifest");
+ process.exit(1);
+}
+
+const packedFiles = new Set(pack.files.map(({ path }) => path));
+for (const file of seen) {
+ const packagePath = relative(process.cwd(), file).split(sep).join("/");
+ if (!packedFiles.has(packagePath)) {
+ missing.push(`npm pack omits ${packagePath}`);
+ }
+}
+
+if (missing.length > 0) {
+ console.error(`missing build artifacts:\n ${missing.join("\n ")}`);
+ process.exit(1);
+}
+
+console.log(
+ `all ${Object.keys(map).length} export subpaths present, ${seen.size} files walked and packed`,
+);
diff --git a/src/_qwp/_core/binds.ts b/src/_qwp/_core/binds.ts
new file mode 100644
index 0000000..fbaeb7b
--- /dev/null
+++ b/src/_qwp/_core/binds.ts
@@ -0,0 +1,482 @@
+import { encodeUtf8, QwpByteWriter } from "./bytes";
+import { QWP_COLUMN_TYPE, QWP_MAX_COLUMNS_PER_TABLE } from "./constants";
+import { writeQwpVarint } from "./varint";
+
+const INT64_MIN = -(1n << 63n);
+const INT64_MAX = (1n << 63n) - 1n;
+const UINT64_MAX = (1n << 64n) - 1n;
+const DECIMAL64_MAX_SCALE = 18;
+const DECIMAL128_MAX_SCALE = 38;
+const DECIMAL256_MAX_SCALE = 76;
+const GEOHASH_MIN_BITS = 1;
+const GEOHASH_MAX_BITS = 60;
+const NULL_FLAG = 0x01;
+const NULL_BITMAP = 0x01;
+const NON_NULL_FLAG = 0x00;
+
+export type QwpInt64 = number | bigint;
+
+/** Phase-1 scalar bind types exposed by the Java reference client. */
+export type QwpBindType =
+ | typeof QWP_COLUMN_TYPE.BOOLEAN
+ | typeof QWP_COLUMN_TYPE.BYTE
+ | typeof QWP_COLUMN_TYPE.SHORT
+ | typeof QWP_COLUMN_TYPE.INT
+ | typeof QWP_COLUMN_TYPE.LONG
+ | typeof QWP_COLUMN_TYPE.FLOAT
+ | typeof QWP_COLUMN_TYPE.DOUBLE
+ | typeof QWP_COLUMN_TYPE.TIMESTAMP
+ | typeof QWP_COLUMN_TYPE.DATE
+ | typeof QWP_COLUMN_TYPE.UUID
+ | typeof QWP_COLUMN_TYPE.LONG256
+ | typeof QWP_COLUMN_TYPE.GEOHASH
+ | typeof QWP_COLUMN_TYPE.VARCHAR
+ | typeof QWP_COLUMN_TYPE.TIMESTAMP_NANOS
+ | typeof QWP_COLUMN_TYPE.DECIMAL64
+ | typeof QWP_COLUMN_TYPE.DECIMAL128
+ | typeof QWP_COLUMN_TYPE.DECIMAL256
+ | typeof QWP_COLUMN_TYPE.CHAR;
+
+export type QwpBindSetter = (binds: QwpBindValues) => void;
+
+export interface QwpEncodedBinds {
+ count: number;
+ payload: Uint8Array;
+}
+
+function checkedIndex(value: number): number {
+ if (!Number.isSafeInteger(value) || value < 0) {
+ throw new RangeError("bind index must be a non-negative safe integer");
+ }
+ return value;
+}
+
+function checkedInteger(
+ value: number,
+ minimum: number,
+ maximum: number,
+ label: string,
+): number {
+ if (!Number.isInteger(value) || value < minimum || value > maximum) {
+ throw new RangeError(
+ `${label} must be an integer between ${minimum} and ${maximum}`,
+ );
+ }
+ return value;
+}
+
+function checkedInt64(value: QwpInt64, label: string): bigint {
+ let integer: bigint;
+ if (typeof value === "number") {
+ if (!Number.isSafeInteger(value)) {
+ throw new RangeError(`${label} must be a safe integer or bigint`);
+ }
+ integer = BigInt(value);
+ } else if (typeof value === "bigint") {
+ integer = value;
+ } else {
+ throw new TypeError(`${label} must be a safe integer or bigint`);
+ }
+ if (integer < INT64_MIN || integer > INT64_MAX) {
+ throw new RangeError(`${label} must fit in int64`);
+ }
+ return integer;
+}
+
+function checkedUint64Bits(value: QwpInt64, label: string): bigint {
+ let integer: bigint;
+ if (typeof value === "number") {
+ if (!Number.isSafeInteger(value)) {
+ throw new RangeError(`${label} must be a safe integer or bigint`);
+ }
+ integer = BigInt(value);
+ } else if (typeof value === "bigint") {
+ integer = value;
+ } else {
+ throw new TypeError(`${label} must be a safe integer or bigint`);
+ }
+ if (integer < INT64_MIN || integer > UINT64_MAX) {
+ throw new RangeError(`${label} must fit in 64 bits`);
+ }
+ return BigInt.asUintN(64, integer);
+}
+
+function checkedScale(value: number, maximum: number, label: string): number {
+ return checkedInteger(value, 0, maximum, `${label} scale`);
+}
+
+/**
+ * Browser-safe typed positional bind encoder.
+ *
+ * Setters must be called in ascending zero-based index order. SQL placeholders
+ * are one-based, so index 0 binds `$1`, index 1 binds `$2`, and so on.
+ */
+export class QwpBindValues {
+ private writer = new QwpByteWriter();
+ private expectedIndex = 0;
+
+ get count(): number {
+ return this.expectedIndex;
+ }
+
+ reset(): this {
+ this.writer = new QwpByteWriter();
+ this.expectedIndex = 0;
+ return this;
+ }
+
+ setBoolean(index: number, value: boolean): this {
+ if (typeof value !== "boolean") {
+ throw new TypeError("BOOLEAN bind value must be a boolean");
+ }
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.BOOLEAN, false);
+ this.writer.writeUint8(value ? 1 : 0);
+ return this;
+ }
+
+ setByte(index: number, value: number): this {
+ const checked = checkedInteger(value, -0x80, 0x7f, "BYTE bind");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.BYTE, false);
+ this.writer.writeInt8(checked);
+ return this;
+ }
+
+ setShort(index: number, value: number): this {
+ const checked = checkedInteger(value, -0x8000, 0x7fff, "SHORT bind");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.SHORT, false);
+ this.writer.writeInt16(checked);
+ return this;
+ }
+
+ setChar(index: number, value: string): this {
+ if (typeof value !== "string" || value.length !== 1) {
+ throw new TypeError("CHAR bind value must be one UTF-16 code unit");
+ }
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.CHAR, false);
+ this.writer.writeUint16(value.charCodeAt(0));
+ return this;
+ }
+
+ setInt(index: number, value: number): this {
+ const checked = checkedInteger(value, -0x80000000, 0x7fffffff, "INT bind");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.INT, false);
+ this.writer.writeInt32(checked);
+ return this;
+ }
+
+ setLong(index: number, value: QwpInt64): this {
+ const checked = checkedInt64(value, "LONG bind");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.LONG, false);
+ this.writer.writeBigInt64(checked);
+ return this;
+ }
+
+ setFloat(index: number, value: number): this {
+ if (typeof value !== "number") {
+ throw new TypeError("FLOAT bind value must be a number");
+ }
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.FLOAT, false);
+ this.writer.writeFloat32(value);
+ return this;
+ }
+
+ setDouble(index: number, value: number): this {
+ if (typeof value !== "number") {
+ throw new TypeError("DOUBLE bind value must be a number");
+ }
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DOUBLE, false);
+ this.writer.writeFloat64(value);
+ return this;
+ }
+
+ /** Binds a DATE expressed as milliseconds since the Unix epoch. */
+ setDate(index: number, millisecondsSinceEpoch: QwpInt64): this {
+ const checked = checkedInt64(millisecondsSinceEpoch, "DATE bind");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DATE, false);
+ this.writer.writeBigInt64(checked);
+ return this;
+ }
+
+ /** Binds a TIMESTAMP expressed as microseconds since the Unix epoch. */
+ setTimestampMicros(index: number, microsecondsSinceEpoch: QwpInt64): this {
+ const checked = checkedInt64(microsecondsSinceEpoch, "TIMESTAMP bind");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.TIMESTAMP, false);
+ this.writer.writeBigInt64(checked);
+ return this;
+ }
+
+ /** Binds a TIMESTAMP_NS expressed as nanoseconds since the Unix epoch. */
+ setTimestampNanos(index: number, nanosecondsSinceEpoch: QwpInt64): this {
+ const checked = checkedInt64(nanosecondsSinceEpoch, "TIMESTAMP_NANOS bind");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.TIMESTAMP_NANOS, false);
+ this.writer.writeBigInt64(checked);
+ return this;
+ }
+
+ setVarchar(index: number, value: string | null): this {
+ if (value === null) return this.setNull(index, QWP_COLUMN_TYPE.VARCHAR);
+ if (typeof value !== "string") {
+ throw new TypeError("VARCHAR bind value must be a string or null");
+ }
+ const utf8 = encodeUtf8(value);
+ if (utf8.length > 0x7fffffff) {
+ throw new RangeError("VARCHAR bind exceeds the int32 wire length limit");
+ }
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.VARCHAR, false);
+ this.writer.writeUint32(0).writeUint32(utf8.length).writeBytes(utf8);
+ return this;
+ }
+
+ setUuid(index: number, value: string | null): this;
+ setUuid(index: number, low: QwpInt64, high: QwpInt64): this;
+ setUuid(
+ index: number,
+ valueOrLow: string | null | QwpInt64,
+ high?: QwpInt64,
+ ): this {
+ if (valueOrLow === null) return this.setNull(index, QWP_COLUMN_TYPE.UUID);
+ let lowBits: bigint;
+ let highBits: bigint;
+ if (typeof valueOrLow === "string") {
+ const match =
+ /^([0-9a-f]{8})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{4})-([0-9a-f]{12})$/i.exec(
+ valueOrLow,
+ );
+ if (!match) {
+ throw new TypeError("UUID bind value must use canonical UUID syntax");
+ }
+ const hex = match.slice(1).join("");
+ highBits = BigInt(`0x${hex.slice(0, 16)}`);
+ lowBits = BigInt(`0x${hex.slice(16)}`);
+ } else {
+ if (high === undefined) {
+ throw new TypeError("UUID limb form requires both low and high limbs");
+ }
+ lowBits = checkedUint64Bits(valueOrLow, "UUID low limb");
+ highBits = checkedUint64Bits(high, "UUID high limb");
+ }
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.UUID, false);
+ this.writer.writeBigUint64(lowBits).writeBigUint64(highBits);
+ return this;
+ }
+
+ setLong256(
+ index: number,
+ word0: QwpInt64,
+ word1: QwpInt64,
+ word2: QwpInt64,
+ word3: QwpInt64,
+ ): this {
+ const words = [word0, word1, word2, word3].map((word, wordIndex) =>
+ checkedInt64(word, `LONG256 word ${wordIndex}`),
+ );
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.LONG256, false);
+ for (const word of words) this.writer.writeBigInt64(word);
+ return this;
+ }
+
+ setGeohash(index: number, precisionBits: number, value: QwpInt64): this {
+ const precision = checkedInteger(
+ precisionBits,
+ GEOHASH_MIN_BITS,
+ GEOHASH_MAX_BITS,
+ "GEOHASH precision",
+ );
+ const mask = (1n << BigInt(precision)) - 1n;
+ let bits = checkedInt64(value, "GEOHASH bind") & mask;
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.GEOHASH, false);
+ writeQwpVarint(this.writer, precision);
+ const byteCount = Math.ceil(precision / 8);
+ for (let byteIndex = 0; byteIndex < byteCount; byteIndex++) {
+ this.writer.writeUint8(Number(bits & 0xffn));
+ bits >>= 8n;
+ }
+ return this;
+ }
+
+ setDecimal64(index: number, scale: number, unscaled: QwpInt64): this {
+ const checked = checkedScale(scale, DECIMAL64_MAX_SCALE, "DECIMAL64");
+ const value = checkedInt64(unscaled, "DECIMAL64 unscaled value");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DECIMAL64, false);
+ this.writer.writeUint8(checked).writeBigInt64(value);
+ return this;
+ }
+
+ setDecimal128(
+ index: number,
+ scale: number,
+ low: QwpInt64,
+ high: QwpInt64,
+ ): this {
+ const checked = checkedScale(scale, DECIMAL128_MAX_SCALE, "DECIMAL128");
+ const lowBits = checkedInt64(low, "DECIMAL128 low limb");
+ const highBits = checkedInt64(high, "DECIMAL128 high limb");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DECIMAL128, false);
+ this.writer
+ .writeUint8(checked)
+ .writeBigInt64(lowBits)
+ .writeBigInt64(highBits);
+ return this;
+ }
+
+ setDecimal256(
+ index: number,
+ scale: number,
+ lowLow: QwpInt64,
+ lowHigh: QwpInt64,
+ highLow: QwpInt64,
+ highHigh: QwpInt64,
+ ): this {
+ const checked = checkedScale(scale, DECIMAL256_MAX_SCALE, "DECIMAL256");
+ const limbs = [lowLow, lowHigh, highLow, highHigh].map((limb, limbIndex) =>
+ checkedInt64(limb, `DECIMAL256 limb ${limbIndex}`),
+ );
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DECIMAL256, false);
+ this.writer.writeUint8(checked);
+ for (const limb of limbs) this.writer.writeBigInt64(limb);
+ return this;
+ }
+
+ setNull(index: number, type: QwpBindType): this {
+ this.assertBindType(type);
+ switch (type) {
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ return this.setNullDecimal64(index, 0);
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ return this.setNullDecimal128(index, 0);
+ case QWP_COLUMN_TYPE.DECIMAL256:
+ return this.setNullDecimal256(index, 0);
+ case QWP_COLUMN_TYPE.GEOHASH:
+ return this.setNullGeohash(index, GEOHASH_MIN_BITS);
+ default:
+ this.advance(index);
+ this.writeHeader(type, true);
+ return this;
+ }
+ }
+
+ setNullDecimal64(index: number, scale: number): this {
+ const checked = checkedScale(scale, DECIMAL64_MAX_SCALE, "DECIMAL64");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DECIMAL64, true);
+ this.writer.writeUint8(checked);
+ return this;
+ }
+
+ setNullDecimal128(index: number, scale: number): this {
+ const checked = checkedScale(scale, DECIMAL128_MAX_SCALE, "DECIMAL128");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DECIMAL128, true);
+ this.writer.writeUint8(checked);
+ return this;
+ }
+
+ setNullDecimal256(index: number, scale: number): this {
+ const checked = checkedScale(scale, DECIMAL256_MAX_SCALE, "DECIMAL256");
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.DECIMAL256, true);
+ this.writer.writeUint8(checked);
+ return this;
+ }
+
+ setNullGeohash(index: number, precisionBits: number): this {
+ const precision = checkedInteger(
+ precisionBits,
+ GEOHASH_MIN_BITS,
+ GEOHASH_MAX_BITS,
+ "GEOHASH precision",
+ );
+ this.advance(index);
+ this.writeHeader(QWP_COLUMN_TYPE.GEOHASH, true);
+ writeQwpVarint(this.writer, precision);
+ return this;
+ }
+
+ toUint8Array(): Uint8Array {
+ return this.writer.toUint8Array();
+ }
+
+ private advance(index: number): void {
+ const checked = checkedIndex(index);
+ if (checked !== this.expectedIndex) {
+ throw new Error(
+ `bind index out of order: expected ${this.expectedIndex}, got ${checked}`,
+ );
+ }
+ if (this.expectedIndex >= QWP_MAX_COLUMNS_PER_TABLE) {
+ throw new RangeError(
+ `too many binds: exceeds ${QWP_MAX_COLUMNS_PER_TABLE}`,
+ );
+ }
+ this.expectedIndex++;
+ }
+
+ private assertBindType(type: number): asserts type is QwpBindType {
+ switch (type) {
+ case QWP_COLUMN_TYPE.BOOLEAN:
+ case QWP_COLUMN_TYPE.BYTE:
+ case QWP_COLUMN_TYPE.SHORT:
+ case QWP_COLUMN_TYPE.CHAR:
+ case QWP_COLUMN_TYPE.INT:
+ case QWP_COLUMN_TYPE.LONG:
+ case QWP_COLUMN_TYPE.FLOAT:
+ case QWP_COLUMN_TYPE.DOUBLE:
+ case QWP_COLUMN_TYPE.DATE:
+ case QWP_COLUMN_TYPE.TIMESTAMP:
+ case QWP_COLUMN_TYPE.TIMESTAMP_NANOS:
+ case QWP_COLUMN_TYPE.UUID:
+ case QWP_COLUMN_TYPE.LONG256:
+ case QWP_COLUMN_TYPE.GEOHASH:
+ case QWP_COLUMN_TYPE.VARCHAR:
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ case QWP_COLUMN_TYPE.DECIMAL256:
+ return;
+ default:
+ throw new RangeError(
+ `unsupported QWP bind type 0x${type.toString(16)}`,
+ );
+ }
+ }
+
+ private writeHeader(type: QwpBindType, isNull: boolean): void {
+ this.writer.writeUint8(type).writeUint8(isNull ? NULL_FLAG : NON_NULL_FLAG);
+ if (isNull) this.writer.writeUint8(NULL_BITMAP);
+ }
+}
+
+/** Runs a setter callback and returns the exact QUERY_REQUEST bind section. */
+export function encodeQwpBinds(setter: QwpBindSetter): QwpEncodedBinds {
+ if (typeof setter !== "function") {
+ throw new TypeError("binds must be a function");
+ }
+ const values = new QwpBindValues();
+ const result = setter(values) as unknown;
+ if (
+ result !== null &&
+ (typeof result === "object" || typeof result === "function") &&
+ "then" in result &&
+ typeof result.then === "function"
+ ) {
+ throw new TypeError("binds callback must be synchronous");
+ }
+ return { count: values.count, payload: values.toUint8Array() };
+}
diff --git a/src/_qwp/_core/bytes.ts b/src/_qwp/_core/bytes.ts
new file mode 100644
index 0000000..b18eebc
--- /dev/null
+++ b/src/_qwp/_core/bytes.ts
@@ -0,0 +1,343 @@
+import { QwpProtocolError } from "./errors";
+
+const UTF8_ENCODER = new TextEncoder();
+const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
+
+export function encodeUtf8(value: string): Uint8Array {
+ return UTF8_ENCODER.encode(value);
+}
+
+// Node's Buffer.byteLength counts UTF-8 bytes natively, ~10x faster than
+// encoding into a Uint8Array only to read .length and discard it (measured
+// 22.7 ns vs 221 ns; the encoder UTF-8-encodes every VARCHAR cell twice --
+// once to size, once to write). Reached through globalThis so the browser
+// build, which has no Node types, still compiles and falls back to the
+// allocation-free scan below. Both count exactly what encodeUtf8() writes,
+// including the 3-byte replacement for an unpaired surrogate, so measured sizes
+// never disagree with the bytes emitted.
+const nodeByteLength = (
+ globalThis as {
+ Buffer?: { byteLength(value: string, encoding: "utf8"): number };
+ }
+).Buffer?.byteLength;
+
+export function utf8Length(value: string): number {
+ if (nodeByteLength) return nodeByteLength(value, "utf8");
+ let bytes = 0;
+ for (let index = 0; index < value.length; index++) {
+ const code = value.charCodeAt(index);
+ if (code < 0x80) {
+ bytes += 1;
+ } else if (code < 0x800) {
+ bytes += 2;
+ } else if (code >= 0xd800 && code <= 0xdbff) {
+ // A high surrogate paired with a low surrogate is one 4-byte code point;
+ // an unpaired one becomes the 3-byte replacement character.
+ const next = value.charCodeAt(index + 1);
+ if (next >= 0xdc00 && next <= 0xdfff) {
+ bytes += 4;
+ index++;
+ } else {
+ bytes += 3;
+ }
+ } else {
+ bytes += 3;
+ }
+ }
+ return bytes;
+}
+
+export function decodeUtf8(value: Uint8Array): string {
+ try {
+ return UTF8_DECODER.decode(value);
+ } catch (error) {
+ throw new QwpProtocolError(
+ `invalid UTF-8 payload: ${(error as Error).message}`,
+ );
+ }
+}
+
+export function concatBytes(parts: readonly Uint8Array[]): Uint8Array {
+ let length = 0;
+ for (const part of parts) length += part.length;
+ const result = new Uint8Array(length);
+ let offset = 0;
+ for (const part of parts) {
+ result.set(part, offset);
+ offset += part.length;
+ }
+ return result;
+}
+
+function checkedLength(value: number, label: string): number {
+ if (!Number.isSafeInteger(value) || value < 0) {
+ throw new RangeError(`${label} must be a non-negative safe integer`);
+ }
+ return value;
+}
+
+/** A growable, runtime-neutral little-endian byte writer. */
+export class QwpByteWriter {
+ private bytes: Uint8Array;
+ private view: DataView;
+ private cursor = 0;
+
+ constructor(initialCapacity = 128) {
+ checkedLength(initialCapacity, "initialCapacity");
+ this.bytes = new Uint8Array(Math.max(initialCapacity, 1));
+ this.view = new DataView(this.bytes.buffer);
+ }
+
+ get length(): number {
+ return this.cursor;
+ }
+
+ private ensure(additional: number): void {
+ checkedLength(additional, "additional byte count");
+ const required = this.cursor + additional;
+ if (required <= this.bytes.length) return;
+ let capacity = this.bytes.length;
+ while (capacity < required) capacity = Math.max(capacity * 2, required);
+ const next = new Uint8Array(capacity);
+ next.set(this.bytes.subarray(0, this.cursor));
+ this.bytes = next;
+ this.view = new DataView(next.buffer);
+ }
+
+ writeUint8(value: number): this {
+ this.ensure(1);
+ this.view.setUint8(this.cursor, value);
+ this.cursor++;
+ return this;
+ }
+
+ writeInt8(value: number): this {
+ this.ensure(1);
+ this.view.setInt8(this.cursor, value);
+ this.cursor++;
+ return this;
+ }
+
+ writeUint16(value: number): this {
+ this.ensure(2);
+ this.view.setUint16(this.cursor, value, true);
+ this.cursor += 2;
+ return this;
+ }
+
+ writeInt16(value: number): this {
+ this.ensure(2);
+ this.view.setInt16(this.cursor, value, true);
+ this.cursor += 2;
+ return this;
+ }
+
+ writeUint32(value: number): this {
+ this.ensure(4);
+ this.view.setUint32(this.cursor, value, true);
+ this.cursor += 4;
+ return this;
+ }
+
+ writeInt32(value: number): this {
+ this.ensure(4);
+ this.view.setInt32(this.cursor, value, true);
+ this.cursor += 4;
+ return this;
+ }
+
+ writeBigUint64(value: bigint): this {
+ this.ensure(8);
+ this.view.setBigUint64(this.cursor, BigInt.asUintN(64, value), true);
+ this.cursor += 8;
+ return this;
+ }
+
+ writeBigInt64(value: bigint): this {
+ this.ensure(8);
+ this.view.setBigInt64(this.cursor, BigInt.asIntN(64, value), true);
+ this.cursor += 8;
+ return this;
+ }
+
+ writeFloat32(value: number): this {
+ this.ensure(4);
+ this.view.setFloat32(this.cursor, value, true);
+ this.cursor += 4;
+ return this;
+ }
+
+ writeFloat64(value: number): this {
+ this.ensure(8);
+ this.view.setFloat64(this.cursor, value, true);
+ this.cursor += 8;
+ return this;
+ }
+
+ writeBytes(value: Uint8Array): this {
+ this.ensure(value.length);
+ this.bytes.set(value, this.cursor);
+ this.cursor += value.length;
+ return this;
+ }
+
+ writeUtf8(value: string): this {
+ return this.writeBytes(encodeUtf8(value));
+ }
+
+ writeZeroes(count: number): this {
+ this.ensure(count);
+ this.bytes.fill(0, this.cursor, this.cursor + count);
+ this.cursor += count;
+ return this;
+ }
+
+ patchUint8(offset: number, value: number): void {
+ if (offset < 0 || offset >= this.cursor) {
+ throw new RangeError(`patch offset ${offset} is outside written bytes`);
+ }
+ this.view.setUint8(offset, value);
+ }
+
+ patchUint16(offset: number, value: number): void {
+ if (offset < 0 || offset + 2 > this.cursor) {
+ throw new RangeError(`patch offset ${offset} is outside written bytes`);
+ }
+ this.view.setUint16(offset, value, true);
+ }
+
+ patchUint32(offset: number, value: number): void {
+ if (offset < 0 || offset + 4 > this.cursor) {
+ throw new RangeError(`patch offset ${offset} is outside written bytes`);
+ }
+ this.view.setUint32(offset, value, true);
+ }
+
+ toUint8Array(): Uint8Array {
+ return this.bytes.slice(0, this.cursor);
+ }
+}
+
+/** A bounds-checked, runtime-neutral little-endian byte reader. */
+export class QwpByteReader {
+ private readonly view: DataView;
+ private cursor: number;
+ private readonly end: number;
+
+ constructor(
+ readonly bytes: Uint8Array,
+ offset = 0,
+ length = bytes.length - offset,
+ ) {
+ checkedLength(offset, "offset");
+ checkedLength(length, "length");
+ if (offset + length > bytes.length) {
+ throw new QwpProtocolError("reader range exceeds payload length");
+ }
+ this.cursor = offset;
+ this.end = offset + length;
+ this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+ }
+
+ get position(): number {
+ return this.cursor;
+ }
+
+ get remaining(): number {
+ return this.end - this.cursor;
+ }
+
+ private ensureAvailable(length: number, label: string): void {
+ if (length < 0 || this.cursor + length > this.end) {
+ throw new QwpProtocolError(
+ `truncated QWP payload while reading ${label}`,
+ );
+ }
+ }
+
+ readUint8(label = "uint8"): number {
+ this.ensureAvailable(1, label);
+ return this.view.getUint8(this.cursor++);
+ }
+
+ readInt8(label = "int8"): number {
+ this.ensureAvailable(1, label);
+ return this.view.getInt8(this.cursor++);
+ }
+
+ readUint16(label = "uint16"): number {
+ this.ensureAvailable(2, label);
+ const value = this.view.getUint16(this.cursor, true);
+ this.cursor += 2;
+ return value;
+ }
+
+ readInt16(label = "int16"): number {
+ this.ensureAvailable(2, label);
+ const value = this.view.getInt16(this.cursor, true);
+ this.cursor += 2;
+ return value;
+ }
+
+ readUint32(label = "uint32"): number {
+ this.ensureAvailable(4, label);
+ const value = this.view.getUint32(this.cursor, true);
+ this.cursor += 4;
+ return value;
+ }
+
+ readInt32(label = "int32"): number {
+ this.ensureAvailable(4, label);
+ const value = this.view.getInt32(this.cursor, true);
+ this.cursor += 4;
+ return value;
+ }
+
+ readBigUint64(label = "uint64"): bigint {
+ this.ensureAvailable(8, label);
+ const value = this.view.getBigUint64(this.cursor, true);
+ this.cursor += 8;
+ return value;
+ }
+
+ readBigInt64(label = "int64"): bigint {
+ this.ensureAvailable(8, label);
+ const value = this.view.getBigInt64(this.cursor, true);
+ this.cursor += 8;
+ return value;
+ }
+
+ readFloat32(label = "float32"): number {
+ this.ensureAvailable(4, label);
+ const value = this.view.getFloat32(this.cursor, true);
+ this.cursor += 4;
+ return value;
+ }
+
+ readFloat64(label = "float64"): number {
+ this.ensureAvailable(8, label);
+ const value = this.view.getFloat64(this.cursor, true);
+ this.cursor += 8;
+ return value;
+ }
+
+ readBytes(length: number, label = "bytes"): Uint8Array {
+ checkedLength(length, "byte length");
+ this.ensureAvailable(length, label);
+ const value = this.bytes.subarray(this.cursor, this.cursor + length);
+ this.cursor += length;
+ return value;
+ }
+
+ readUtf8(length: number, label = "UTF-8 string"): string {
+ return decodeUtf8(this.readBytes(length, label));
+ }
+
+ expectEnd(label = "QWP payload"): void {
+ if (this.remaining !== 0) {
+ throw new QwpProtocolError(
+ `${label} has ${this.remaining} unexpected trailing byte(s)`,
+ );
+ }
+ }
+}
diff --git a/src/_qwp/_core/compression.ts b/src/_qwp/_core/compression.ts
new file mode 100644
index 0000000..892fca5
--- /dev/null
+++ b/src/_qwp/_core/compression.ts
@@ -0,0 +1,63 @@
+export const QWP_ZSTD_MIN_COMPRESSION_LEVEL = 1;
+export const QWP_ZSTD_MAX_COMPRESSION_LEVEL = 22;
+
+export type QwpEgressCompression = "raw" | "zstd" | "auto";
+
+export type QwpNegotiatedEgressCompression =
+ | {
+ readonly codec: "raw";
+ readonly level: 0;
+ }
+ | {
+ readonly codec: "zstd";
+ readonly level: number;
+ }
+ | {
+ readonly codec: "unknown";
+ readonly level: 0;
+ readonly contentEncoding: string;
+ };
+
+/** Builds the Node upgrade header for an egress compression preference. */
+export function encodeQwpAcceptEncoding(
+ preference: QwpEgressCompression,
+ level = QWP_ZSTD_MIN_COMPRESSION_LEVEL,
+): string | undefined {
+ if (preference !== "raw" && preference !== "zstd" && preference !== "auto") {
+ throw new RangeError("compression must be one of raw, zstd, or auto");
+ }
+ if (
+ !Number.isSafeInteger(level) ||
+ level < QWP_ZSTD_MIN_COMPRESSION_LEVEL ||
+ level > QWP_ZSTD_MAX_COMPRESSION_LEVEL
+ ) {
+ throw new RangeError(
+ `compressionLevel must be an integer between ${QWP_ZSTD_MIN_COMPRESSION_LEVEL} and ${QWP_ZSTD_MAX_COMPRESSION_LEVEL}`,
+ );
+ }
+ return preference === "raw" ? undefined : `zstd;level=${level},raw`;
+}
+
+/**
+ * Parses the server's `X-QWP-Content-Encoding` response. Unknown values remain
+ * observable but do not claim that Zstd was negotiated; RESULT_BATCH flags
+ * remain authoritative for each individual batch.
+ */
+export function decodeQwpContentEncoding(
+ value: string | undefined,
+): QwpNegotiatedEgressCompression {
+ const contentEncoding = value?.trim();
+ if (!contentEncoding) return { codec: "raw", level: 0 };
+ if (/^(?:raw|identity)$/i.test(contentEncoding)) {
+ return { codec: "raw", level: 0 };
+ }
+
+ const match = /^zstd\s*;\s*level\s*=\s*(\d+)$/i.exec(contentEncoding);
+ if (match) {
+ const level = Number(match[1]);
+ if (Number.isSafeInteger(level) && level > 0) {
+ return { codec: "zstd", level };
+ }
+ }
+ return { codec: "unknown", level: 0, contentEncoding };
+}
diff --git a/src/_qwp/_core/constants.ts b/src/_qwp/_core/constants.ts
new file mode 100644
index 0000000..ffd2fc2
--- /dev/null
+++ b/src/_qwp/_core/constants.ts
@@ -0,0 +1,130 @@
+/** ASCII `QWP1`, represented as its little-endian uint32 value. */
+export const QWP_MAGIC = 0x31505751;
+export const QWP_VERSION = 1;
+export const QWP_HEADER_SIZE = 12;
+
+export const QWP_FLAG_DEFER_COMMIT = 0x01;
+/** Table-less ingress control frame that polls negotiated durable-ACK progress. */
+export const QWP_FLAG_DURABLE_ACK_POLL = 0x02;
+export const QWP_FLAG_GORILLA = 0x04;
+export const QWP_FLAG_DELTA_SYMBOL_DICTIONARY = 0x08;
+export const QWP_FLAG_ZSTD = 0x10;
+
+export const QWP_COLUMN_TYPE = {
+ BOOLEAN: 0x01,
+ BYTE: 0x02,
+ SHORT: 0x03,
+ INT: 0x04,
+ LONG: 0x05,
+ FLOAT: 0x06,
+ DOUBLE: 0x07,
+ SYMBOL: 0x09,
+ TIMESTAMP: 0x0a,
+ DATE: 0x0b,
+ UUID: 0x0c,
+ LONG256: 0x0d,
+ GEOHASH: 0x0e,
+ VARCHAR: 0x0f,
+ TIMESTAMP_NANOS: 0x10,
+ DOUBLE_ARRAY: 0x11,
+ LONG_ARRAY: 0x12,
+ DECIMAL64: 0x13,
+ DECIMAL128: 0x14,
+ DECIMAL256: 0x15,
+ CHAR: 0x16,
+ BINARY: 0x17,
+ IPV4: 0x18,
+} as const;
+
+export type QwpColumnType =
+ (typeof QWP_COLUMN_TYPE)[keyof typeof QWP_COLUMN_TYPE];
+
+export const QWP_ENCODING_UNCOMPRESSED = 0x00;
+export const QWP_ENCODING_GORILLA = 0x01;
+
+export const QWP_STATUS = {
+ OK: 0x00,
+ SERVER_INFO: 0x01,
+ DURABLE_ACK: 0x02,
+ SCHEMA_MISMATCH: 0x03,
+ PARSE_ERROR: 0x05,
+ INTERNAL_ERROR: 0x06,
+ SECURITY_ERROR: 0x08,
+ WRITE_ERROR: 0x09,
+ CANCELLED: 0x0a,
+ LIMIT_EXCEEDED: 0x0b,
+ NOT_WRITABLE: 0x0c,
+ DICTIONARY_GAP: 0x0d,
+} as const;
+
+export const QWP_EGRESS_MESSAGE = {
+ QUERY_REQUEST: 0x10,
+ RESULT_BATCH: 0x11,
+ RESULT_END: 0x12,
+ QUERY_ERROR: 0x13,
+ CANCEL: 0x14,
+ CREDIT: 0x15,
+ EXEC_DONE: 0x16,
+ CACHE_RESET: 0x17,
+ SERVER_INFO: 0x18,
+} as const;
+
+export const QWP_EGRESS_CAPABILITY = {
+ ZONE: 0x00000001,
+ QUERY_FLAGS: 0x00000002,
+ COMPRESSION: 0x00000004,
+} as const;
+
+export const QWP_COMPRESSION_CODEC = {
+ RAW: 0,
+ ZSTD: 1,
+} as const;
+
+export const QWP_QUERY_FLAG_RESET_DICTIONARY = 0x01;
+export const QWP_RESET_MASK_DICTIONARY = 0x01;
+
+export const QWP_SERVER_ROLE = {
+ STANDALONE: 0,
+ PRIMARY: 1,
+ REPLICA: 2,
+ PRIMARY_CATCHUP: 3,
+} as const;
+
+export const QWP_MAX_COLUMNS_PER_TABLE = 2048;
+/** Default QWP ingress identifier limits, in UTF-8 wire bytes. */
+export const QWP_MAX_COLUMN_NAME_LENGTH = 127;
+export const QWP_MAX_TABLE_NAME_LENGTH = 127;
+/**
+ * Defensive byte bound for identifiers decoded from query results.
+ *
+ * Existing tables may have names created through APIs that apply Java's
+ * 127-UTF-16-code-unit metadata limit. One code unit takes at most three UTF-8
+ * bytes, so query decoding accepts that larger representation even though QWP
+ * ingress enforces its 127-byte protocol limit.
+ */
+export const QWP_MAX_IDENTIFIER_BYTES = QWP_MAX_TABLE_NAME_LENGTH * 3;
+export const QWP_MAX_ROWS_PER_TABLE = 1_000_000;
+export const QWP_MAX_SYMBOL_DICTIONARY_SIZE = 1_000_000;
+export const QWP_MAX_ERROR_MESSAGE_LENGTH = 1024;
+/** Largest client-requested egress RESULT_BATCH row cap. */
+export const QWP_MAX_BATCH_ROWS_UPPER_BOUND = 1_048_576;
+/**
+ * Largest `rowCount * columnCount` a single RESULT_BATCH may declare.
+ *
+ * The row and column caps above bound each dimension on its own, and their
+ * product does not have to be reachable: 1,048,576 rows of 2,048 columns is
+ * 2.1 billion cells. Decoding materializes two `rowCount`-length arrays per
+ * column, measured at 16 bytes per cell, so the product is what decides how
+ * much memory a response can cost. It is also the dimension a compressed body
+ * detaches from the wire: an all-NULL column is one bit per cell before zstd,
+ * so without this bound a few kilobytes of RLE-compressed bitmap declares a
+ * grid no heap can hold.
+ *
+ * 32Mi cells is roughly 512 MB decoded. That is far above any plausible
+ * result -- the widest supported table at 16k rows, or a full 1,048,576-row
+ * batch at 32 columns -- and far below what the caps alone would permit.
+ */
+export const QWP_MAX_CELLS_PER_BATCH = 33_554_432;
+
+export const QWP_INGRESS_PATH = "/write/v4";
+export const QWP_EGRESS_PATH = "/read/v1";
diff --git a/src/_qwp/_core/durable-ack.ts b/src/_qwp/_core/durable-ack.ts
new file mode 100644
index 0000000..0aba5f8
--- /dev/null
+++ b/src/_qwp/_core/durable-ack.ts
@@ -0,0 +1,27 @@
+/**
+ * Browser-visible WebSocket subprotocol used to request and confirm durable
+ * ingress acknowledgements. Browsers cannot set or inspect X-QWP-* headers.
+ */
+export const QWP_DURABLE_ACK_WEBSOCKET_PROTOCOL = "questdb.qwp.durable-ack.v1";
+
+/** Adds the durable-ACK capability token without mutating user options. */
+export function addQwpDurableAckWebSocketProtocol(
+ protocols: string | readonly string[] | undefined,
+): string | string[] {
+ if (protocols === undefined) return QWP_DURABLE_ACK_WEBSOCKET_PROTOCOL;
+ if (typeof protocols === "string") {
+ return protocols === QWP_DURABLE_ACK_WEBSOCKET_PROTOCOL
+ ? protocols
+ : [protocols, QWP_DURABLE_ACK_WEBSOCKET_PROTOCOL];
+ }
+ return protocols.includes(QWP_DURABLE_ACK_WEBSOCKET_PROTOCOL)
+ ? [...protocols]
+ : [...protocols, QWP_DURABLE_ACK_WEBSOCKET_PROTOCOL];
+}
+
+/** True when the server selected the browser durable-ACK subprotocol. */
+export function isQwpDurableAckWebSocketProtocol(
+ protocol: string | undefined,
+): boolean {
+ return protocol === QWP_DURABLE_ACK_WEBSOCKET_PROTOCOL;
+}
diff --git a/src/_qwp/_core/egress.ts b/src/_qwp/_core/egress.ts
new file mode 100644
index 0000000..a214c8a
--- /dev/null
+++ b/src/_qwp/_core/egress.ts
@@ -0,0 +1,277 @@
+import { encodeUtf8, QwpByteReader, QwpByteWriter } from "./bytes";
+import { encodeQwpBinds, QwpBindSetter } from "./binds";
+import {
+ QWP_EGRESS_CAPABILITY,
+ QWP_EGRESS_MESSAGE,
+ QWP_MAX_COLUMNS_PER_TABLE,
+} from "./constants";
+import { decodeQwpFrame, QwpFrameHeader } from "./frame";
+import { QwpProtocolError } from "./errors";
+import { readQwpVarint, writeQwpVarint } from "./varint";
+
+export interface QwpQueryRequest {
+ requestId: number | bigint;
+ sql: string;
+ /** Zero means unbounded. */
+ initialCredit?: number | bigint;
+ /** Browser-safe typed positional binds. */
+ binds?: QwpBindSetter;
+ /** Advanced escape hatch for an already encoded bind section. */
+ bindCount?: number;
+ /** Advanced escape hatch for an already encoded bind section. */
+ bindPayload?: Uint8Array;
+ /** Append only after SERVER_INFO advertises QUERY_FLAGS. */
+ queryFlags?: number | bigint;
+}
+
+/** Immutable endpoint metadata from the most recent successful egress bind. */
+export interface QwpServerInfoMessage extends QwpFrameHeader {
+ kind: "server-info";
+ role: number;
+ epoch: bigint;
+ capabilities: number;
+ serverWallNanoseconds: bigint;
+ clusterId: string;
+ nodeId: string;
+ zoneId: string | null;
+ compressionCodec: number | null;
+ compressionLevel: number | null;
+}
+
+export interface QwpResultBatchMessage extends QwpFrameHeader {
+ kind: "result-batch";
+ requestId: bigint;
+ batchSequence: bigint;
+ /**
+ * Raw or Zstd-compressed delta dictionary and columnar table block; decoded
+ * by the batch decoder according to the frame flags.
+ */
+ body: Uint8Array;
+}
+
+export interface QwpResultEndMessage extends QwpFrameHeader {
+ kind: "result-end";
+ requestId: bigint;
+ finalSequence: bigint;
+ totalRows: bigint;
+}
+
+export interface QwpQueryErrorMessage extends QwpFrameHeader {
+ kind: "query-error";
+ requestId: bigint;
+ status: number;
+ message: string;
+}
+
+export interface QwpExecDoneMessage extends QwpFrameHeader {
+ kind: "exec-done";
+ requestId: bigint;
+ operationType: number;
+ rowsAffected: bigint;
+}
+
+export interface QwpCacheResetMessage extends QwpFrameHeader {
+ kind: "cache-reset";
+ resetMask: number;
+}
+
+export type QwpEgressMessage =
+ | QwpServerInfoMessage
+ | QwpResultBatchMessage
+ | QwpResultEndMessage
+ | QwpQueryErrorMessage
+ | QwpExecDoneMessage
+ | QwpCacheResetMessage;
+
+function requestId(value: number | bigint): bigint {
+ if (typeof value === "number") {
+ if (!Number.isSafeInteger(value) || value < 0) {
+ throw new RangeError("requestId must be a non-negative safe integer");
+ }
+ return BigInt(value);
+ }
+ if (value < 0n || value > 0xffffffffffffffffn) {
+ throw new RangeError("requestId must fit in uint64");
+ }
+ return value;
+}
+
+/** Encodes the unframed client-to-server QUERY_REQUEST payload. */
+export function encodeQwpQueryRequest(request: QwpQueryRequest): Uint8Array {
+ if (
+ request.binds !== undefined &&
+ (request.bindCount !== undefined || request.bindPayload !== undefined)
+ ) {
+ throw new Error(
+ "typed binds cannot be mixed with raw bindCount/bindPayload",
+ );
+ }
+ const encodedBinds = request.binds
+ ? encodeQwpBinds(request.binds)
+ : undefined;
+ const bindCount = encodedBinds?.count ?? request.bindCount ?? 0;
+ if (
+ !Number.isSafeInteger(bindCount) ||
+ bindCount < 0 ||
+ bindCount > QWP_MAX_COLUMNS_PER_TABLE
+ ) {
+ throw new RangeError(
+ `bindCount must be an integer between 0 and ${QWP_MAX_COLUMNS_PER_TABLE}`,
+ );
+ }
+ const bindPayload =
+ encodedBinds?.payload ?? request.bindPayload ?? new Uint8Array();
+ if (bindCount === 0 && bindPayload.length !== 0) {
+ throw new Error("bindPayload requires a non-zero bindCount");
+ }
+
+ const sql = encodeUtf8(request.sql);
+ const writer = new QwpByteWriter(32 + sql.length + bindPayload.length);
+ writer.writeUint8(QWP_EGRESS_MESSAGE.QUERY_REQUEST);
+ writer.writeBigUint64(requestId(request.requestId));
+ writeQwpVarint(writer, sql.length);
+ writer.writeBytes(sql);
+ writeQwpVarint(writer, request.initialCredit ?? 0);
+ writeQwpVarint(writer, bindCount);
+ writer.writeBytes(bindPayload);
+ if ((request.queryFlags ?? 0) !== 0) {
+ writeQwpVarint(writer, request.queryFlags!);
+ }
+ return writer.toUint8Array();
+}
+
+/** Encodes the unframed client-to-server CANCEL payload. */
+export function encodeQwpCancel(request: number | bigint): Uint8Array {
+ const writer = new QwpByteWriter(9);
+ writer.writeUint8(QWP_EGRESS_MESSAGE.CANCEL);
+ writer.writeBigUint64(requestId(request));
+ return writer.toUint8Array();
+}
+
+/** Encodes the unframed client-to-server CREDIT payload. */
+export function encodeQwpCredit(
+ request: number | bigint,
+ additionalBytes: number | bigint,
+): Uint8Array {
+ const writer = new QwpByteWriter(19);
+ writer.writeUint8(QWP_EGRESS_MESSAGE.CREDIT);
+ writer.writeBigUint64(requestId(request));
+ writeQwpVarint(writer, additionalBytes);
+ return writer.toUint8Array();
+}
+
+function readUint16Utf8(reader: QwpByteReader, label: string): string {
+ const length = reader.readUint16(`${label} length`);
+ return reader.readUtf8(length, label);
+}
+
+/** Decodes one QWP-framed server-to-client egress message. */
+export function decodeQwpEgressMessage(bytes: Uint8Array): QwpEgressMessage {
+ const frame = decodeQwpFrame(bytes);
+ const reader = new QwpByteReader(frame.payload);
+ const messageKind = reader.readUint8("egress message kind");
+ const header: QwpFrameHeader = {
+ version: frame.version,
+ flags: frame.flags,
+ tableCount: frame.tableCount,
+ payloadLength: frame.payloadLength,
+ };
+
+ switch (messageKind) {
+ case QWP_EGRESS_MESSAGE.SERVER_INFO: {
+ const role = reader.readUint8("server role");
+ const epoch = reader.readBigUint64("server epoch");
+ const capabilities = reader.readUint32("server capabilities");
+ const serverWallNanoseconds = reader.readBigInt64("server wall clock");
+ const clusterId = readUint16Utf8(reader, "cluster ID");
+ const nodeId = readUint16Utf8(reader, "node ID");
+ const zoneId =
+ (capabilities & QWP_EGRESS_CAPABILITY.ZONE) !== 0
+ ? readUint16Utf8(reader, "zone ID")
+ : null;
+ const compressionCodec =
+ (capabilities & QWP_EGRESS_CAPABILITY.COMPRESSION) !== 0
+ ? reader.readUint8("egress compression codec")
+ : null;
+ const compressionLevel =
+ compressionCodec !== null
+ ? reader.readUint8("egress compression level")
+ : null;
+ reader.expectEnd("SERVER_INFO");
+ return Object.freeze({
+ ...header,
+ kind: "server-info",
+ role,
+ epoch,
+ capabilities,
+ serverWallNanoseconds,
+ clusterId,
+ nodeId,
+ zoneId,
+ compressionCodec,
+ compressionLevel,
+ });
+ }
+ case QWP_EGRESS_MESSAGE.RESULT_BATCH: {
+ const requestId = reader.readBigUint64("result request ID");
+ const batchSequence = readQwpVarint(reader);
+ const body = reader.readBytes(reader.remaining, "result batch body");
+ return {
+ ...header,
+ kind: "result-batch",
+ requestId,
+ batchSequence,
+ body,
+ };
+ }
+ case QWP_EGRESS_MESSAGE.RESULT_END: {
+ const requestId = reader.readBigUint64("result request ID");
+ const finalSequence = readQwpVarint(reader);
+ const totalRows = readQwpVarint(reader);
+ reader.expectEnd("RESULT_END");
+ return {
+ ...header,
+ kind: "result-end",
+ requestId,
+ finalSequence,
+ totalRows,
+ };
+ }
+ case QWP_EGRESS_MESSAGE.QUERY_ERROR: {
+ const requestId = reader.readBigUint64("query error request ID");
+ const status = reader.readUint8("query error status");
+ const length = reader.readUint16("query error message length");
+ const message = reader.readUtf8(length, "query error message");
+ reader.expectEnd("QUERY_ERROR");
+ return {
+ ...header,
+ kind: "query-error",
+ requestId,
+ status,
+ message,
+ };
+ }
+ case QWP_EGRESS_MESSAGE.EXEC_DONE: {
+ const requestId = reader.readBigUint64("exec request ID");
+ const operationType = reader.readUint8("operation type");
+ const rowsAffected = readQwpVarint(reader);
+ reader.expectEnd("EXEC_DONE");
+ return {
+ ...header,
+ kind: "exec-done",
+ requestId,
+ operationType,
+ rowsAffected,
+ };
+ }
+ case QWP_EGRESS_MESSAGE.CACHE_RESET: {
+ const resetMask = reader.readUint8("cache reset mask");
+ reader.expectEnd("CACHE_RESET");
+ return { ...header, kind: "cache-reset", resetMask };
+ }
+ default:
+ throw new QwpProtocolError(
+ `unsupported QWP egress message kind 0x${messageKind.toString(16)}`,
+ );
+ }
+}
diff --git a/src/_qwp/_core/errors.ts b/src/_qwp/_core/errors.ts
new file mode 100644
index 0000000..58bd697
--- /dev/null
+++ b/src/_qwp/_core/errors.ts
@@ -0,0 +1,7 @@
+/** Raised when a QWP payload is malformed, truncated, or unsupported. */
+export class QwpProtocolError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = "QwpProtocolError";
+ }
+}
diff --git a/src/_qwp/_core/frame.ts b/src/_qwp/_core/frame.ts
new file mode 100644
index 0000000..b00a5b5
--- /dev/null
+++ b/src/_qwp/_core/frame.ts
@@ -0,0 +1,72 @@
+import { QwpByteReader, QwpByteWriter } from "./bytes";
+import { QWP_HEADER_SIZE, QWP_MAGIC, QWP_VERSION } from "./constants";
+import { QwpProtocolError } from "./errors";
+
+export interface QwpFrameHeader {
+ version: number;
+ flags: number;
+ tableCount: number;
+ payloadLength: number;
+}
+
+export interface QwpFrame extends QwpFrameHeader {
+ payload: Uint8Array;
+}
+
+export function writeQwpFrameHeader(
+ writer: QwpByteWriter,
+ header: Omit & { version?: number },
+): void {
+ writer.writeUint32(QWP_MAGIC);
+ writer.writeUint8(header.version ?? QWP_VERSION);
+ writer.writeUint8(header.flags);
+ writer.writeUint16(header.tableCount);
+ writer.writeUint32(header.payloadLength);
+}
+
+export function encodeQwpFrame(
+ payload: Uint8Array,
+ flags = 0,
+ tableCount = 0,
+): Uint8Array {
+ const writer = new QwpByteWriter(QWP_HEADER_SIZE + payload.length);
+ writeQwpFrameHeader(writer, {
+ flags,
+ tableCount,
+ payloadLength: payload.length,
+ });
+ writer.writeBytes(payload);
+ return writer.toUint8Array();
+}
+
+export function decodeQwpFrame(bytes: Uint8Array): QwpFrame {
+ if (bytes.length < QWP_HEADER_SIZE) {
+ throw new QwpProtocolError("QWP frame is shorter than its 12-byte header");
+ }
+ const reader = new QwpByteReader(bytes);
+ const magic = reader.readUint32("QWP magic");
+ if (magic !== QWP_MAGIC) {
+ throw new QwpProtocolError(
+ `invalid QWP magic 0x${magic.toString(16).padStart(8, "0")}`,
+ );
+ }
+ const version = reader.readUint8("QWP version");
+ if (version !== QWP_VERSION) {
+ throw new QwpProtocolError(`unsupported QWP version ${version}`);
+ }
+ const flags = reader.readUint8("QWP flags");
+ const tableCount = reader.readUint16("QWP table count");
+ const payloadLength = reader.readUint32("QWP payload length");
+ if (payloadLength !== reader.remaining) {
+ throw new QwpProtocolError(
+ `QWP payload length mismatch [declared=${payloadLength}, actual=${reader.remaining}]`,
+ );
+ }
+ return {
+ version,
+ flags,
+ tableCount,
+ payloadLength,
+ payload: reader.readBytes(payloadLength, "QWP payload"),
+ };
+}
diff --git a/src/_qwp/_core/gorilla.ts b/src/_qwp/_core/gorilla.ts
new file mode 100644
index 0000000..7e8d138
--- /dev/null
+++ b/src/_qwp/_core/gorilla.ts
@@ -0,0 +1,101 @@
+import { QwpByteWriter } from "./bytes";
+
+const INT32_MIN = -2147483648n;
+const INT32_MAX = 2147483647n;
+
+class QwpBitWriter {
+ private readonly bytes: Uint8Array;
+ private byteIndex = 0;
+ private bitIndex = 0;
+
+ constructor(capacity: number) {
+ this.bytes = new Uint8Array(capacity);
+ }
+
+ writeBits(value: number, count: number): void {
+ for (let index = 0; index < count; index++) {
+ if ((value >>> index) & 1) {
+ this.bytes[this.byteIndex] |= 1 << this.bitIndex;
+ }
+ this.bitIndex++;
+ if (this.bitIndex === 8) {
+ this.bitIndex = 0;
+ this.byteIndex++;
+ }
+ }
+ }
+
+ finish(): Uint8Array {
+ const length = this.byteIndex + (this.bitIndex > 0 ? 1 : 0);
+ return this.bytes.slice(0, length);
+ }
+}
+
+function encodedDeltaBits(deltaOfDelta: bigint): number {
+ if (deltaOfDelta === 0n) return 1;
+ if (deltaOfDelta >= -64n && deltaOfDelta <= 63n) return 9;
+ if (deltaOfDelta >= -256n && deltaOfDelta <= 255n) return 12;
+ if (deltaOfDelta >= -2048n && deltaOfDelta <= 2047n) return 16;
+ return 36;
+}
+
+/** Encoded byte count, or -1 when a delta-of-delta leaves int32 range. */
+export function qwpGorillaSize(timestamps: readonly bigint[]): number {
+ if (timestamps.length === 0) return 0;
+ if (timestamps.length === 1) return 8;
+ if (timestamps.length === 2) return 16;
+ let previousTimestamp = timestamps[1];
+ let previousDelta = timestamps[1] - timestamps[0];
+ let bits = 0;
+ for (let index = 2; index < timestamps.length; index++) {
+ const delta = timestamps[index] - previousTimestamp;
+ const deltaOfDelta = delta - previousDelta;
+ if (deltaOfDelta < INT32_MIN || deltaOfDelta > INT32_MAX) return -1;
+ bits += encodedDeltaBits(deltaOfDelta);
+ previousDelta = delta;
+ previousTimestamp = timestamps[index];
+ }
+ return 16 + Math.ceil(bits / 8);
+}
+
+/** Encodes timestamps with the QWP LSB-first Gorilla variant. */
+export function encodeQwpGorilla(timestamps: readonly bigint[]): Uint8Array {
+ const size = qwpGorillaSize(timestamps);
+ if (size < 0) {
+ throw new Error("Gorilla delta-of-delta is outside the int32 range");
+ }
+ const writer = new QwpByteWriter(Math.max(size, 1));
+ if (timestamps.length === 0) return writer.toUint8Array();
+ writer.writeBigInt64(timestamps[0]);
+ if (timestamps.length === 1) return writer.toUint8Array();
+ writer.writeBigInt64(timestamps[1]);
+ if (timestamps.length === 2) return writer.toUint8Array();
+
+ const bits = new QwpBitWriter(size - 16);
+ let previousTimestamp = timestamps[1];
+ let previousDelta = timestamps[1] - timestamps[0];
+ for (let index = 2; index < timestamps.length; index++) {
+ const delta = timestamps[index] - previousTimestamp;
+ const deltaOfDelta = delta - previousDelta;
+ // Prefixes are bit-reversed because QWP packs bits least-significant first.
+ if (deltaOfDelta === 0n) {
+ bits.writeBits(0, 1);
+ } else if (deltaOfDelta >= -64n && deltaOfDelta <= 63n) {
+ bits.writeBits(0b01, 2);
+ bits.writeBits(Number(deltaOfDelta & 0x7fn), 7);
+ } else if (deltaOfDelta >= -256n && deltaOfDelta <= 255n) {
+ bits.writeBits(0b011, 3);
+ bits.writeBits(Number(deltaOfDelta & 0x1ffn), 9);
+ } else if (deltaOfDelta >= -2048n && deltaOfDelta <= 2047n) {
+ bits.writeBits(0b0111, 4);
+ bits.writeBits(Number(deltaOfDelta & 0xfffn), 12);
+ } else {
+ bits.writeBits(0b1111, 4);
+ bits.writeBits(Number(deltaOfDelta & 0xffffffffn), 32);
+ }
+ previousDelta = delta;
+ previousTimestamp = timestamps[index];
+ }
+ writer.writeBytes(bits.finish());
+ return writer.toUint8Array();
+}
diff --git a/src/_qwp/_core/identifiers.ts b/src/_qwp/_core/identifiers.ts
new file mode 100644
index 0000000..017592e
--- /dev/null
+++ b/src/_qwp/_core/identifiers.ts
@@ -0,0 +1,102 @@
+import { utf8Length } from "./bytes";
+
+function isIllegalCommonIdentifierCharacter(
+ character: string,
+ codeUnit: number,
+): boolean {
+ if (codeUnit <= 0x0f || codeUnit === 0x7f || codeUnit === 0xfeff) {
+ return true;
+ }
+ switch (character) {
+ case "?":
+ case ",":
+ case "'":
+ case '"':
+ case "\\":
+ case "/":
+ case ":":
+ case ")":
+ case "(":
+ case "+":
+ case "*":
+ case "%":
+ case "~":
+ return true;
+ default:
+ return false;
+ }
+}
+
+/** @internal Applies Java TableUtils rules and the QWP UTF-8 byte limit. */
+export function validateQwpTableName(
+ name: string,
+ maxNameLength: number,
+): void {
+ if (name.length === 0) throw new Error("table name cannot be empty");
+ if (utf8Length(name) > maxNameLength) {
+ throw new Error(`table name too long [maxLength=${maxNameLength}]`);
+ }
+ if (name.charAt(0) === " " || name.charAt(name.length - 1) === " ") {
+ throw new Error(`table name contains illegal characters: ${name}`);
+ }
+ for (let index = 0; index < name.length; index++) {
+ const character = name.charAt(index);
+ if (
+ (character === "." &&
+ (index === 0 ||
+ index === name.length - 1 ||
+ name.charAt(index - 1) === ".")) ||
+ isIllegalCommonIdentifierCharacter(character, name.charCodeAt(index))
+ ) {
+ throw new Error(`table name contains illegal characters: ${name}`);
+ }
+ }
+}
+
+/** @internal Applies Java TableUtils rules and the QWP UTF-8 byte limit. */
+export function validateQwpColumnName(
+ name: string,
+ maxNameLength: number,
+): void {
+ if (name.length === 0) throw new Error("column name cannot be empty");
+ if (utf8Length(name) > maxNameLength) {
+ throw new Error(`column name too long [maxLength=${maxNameLength}]`);
+ }
+ for (let index = 0; index < name.length; index++) {
+ const character = name.charAt(index);
+ if (
+ character === "." ||
+ character === "-" ||
+ isIllegalCommonIdentifierCharacter(character, name.charCodeAt(index))
+ ) {
+ throw new Error(`column name contains illegal characters: ${name}`);
+ }
+ }
+}
+
+/**
+ * @internal Java's LowerCaseCharSequenceIntHashMap lowercases each UTF-16 code
+ * unit independently. Taking the first code unit avoids JavaScript's one
+ * expanding lowercase mapping (U+0130) and gives the same simple mapping.
+ */
+export function qwpColumnNameKey(name: string): string {
+ // Fast path: a name of only lower-case-stable code units -- ASCII other than
+ // A-Z -- already equals its key, so it is returned without rebuilding. The
+ // first upper-case ASCII letter or non-ASCII code unit (which may lower-case
+ // or expand) drops to the per-code-unit mapping below, resuming from the
+ // stable prefix. This runs once per cell on the ingest path, so the common
+ // all-lower-case name skips the character-by-character rebuild entirely.
+ let index = 0;
+ for (; index < name.length; index++) {
+ const code = name.charCodeAt(index);
+ if (code >= 0x80 || (code >= 0x41 && code <= 0x5a)) break;
+ }
+ if (index === name.length) return name;
+
+ let key = name.slice(0, index);
+ for (; index < name.length; index++) {
+ const character = name.charAt(index);
+ key += character.toLowerCase().charAt(0);
+ }
+ return key;
+}
diff --git a/src/_qwp/_core/index.ts b/src/_qwp/_core/index.ts
new file mode 100644
index 0000000..e7c3db3
--- /dev/null
+++ b/src/_qwp/_core/index.ts
@@ -0,0 +1,15 @@
+export * from "./bytes";
+export * from "./binds";
+export * from "./compression";
+export * from "./constants";
+export * from "./durable-ack";
+export * from "./egress";
+export * from "./errors";
+export * from "./frame";
+export * from "./gorilla";
+export * from "./ingress";
+export * from "./result-batch";
+export * from "./symbol-dictionary";
+export * from "./table";
+export * from "./varint";
+export * from "./zstd";
diff --git a/src/_qwp/_core/ingress.ts b/src/_qwp/_core/ingress.ts
new file mode 100644
index 0000000..5aa4331
--- /dev/null
+++ b/src/_qwp/_core/ingress.ts
@@ -0,0 +1,757 @@
+import { encodeUtf8, QwpByteReader, QwpByteWriter, utf8Length } from "./bytes";
+import {
+ QWP_COLUMN_TYPE,
+ QWP_ENCODING_GORILLA,
+ QWP_ENCODING_UNCOMPRESSED,
+ QWP_FLAG_DEFER_COMMIT,
+ QWP_FLAG_DELTA_SYMBOL_DICTIONARY,
+ QWP_FLAG_DURABLE_ACK_POLL,
+ QWP_FLAG_GORILLA,
+ QWP_HEADER_SIZE,
+ QWP_MAX_ERROR_MESSAGE_LENGTH,
+ QWP_MAX_ROWS_PER_TABLE,
+ QWP_MAX_SYMBOL_DICTIONARY_SIZE,
+ QWP_STATUS,
+ QwpColumnType,
+} from "./constants";
+import { decodeQwpFrame, writeQwpFrameHeader } from "./frame";
+import { QwpProtocolError } from "./errors";
+import { encodeQwpGorilla, qwpGorillaSize } from "./gorilla";
+import { QwpSymbolDictionary } from "./symbol-dictionary";
+import {
+ QwpArrayValue,
+ QwpColumnBuffer,
+ QwpSymbolValue,
+ QwpTableBuffer,
+} from "./table";
+import { qwpVarintSize, readQwpVarintNumber, writeQwpVarint } from "./varint";
+
+export interface QwpIngressEncodeOptions {
+ gorilla?: boolean;
+ /** Present means connection-scoped delta dictionary mode. */
+ dictionary?: QwpSymbolDictionary;
+ /** Highest global symbol ID already published on this logical connection. */
+ confirmedMaxSymbolId?: number;
+ deferCommit?: boolean;
+}
+
+interface ColumnEncodeOptions {
+ gorilla: boolean;
+ deltaSymbols: boolean;
+ dictionary?: QwpSymbolDictionary;
+}
+
+export interface QwpIngressTableResult {
+ name: string;
+ sequenceTransaction: bigint;
+}
+
+export interface QwpIngressResponse {
+ status: number;
+ sequence: bigint | null;
+ tables: QwpIngressTableResult[];
+ errorMessage?: string;
+}
+
+/** Decodes the browser-requested ingress SERVER_INFO payload when present. */
+export function decodeQwpIngressServerInfo(
+ payload: Uint8Array,
+): number | undefined {
+ if (payload[0] !== QWP_STATUS.SERVER_INFO) return undefined;
+ if (payload.byteLength !== 5) {
+ throw new QwpProtocolError("invalid QWP ingress SERVER_INFO length");
+ }
+ const maxBatchSizeBytes = new DataView(
+ payload.buffer,
+ payload.byteOffset,
+ payload.byteLength,
+ ).getUint32(1, true);
+ if (maxBatchSizeBytes === 0) {
+ throw new QwpProtocolError("invalid QWP ingress SERVER_INFO batch cap");
+ }
+ return maxBatchSizeBytes;
+}
+
+function symbolText(value: unknown): string {
+ if (typeof value === "string") return value;
+ // A bare dictionary ID carries no text, and this encoder builds its inline
+ // dictionary out of the texts, so there is nothing to resolve it against.
+ // Reading `.text` off a number yields undefined, which TextEncoder happily
+ // encodes as zero bytes -- every symbol in the frame would collapse into one
+ // empty-string entry and be acknowledged as if it were correct. Say so
+ // instead. symbolId() accepts the numeric form because the delta encoder is
+ // given the dictionary that gives it meaning.
+ if (typeof value === "number") {
+ throw new Error(
+ `QWP symbol ID ${value} needs a symbol dictionary; pass one to encode a delta frame, or supply the symbol as a string or {id, text}`,
+ );
+ }
+ const text = (value as QwpSymbolValue)?.text;
+ if (typeof text !== "string") {
+ throw new Error(
+ "QWP symbol value must be a string or a {id, text} pair, received " +
+ (value === null ? "null" : typeof value),
+ );
+ }
+ return text;
+}
+
+function symbolId(value: unknown, dictionary: QwpSymbolDictionary): number {
+ if (typeof value === "string") return dictionary.getOrAdd(value);
+ const id = typeof value === "number" ? value : (value as QwpSymbolValue).id;
+ if (!Number.isSafeInteger(id) || id < 0 || id >= dictionary.size) {
+ throw new Error(`QWP symbol ID is outside the dictionary: ${id}`);
+ }
+ if (typeof value !== "number") {
+ const symbol = value as QwpSymbolValue;
+ if (dictionary.valueAt(id) !== symbol.text) {
+ throw new Error(
+ `QWP symbol value does not match dictionary ID ${id}: '${symbol.text}'`,
+ );
+ }
+ }
+ return id;
+}
+
+interface InlineSymbolDictionary {
+ /** Distinct symbol texts in first-seen order, matching Set iteration. */
+ readonly entries: readonly string[];
+ /** The dictionary index of each row's value, in row order. */
+ readonly rowIds: readonly number[];
+}
+
+// A non-delta ("full") symbol column carries its own inline dictionary.
+// Resolving each row against it with Array.prototype.indexOf is O(rows x
+// distinct) -- measured quadratic, 67x slower than delta mode at 32k rows. A
+// Map keyed by text makes each lookup O(1), the same fix
+// QwpSymbolDictionary.getOrAdd already applies in delta mode. symbolText() runs
+// once per value here, so measureColumn and writeColumn no longer resolve each
+// value twice.
+function inlineSymbolDictionary(
+ values: readonly unknown[],
+): InlineSymbolDictionary {
+ const entries: string[] = [];
+ const indexByText = new Map();
+ const rowIds = new Array(values.length);
+ for (let row = 0; row < values.length; row++) {
+ const text = symbolText(values[row]);
+ let id = indexByText.get(text);
+ if (id === undefined) {
+ id = entries.length;
+ indexByText.set(text, id);
+ entries.push(text);
+ }
+ rowIds[row] = id;
+ }
+ return { entries, rowIds };
+}
+
+function nullCount(column: QwpColumnBuffer): number {
+ let count = 0;
+ for (const value of column.nulls) if (value) count++;
+ return count;
+}
+
+function fixedWidth(type: QwpColumnType): number | undefined {
+ switch (type) {
+ case QWP_COLUMN_TYPE.BYTE:
+ return 1;
+ case QWP_COLUMN_TYPE.SHORT:
+ case QWP_COLUMN_TYPE.CHAR:
+ return 2;
+ case QWP_COLUMN_TYPE.INT:
+ case QWP_COLUMN_TYPE.FLOAT:
+ case QWP_COLUMN_TYPE.IPV4:
+ return 4;
+ case QWP_COLUMN_TYPE.LONG:
+ case QWP_COLUMN_TYPE.DOUBLE:
+ case QWP_COLUMN_TYPE.DATE:
+ return 8;
+ case QWP_COLUMN_TYPE.UUID:
+ return 16;
+ case QWP_COLUMN_TYPE.LONG256:
+ return 32;
+ default:
+ return undefined;
+ }
+}
+
+function qwpStringSize(value: string): number {
+ const length = utf8Length(value);
+ return qwpVarintSize(length) + length;
+}
+
+function writeQwpString(writer: QwpByteWriter, value: string): void {
+ const bytes = encodeUtf8(value);
+ writeQwpVarint(writer, bytes.length);
+ writer.writeBytes(bytes);
+}
+
+function binaryValue(value: unknown, width?: number): Uint8Array {
+ if (!(value instanceof Uint8Array)) {
+ throw new Error("QWP binary values must be Uint8Array instances");
+ }
+ if (width !== undefined && value.length !== width) {
+ throw new Error(
+ `QWP binary value has length ${value.length}; expected ${width}`,
+ );
+ }
+ return value;
+}
+
+function columnPayloadSize(
+ column: QwpColumnBuffer,
+ rowCount: number,
+ options: ColumnEncodeOptions,
+): number {
+ let size = 1;
+ if (nullCount(column) > 0) size += Math.ceil(rowCount / 8);
+ const valueCount = column.values.length;
+
+ if (column.type === QWP_COLUMN_TYPE.BOOLEAN) {
+ return size + Math.ceil(valueCount / 8);
+ }
+
+ // DATE is deliberately absent here. The protocol is asymmetric for it: on
+ // ingress the server parses DATE as a plain fixed-width int64
+ // (QwpTableBlockCursor dispatches TYPE_DATE to QwpFixedWidthColumnCursor,
+ // alongside LONG and UUID), while on egress it emits DATE through
+ // emitTimestampSlice with a per-column encoding byte. The result decoder in
+ // this package matches the egress side, so the two directions genuinely
+ // differ. Adding DATE to this branch makes every ingress frame carrying a
+ // DATE column misparse server-side.
+ if (
+ column.type === QWP_COLUMN_TYPE.TIMESTAMP ||
+ column.type === QWP_COLUMN_TYPE.TIMESTAMP_NANOS
+ ) {
+ if (!options.gorilla) return size + valueCount * 8;
+ const timestamps = column.values.map((value) => BigInt(value as bigint));
+ const gorillaSize = timestamps.length > 2 ? qwpGorillaSize(timestamps) : -1;
+ return size + 1 + (gorillaSize > 0 ? gorillaSize : valueCount * 8);
+ }
+
+ const width = fixedWidth(column.type);
+ if (width !== undefined) return size + valueCount * width;
+
+ if (column.type === QWP_COLUMN_TYPE.SYMBOL) {
+ if (options.deltaSymbols) {
+ for (const value of column.values) {
+ size += qwpVarintSize(symbolId(value, options.dictionary!));
+ }
+ return size;
+ }
+ const { entries, rowIds } = inlineSymbolDictionary(column.values);
+ size += qwpVarintSize(entries.length);
+ for (const entry of entries) size += qwpStringSize(entry);
+ for (const id of rowIds) size += qwpVarintSize(id);
+ return size;
+ }
+
+ if (
+ column.type === QWP_COLUMN_TYPE.VARCHAR ||
+ column.type === QWP_COLUMN_TYPE.BINARY
+ ) {
+ let dataLength = 0;
+ for (const value of column.values) {
+ dataLength +=
+ column.type === QWP_COLUMN_TYPE.VARCHAR
+ ? utf8Length(value as string)
+ : binaryValue(value).length;
+ }
+ return size + (valueCount + 1) * 4 + dataLength;
+ }
+
+ if (
+ column.type === QWP_COLUMN_TYPE.DOUBLE_ARRAY ||
+ column.type === QWP_COLUMN_TYPE.LONG_ARRAY
+ ) {
+ for (const value of column.values) {
+ const array = value as QwpArrayValue;
+ size += 1 + array.dimensions.length * 4 + array.values.length * 8;
+ }
+ return size;
+ }
+
+ if (column.type === QWP_COLUMN_TYPE.GEOHASH) {
+ const precision = column.geohashPrecision ?? 1;
+ return (
+ size + qwpVarintSize(precision) + valueCount * Math.ceil(precision / 8)
+ );
+ }
+
+ if (column.type === QWP_COLUMN_TYPE.DECIMAL64) {
+ return size + 1 + valueCount * 8;
+ }
+ if (column.type === QWP_COLUMN_TYPE.DECIMAL128) {
+ return size + 1 + valueCount * 16;
+ }
+ if (column.type === QWP_COLUMN_TYPE.DECIMAL256) {
+ return size + 1 + valueCount * 32;
+ }
+
+ throw new Error(`unsupported QWP column type 0x${column.type.toString(16)}`);
+}
+
+function writeNullHeader(
+ writer: QwpByteWriter,
+ column: QwpColumnBuffer,
+ rowCount: number,
+): void {
+ if (nullCount(column) === 0) {
+ writer.writeUint8(0);
+ return;
+ }
+ writer.writeUint8(1);
+ const bitmap = new Uint8Array(Math.ceil(rowCount / 8));
+ for (let row = 0; row < rowCount; row++) {
+ if (column.nulls[row]) bitmap[row >>> 3] |= 1 << (row & 7);
+ }
+ writer.writeBytes(bitmap);
+}
+
+function writeSignedLittleEndian(
+ writer: QwpByteWriter,
+ value: bigint,
+ width: number,
+): void {
+ let remaining = BigInt.asIntN(width * 8, value);
+ for (let index = 0; index < width; index++) {
+ writer.writeUint8(Number(remaining & 0xffn));
+ remaining >>= 8n;
+ }
+}
+
+function writeColumn(
+ writer: QwpByteWriter,
+ column: QwpColumnBuffer,
+ rowCount: number,
+ options: ColumnEncodeOptions,
+): void {
+ writeNullHeader(writer, column, rowCount);
+
+ switch (column.type) {
+ case QWP_COLUMN_TYPE.BOOLEAN: {
+ const bitmap = new Uint8Array(Math.ceil(column.values.length / 8));
+ column.values.forEach((value, index) => {
+ if (value) bitmap[index >>> 3] |= 1 << (index & 7);
+ });
+ writer.writeBytes(bitmap);
+ return;
+ }
+ case QWP_COLUMN_TYPE.BYTE:
+ for (const value of column.values) writer.writeInt8(Number(value));
+ return;
+ case QWP_COLUMN_TYPE.SHORT:
+ for (const value of column.values) writer.writeInt16(Number(value));
+ return;
+ case QWP_COLUMN_TYPE.CHAR:
+ for (const value of column.values) {
+ const text = value as string;
+ if (text.length !== 1) {
+ throw new Error("QWP CHAR values must contain one UTF-16 code unit");
+ }
+ writer.writeUint16(text.charCodeAt(0));
+ }
+ return;
+ case QWP_COLUMN_TYPE.INT:
+ for (const value of column.values) writer.writeInt32(Number(value));
+ return;
+ case QWP_COLUMN_TYPE.IPV4:
+ for (const value of column.values)
+ writer.writeUint32(Number(value) >>> 0);
+ return;
+ case QWP_COLUMN_TYPE.FLOAT:
+ for (const value of column.values) writer.writeFloat32(Number(value));
+ return;
+ // DATE joins LONG here: raw int64s, no per-column encoding byte.
+ // See columnPayloadSize() for why it is not a timestamp on ingress.
+ case QWP_COLUMN_TYPE.LONG:
+ case QWP_COLUMN_TYPE.DATE:
+ for (const value of column.values) {
+ writer.writeBigInt64(BigInt(value as number | bigint));
+ }
+ return;
+ case QWP_COLUMN_TYPE.TIMESTAMP:
+ case QWP_COLUMN_TYPE.TIMESTAMP_NANOS: {
+ const timestamps = column.values.map((value) => BigInt(value as bigint));
+ if (!options.gorilla) {
+ for (const timestamp of timestamps) writer.writeBigInt64(timestamp);
+ return;
+ }
+ const gorillaSize =
+ timestamps.length > 2 ? qwpGorillaSize(timestamps) : -1;
+ if (gorillaSize > 0) {
+ writer.writeUint8(QWP_ENCODING_GORILLA);
+ writer.writeBytes(encodeQwpGorilla(timestamps));
+ } else {
+ writer.writeUint8(QWP_ENCODING_UNCOMPRESSED);
+ for (const timestamp of timestamps) writer.writeBigInt64(timestamp);
+ }
+ return;
+ }
+ case QWP_COLUMN_TYPE.DOUBLE:
+ for (const value of column.values) writer.writeFloat64(Number(value));
+ return;
+ case QWP_COLUMN_TYPE.UUID:
+ for (const value of column.values) {
+ writer.writeBytes(binaryValue(value, 16));
+ }
+ return;
+ case QWP_COLUMN_TYPE.LONG256:
+ for (const value of column.values) {
+ writer.writeBytes(binaryValue(value, 32));
+ }
+ return;
+ case QWP_COLUMN_TYPE.SYMBOL: {
+ if (options.deltaSymbols) {
+ for (const value of column.values) {
+ writeQwpVarint(writer, symbolId(value, options.dictionary!));
+ }
+ return;
+ }
+ const { entries, rowIds } = inlineSymbolDictionary(column.values);
+ writeQwpVarint(writer, entries.length);
+ for (const entry of entries) writeQwpString(writer, entry);
+ for (const id of rowIds) writeQwpVarint(writer, id);
+ return;
+ }
+ case QWP_COLUMN_TYPE.VARCHAR:
+ case QWP_COLUMN_TYPE.BINARY: {
+ const parts = column.values.map((value) =>
+ column.type === QWP_COLUMN_TYPE.VARCHAR
+ ? encodeUtf8(value as string)
+ : binaryValue(value),
+ );
+ let cumulative = 0;
+ writer.writeUint32(0);
+ for (const part of parts) {
+ cumulative += part.length;
+ writer.writeUint32(cumulative);
+ }
+ for (const part of parts) writer.writeBytes(part);
+ return;
+ }
+ case QWP_COLUMN_TYPE.DOUBLE_ARRAY:
+ case QWP_COLUMN_TYPE.LONG_ARRAY:
+ for (const value of column.values) {
+ const array = value as QwpArrayValue;
+ writer.writeUint8(array.dimensions.length);
+ for (const dimension of array.dimensions) writer.writeUint32(dimension);
+ for (const item of array.values) {
+ if (column.type === QWP_COLUMN_TYPE.DOUBLE_ARRAY) {
+ writer.writeFloat64(Number(item));
+ } else {
+ writer.writeBigInt64(BigInt(item));
+ }
+ }
+ }
+ return;
+ case QWP_COLUMN_TYPE.GEOHASH: {
+ const precision = column.geohashPrecision ?? 1;
+ writeQwpVarint(writer, precision);
+ const width = Math.ceil(precision / 8);
+ for (const value of column.values) {
+ let remaining = BigInt(value as bigint);
+ for (let index = 0; index < width; index++) {
+ writer.writeUint8(Number(remaining & 0xffn));
+ remaining >>= 8n;
+ }
+ }
+ return;
+ }
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ case QWP_COLUMN_TYPE.DECIMAL256: {
+ writer.writeUint8(column.decimalScale ?? 0);
+ const width =
+ column.type === QWP_COLUMN_TYPE.DECIMAL64
+ ? 8
+ : column.type === QWP_COLUMN_TYPE.DECIMAL128
+ ? 16
+ : 32;
+ for (const value of column.values) {
+ writeSignedLittleEndian(writer, BigInt(value as bigint), width);
+ }
+ return;
+ }
+ default:
+ throw new Error("unsupported QWP column type");
+ }
+}
+
+function tableSize(
+ table: QwpTableBuffer,
+ options: ColumnEncodeOptions,
+): number {
+ let size =
+ qwpStringSize(table.name) +
+ qwpVarintSize(table.rowCount) +
+ qwpVarintSize(table.columns.length);
+ for (const column of table.columns) size += qwpStringSize(column.name) + 1;
+ for (const column of table.columns) {
+ size += columnPayloadSize(column, table.rowCount, options);
+ }
+ return size;
+}
+
+function validateTableForEncoding(table: QwpTableBuffer): void {
+ for (const column of table.columns) {
+ if (
+ column.size !== table.rowCount ||
+ column.nulls.length !== table.rowCount
+ ) {
+ throw new Error(
+ `table '${table.name}' has an unfinished row in column '${column.name}'`,
+ );
+ }
+ let nonNullCount = 0;
+ for (const isNull of column.nulls) if (!isNull) nonNullCount++;
+ if (nonNullCount !== column.values.length) {
+ throw new Error(
+ `table '${table.name}' column '${column.name}' has ${nonNullCount} non-null row(s) but ${column.values.length} value(s)`,
+ );
+ }
+ }
+}
+
+/** Encodes one QWP v1 ingress message. */
+export function encodeQwpIngressFrame(
+ tables: readonly QwpTableBuffer[],
+ options: QwpIngressEncodeOptions = {},
+): Uint8Array {
+ const dictionarySize = options.dictionary?.size;
+ try {
+ return encodeQwpIngressFrameInternal(tables, options);
+ } catch (error) {
+ if (dictionarySize !== undefined)
+ options.dictionary!.truncate(dictionarySize);
+ throw error;
+ }
+}
+
+function encodeQwpIngressFrameInternal(
+ tables: readonly QwpTableBuffer[],
+ options: QwpIngressEncodeOptions,
+): Uint8Array {
+ if (tables.length > 0xffff) {
+ throw new Error("QWP frame contains more than 65535 tables");
+ }
+ for (const table of tables) {
+ validateTableForEncoding(table);
+ if (table.rowCount > QWP_MAX_ROWS_PER_TABLE) {
+ throw new Error(
+ `table '${table.name}' contains ${table.rowCount} rows; maximum is ${QWP_MAX_ROWS_PER_TABLE}`,
+ );
+ }
+ }
+
+ const gorilla = options.gorilla ?? true;
+ const deltaSymbols = options.dictionary !== undefined;
+ if (deltaSymbols) {
+ const published = options.confirmedMaxSymbolId ?? -1;
+ if (
+ !Number.isSafeInteger(published) ||
+ published < -1 ||
+ published >= options.dictionary!.size
+ ) {
+ throw new RangeError(
+ `published symbol dictionary ID is out of range [id=${published}, size=${options.dictionary!.size}]`,
+ );
+ }
+ }
+ if (deltaSymbols) {
+ // Resolve string values before calculating the delta prefix and frame size.
+ for (const table of tables) {
+ for (const column of table.columns) {
+ if (column.type !== QWP_COLUMN_TYPE.SYMBOL) continue;
+ for (const value of column.values) {
+ if (typeof value === "string") options.dictionary!.getOrAdd(value);
+ }
+ }
+ }
+ }
+ const deltaStart = deltaSymbols
+ ? (options.confirmedMaxSymbolId ?? -1) + 1
+ : 0;
+ const dictionaryEntries = deltaSymbols
+ ? options.dictionary!.entriesFrom(deltaStart)
+ : [];
+ const columnOptions = {
+ gorilla,
+ deltaSymbols,
+ dictionary: options.dictionary,
+ };
+
+ let flags = 0;
+ if (gorilla) flags |= QWP_FLAG_GORILLA;
+ if (deltaSymbols) flags |= QWP_FLAG_DELTA_SYMBOL_DICTIONARY;
+ if (options.deferCommit) flags |= QWP_FLAG_DEFER_COMMIT;
+
+ let payloadLength = 0;
+ if (deltaSymbols) {
+ payloadLength +=
+ qwpVarintSize(deltaStart) + qwpVarintSize(dictionaryEntries.length);
+ for (const entry of dictionaryEntries)
+ payloadLength += qwpStringSize(entry);
+ }
+ for (const table of tables) payloadLength += tableSize(table, columnOptions);
+
+ const writer = new QwpByteWriter(QWP_HEADER_SIZE + payloadLength);
+ writeQwpFrameHeader(writer, {
+ flags,
+ tableCount: tables.length,
+ payloadLength,
+ });
+ if (deltaSymbols) {
+ writeQwpVarint(writer, deltaStart);
+ writeQwpVarint(writer, dictionaryEntries.length);
+ for (const entry of dictionaryEntries) writeQwpString(writer, entry);
+ }
+ for (const table of tables) {
+ writeQwpString(writer, table.name);
+ writeQwpVarint(writer, table.rowCount);
+ writeQwpVarint(writer, table.columns.length);
+ for (const column of table.columns) {
+ writeQwpString(writer, column.name);
+ writer.writeUint8(column.type);
+ }
+ for (const column of table.columns) {
+ writeColumn(writer, column, table.rowCount, columnOptions);
+ }
+ }
+ const result = writer.toUint8Array();
+ if (result.length !== QWP_HEADER_SIZE + payloadLength) {
+ throw new Error(
+ `QWP frame size mismatch [expected=${QWP_HEADER_SIZE + payloadLength}, actual=${result.length}]`,
+ );
+ }
+ return result;
+}
+
+export interface QwpIngressSymbolDictionaryDelta {
+ readonly startId: number;
+ readonly entries: readonly string[];
+}
+
+/** Reads the connection-scoped dictionary prefix from a delta ingress frame. */
+export function decodeQwpIngressSymbolDictionaryDelta(
+ bytes: Uint8Array,
+): QwpIngressSymbolDictionaryDelta | undefined {
+ const frame = decodeQwpFrame(bytes);
+ if ((frame.flags & QWP_FLAG_DELTA_SYMBOL_DICTIONARY) === 0) return undefined;
+ const reader = new QwpByteReader(frame.payload);
+ const startId = readQwpVarintNumber(reader, "symbol dictionary start ID");
+ const count = readQwpVarintNumber(reader, "symbol dictionary entry count");
+ if (startId + count > QWP_MAX_SYMBOL_DICTIONARY_SIZE) {
+ throw new QwpProtocolError(
+ `QWP symbol dictionary exceeds maximum size ${QWP_MAX_SYMBOL_DICTIONARY_SIZE}`,
+ );
+ }
+ const entries: string[] = [];
+ for (let index = 0; index < count; index++) {
+ const length = readQwpVarintNumber(
+ reader,
+ "symbol dictionary entry length",
+ );
+ entries.push(reader.readUtf8(length, "symbol dictionary entry"));
+ }
+ return { startId, entries };
+}
+
+/** Encodes a table-less committed dictionary catch-up frame. */
+export function encodeQwpIngressSymbolDictionaryFrame(
+ startId: number,
+ entries: readonly string[],
+): Uint8Array {
+ if (!Number.isSafeInteger(startId) || startId < 0) {
+ throw new RangeError("symbol dictionary start ID must be non-negative");
+ }
+ if (startId + entries.length > QWP_MAX_SYMBOL_DICTIONARY_SIZE) {
+ throw new RangeError(
+ `symbol dictionary exceeds maximum size ${QWP_MAX_SYMBOL_DICTIONARY_SIZE}`,
+ );
+ }
+ let payloadLength = qwpVarintSize(startId) + qwpVarintSize(entries.length);
+ for (const entry of entries) payloadLength += qwpStringSize(entry);
+ const writer = new QwpByteWriter(QWP_HEADER_SIZE + payloadLength);
+ writeQwpFrameHeader(writer, {
+ flags: QWP_FLAG_DELTA_SYMBOL_DICTIONARY,
+ tableCount: 0,
+ payloadLength,
+ });
+ writeQwpVarint(writer, startId);
+ writeQwpVarint(writer, entries.length);
+ for (const entry of entries) writeQwpString(writer, entry);
+ return writer.toUint8Array();
+}
+
+export function encodeQwpIngressCommitFrame(
+ dictionary?: QwpSymbolDictionary,
+ confirmedMaxSymbolId = -1,
+): Uint8Array {
+ return encodeQwpIngressFrame([], {
+ gorilla: false,
+ dictionary,
+ confirmedMaxSymbolId,
+ });
+}
+
+/** Encodes a negotiated, side-effect-free durable-ACK progress poll. */
+export function encodeQwpDurableAckPollFrame(): Uint8Array {
+ const writer = new QwpByteWriter(QWP_HEADER_SIZE);
+ writeQwpFrameHeader(writer, {
+ flags: QWP_FLAG_DURABLE_ACK_POLL,
+ tableCount: 0,
+ payloadLength: 0,
+ });
+ return writer.toUint8Array();
+}
+
+function readIngressTables(
+ reader: QwpByteReader,
+ count: number,
+): QwpIngressTableResult[] {
+ const tables: QwpIngressTableResult[] = [];
+ for (let index = 0; index < count; index++) {
+ const nameLength = reader.readUint16("ingress table name length");
+ const name = reader.readUtf8(nameLength, "ingress table name");
+ const sequenceTransaction = reader.readBigInt64(
+ "ingress table sequence transaction",
+ );
+ tables.push({ name, sequenceTransaction });
+ }
+ return tables;
+}
+
+/** Decodes an ingress ACK, durable ACK, or NACK WebSocket payload. */
+export function decodeQwpIngressResponse(
+ payload: Uint8Array,
+): QwpIngressResponse {
+ const reader = new QwpByteReader(payload);
+ const status = reader.readUint8("ingress response status");
+
+ if (status === QWP_STATUS.DURABLE_ACK) {
+ const count = reader.readUint16("durable ACK table count");
+ const tables = readIngressTables(reader, count);
+ reader.expectEnd("durable ACK");
+ return { status, sequence: null, tables };
+ }
+
+ const sequence = reader.readBigUint64("ingress response sequence");
+ if (status === QWP_STATUS.OK) {
+ const count = reader.readUint16("ACK table count");
+ const tables = readIngressTables(reader, count);
+ reader.expectEnd("ingress ACK");
+ return { status, sequence, tables };
+ }
+
+ const messageLength = reader.readUint16("NACK message length");
+ if (messageLength > QWP_MAX_ERROR_MESSAGE_LENGTH) {
+ throw new QwpProtocolError(
+ `QWP error message exceeds ${QWP_MAX_ERROR_MESSAGE_LENGTH} bytes`,
+ );
+ }
+ const errorMessage = reader.readUtf8(messageLength, "NACK message");
+ reader.expectEnd("ingress NACK");
+ return { status, sequence, tables: [], errorMessage };
+}
diff --git a/src/_qwp/_core/result-batch.ts b/src/_qwp/_core/result-batch.ts
new file mode 100644
index 0000000..7c4a309
--- /dev/null
+++ b/src/_qwp/_core/result-batch.ts
@@ -0,0 +1,1956 @@
+import { decodeUtf8, QwpByteReader } from "./bytes";
+import {
+ QWP_COLUMN_TYPE,
+ QWP_FLAG_DELTA_SYMBOL_DICTIONARY,
+ QWP_FLAG_GORILLA,
+ QWP_FLAG_ZSTD,
+ QWP_MAX_CELLS_PER_BATCH,
+ QWP_MAX_COLUMNS_PER_TABLE,
+ QWP_MAX_IDENTIFIER_BYTES,
+ QWP_RESET_MASK_DICTIONARY,
+ QwpColumnType,
+} from "./constants";
+import { QwpResultBatchMessage } from "./egress";
+import { QwpProtocolError } from "./errors";
+import { readQwpVarint } from "./varint";
+import { decompressQwpZstdFrame } from "./zstd";
+
+const MAX_ARRAY_DIMENSION_LENGTH = (1 << 28) - 1;
+const MAX_ARRAY_ELEMENTS = 268_435_327;
+// Matches QuestDB's connection-scoped symbol dictionary limit.
+const MAX_CONNECTION_SYMBOLS = 2_000_000;
+const MAX_ROWS_PER_BATCH = 1_048_576;
+
+export interface QwpDecimalValue {
+ unscaled: bigint;
+ scale: number;
+}
+
+export interface QwpUuidValue {
+ low: bigint;
+ high: bigint;
+}
+
+export interface QwpLong256Value {
+ /** Little-endian 64-bit words; word 0 is least significant. */
+ words: readonly [bigint, bigint, bigint, bigint];
+}
+
+export interface QwpGeohashValue {
+ bits: bigint;
+ precisionBits: number;
+}
+
+export interface QwpResultArrayValue {
+ dimensions: readonly number[];
+ values: readonly number[] | readonly bigint[];
+}
+
+export type QwpResultValue =
+ | boolean
+ | number
+ | bigint
+ | string
+ | Uint8Array
+ | QwpDecimalValue
+ | QwpUuidValue
+ | QwpLong256Value
+ | QwpGeohashValue
+ | QwpResultArrayValue
+ | null;
+
+export interface QwpResultColumnSchema {
+ name: string;
+ type: QwpColumnType;
+}
+
+export interface QwpResultColumn extends QwpResultColumnSchema {
+ values: readonly QwpResultValue[];
+ scale?: number;
+ precisionBits?: number;
+}
+
+export class QwpResultBatch {
+ constructor(
+ readonly requestId: bigint,
+ readonly batchSequence: bigint,
+ readonly tableName: string,
+ readonly rowCount: number,
+ readonly columns: readonly QwpResultColumn[],
+ ) {}
+
+ get(rowIndex: number, columnIndex: number): QwpResultValue {
+ if (
+ !Number.isInteger(rowIndex) ||
+ rowIndex < 0 ||
+ rowIndex >= this.rowCount
+ ) {
+ throw new RangeError(`row index out of range: ${rowIndex}`);
+ }
+ const column = this.columns[columnIndex];
+ if (!column)
+ throw new RangeError(`column index out of range: ${columnIndex}`);
+ return column.values[rowIndex];
+ }
+
+ *rows(): IterableIterator {
+ for (let row = 0; row < this.rowCount; row++) {
+ yield this.columns.map((column) => column.values[row]);
+ }
+ }
+}
+
+class QwpResultColumnViewLayout {
+ schema!: QwpResultColumnSchema;
+ rowCount = 0;
+ nonNullCount = 0;
+ nullBitmap?: Uint8Array;
+ nonNullIndexes?: Int32Array;
+ values?: Uint8Array;
+ valuesView?: DataView;
+ stringBytes?: Uint8Array;
+ symbolDictionary?: readonly string[];
+ symbolRowIds?: Int32Array;
+ arrayOffsets?: Int32Array;
+ arrayLengths?: Int32Array;
+ scale?: number;
+ precisionBits?: number;
+ private timestampStorage?: Uint8Array;
+ readonly localSymbols: string[] = [];
+
+ reset(schema: QwpResultColumnSchema, rowCount: number): void {
+ this.schema = schema;
+ this.rowCount = rowCount;
+ this.nonNullCount = 0;
+ this.nullBitmap = undefined;
+ this.values = undefined;
+ this.valuesView = undefined;
+ this.stringBytes = undefined;
+ this.symbolDictionary = undefined;
+ this.scale = undefined;
+ this.precisionBits = undefined;
+ this.localSymbols.length = 0;
+ }
+
+ release(): void {
+ // Drop frame-backed references immediately. Capacity-bearing scratch
+ // arrays remain attached to the layout for the next batch.
+ this.nullBitmap = undefined;
+ this.values = undefined;
+ this.valuesView = undefined;
+ this.stringBytes = undefined;
+ this.symbolDictionary = undefined;
+ this.localSymbols.length = 0;
+ }
+
+ setValues(bytes: Uint8Array): void {
+ this.values = bytes;
+ this.valuesView = new DataView(
+ bytes.buffer,
+ bytes.byteOffset,
+ bytes.byteLength,
+ );
+ }
+
+ ensureNonNullIndexes(size: number): Int32Array {
+ this.nonNullIndexes = ensureInt32Capacity(this.nonNullIndexes, size);
+ return this.nonNullIndexes;
+ }
+
+ ensureSymbolRowIds(size: number): Int32Array {
+ this.symbolRowIds = ensureInt32Capacity(this.symbolRowIds, size);
+ return this.symbolRowIds;
+ }
+
+ ensureArrayOffsets(size: number): Int32Array {
+ this.arrayOffsets = ensureInt32Capacity(this.arrayOffsets, size);
+ return this.arrayOffsets;
+ }
+
+ ensureArrayLengths(size: number): Int32Array {
+ this.arrayLengths = ensureInt32Capacity(this.arrayLengths, size);
+ return this.arrayLengths;
+ }
+
+ timestampBytes(size: number): Uint8Array {
+ if (!this.timestampStorage || this.timestampStorage.byteLength < size) {
+ let capacity = Math.max(64, this.timestampStorage?.byteLength ?? 0);
+ while (capacity < size) capacity *= 2;
+ this.timestampStorage = new Uint8Array(capacity);
+ }
+ return this.timestampStorage.subarray(0, size);
+ }
+
+ isNull(row: number): boolean {
+ const bitmap = this.nullBitmap;
+ return bitmap !== undefined && (bitmap[row >>> 3] & (1 << (row & 7))) !== 0;
+ }
+
+ denseIndex(row: number): number {
+ return this.nullBitmap ? this.nonNullIndexes![row] : row;
+ }
+}
+
+function ensureInt32Capacity(
+ current: Int32Array | undefined,
+ size: number,
+): Int32Array {
+ if (current && current.length >= size) return current;
+ let capacity = Math.max(16, current?.length ?? 0);
+ while (capacity < size) capacity *= 2;
+ return new Int32Array(capacity);
+}
+
+/**
+ * Reusable, zero-copy view over one QWP result column.
+ *
+ * The view and every byte slice returned from it are valid only while the
+ * surrounding queryViews() callback is running. Copy data that must outlive
+ * the callback.
+ */
+export class QwpResultColumnView {
+ /** @internal */
+ constructor(
+ private readonly batch: QwpResultBatchView,
+ readonly columnIndex: number,
+ ) {}
+
+ get name(): string {
+ return this.layout().schema.name;
+ }
+
+ get type(): QwpColumnType {
+ return this.layout().schema.type;
+ }
+
+ get rowCount(): number {
+ return this.layout().rowCount;
+ }
+
+ get nonNullCount(): number {
+ return this.layout().nonNullCount;
+ }
+
+ get scale(): number | undefined {
+ return this.layout().scale;
+ }
+
+ get precisionBits(): number | undefined {
+ return this.layout().precisionBits;
+ }
+
+ /** Fixed-width stride, zero for bit-packed BOOLEAN, or -1 when variable. */
+ get bytesPerValue(): number {
+ const layout = this.layout();
+ switch (layout.schema.type) {
+ case QWP_COLUMN_TYPE.BOOLEAN:
+ return 0;
+ case QWP_COLUMN_TYPE.BYTE:
+ return 1;
+ case QWP_COLUMN_TYPE.SHORT:
+ case QWP_COLUMN_TYPE.CHAR:
+ return 2;
+ case QWP_COLUMN_TYPE.INT:
+ case QWP_COLUMN_TYPE.FLOAT:
+ case QWP_COLUMN_TYPE.IPV4:
+ return 4;
+ case QWP_COLUMN_TYPE.LONG:
+ case QWP_COLUMN_TYPE.DOUBLE:
+ case QWP_COLUMN_TYPE.DATE:
+ case QWP_COLUMN_TYPE.TIMESTAMP:
+ case QWP_COLUMN_TYPE.TIMESTAMP_NANOS:
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ return 8;
+ case QWP_COLUMN_TYPE.UUID:
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ return 16;
+ case QWP_COLUMN_TYPE.LONG256:
+ case QWP_COLUMN_TYPE.DECIMAL256:
+ return 32;
+ case QWP_COLUMN_TYPE.GEOHASH:
+ return Math.ceil(layout.precisionBits! / 8);
+ default:
+ return -1;
+ }
+ }
+
+ isNull(rowIndex: number): boolean {
+ const layout = this.checkedLayout(rowIndex);
+ return layout.isNull(rowIndex);
+ }
+
+ nonNullIndex(rowIndex: number): number {
+ const layout = this.checkedLayout(rowIndex);
+ return layout.isNull(rowIndex) ? -1 : layout.denseIndex(rowIndex);
+ }
+
+ /** Raw per-row NULL bitmap, without copying. Undefined means no NULLs. */
+ nullBitmapBytes(): Uint8Array | undefined {
+ return this.layout().nullBitmap;
+ }
+
+ /**
+ * Raw packed non-null values. Fixed-width values use QWP little-endian
+ * layout; booleans are bit-packed and variable-width columns contain their
+ * uint32 offset table. SYMBOL returns undefined because IDs are varints.
+ */
+ valuesBytes(): Uint8Array | undefined {
+ return this.layout().values;
+ }
+
+ /** Concatenated VARCHAR/BINARY payload bytes, without copying. */
+ stringBytes(): Uint8Array | undefined {
+ return this.layout().stringBytes;
+ }
+
+ /** Reusable dense-index table; only the first rowCount entries are valid. */
+ nonNullIndexView(): Int32Array | undefined {
+ const layout = this.layout();
+ return layout.nullBitmap
+ ? layout.nonNullIndexes!.subarray(0, layout.rowCount)
+ : undefined;
+ }
+
+ /** Reusable per-row SYMBOL IDs; NULL-row entries are unspecified. */
+ symbolIdView(): Int32Array | undefined {
+ const layout = this.layout();
+ this.requireType(layout, QWP_COLUMN_TYPE.SYMBOL);
+ return layout.symbolRowIds?.subarray(0, layout.rowCount);
+ }
+
+ getBoolean(rowIndex: number): boolean {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.BOOLEAN,
+ );
+ if (dense < 0) return false;
+ return (layout.values![dense >>> 3] & (1 << (dense & 7))) !== 0;
+ }
+
+ getByte(rowIndex: number): number {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.BYTE,
+ );
+ return dense < 0 ? 0 : layout.valuesView!.getInt8(dense);
+ }
+
+ getShort(rowIndex: number): number {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.SHORT,
+ );
+ return dense < 0 ? 0 : layout.valuesView!.getInt16(dense * 2, true);
+ }
+
+ getChar(rowIndex: number): string {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.CHAR,
+ );
+ return dense < 0
+ ? "\0"
+ : String.fromCharCode(layout.valuesView!.getUint16(dense * 2, true));
+ }
+
+ getInt(rowIndex: number): number {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.INT,
+ QWP_COLUMN_TYPE.IPV4,
+ );
+ return dense < 0 ? 0 : layout.valuesView!.getInt32(dense * 4, true);
+ }
+
+ getFloat(rowIndex: number): number {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.FLOAT,
+ );
+ return dense < 0
+ ? Number.NaN
+ : layout.valuesView!.getFloat32(dense * 4, true);
+ }
+
+ getDouble(rowIndex: number): number {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.DOUBLE,
+ );
+ return dense < 0
+ ? Number.NaN
+ : layout.valuesView!.getFloat64(dense * 8, true);
+ }
+
+ getLong(rowIndex: number): bigint {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.LONG,
+ QWP_COLUMN_TYPE.DATE,
+ QWP_COLUMN_TYPE.TIMESTAMP,
+ QWP_COLUMN_TYPE.TIMESTAMP_NANOS,
+ );
+ return dense < 0 ? 0n : layout.valuesView!.getBigInt64(dense * 8, true);
+ }
+
+ /** Zero-copy UTF-8 bytes for a VARCHAR value. */
+ getUtf8View(rowIndex: number): Uint8Array | null {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.VARCHAR,
+ );
+ return dense < 0 ? null : variableWidthValue(layout, dense);
+ }
+
+ getString(rowIndex: number): string | null {
+ const layout = this.checkedLayout(rowIndex);
+ if (layout.schema.type === QWP_COLUMN_TYPE.SYMBOL) {
+ return this.getSymbol(rowIndex);
+ }
+ this.requireType(layout, QWP_COLUMN_TYPE.VARCHAR);
+ if (layout.isNull(rowIndex)) return null;
+ return decodeUtf8(variableWidthValue(layout, layout.denseIndex(rowIndex)));
+ }
+
+ /** Zero-copy BINARY bytes. */
+ getBinaryView(rowIndex: number): Uint8Array | null {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.BINARY,
+ );
+ return dense < 0 ? null : variableWidthValue(layout, dense);
+ }
+
+ getSymbolId(rowIndex: number): number {
+ const layout = this.checkedLayout(rowIndex);
+ this.requireType(layout, QWP_COLUMN_TYPE.SYMBOL);
+ return layout.isNull(rowIndex) ? -1 : layout.symbolRowIds![rowIndex];
+ }
+
+ getSymbol(rowIndex: number): string | null {
+ const layout = this.checkedLayout(rowIndex);
+ this.requireType(layout, QWP_COLUMN_TYPE.SYMBOL);
+ return layout.isNull(rowIndex)
+ ? null
+ : layout.symbolDictionary![layout.symbolRowIds![rowIndex]];
+ }
+
+ getSymbolForId(symbolId: number): string {
+ const layout = this.layout();
+ this.requireType(layout, QWP_COLUMN_TYPE.SYMBOL);
+ const dictionary = layout.symbolDictionary!;
+ if (
+ !Number.isInteger(symbolId) ||
+ symbolId < 0 ||
+ symbolId >= dictionary.length
+ ) {
+ throw new RangeError(`symbol ID out of range: ${symbolId}`);
+ }
+ return dictionary[symbolId];
+ }
+
+ get symbolDictionarySize(): number {
+ const layout = this.layout();
+ this.requireType(layout, QWP_COLUMN_TYPE.SYMBOL);
+ return layout.symbolDictionary!.length;
+ }
+
+ getUuidLow(rowIndex: number): bigint {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.UUID,
+ );
+ return dense < 0 ? 0n : layout.valuesView!.getBigUint64(dense * 16, true);
+ }
+
+ getUuidHigh(rowIndex: number): bigint {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.UUID,
+ );
+ return dense < 0
+ ? 0n
+ : layout.valuesView!.getBigUint64(dense * 16 + 8, true);
+ }
+
+ getLong256Word(rowIndex: number, wordIndex: number): bigint {
+ if (!Number.isInteger(wordIndex) || wordIndex < 0 || wordIndex > 3) {
+ throw new RangeError(`LONG256 word index out of range: ${wordIndex}`);
+ }
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.LONG256,
+ );
+ return dense < 0
+ ? 0n
+ : layout.valuesView!.getBigInt64(dense * 32 + wordIndex * 8, true);
+ }
+
+ getDecimalUnscaled(rowIndex: number): bigint {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.DECIMAL64,
+ QWP_COLUMN_TYPE.DECIMAL128,
+ QWP_COLUMN_TYPE.DECIMAL256,
+ );
+ if (dense < 0) return 0n;
+ const width = fixedTypeWidth(layout.schema.type);
+ return signedLittleEndianValue(layout.values!, dense * width, width);
+ }
+
+ getGeohashBits(rowIndex: number): bigint {
+ const { layout, dense } = this.valuePosition(
+ rowIndex,
+ QWP_COLUMN_TYPE.GEOHASH,
+ );
+ if (dense < 0) return 0n;
+ const width = Math.ceil(layout.precisionBits! / 8);
+ return unsignedLittleEndianValue(layout.values!, dense * width, width);
+ }
+
+ /** Zero-copy encoded ARRAY row, including dimension header. */
+ getArrayView(rowIndex: number): Uint8Array | null {
+ const layout = this.checkedLayout(rowIndex);
+ this.requireType(
+ layout,
+ QWP_COLUMN_TYPE.DOUBLE_ARRAY,
+ QWP_COLUMN_TYPE.LONG_ARRAY,
+ );
+ if (layout.isNull(rowIndex)) return null;
+ const offset = layout.arrayOffsets![rowIndex];
+ return layout.values!.subarray(
+ offset,
+ offset + layout.arrayLengths![rowIndex],
+ );
+ }
+
+ getArrayDimensionCount(rowIndex: number): number {
+ const layout = this.checkedLayout(rowIndex);
+ this.requireType(
+ layout,
+ QWP_COLUMN_TYPE.DOUBLE_ARRAY,
+ QWP_COLUMN_TYPE.LONG_ARRAY,
+ );
+ return layout.isNull(rowIndex)
+ ? 0
+ : layout.values![layout.arrayOffsets![rowIndex]];
+ }
+
+ /** Lazily materializes one cell; prefer typed/raw accessors on hot paths. */
+ get(rowIndex: number): QwpResultValue {
+ const layout = this.checkedLayout(rowIndex);
+ if (layout.isNull(rowIndex)) return null;
+ const dense = layout.denseIndex(rowIndex);
+ const view = layout.valuesView;
+ switch (layout.schema.type) {
+ case QWP_COLUMN_TYPE.BOOLEAN:
+ return (layout.values![dense >>> 3] & (1 << (dense & 7))) !== 0;
+ case QWP_COLUMN_TYPE.BYTE:
+ return view!.getInt8(dense);
+ case QWP_COLUMN_TYPE.SHORT:
+ return view!.getInt16(dense * 2, true);
+ case QWP_COLUMN_TYPE.CHAR:
+ return String.fromCharCode(view!.getUint16(dense * 2, true));
+ case QWP_COLUMN_TYPE.INT:
+ case QWP_COLUMN_TYPE.IPV4:
+ return view!.getInt32(dense * 4, true);
+ case QWP_COLUMN_TYPE.FLOAT:
+ return view!.getFloat32(dense * 4, true);
+ case QWP_COLUMN_TYPE.DOUBLE:
+ return view!.getFloat64(dense * 8, true);
+ case QWP_COLUMN_TYPE.LONG:
+ case QWP_COLUMN_TYPE.DATE:
+ case QWP_COLUMN_TYPE.TIMESTAMP:
+ case QWP_COLUMN_TYPE.TIMESTAMP_NANOS:
+ return view!.getBigInt64(dense * 8, true);
+ case QWP_COLUMN_TYPE.VARCHAR:
+ return decodeUtf8(variableWidthValue(layout, dense));
+ case QWP_COLUMN_TYPE.BINARY:
+ return variableWidthValue(layout, dense);
+ case QWP_COLUMN_TYPE.SYMBOL:
+ return layout.symbolDictionary![layout.symbolRowIds![rowIndex]];
+ case QWP_COLUMN_TYPE.UUID:
+ return {
+ low: view!.getBigUint64(dense * 16, true),
+ high: view!.getBigUint64(dense * 16 + 8, true),
+ };
+ case QWP_COLUMN_TYPE.LONG256:
+ return {
+ words: [
+ view!.getBigInt64(dense * 32, true),
+ view!.getBigInt64(dense * 32 + 8, true),
+ view!.getBigInt64(dense * 32 + 16, true),
+ view!.getBigInt64(dense * 32 + 24, true),
+ ],
+ };
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ case QWP_COLUMN_TYPE.DECIMAL256: {
+ const width = fixedTypeWidth(layout.schema.type);
+ return {
+ unscaled: signedLittleEndianValue(
+ layout.values!,
+ dense * width,
+ width,
+ ),
+ scale: layout.scale!,
+ };
+ }
+ case QWP_COLUMN_TYPE.GEOHASH: {
+ const width = Math.ceil(layout.precisionBits! / 8);
+ return {
+ bits: unsignedLittleEndianValue(layout.values!, dense * width, width),
+ precisionBits: layout.precisionBits!,
+ };
+ }
+ case QWP_COLUMN_TYPE.DOUBLE_ARRAY:
+ case QWP_COLUMN_TYPE.LONG_ARRAY:
+ return readArrayValue(
+ new QwpByteReader(this.getArrayView(rowIndex)!),
+ layout.schema.type,
+ );
+ default:
+ throw new QwpProtocolError(
+ `unsupported QWP result column type: ${String(layout.schema.type)}`,
+ );
+ }
+ }
+
+ private layout(): QwpResultColumnViewLayout {
+ return this.batch.layout(this.columnIndex);
+ }
+
+ private checkedLayout(rowIndex: number): QwpResultColumnViewLayout {
+ const layout = this.layout();
+ if (
+ !Number.isInteger(rowIndex) ||
+ rowIndex < 0 ||
+ rowIndex >= layout.rowCount
+ ) {
+ throw new RangeError(`row index out of range: ${rowIndex}`);
+ }
+ return layout;
+ }
+
+ private requireType(
+ layout: QwpResultColumnViewLayout,
+ type1: QwpColumnType,
+ type2?: QwpColumnType,
+ type3?: QwpColumnType,
+ type4?: QwpColumnType,
+ ): void {
+ const actual = layout.schema.type;
+ if (
+ actual !== type1 &&
+ actual !== type2 &&
+ actual !== type3 &&
+ actual !== type4
+ ) {
+ throw new TypeError(
+ `column '${layout.schema.name}' has QWP type 0x${actual.toString(16)}`,
+ );
+ }
+ }
+
+ private valuePosition(
+ rowIndex: number,
+ type1: QwpColumnType,
+ type2?: QwpColumnType,
+ type3?: QwpColumnType,
+ type4?: QwpColumnType,
+ ): { layout: QwpResultColumnViewLayout; dense: number } {
+ const layout = this.checkedLayout(rowIndex);
+ this.requireType(layout, type1, type2, type3, type4);
+ return {
+ layout,
+ dense: layout.isNull(rowIndex) ? -1 : layout.denseIndex(rowIndex),
+ };
+ }
+}
+
+/** Callback invoked by QwpResultBatchView.forEachRow(). */
+export type QwpResultRowViewCallback = (row: QwpResultRowView) => void;
+
+/**
+ * Reusable row-pinned facade over a QwpResultBatchView.
+ *
+ * The batch owns one instance and re-points it in place. It is valid only
+ * while the surrounding queryViews() callback is running, and must not be
+ * retained across forEachRow() iterations. Byte and array views returned by
+ * its accessors remain zero-copy and have the same lifetime.
+ */
+export class QwpResultRowView {
+ private _rowIndex = -1;
+
+ /** @internal */
+ constructor(private readonly parent: QwpResultBatchView) {}
+
+ /** Parent batch, primarily for column metadata. */
+ get batch(): QwpResultBatchView {
+ // Validate the shared batch before exposing it through a retained row.
+ void this.parent.rowCount;
+ return this.parent;
+ }
+
+ /** Zero-based row currently pinned by this reusable view. */
+ get rowIndex(): number {
+ void this.parent.rowCount;
+ return this._rowIndex;
+ }
+
+ /** Re-points this flyweight at a row and returns the same instance. */
+ of(rowIndex: number): this {
+ const rowCount = this.parent.rowCount;
+ if (!Number.isInteger(rowIndex) || rowIndex < 0 || rowIndex >= rowCount) {
+ throw new RangeError(`row index out of range: ${rowIndex}`);
+ }
+ this._rowIndex = rowIndex;
+ return this;
+ }
+
+ isNull(columnIndex: number): boolean {
+ return this.column(columnIndex).isNull(this._rowIndex);
+ }
+
+ get(columnIndex: number): QwpResultValue {
+ return this.column(columnIndex).get(this._rowIndex);
+ }
+
+ getBoolean(columnIndex: number): boolean {
+ return this.column(columnIndex).getBoolean(this._rowIndex);
+ }
+
+ getByte(columnIndex: number): number {
+ return this.column(columnIndex).getByte(this._rowIndex);
+ }
+
+ getShort(columnIndex: number): number {
+ return this.column(columnIndex).getShort(this._rowIndex);
+ }
+
+ getChar(columnIndex: number): string {
+ return this.column(columnIndex).getChar(this._rowIndex);
+ }
+
+ getInt(columnIndex: number): number {
+ return this.column(columnIndex).getInt(this._rowIndex);
+ }
+
+ getFloat(columnIndex: number): number {
+ return this.column(columnIndex).getFloat(this._rowIndex);
+ }
+
+ getDouble(columnIndex: number): number {
+ return this.column(columnIndex).getDouble(this._rowIndex);
+ }
+
+ getLong(columnIndex: number): bigint {
+ return this.column(columnIndex).getLong(this._rowIndex);
+ }
+
+ /** Zero-copy UTF-8 bytes for a VARCHAR value. */
+ getUtf8View(columnIndex: number): Uint8Array | null {
+ return this.column(columnIndex).getUtf8View(this._rowIndex);
+ }
+
+ getString(columnIndex: number): string | null {
+ return this.column(columnIndex).getString(this._rowIndex);
+ }
+
+ /** Zero-copy BINARY bytes. */
+ getBinaryView(columnIndex: number): Uint8Array | null {
+ return this.column(columnIndex).getBinaryView(this._rowIndex);
+ }
+
+ getSymbolId(columnIndex: number): number {
+ return this.column(columnIndex).getSymbolId(this._rowIndex);
+ }
+
+ getSymbol(columnIndex: number): string | null {
+ return this.column(columnIndex).getSymbol(this._rowIndex);
+ }
+
+ getUuidLow(columnIndex: number): bigint {
+ return this.column(columnIndex).getUuidLow(this._rowIndex);
+ }
+
+ getUuidHigh(columnIndex: number): bigint {
+ return this.column(columnIndex).getUuidHigh(this._rowIndex);
+ }
+
+ getLong256Word(columnIndex: number, wordIndex: number): bigint {
+ return this.column(columnIndex).getLong256Word(this._rowIndex, wordIndex);
+ }
+
+ getDecimalUnscaled(columnIndex: number): bigint {
+ return this.column(columnIndex).getDecimalUnscaled(this._rowIndex);
+ }
+
+ getGeohashBits(columnIndex: number): bigint {
+ return this.column(columnIndex).getGeohashBits(this._rowIndex);
+ }
+
+ /** Zero-copy encoded ARRAY row, including its dimension header. */
+ getArrayView(columnIndex: number): Uint8Array | null {
+ return this.column(columnIndex).getArrayView(this._rowIndex);
+ }
+
+ getArrayDimensionCount(columnIndex: number): number {
+ return this.column(columnIndex).getArrayDimensionCount(this._rowIndex);
+ }
+
+ private column(columnIndex: number): QwpResultColumnView {
+ return this.parent.column(columnIndex);
+ }
+}
+
+/**
+ * Batch-owned reusable view delivered by QwpEgressSession.queryViews().
+ * Access is invalid after the callback returns. materialize() creates an
+ * independently owned QwpResultBatch when retention is required.
+ */
+export class QwpResultBatchView {
+ private active = false;
+ private _requestId = -1n;
+ private _batchSequence = -1n;
+ private _tableName = "";
+ private _rowCount = 0;
+ private layouts: QwpResultColumnViewLayout[] = [];
+ private readonly columnViews: QwpResultColumnView[] = [];
+ private readonly columnViewPool: QwpResultColumnView[] = [];
+ private rowView?: QwpResultRowView;
+
+ get valid(): boolean {
+ return this.active;
+ }
+
+ get requestId(): bigint {
+ this.assertValid();
+ return this._requestId;
+ }
+
+ get batchSequence(): bigint {
+ this.assertValid();
+ return this._batchSequence;
+ }
+
+ get tableName(): string {
+ this.assertValid();
+ return this._tableName;
+ }
+
+ get rowCount(): number {
+ this.assertValid();
+ return this._rowCount;
+ }
+
+ get columnCount(): number {
+ this.assertValid();
+ return this.layouts.length;
+ }
+
+ get columns(): readonly QwpResultColumnView[] {
+ this.assertValid();
+ return this.columnViews;
+ }
+
+ column(columnIndex: number): QwpResultColumnView {
+ this.assertValid();
+ const column = this.columnViews[columnIndex];
+ if (!column) {
+ throw new RangeError(`column index out of range: ${columnIndex}`);
+ }
+ return column;
+ }
+
+ get(rowIndex: number, columnIndex: number): QwpResultValue {
+ return this.column(columnIndex).get(rowIndex);
+ }
+
+ /**
+ * Returns the batch-owned reusable row view pinned to rowIndex. Every call
+ * returns the same object re-pointed at the requested row.
+ */
+ row(rowIndex: number): QwpResultRowView {
+ this.assertValid();
+ return this.reusableRowView().of(rowIndex);
+ }
+
+ /**
+ * Visits rows in index order with one re-pointed row view. The callback is
+ * synchronous; copy values that must survive the current invocation.
+ */
+ forEachRow(callback: QwpResultRowViewCallback): void {
+ this.assertValid();
+ if (this._rowCount === 0) return;
+ const rowView = this.reusableRowView();
+ for (let rowIndex = 0; rowIndex < this._rowCount; rowIndex++) {
+ callback(rowView.of(rowIndex));
+ }
+ }
+
+ materialize(): QwpResultBatch {
+ this.assertValid();
+ return new QwpResultBatch(
+ this._requestId,
+ this._batchSequence,
+ this._tableName,
+ this._rowCount,
+ this.columnViews.map((column) => ({
+ name: column.name,
+ type: column.type,
+ values: Array.from({ length: this._rowCount }, (_, row) => {
+ const value = column.get(row);
+ // Binary values are zero-copy slices in the view API. materialize()
+ // promises independently owned data, so detach those slices here.
+ return value instanceof Uint8Array ? value.slice() : value;
+ }),
+ ...(column.scale === undefined ? {} : { scale: column.scale }),
+ ...(column.precisionBits === undefined
+ ? {}
+ : { precisionBits: column.precisionBits }),
+ })),
+ );
+ }
+
+ /** Invalidates the view. Normally called automatically after queryViews(). */
+ release(): void {
+ if (!this.active) return;
+ this.active = false;
+ for (const layout of this.layouts) layout.release();
+ }
+
+ /** @internal */
+ reset(
+ requestId: bigint,
+ batchSequence: bigint,
+ tableName: string,
+ rowCount: number,
+ layouts: QwpResultColumnViewLayout[],
+ ): this {
+ this._requestId = requestId;
+ this._batchSequence = batchSequence;
+ this._tableName = tableName;
+ this._rowCount = rowCount;
+ this.layouts = layouts;
+ while (this.columnViewPool.length < layouts.length) {
+ this.columnViewPool.push(
+ new QwpResultColumnView(this, this.columnViewPool.length),
+ );
+ }
+ this.columnViews.length = layouts.length;
+ for (let index = 0; index < layouts.length; index++) {
+ this.columnViews[index] = this.columnViewPool[index];
+ }
+ this.active = true;
+ return this;
+ }
+
+ /** @internal */
+ layout(columnIndex: number): QwpResultColumnViewLayout {
+ this.assertValid();
+ const layout = this.layouts[columnIndex];
+ if (!layout) {
+ throw new RangeError(`column index out of range: ${columnIndex}`);
+ }
+ return layout;
+ }
+
+ private assertValid(): void {
+ if (!this.active) {
+ throw new Error(
+ "QWP result batch view is no longer valid; copy or materialize values inside the queryViews callback",
+ );
+ }
+ }
+
+ private reusableRowView(): QwpResultRowView {
+ return (this.rowView ??= new QwpResultRowView(this));
+ }
+}
+
+function variableWidthValue(
+ layout: QwpResultColumnViewLayout,
+ denseIndex: number,
+): Uint8Array {
+ const offsets = layout.valuesView!;
+ const start = offsets.getUint32(denseIndex * 4, true);
+ const end = offsets.getUint32((denseIndex + 1) * 4, true);
+ return layout.stringBytes!.subarray(start, end);
+}
+
+function fixedTypeWidth(type: QwpColumnType): number {
+ switch (type) {
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ return 8;
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ return 16;
+ case QWP_COLUMN_TYPE.DECIMAL256:
+ return 32;
+ default:
+ throw new TypeError(`QWP type 0x${type.toString(16)} is not decimal`);
+ }
+}
+
+// The scale the server sends is a single byte, so it must be bounded like the
+// encoder bounds it (QwpTableBuffer.setDecimalScale) and QWP_DECIMAL_MAX_SCALE
+// exports it. An unchecked 255 decodes to a value off by up to 10^237.
+function decimalMaxScale(type: QwpColumnType): number {
+ switch (type) {
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ return 18;
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ return 38;
+ case QWP_COLUMN_TYPE.DECIMAL256:
+ return 76;
+ default:
+ throw new TypeError(`QWP type 0x${type.toString(16)} is not decimal`);
+ }
+}
+
+function readDecimalScale(reader: QwpByteReader, type: QwpColumnType): number {
+ const scale = reader.readUint8("decimal scale");
+ const maximum = decimalMaxScale(type);
+ if (scale > maximum) {
+ throw new QwpProtocolError(
+ `decimal scale out of range: ${scale} (max ${maximum})`,
+ );
+ }
+ return scale;
+}
+
+function unsignedLittleEndianValue(
+ bytes: Uint8Array,
+ offset = 0,
+ length = bytes.length - offset,
+): bigint {
+ let value = 0n;
+ for (let index = 0; index < length; index++) {
+ value |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ }
+ return value;
+}
+
+function signedLittleEndianValue(
+ bytes: Uint8Array,
+ offset = 0,
+ length = bytes.length - offset,
+): bigint {
+ const value = unsignedLittleEndianValue(bytes, offset, length);
+ const bits = BigInt(length * 8);
+ const sign = 1n << (bits - 1n);
+ return (value & sign) === 0n ? value : value - (1n << bits);
+}
+
+interface NullLayout {
+ nulls: boolean[];
+ nonNullCount: number;
+}
+
+class QwpBitReader {
+ private bitPosition = 0;
+
+ constructor(private readonly bytes: Uint8Array) {}
+
+ get bytesConsumed(): number {
+ return Math.ceil(this.bitPosition / 8);
+ }
+
+ readBit(): number {
+ if (this.bitPosition >= this.bytes.length * 8) {
+ throw new QwpProtocolError("truncated QWP Gorilla bitstream");
+ }
+ const result =
+ (this.bytes[this.bitPosition >>> 3] >>> (this.bitPosition & 7)) & 1;
+ this.bitPosition++;
+ return result;
+ }
+
+ readSigned(bitCount: number): bigint {
+ let value = 0n;
+ for (let bit = 0; bit < bitCount; bit++) {
+ if (this.readBit() !== 0) value |= 1n << BigInt(bit);
+ }
+ const sign = 1n << BigInt(bitCount - 1);
+ return (value & sign) === 0n ? value : value - (1n << BigInt(bitCount));
+ }
+}
+
+function readCount(
+ reader: QwpByteReader,
+ maximum: number,
+ label: string,
+): number {
+ const value = readQwpVarint(reader);
+ if (value > BigInt(maximum)) {
+ throw new QwpProtocolError(`${label} out of range: ${value}`);
+ }
+ return Number(value);
+}
+
+function readNullLayout(reader: QwpByteReader, rowCount: number): NullLayout {
+ const flag = reader.readUint8("column null flag");
+ if (flag !== 0 && flag !== 1) {
+ throw new QwpProtocolError(`invalid column null flag: ${flag}`);
+ }
+ const nulls = new Array(rowCount).fill(false);
+ if (flag === 0) return { nulls, nonNullCount: rowCount };
+
+ const bitmap = reader.readBytes(
+ Math.ceil(rowCount / 8),
+ "column null bitmap",
+ );
+ let nonNullCount = rowCount;
+ for (let row = 0; row < rowCount; row++) {
+ if ((bitmap[row >>> 3] & (1 << (row & 7))) !== 0) {
+ nulls[row] = true;
+ nonNullCount--;
+ }
+ }
+ return { nulls, nonNullCount };
+}
+
+function expandNulls(
+ dense: readonly T[],
+ layout: NullLayout,
+): QwpResultValue[] {
+ const values = new Array(layout.nulls.length);
+ let denseIndex = 0;
+ for (let row = 0; row < layout.nulls.length; row++) {
+ values[row] = layout.nulls[row] ? null : dense[denseIndex++];
+ }
+ return values;
+}
+
+function readSignedLittleEndian(
+ reader: QwpByteReader,
+ byteCount: number,
+ label: string,
+): bigint {
+ const bytes = reader.readBytes(byteCount, label);
+ let value = 0n;
+ for (let index = 0; index < byteCount; index++) {
+ value |= BigInt(bytes[index]) << BigInt(index * 8);
+ }
+ const bits = BigInt(byteCount * 8);
+ const sign = 1n << (bits - 1n);
+ return (value & sign) === 0n ? value : value - (1n << bits);
+}
+
+function readStringValues(
+ reader: QwpByteReader,
+ count: number,
+ binary: boolean,
+): (string | Uint8Array)[] {
+ const offsets = new Array(count + 1);
+ for (let index = 0; index <= count; index++) {
+ offsets[index] = reader.readUint32("variable-width column offset");
+ }
+ if (offsets[0] !== 0) {
+ throw new QwpProtocolError(
+ "variable-width column must start at offset zero",
+ );
+ }
+ for (let index = 1; index < offsets.length; index++) {
+ if (offsets[index] < offsets[index - 1]) {
+ throw new QwpProtocolError(
+ `variable-width column offsets are not monotonic at index ${index}`,
+ );
+ }
+ }
+ const bytes = reader.readBytes(offsets[count], "variable-width column data");
+ const values = new Array(count);
+ for (let index = 0; index < count; index++) {
+ const value = bytes.subarray(offsets[index], offsets[index + 1]);
+ values[index] = binary ? value.slice() : decodeUtf8(value);
+ }
+ return values;
+}
+
+function decodeGorillaValues(reader: QwpByteReader, count: number): bigint[] {
+ if (count < 3) {
+ throw new QwpProtocolError(
+ `Gorilla-encoded column has fewer than three values: ${count}`,
+ );
+ }
+ const first = reader.readBigInt64("first Gorilla timestamp");
+ const second = reader.readBigInt64("second Gorilla timestamp");
+ const values = [first, second];
+ const bits = new QwpBitReader(
+ reader.bytes.subarray(reader.position, reader.position + reader.remaining),
+ );
+ let previousTimestamp = second;
+ let previousDelta = BigInt.asIntN(64, second - first);
+ for (let index = 2; index < count; index++) {
+ let deltaOfDelta: bigint;
+ let prefixOnes = 0;
+ while (prefixOnes < 4 && bits.readBit() !== 0) prefixOnes++;
+ switch (prefixOnes) {
+ case 0:
+ deltaOfDelta = 0n;
+ break;
+ case 1:
+ deltaOfDelta = bits.readSigned(7);
+ break;
+ case 2:
+ deltaOfDelta = bits.readSigned(9);
+ break;
+ case 3:
+ deltaOfDelta = bits.readSigned(12);
+ break;
+ default:
+ deltaOfDelta = bits.readSigned(32);
+ }
+ const delta = BigInt.asIntN(64, previousDelta + deltaOfDelta);
+ const timestamp = BigInt.asIntN(64, previousTimestamp + delta);
+ values.push(timestamp);
+ previousDelta = delta;
+ previousTimestamp = timestamp;
+ }
+ reader.readBytes(bits.bytesConsumed, "Gorilla bitstream");
+ return values;
+}
+
+function readTimestampValues(
+ reader: QwpByteReader,
+ count: number,
+ gorilla: boolean,
+): bigint[] {
+ if (!gorilla) {
+ return Array.from({ length: count }, () =>
+ reader.readBigInt64("timestamp value"),
+ );
+ }
+ const encoding = reader.readUint8("timestamp encoding");
+ if (encoding === 0) {
+ return Array.from({ length: count }, () =>
+ reader.readBigInt64("timestamp value"),
+ );
+ }
+ if (encoding !== 1) {
+ throw new QwpProtocolError(`unknown timestamp encoding: ${encoding}`);
+ }
+ return decodeGorillaValues(reader, count);
+}
+
+function readArrayValue(
+ reader: QwpByteReader,
+ type: QwpColumnType,
+): QwpResultArrayValue {
+ const dimensions = reader.readUint8("array dimension count");
+ if (dimensions < 1 || dimensions > 32) {
+ throw new QwpProtocolError(
+ `array dimension count out of range: ${dimensions}`,
+ );
+ }
+ const shape = new Array(dimensions);
+ let elementCount = 1;
+ for (let index = 0; index < dimensions; index++) {
+ const length = reader.readInt32("array dimension length");
+ if (length < 0 || length > MAX_ARRAY_DIMENSION_LENGTH) {
+ throw new QwpProtocolError(
+ `array dimension length out of range: ${length}`,
+ );
+ }
+ shape[index] = length;
+ elementCount *= length;
+ if (elementCount > MAX_ARRAY_ELEMENTS) {
+ throw new QwpProtocolError(
+ `array element count exceeds ${MAX_ARRAY_ELEMENTS}`,
+ );
+ }
+ }
+ if (elementCount > Math.floor(reader.remaining / 8)) {
+ throw new QwpProtocolError("truncated array payload");
+ }
+ if (type === QWP_COLUMN_TYPE.DOUBLE_ARRAY) {
+ return {
+ dimensions: shape,
+ values: Array.from({ length: elementCount }, () =>
+ reader.readFloat64("double array element"),
+ ),
+ };
+ }
+ return {
+ dimensions: shape,
+ values: Array.from({ length: elementCount }, () =>
+ reader.readBigInt64("long array element"),
+ ),
+ };
+}
+
+interface PreparedResultBatch {
+ readonly reader: QwpByteReader;
+ readonly tableName: string;
+ readonly rowCount: number;
+ readonly deltaMode: boolean;
+}
+
+/** Stateful decoder for connection-scoped QWP result batches. */
+export class QwpResultBatchDecoder {
+ private readonly symbolDictionary: string[] = [];
+ private readonly viewBatches: QwpResultBatchView[] = [];
+ private readonly viewLayouts: QwpResultColumnViewLayout[][] = [];
+ private readonly viewLayoutPools: QwpResultColumnViewLayout[][] = [];
+ private schema?: QwpResultColumnSchema[];
+ private expectedBatchSequence = 0n;
+
+ resetQuerySchema(): void {
+ for (const batch of this.viewBatches) batch.release();
+ this.schema = undefined;
+ this.expectedBatchSequence = 0n;
+ }
+
+ applyCacheReset(resetMask: number): void {
+ if ((resetMask & QWP_RESET_MASK_DICTIONARY) !== 0) {
+ this.symbolDictionary.length = 0;
+ }
+ }
+
+ /** @internal Drops frame-backed references after a failed slot decode. */
+ releaseView(slot: number): void {
+ this.viewBatches[slot]?.release();
+ for (const layout of this.viewLayoutPools[slot] ?? []) layout.release();
+ }
+
+ decode(message: QwpResultBatchMessage): QwpResultBatch {
+ const { reader, tableName, rowCount, deltaMode } = this.prepare(message);
+
+ const columns = this.schema!.map((column) =>
+ this.readColumn(reader, column, rowCount, deltaMode, message.flags),
+ );
+ reader.expectEnd("RESULT_BATCH");
+ this.expectedBatchSequence++;
+ return new QwpResultBatch(
+ message.requestId,
+ message.batchSequence,
+ tableName,
+ rowCount,
+ columns,
+ );
+ }
+
+ /**
+ * Decodes into one slot from a reusable batch/column-view pool without
+ * materializing a JavaScript value array. Reusing the same slot invalidates
+ * its prior view; callers must not reuse a slot until its consumer releases
+ * the preceding batch.
+ */
+ decodeView(message: QwpResultBatchMessage, slot = 0): QwpResultBatchView {
+ if (!Number.isSafeInteger(slot) || slot < 0) {
+ throw new RangeError(
+ "QWP result view slot must be a non-negative integer",
+ );
+ }
+ const viewBatch = (this.viewBatches[slot] ??= new QwpResultBatchView());
+ const viewLayouts = (this.viewLayouts[slot] ??= []);
+ const viewLayoutPool = (this.viewLayoutPools[slot] ??= []);
+ viewBatch.release();
+ const { reader, tableName, rowCount, deltaMode } = this.prepare(message);
+ const schema = this.schema!;
+ while (viewLayoutPool.length < schema.length) {
+ viewLayoutPool.push(new QwpResultColumnViewLayout());
+ }
+ viewLayouts.length = schema.length;
+ for (let index = 0; index < schema.length; index++) {
+ const layout = viewLayoutPool[index];
+ viewLayouts[index] = layout;
+ layout.reset(schema[index], rowCount);
+ this.readColumnView(reader, layout, deltaMode, message.flags);
+ }
+ reader.expectEnd("RESULT_BATCH");
+ this.expectedBatchSequence++;
+ return viewBatch.reset(
+ message.requestId,
+ message.batchSequence,
+ tableName,
+ rowCount,
+ viewLayouts,
+ );
+ }
+
+ private prepare(message: QwpResultBatchMessage): PreparedResultBatch {
+ if (message.tableCount !== 1) {
+ throw new QwpProtocolError(
+ `RESULT_BATCH must contain exactly one table, got ${message.tableCount}`,
+ );
+ }
+ if (message.batchSequence !== this.expectedBatchSequence) {
+ throw new QwpProtocolError(
+ `unexpected RESULT_BATCH sequence [expected=${this.expectedBatchSequence}, actual=${message.batchSequence}]`,
+ );
+ }
+
+ const body =
+ (message.flags & QWP_FLAG_ZSTD) !== 0
+ ? decompressQwpZstdFrame(message.body)
+ : message.body;
+ const reader = new QwpByteReader(body);
+ const deltaMode = (message.flags & QWP_FLAG_DELTA_SYMBOL_DICTIONARY) !== 0;
+ if (deltaMode) this.readDeltaDictionary(reader, body.length);
+
+ const tableNameLength = readCount(
+ reader,
+ QWP_MAX_IDENTIFIER_BYTES,
+ "table name length",
+ );
+ const tableName = reader.readUtf8(tableNameLength, "table name");
+ const rowCount = readCount(reader, MAX_ROWS_PER_BATCH, "result row count");
+
+ if (message.batchSequence === 0n) {
+ const columnCount = readCount(
+ reader,
+ QWP_MAX_COLUMNS_PER_TABLE,
+ "result column count",
+ );
+ this.schema = Array.from({ length: columnCount }, () => {
+ const nameLength = readCount(
+ reader,
+ QWP_MAX_IDENTIFIER_BYTES,
+ "column name length",
+ );
+ const name = reader.readUtf8(nameLength, "column name");
+ const type = reader.readUint8("column type") as QwpColumnType;
+ if (!Object.values(QWP_COLUMN_TYPE).includes(type)) {
+ throw new QwpProtocolError(
+ `unsupported QWP result column type: 0x${type.toString(16)}`,
+ );
+ }
+ return { name, type };
+ });
+ } else if (!this.schema) {
+ throw new QwpProtocolError(
+ "continuation RESULT_BATCH arrived before its schema-bearing batch",
+ );
+ }
+ // Each dimension passed its own cap; the grid they describe still has to
+ // be one this client will allocate. Checked before any column is read,
+ // because reading one is what allocates.
+ const cells = rowCount * this.schema.length;
+ if (cells > QWP_MAX_CELLS_PER_BATCH) {
+ throw new QwpProtocolError(
+ `RESULT_BATCH declares ${cells} cells, above the client cap ${QWP_MAX_CELLS_PER_BATCH} [rows=${rowCount}, columns=${this.schema.length}]`,
+ );
+ }
+ return { reader, tableName, rowCount, deltaMode };
+ }
+
+ private readColumnView(
+ reader: QwpByteReader,
+ layout: QwpResultColumnViewLayout,
+ deltaMode: boolean,
+ flags: number,
+ ): void {
+ this.readNullView(reader, layout);
+ const count = layout.nonNullCount;
+ const type = layout.schema.type;
+ switch (type) {
+ case QWP_COLUMN_TYPE.BOOLEAN:
+ layout.setValues(
+ reader.readBytes(Math.ceil(count / 8), "boolean values"),
+ );
+ return;
+ case QWP_COLUMN_TYPE.BYTE:
+ this.readFixedView(reader, layout, count, 1, "byte values");
+ return;
+ case QWP_COLUMN_TYPE.SHORT:
+ case QWP_COLUMN_TYPE.CHAR:
+ this.readFixedView(reader, layout, count, 2, "short values");
+ return;
+ case QWP_COLUMN_TYPE.INT:
+ case QWP_COLUMN_TYPE.FLOAT:
+ case QWP_COLUMN_TYPE.IPV4:
+ this.readFixedView(reader, layout, count, 4, "int values");
+ return;
+ case QWP_COLUMN_TYPE.LONG:
+ case QWP_COLUMN_TYPE.DOUBLE:
+ this.readFixedView(reader, layout, count, 8, "long values");
+ return;
+ case QWP_COLUMN_TYPE.DATE:
+ case QWP_COLUMN_TYPE.TIMESTAMP:
+ case QWP_COLUMN_TYPE.TIMESTAMP_NANOS:
+ this.readTimestampView(reader, layout, flags);
+ return;
+ case QWP_COLUMN_TYPE.VARCHAR:
+ case QWP_COLUMN_TYPE.BINARY:
+ this.readVariableWidthView(reader, layout);
+ return;
+ case QWP_COLUMN_TYPE.SYMBOL:
+ this.readSymbolView(reader, layout, deltaMode);
+ return;
+ case QWP_COLUMN_TYPE.UUID:
+ this.readFixedView(reader, layout, count, 16, "UUID values");
+ return;
+ case QWP_COLUMN_TYPE.LONG256:
+ this.readFixedView(reader, layout, count, 32, "LONG256 values");
+ return;
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ case QWP_COLUMN_TYPE.DECIMAL256: {
+ layout.scale = readDecimalScale(reader, type);
+ this.readFixedView(
+ reader,
+ layout,
+ count,
+ fixedTypeWidth(type),
+ "decimal values",
+ );
+ return;
+ }
+ case QWP_COLUMN_TYPE.GEOHASH: {
+ layout.precisionBits = readCount(reader, 60, "geohash precision");
+ if (layout.precisionBits < 1) {
+ throw new QwpProtocolError(
+ `geohash precision out of range: ${layout.precisionBits}`,
+ );
+ }
+ this.readFixedView(
+ reader,
+ layout,
+ count,
+ Math.ceil(layout.precisionBits / 8),
+ "geohash values",
+ );
+ return;
+ }
+ case QWP_COLUMN_TYPE.DOUBLE_ARRAY:
+ case QWP_COLUMN_TYPE.LONG_ARRAY:
+ this.readArrayView(reader, layout);
+ return;
+ default:
+ throw new QwpProtocolError(
+ `unsupported QWP result column type: ${String(type)}`,
+ );
+ }
+ }
+
+ private readNullView(
+ reader: QwpByteReader,
+ layout: QwpResultColumnViewLayout,
+ ): void {
+ const flag = reader.readUint8("column null flag");
+ if (flag !== 0 && flag !== 1) {
+ throw new QwpProtocolError(`invalid column null flag: ${flag}`);
+ }
+ if (flag === 0) {
+ layout.nonNullCount = layout.rowCount;
+ return;
+ }
+ const bitmap = reader.readBytes(
+ Math.ceil(layout.rowCount / 8),
+ "column null bitmap",
+ );
+ layout.nullBitmap = bitmap;
+ const indexes = layout.ensureNonNullIndexes(layout.rowCount);
+ let dense = 0;
+ for (let row = 0; row < layout.rowCount; row++) {
+ if ((bitmap[row >>> 3] & (1 << (row & 7))) !== 0) {
+ indexes[row] = -1;
+ } else {
+ indexes[row] = dense++;
+ }
+ }
+ layout.nonNullCount = dense;
+ }
+
+ private readFixedView(
+ reader: QwpByteReader,
+ layout: QwpResultColumnViewLayout,
+ count: number,
+ width: number,
+ label: string,
+ ): void {
+ layout.setValues(reader.readBytes(count * width, label));
+ }
+
+ private readVariableWidthView(
+ reader: QwpByteReader,
+ layout: QwpResultColumnViewLayout,
+ ): void {
+ const count = layout.nonNullCount;
+ const offsets = reader.readBytes(
+ (count + 1) * 4,
+ "variable-width column offsets",
+ );
+ const view = new DataView(
+ offsets.buffer,
+ offsets.byteOffset,
+ offsets.byteLength,
+ );
+ if (view.getUint32(0, true) !== 0) {
+ throw new QwpProtocolError(
+ "variable-width column must start at offset zero",
+ );
+ }
+ let previous = 0;
+ for (let index = 1; index <= count; index++) {
+ const offset = view.getUint32(index * 4, true);
+ if (offset < previous) {
+ throw new QwpProtocolError(
+ `variable-width column offsets are not monotonic at index ${index}`,
+ );
+ }
+ previous = offset;
+ }
+ layout.setValues(offsets);
+ layout.stringBytes = reader.readBytes(
+ previous,
+ "variable-width column data",
+ );
+ }
+
+ private readSymbolView(
+ reader: QwpByteReader,
+ layout: QwpResultColumnViewLayout,
+ deltaMode: boolean,
+ ): void {
+ let dictionary: readonly string[];
+ if (deltaMode) {
+ dictionary = this.symbolDictionary;
+ } else {
+ const size = readCount(reader, layout.rowCount, "symbol dictionary size");
+ const local = layout.localSymbols;
+ for (let index = 0; index < size; index++) {
+ const length = readCount(reader, reader.remaining, "symbol length");
+ local.push(reader.readUtf8(length, "symbol"));
+ }
+ dictionary = local;
+ }
+ layout.symbolDictionary = dictionary;
+ const ids = layout.ensureSymbolRowIds(layout.rowCount);
+ for (let row = 0; row < layout.rowCount; row++) {
+ if (layout.isNull(row)) continue;
+ const id = readCount(reader, dictionary.length, "symbol ID");
+ if (id >= dictionary.length) {
+ throw new QwpProtocolError(`symbol ID out of range: ${id}`);
+ }
+ ids[row] = id;
+ }
+ }
+
+ private readArrayView(
+ reader: QwpByteReader,
+ layout: QwpResultColumnViewLayout,
+ ): void {
+ const start = reader.position;
+ const offsets = layout.ensureArrayOffsets(layout.rowCount);
+ const lengths = layout.ensureArrayLengths(layout.rowCount);
+ for (let row = 0; row < layout.rowCount; row++) {
+ if (layout.isNull(row)) {
+ offsets[row] = 0;
+ lengths[row] = 0;
+ continue;
+ }
+ const rowStart = reader.position;
+ const dimensions = reader.readUint8("array dimension count");
+ if (dimensions < 1 || dimensions > 32) {
+ throw new QwpProtocolError(
+ `array dimension count out of range: ${dimensions}`,
+ );
+ }
+ let elementCount = 1;
+ for (let index = 0; index < dimensions; index++) {
+ const length = reader.readInt32("array dimension length");
+ if (length < 0 || length > MAX_ARRAY_DIMENSION_LENGTH) {
+ throw new QwpProtocolError(
+ `array dimension length out of range: ${length}`,
+ );
+ }
+ elementCount *= length;
+ if (elementCount > MAX_ARRAY_ELEMENTS) {
+ throw new QwpProtocolError(
+ `array element count exceeds ${MAX_ARRAY_ELEMENTS}`,
+ );
+ }
+ }
+ reader.readBytes(elementCount * 8, "array payload");
+ offsets[row] = rowStart - start;
+ lengths[row] = reader.position - rowStart;
+ }
+ layout.setValues(reader.bytes.subarray(start, reader.position));
+ }
+
+ private readTimestampView(
+ reader: QwpByteReader,
+ layout: QwpResultColumnViewLayout,
+ flags: number,
+ ): void {
+ const count = layout.nonNullCount;
+ if ((flags & QWP_FLAG_GORILLA) === 0) {
+ this.readFixedView(reader, layout, count, 8, "timestamp values");
+ return;
+ }
+ const encoding = reader.readUint8("timestamp encoding");
+ if (encoding === 0) {
+ this.readFixedView(reader, layout, count, 8, "timestamp values");
+ return;
+ }
+ if (encoding !== 1) {
+ throw new QwpProtocolError(`unknown timestamp encoding: ${encoding}`);
+ }
+ if (count < 3) {
+ throw new QwpProtocolError(
+ `Gorilla-encoded column has fewer than three values: ${count}`,
+ );
+ }
+ const bytes = layout.timestampBytes(count * 8);
+ const decoded = new DataView(
+ bytes.buffer,
+ bytes.byteOffset,
+ bytes.byteLength,
+ );
+ const first = reader.readBigInt64("first Gorilla timestamp");
+ const second = reader.readBigInt64("second Gorilla timestamp");
+ decoded.setBigInt64(0, first, true);
+ decoded.setBigInt64(8, second, true);
+ const bits = new QwpBitReader(
+ reader.bytes.subarray(
+ reader.position,
+ reader.position + reader.remaining,
+ ),
+ );
+ let previousTimestamp = second;
+ let previousDelta = BigInt.asIntN(64, second - first);
+ for (let index = 2; index < count; index++) {
+ let deltaOfDelta: bigint;
+ let prefixOnes = 0;
+ while (prefixOnes < 4 && bits.readBit() !== 0) prefixOnes++;
+ switch (prefixOnes) {
+ case 0:
+ deltaOfDelta = 0n;
+ break;
+ case 1:
+ deltaOfDelta = bits.readSigned(7);
+ break;
+ case 2:
+ deltaOfDelta = bits.readSigned(9);
+ break;
+ case 3:
+ deltaOfDelta = bits.readSigned(12);
+ break;
+ default:
+ deltaOfDelta = bits.readSigned(32);
+ }
+ const delta = BigInt.asIntN(64, previousDelta + deltaOfDelta);
+ const timestamp = BigInt.asIntN(64, previousTimestamp + delta);
+ decoded.setBigInt64(index * 8, timestamp, true);
+ previousDelta = delta;
+ previousTimestamp = timestamp;
+ }
+ reader.readBytes(bits.bytesConsumed, "Gorilla bitstream");
+ layout.setValues(bytes);
+ }
+
+ private readColumn(
+ reader: QwpByteReader,
+ schema: QwpResultColumnSchema,
+ rowCount: number,
+ deltaMode: boolean,
+ flags: number,
+ ): QwpResultColumn {
+ const layout = readNullLayout(reader, rowCount);
+ const count = layout.nonNullCount;
+ let dense: QwpResultValue[];
+ let scale: number | undefined;
+ let precisionBits: number | undefined;
+
+ switch (schema.type) {
+ case QWP_COLUMN_TYPE.BOOLEAN: {
+ const bytes = reader.readBytes(Math.ceil(count / 8), "boolean values");
+ dense = Array.from(
+ { length: count },
+ (_, index) => (bytes[index >>> 3] & (1 << (index & 7))) !== 0,
+ );
+ break;
+ }
+ case QWP_COLUMN_TYPE.BYTE:
+ dense = Array.from({ length: count }, () =>
+ reader.readInt8("byte value"),
+ );
+ break;
+ case QWP_COLUMN_TYPE.SHORT:
+ dense = Array.from({ length: count }, () =>
+ reader.readInt16("short value"),
+ );
+ break;
+ case QWP_COLUMN_TYPE.CHAR:
+ dense = Array.from({ length: count }, () =>
+ String.fromCharCode(reader.readUint16("char value")),
+ );
+ break;
+ case QWP_COLUMN_TYPE.INT:
+ case QWP_COLUMN_TYPE.IPV4:
+ dense = Array.from({ length: count }, () =>
+ reader.readInt32("int value"),
+ );
+ break;
+ case QWP_COLUMN_TYPE.FLOAT:
+ dense = Array.from({ length: count }, () =>
+ reader.readFloat32("float value"),
+ );
+ break;
+ case QWP_COLUMN_TYPE.DOUBLE:
+ dense = Array.from({ length: count }, () =>
+ reader.readFloat64("double value"),
+ );
+ break;
+ case QWP_COLUMN_TYPE.LONG:
+ dense = Array.from({ length: count }, () =>
+ reader.readBigInt64("long value"),
+ );
+ break;
+ case QWP_COLUMN_TYPE.DATE:
+ case QWP_COLUMN_TYPE.TIMESTAMP:
+ case QWP_COLUMN_TYPE.TIMESTAMP_NANOS:
+ dense = readTimestampValues(
+ reader,
+ count,
+ (flags & QWP_FLAG_GORILLA) !== 0,
+ );
+ break;
+ case QWP_COLUMN_TYPE.VARCHAR:
+ dense = readStringValues(reader, count, false);
+ break;
+ case QWP_COLUMN_TYPE.BINARY:
+ dense = readStringValues(reader, count, true);
+ break;
+ case QWP_COLUMN_TYPE.SYMBOL:
+ dense = this.readSymbols(reader, count, rowCount, deltaMode);
+ break;
+ case QWP_COLUMN_TYPE.UUID:
+ dense = Array.from({ length: count }, () => ({
+ low: reader.readBigUint64("UUID low bits"),
+ high: reader.readBigUint64("UUID high bits"),
+ }));
+ break;
+ case QWP_COLUMN_TYPE.LONG256:
+ dense = Array.from({ length: count }, () => ({
+ words: [
+ reader.readBigInt64("LONG256 word 0"),
+ reader.readBigInt64("LONG256 word 1"),
+ reader.readBigInt64("LONG256 word 2"),
+ reader.readBigInt64("LONG256 word 3"),
+ ] as const,
+ }));
+ break;
+ case QWP_COLUMN_TYPE.DECIMAL64:
+ case QWP_COLUMN_TYPE.DECIMAL128:
+ case QWP_COLUMN_TYPE.DECIMAL256: {
+ scale = readDecimalScale(reader, schema.type);
+ const bytes =
+ schema.type === QWP_COLUMN_TYPE.DECIMAL64
+ ? 8
+ : schema.type === QWP_COLUMN_TYPE.DECIMAL128
+ ? 16
+ : 32;
+ dense = Array.from({ length: count }, () => ({
+ unscaled: readSignedLittleEndian(reader, bytes, "decimal value"),
+ scale: scale!,
+ }));
+ break;
+ }
+ case QWP_COLUMN_TYPE.GEOHASH: {
+ precisionBits = readCount(reader, 60, "geohash precision");
+ if (precisionBits < 1) {
+ throw new QwpProtocolError(
+ `geohash precision out of range: ${precisionBits}`,
+ );
+ }
+ const byteCount = Math.ceil(precisionBits / 8);
+ dense = Array.from({ length: count }, () => {
+ const bytes = reader.readBytes(byteCount, "geohash value");
+ let bits = 0n;
+ for (let index = 0; index < bytes.length; index++) {
+ bits |= BigInt(bytes[index]) << BigInt(index * 8);
+ }
+ return { bits, precisionBits: precisionBits! };
+ });
+ break;
+ }
+ case QWP_COLUMN_TYPE.DOUBLE_ARRAY:
+ case QWP_COLUMN_TYPE.LONG_ARRAY:
+ dense = Array.from({ length: count }, () =>
+ readArrayValue(reader, schema.type),
+ );
+ break;
+ default:
+ throw new QwpProtocolError(
+ `unsupported QWP result column type: ${String(schema.type)}`,
+ );
+ }
+
+ return {
+ ...schema,
+ values: expandNulls(dense, layout),
+ ...(scale === undefined ? {} : { scale }),
+ ...(precisionBits === undefined ? {} : { precisionBits }),
+ };
+ }
+
+ private readDeltaDictionary(
+ reader: QwpByteReader,
+ decompressedPayloadBytes: number,
+ ): void {
+ const start = readCount(
+ reader,
+ MAX_CONNECTION_SYMBOLS,
+ "delta dictionary start",
+ );
+ const count = readCount(
+ reader,
+ MAX_CONNECTION_SYMBOLS,
+ "delta dictionary count",
+ );
+ if (start !== this.symbolDictionary.length) {
+ throw new QwpProtocolError(
+ `delta symbol dictionary is out of sync [expected=${this.symbolDictionary.length}, actual=${start}]`,
+ );
+ }
+ if (start + count > MAX_CONNECTION_SYMBOLS) {
+ throw new QwpProtocolError(
+ `symbol dictionary exceeds ${MAX_CONNECTION_SYMBOLS} entries`,
+ );
+ }
+ // Each declared entry occupies at least one length byte in the decompressed
+ // body. Check that structural lower bound before the loop, because reading
+ // an entry is what allocates. The compressed length is not a valid bound:
+ // a legitimate dictionary with repetitive symbols may compress below its
+ // entry count.
+ if (count > decompressedPayloadBytes) {
+ throw new QwpProtocolError(
+ `delta symbol dictionary declares ${count} entries, above the ${decompressedPayloadBytes}-byte decompressed payload`,
+ );
+ }
+ for (let index = 0; index < count; index++) {
+ const length = readCount(reader, reader.remaining, "symbol length");
+ this.symbolDictionary.push(reader.readUtf8(length, "symbol"));
+ }
+ }
+
+ private readSymbols(
+ reader: QwpByteReader,
+ count: number,
+ rowCount: number,
+ deltaMode: boolean,
+ ): string[] {
+ let dictionary: readonly string[];
+ if (deltaMode) {
+ dictionary = this.symbolDictionary;
+ } else {
+ const size = readCount(reader, rowCount, "symbol dictionary size");
+ const local = new Array(size);
+ for (let index = 0; index < size; index++) {
+ const length = readCount(reader, reader.remaining, "symbol length");
+ local[index] = reader.readUtf8(length, "symbol");
+ }
+ dictionary = local;
+ }
+ return Array.from({ length: count }, () => {
+ const id = readCount(reader, dictionary.length, "symbol ID");
+ if (id >= dictionary.length) {
+ throw new QwpProtocolError(`symbol ID out of range: ${id}`);
+ }
+ return dictionary[id];
+ });
+ }
+}
diff --git a/src/_qwp/_core/symbol-dictionary.ts b/src/_qwp/_core/symbol-dictionary.ts
new file mode 100644
index 0000000..cfa70b0
--- /dev/null
+++ b/src/_qwp/_core/symbol-dictionary.ts
@@ -0,0 +1,62 @@
+import { QWP_MAX_SYMBOL_DICTIONARY_SIZE } from "./constants";
+
+/** Connection-scoped QWP symbol dictionary. IDs are dense from zero. */
+export class QwpSymbolDictionary {
+ private readonly ids = new Map();
+ private readonly values: string[] = [];
+
+ get size(): number {
+ return this.values.length;
+ }
+
+ getOrAdd(value: string): number {
+ const existing = this.ids.get(value);
+ if (existing !== undefined) return existing;
+ if (this.values.length >= QWP_MAX_SYMBOL_DICTIONARY_SIZE) {
+ throw new Error(
+ `symbol dictionary exceeds maximum size ${QWP_MAX_SYMBOL_DICTIONARY_SIZE}`,
+ );
+ }
+ const id = this.values.length;
+ this.ids.set(value, id);
+ this.values.push(value);
+ return id;
+ }
+
+ valueAt(id: number): string | undefined {
+ return this.values[id];
+ }
+
+ /** Appends positionally without de-duplicating recovered entries. */
+ addRecovered(value: string): number {
+ if (this.values.length >= QWP_MAX_SYMBOL_DICTIONARY_SIZE) {
+ throw new Error(
+ `symbol dictionary exceeds maximum size ${QWP_MAX_SYMBOL_DICTIONARY_SIZE}`,
+ );
+ }
+ const id = this.values.length;
+ this.values.push(value);
+ this.ids.set(value, id);
+ return id;
+ }
+
+ entriesFrom(startId: number): string[] {
+ return this.values.slice(Math.max(0, startId));
+ }
+
+ /** Rolls back entries added while preparing a frame that was not published. */
+ truncate(size: number): void {
+ if (!Number.isSafeInteger(size) || size < 0 || size > this.values.length) {
+ throw new RangeError(`invalid symbol dictionary size ${size}`);
+ }
+ if (size === this.values.length) return;
+ this.values.length = size;
+ this.ids.clear();
+ this.values.forEach((value, id) => this.ids.set(value, id));
+ }
+
+ reset(): void {
+ this.ids.clear();
+ this.values.length = 0;
+ }
+}
diff --git a/src/_qwp/_core/table.ts b/src/_qwp/_core/table.ts
new file mode 100644
index 0000000..b1a793c
--- /dev/null
+++ b/src/_qwp/_core/table.ts
@@ -0,0 +1,315 @@
+import {
+ QWP_COLUMN_TYPE,
+ QWP_MAX_COLUMNS_PER_TABLE,
+ QWP_MAX_TABLE_NAME_LENGTH,
+ QwpColumnType,
+} from "./constants";
+import {
+ qwpColumnNameKey,
+ validateQwpColumnName,
+ validateQwpTableName,
+} from "./identifiers";
+
+export interface QwpSymbolValue {
+ id: number;
+ text: string;
+}
+
+export interface QwpArrayValue {
+ dimensions: number[];
+ values: (number | bigint)[];
+}
+
+export interface QwpColumnBuffer {
+ name: string;
+ type: QwpColumnType;
+ /** Non-null values only; QWP compacts values around the null bitmap. */
+ values: unknown[];
+ /** One entry per row; true means NULL. */
+ nulls: boolean[];
+ /** Rows accounted for so far, including nulls. */
+ size: number;
+ geohashPrecision?: number;
+ decimalScale?: number;
+}
+
+/** Mutable columnar staging area for one QWP ingress table. */
+export class QwpTableBuffer {
+ readonly name: string;
+ private readonly maxNameLength: number;
+ private readonly columnList: QwpColumnBuffer[] = [];
+ private readonly columnsByName = new Map();
+ private rows = 0;
+ // Memoizes the non-null value offset each column's slice starts from, reused
+ // while a caller walks the table in ascending `start` slices. See sliceRows().
+ private sliceValueOffsets?: {
+ rows: number;
+ start: number;
+ offsets: number[];
+ };
+
+ constructor(name: string, maxNameLength = QWP_MAX_TABLE_NAME_LENGTH) {
+ if (!Number.isSafeInteger(maxNameLength) || maxNameLength < 1) {
+ throw new RangeError("maxNameLength must be a positive safe integer");
+ }
+ validateQwpTableName(name, maxNameLength);
+ this.name = name;
+ this.maxNameLength = maxNameLength;
+ }
+
+ get rowCount(): number {
+ return this.rows;
+ }
+
+ get columns(): readonly QwpColumnBuffer[] {
+ return this.columnList;
+ }
+
+ /**
+ * Returns null when the current row already contains this column. The first
+ * value wins, matching the existing Sender API.
+ */
+ getOrCreateColumn(
+ name: string,
+ type: QwpColumnType,
+ // The caller may pass the key it already holds -- the flush path iterates a
+ // Map already keyed by it -- to skip a per-cell rebuild. It must equal
+ // qwpColumnNameKey(name); it defaults to it when omitted.
+ nameKey: string = qwpColumnNameKey(name),
+ ): QwpColumnBuffer | null {
+ const designatedTimestamp =
+ name.length === 0 &&
+ (type === QWP_COLUMN_TYPE.TIMESTAMP ||
+ type === QWP_COLUMN_TYPE.TIMESTAMP_NANOS);
+ if (!name && !designatedTimestamp) {
+ throw new Error("column name cannot be empty");
+ }
+
+ const existing = this.columnsByName.get(nameKey);
+ if (existing) {
+ if (existing.type !== type) {
+ throw new Error(
+ `column type mismatch for '${name}' [existing=${existing.type}, received=${type}]`,
+ );
+ }
+ if (existing.size > this.rows) return null;
+ existing.nulls.push(false);
+ existing.size++;
+ return existing;
+ }
+
+ if (!designatedTimestamp) validateQwpColumnName(name, this.maxNameLength);
+ if (this.columnList.length >= QWP_MAX_COLUMNS_PER_TABLE) {
+ throw new Error(
+ `column count exceeds maximum ${QWP_MAX_COLUMNS_PER_TABLE}`,
+ );
+ }
+
+ const column: QwpColumnBuffer = {
+ name,
+ type,
+ values: [],
+ nulls: new Array(this.rows).fill(true),
+ size: this.rows,
+ };
+ column.nulls.push(false);
+ column.size++;
+ this.columnList.push(column);
+ this.columnsByName.set(nameKey, column);
+ return column;
+ }
+
+ /** Closes the current row and back-fills missing columns with nulls. */
+ nextRow(): void {
+ this.rows++;
+ for (const column of this.columnList) {
+ while (column.size < this.rows) {
+ column.nulls.push(true);
+ column.size++;
+ }
+ }
+ }
+
+ setGeohashPrecision(column: QwpColumnBuffer, precision: number): void {
+ if (column.type !== QWP_COLUMN_TYPE.GEOHASH) {
+ throw new Error("geohash precision can only be set on a GEOHASH column");
+ }
+ if (!Number.isInteger(precision) || precision < 1 || precision > 60) {
+ throw new Error(
+ `invalid geohash precision ${precision}; expected 1 through 60`,
+ );
+ }
+ if (column.geohashPrecision === undefined) {
+ column.geohashPrecision = precision;
+ } else if (column.geohashPrecision !== precision) {
+ throw new Error(
+ `geohash precision mismatch [existing=${column.geohashPrecision}, received=${precision}]`,
+ );
+ }
+ }
+
+ setDecimalScale(column: QwpColumnBuffer, scale: number): number {
+ const maximum =
+ column.type === QWP_COLUMN_TYPE.DECIMAL64
+ ? 18
+ : column.type === QWP_COLUMN_TYPE.DECIMAL128
+ ? 38
+ : column.type === QWP_COLUMN_TYPE.DECIMAL256
+ ? 76
+ : undefined;
+ if (maximum === undefined) {
+ throw new Error("decimal scale can only be set on a DECIMAL column");
+ }
+ if (!Number.isInteger(scale) || scale < 0 || scale > maximum) {
+ throw new Error(
+ `invalid decimal scale ${scale}; expected 0 through ${maximum}`,
+ );
+ }
+ if (column.decimalScale === undefined) column.decimalScale = scale;
+ return column.decimalScale;
+ }
+
+ /** Truncates every column back to the last completed row. */
+ rollbackRow(): void {
+ for (const column of this.columnList) {
+ while (column.size > this.rows) {
+ const wasNull = column.nulls.pop();
+ column.size--;
+ if (wasNull === false) column.values.pop();
+ }
+ }
+ for (let index = this.columnList.length - 1; index >= 0; index--) {
+ const column = this.columnList[index];
+ if (this.rows === 0 && column.size === 0) {
+ this.columnsByName.delete(qwpColumnNameKey(column.name));
+ this.columnList.splice(index, 1);
+ }
+ }
+ }
+
+ /**
+ * Copies a completed half-open row range into an independent table buffer.
+ * Compact column values and their null bitmaps are sliced together, so the
+ * result can be encoded without materialising rows first.
+ */
+ sliceRows(start: number, end: number): QwpTableBuffer {
+ if (
+ !Number.isSafeInteger(start) ||
+ !Number.isSafeInteger(end) ||
+ start < 0 ||
+ end < start ||
+ end > this.rows
+ ) {
+ throw new RangeError(
+ `invalid QWP table row range [start=${start}, end=${end}, rows=${this.rows}]`,
+ );
+ }
+
+ const result = new QwpTableBuffer(this.name, this.maxNameLength);
+ result.rows = end - start;
+ // `values` holds non-null entries only, so a row index becomes a value
+ // index by skipping the nulls before it. A column with no nulls at all
+ // needs no scan (the common case), and for a sparse one the offset before
+ // `start` is memoized and advanced across slices rather than recounted from
+ // row 0 -- otherwise a caller walking the table in ascending slices
+ // (encodeUdpDatagrams, the ingress batch-cap search) is quadratic in its
+ // row count all over again.
+ const valueStarts = this.nonNullValueOffsets(start);
+ for (let index = 0; index < this.columnList.length; index++) {
+ const column = this.columnList[index];
+ const valueStart = valueStarts[index];
+ let valueEnd: number;
+ if (column.values.length === column.size) {
+ valueEnd = end;
+ } else {
+ valueEnd = valueStart;
+ for (let row = start; row < end; row++) {
+ if (!column.nulls[row]) valueEnd++;
+ }
+ }
+ const sliced: QwpColumnBuffer = {
+ name: column.name,
+ type: column.type,
+ values: column.values.slice(valueStart, valueEnd),
+ nulls: column.nulls.slice(start, end),
+ size: end - start,
+ geohashPrecision: column.geohashPrecision,
+ decimalScale: column.decimalScale,
+ };
+ result.columnList.push(sliced);
+ result.columnsByName.set(qwpColumnNameKey(sliced.name), sliced);
+ }
+ return result;
+ }
+
+ /**
+ * The non-null value count in rows `[0, start)` for each column -- the value
+ * index at which a slice starting at `start` begins. Recomputing this from
+ * row 0 on every call makes sliceRows() O(start), so the previous result is
+ * reused and advanced only over the newly covered rows when `start` moves
+ * forward, keeping an ascending walk linear. A dense column needs no scan;
+ * its value index equals the row index.
+ */
+ private nonNullValueOffsets(start: number): number[] {
+ const columns = this.columnList;
+ const cache = this.sliceValueOffsets;
+ const reuse =
+ cache !== undefined &&
+ cache.rows === this.rows &&
+ cache.offsets.length === columns.length &&
+ cache.start <= start;
+ const from = reuse ? cache.start : 0;
+ const offsets = reuse ? cache.offsets : new Array(columns.length);
+ for (let index = 0; index < columns.length; index++) {
+ const column = columns[index];
+ if (column.values.length === column.size) {
+ offsets[index] = start;
+ continue;
+ }
+ const nulls = column.nulls;
+ let offset = reuse ? offsets[index] : 0;
+ for (let row = from; row < start; row++) {
+ if (!nulls[row]) offset++;
+ }
+ offsets[index] = offset;
+ }
+ this.sliceValueOffsets = { rows: this.rows, start, offsets };
+ return offsets;
+ }
+
+ reset(): void {
+ this.columnList.length = 0;
+ this.columnsByName.clear();
+ this.rows = 0;
+ this.sliceValueOffsets = undefined;
+ }
+}
+
+export function flattenQwpArray(value: unknown[]): QwpArrayValue {
+ const dimensions: number[] = [];
+ let level: unknown = value;
+ while (Array.isArray(level)) {
+ dimensions.push(level.length);
+ level = level[0];
+ }
+ if (dimensions.length === 0 || dimensions.length > 255) {
+ throw new Error("QWP array must have between 1 and 255 dimensions");
+ }
+
+ const values: (number | bigint)[] = [];
+ const walk = (node: unknown, depth: number): void => {
+ if (depth === dimensions.length) {
+ if (typeof node !== "number" && typeof node !== "bigint") {
+ throw new Error("QWP array elements must be numbers or bigints");
+ }
+ values.push(node);
+ return;
+ }
+ if (!Array.isArray(node) || node.length !== dimensions[depth]) {
+ throw new Error("irregular QWP array shape");
+ }
+ for (const child of node) walk(child, depth + 1);
+ };
+ walk(value, 0);
+ return { dimensions, values };
+}
diff --git a/src/_qwp/_core/varint.ts b/src/_qwp/_core/varint.ts
new file mode 100644
index 0000000..bb73d3f
--- /dev/null
+++ b/src/_qwp/_core/varint.ts
@@ -0,0 +1,85 @@
+import { QwpByteReader, QwpByteWriter } from "./bytes";
+import { QwpProtocolError } from "./errors";
+
+const MAX_UINT64 = 0xffffffffffffffffn;
+
+function toBigInt(value: number | bigint): bigint {
+ if (typeof value === "number") {
+ if (!Number.isSafeInteger(value) || value < 0) {
+ throw new RangeError(
+ `varint requires a non-negative safe integer, got ${value}`,
+ );
+ }
+ return BigInt(value);
+ }
+ if (value < 0n || value > MAX_UINT64) {
+ throw new RangeError(`varint is outside the uint64 range: ${value}`);
+ }
+ return value;
+}
+
+/** Returns the encoded byte count of an unsigned LEB128 uint64. */
+export function qwpVarintSize(value: number | bigint): number {
+ let remaining = toBigInt(value);
+ let size = 1;
+ while (remaining >= 0x80n) {
+ remaining >>= 7n;
+ size++;
+ }
+ return size;
+}
+
+/** Writes an unsigned LEB128 uint64. */
+export function writeQwpVarint(
+ writer: QwpByteWriter,
+ value: number | bigint,
+): void {
+ let remaining = toBigInt(value);
+ while (remaining >= 0x80n) {
+ writer.writeUint8(Number(remaining & 0x7fn) | 0x80);
+ remaining >>= 7n;
+ }
+ writer.writeUint8(Number(remaining));
+}
+
+/** Reads an unsigned LEB128 uint64. */
+export function readQwpVarint(reader: QwpByteReader): bigint {
+ let value = 0n;
+ for (let index = 0; index < 10; index++) {
+ const byte = reader.readUint8("varint");
+ if (index === 9 && (byte & 0xfe) !== 0) {
+ throw new QwpProtocolError("QWP varint exceeds uint64 range");
+ }
+ value |= BigInt(byte & 0x7f) << BigInt(index * 7);
+ if ((byte & 0x80) === 0) return value;
+ }
+ throw new QwpProtocolError("QWP varint exceeds 10 bytes");
+}
+
+export function readQwpVarintNumber(
+ reader: QwpByteReader,
+ label = "varint",
+): number {
+ const value = readQwpVarint(reader);
+ if (value > BigInt(Number.MAX_SAFE_INTEGER)) {
+ throw new QwpProtocolError(
+ `${label} exceeds JavaScript's safe integer range`,
+ );
+ }
+ return Number(value);
+}
+
+export function encodeQwpVarint(value: number | bigint): Uint8Array {
+ const writer = new QwpByteWriter(qwpVarintSize(value));
+ writeQwpVarint(writer, value);
+ return writer.toUint8Array();
+}
+
+export function decodeQwpVarint(
+ bytes: Uint8Array,
+ offset = 0,
+): { value: bigint; offset: number } {
+ const reader = new QwpByteReader(bytes, offset);
+ const value = readQwpVarint(reader);
+ return { value, offset: reader.position };
+}
diff --git a/src/_qwp/_core/zstd.ts b/src/_qwp/_core/zstd.ts
new file mode 100644
index 0000000..3afa49f
--- /dev/null
+++ b/src/_qwp/_core/zstd.ts
@@ -0,0 +1,316 @@
+import { decompress } from "fzstd";
+import { QwpProtocolError } from "./errors";
+
+/** Matches the Java client's per-connection decompression safety cap. */
+export const QWP_MAX_ZSTD_DECOMPRESSED_SIZE = 64 * 1024 * 1024;
+
+const ZSTD_MAGIC = 0xfd2fb528;
+const ZSTD_MAX_BLOCK_SIZE = 128 * 1024;
+
+/**
+ * Bytes of marker appended after the declared content, and the value they
+ * carry. fzstd decodes into the output buffer without reporting how far it
+ * got, so the marker is how the decoded length is observed: a run this long
+ * cannot be faked by a frame that stops early, because everything past what
+ * the frame wrote is the untouched zero tail of the buffer.
+ */
+const ZSTD_SIZE_MARKER_BYTES = 8;
+/**
+ * The marker written past a frame's declared content size, as an eight-byte
+ * raw block rather than a repeated byte.
+ *
+ * A run of one byte cannot say where it starts. A frame that ran long by k
+ * bytes pushes the marker to contentSize + k, leaving its own k bytes in front
+ * of it -- and a repeated-byte marker still matched at contentSize whenever
+ * those k bytes happened to be that byte. Only min(k, 8) of them had to,
+ * so overshooting by one needed a single byte with probability 1/256, and
+ * 0xa5 is a legal UTF-8 continuation byte, so a VARCHAR ending in one collided
+ * by accident.
+ *
+ * These eight bytes are distinct, so no proper prefix of the pattern equals a
+ * proper suffix of it and no shift can reproduce it. The last byte is non-zero
+ * so the scan for a short frame's marker still stops at the marker.
+ */
+const ZSTD_SIZE_MARKER = Uint8Array.of(
+ 0xa5,
+ 0x5a,
+ 0xc3,
+ 0x3c,
+ 0x69,
+ 0x96,
+ 0x0f,
+ 0xf0,
+);
+/**
+ * Room past the marker, so a frame that overshoots by up to this much still
+ * lands its marker inside the buffer and gets a report of the size it really
+ * decoded rather than a bare decompression failure.
+ */
+const ZSTD_SIZE_SLACK_BYTES = 8;
+/** How far back the marker is looked for when reporting a size mismatch. */
+const ZSTD_SIZE_SEARCH_BYTES = 64 * 1024;
+
+interface ZstdFrameInfo {
+ readonly contentSize: number;
+ readonly dataOffset: number;
+ readonly checksum: boolean;
+}
+
+interface ZstdBlockLayout {
+ /** Offset of the header of the block flagged last. */
+ readonly lastBlockOffset: number;
+ /** First byte after the last block, so before any content checksum. */
+ readonly blocksEnd: number;
+}
+
+function requireAvailable(
+ bytes: Uint8Array,
+ offset: number,
+ length: number,
+ label: string,
+): void {
+ if (offset < 0 || length < 0 || offset + length > bytes.byteLength) {
+ throw new QwpProtocolError(`truncated zstd ${label}`);
+ }
+}
+
+function readLittleEndian(
+ bytes: Uint8Array,
+ offset: number,
+ length: number,
+): bigint {
+ requireAvailable(bytes, offset, length, "frame header");
+ let value = 0n;
+ for (let index = 0; index < length; index++) {
+ value |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ }
+ return value;
+}
+
+function inspectZstdFrame(frame: Uint8Array): ZstdFrameInfo {
+ if (frame.byteLength > QWP_MAX_ZSTD_DECOMPRESSED_SIZE) {
+ throw new QwpProtocolError(
+ `zstd frame size ${frame.byteLength} exceeds client cap ${QWP_MAX_ZSTD_DECOMPRESSED_SIZE}`,
+ );
+ }
+ requireAvailable(frame, 0, 5, "frame header");
+ if (Number(readLittleEndian(frame, 0, 4)) !== ZSTD_MAGIC) {
+ throw new QwpProtocolError("invalid zstd frame magic");
+ }
+
+ const descriptor = frame[4];
+ if ((descriptor & 0x08) !== 0) {
+ throw new QwpProtocolError("zstd frame uses its reserved descriptor bit");
+ }
+ const singleSegment = (descriptor & 0x20) !== 0;
+ const checksum = (descriptor & 0x04) !== 0;
+ const dictionaryIdFlag = descriptor & 0x03;
+ const contentSizeFlag = descriptor >>> 6;
+ let offset = 5;
+
+ let windowSize: bigint | undefined;
+ if (!singleSegment) {
+ requireAvailable(frame, offset, 1, "window descriptor");
+ const windowDescriptor = frame[offset++];
+ const base = 1n << BigInt(10 + (windowDescriptor >>> 3));
+ windowSize = base + (base >> 3n) * BigInt(windowDescriptor & 0x07);
+ }
+
+ const dictionaryIdSize = dictionaryIdFlag === 3 ? 4 : dictionaryIdFlag;
+ requireAvailable(frame, offset, dictionaryIdSize, "dictionary ID");
+ if (dictionaryIdSize !== 0) {
+ throw new QwpProtocolError(
+ "zstd frames using an external dictionary are not supported",
+ );
+ }
+ offset += dictionaryIdSize;
+
+ const contentSizeBytes =
+ contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag;
+ if (contentSizeBytes === 0) {
+ throw new QwpProtocolError(
+ "zstd frame is missing its declared content size",
+ );
+ }
+ let contentSize = readLittleEndian(frame, offset, contentSizeBytes);
+ offset += contentSizeBytes;
+ if (contentSizeFlag === 1) contentSize += 256n;
+
+ const cap = BigInt(QWP_MAX_ZSTD_DECOMPRESSED_SIZE);
+ if (contentSize > cap) {
+ throw new QwpProtocolError(
+ `zstd frame content size ${contentSize} exceeds client cap ${cap}`,
+ );
+ }
+ if (windowSize !== undefined && windowSize > cap) {
+ throw new QwpProtocolError(
+ `zstd frame window size ${windowSize} exceeds client cap ${cap}`,
+ );
+ }
+ return { contentSize: Number(contentSize), dataOffset: offset, checksum };
+}
+
+function validateSingleZstdFrame(
+ frame: Uint8Array,
+ info: ZstdFrameInfo,
+): ZstdBlockLayout {
+ let offset = info.dataOffset;
+ let lastBlockOffset = info.dataOffset;
+ let lastBlock = false;
+ while (!lastBlock) {
+ requireAvailable(frame, offset, 3, "block header");
+ const header =
+ frame[offset] | (frame[offset + 1] << 8) | (frame[offset + 2] << 16);
+ lastBlockOffset = offset;
+ offset += 3;
+ lastBlock = (header & 1) !== 0;
+ const blockType = (header >>> 1) & 0x03;
+ if (blockType === 3) {
+ throw new QwpProtocolError("zstd frame contains a reserved block type");
+ }
+ const blockSize = header >>> 3;
+ if (blockSize > ZSTD_MAX_BLOCK_SIZE) {
+ throw new QwpProtocolError(
+ `zstd block size ${blockSize} exceeds format maximum ${ZSTD_MAX_BLOCK_SIZE}`,
+ );
+ }
+ const encodedSize = blockType === 1 ? 1 : blockSize;
+ requireAvailable(frame, offset, encodedSize, "block body");
+ offset += encodedSize;
+ }
+ const blocksEnd = offset;
+ if (info.checksum) {
+ requireAvailable(frame, offset, 4, "content checksum");
+ offset += 4;
+ }
+ if (offset !== frame.byteLength) {
+ throw new QwpProtocolError(
+ `zstd body must contain exactly one frame [frameBytes=${offset}, actual=${frame.byteLength}]`,
+ );
+ }
+ return { lastBlockOffset, blocksEnd };
+}
+
+/**
+ * Reframes the blocks with a single-segment header, an eight-byte size marker
+ * appended as a final RLE block, and the content size that marker needs.
+ *
+ * fzstd sizes its output from the declared content size and never reports how
+ * far it actually got, so the marker is what makes the decoded length
+ * observable: it lands wherever the frame's own output ends, which is the
+ * declared content size and nowhere else for a frame that means what its
+ * header says. Those bytes are also the headroom that lets an over-long frame
+ * write past the declared size instead of being silently truncated into it.
+ *
+ * A single-segment window of the output's size is sufficient for all valid
+ * frames because no match can refer before the decoded content, and it is what
+ * makes fzstd decode in place: given a window that spans the whole output, it
+ * resolves matches against the output itself instead of shifting a separate
+ * window buffer down after every block, which is quadratic in the content
+ * size. That shift cost a 4 KB frame declaring 64 MiB about 1.5 seconds.
+ */
+function frameWithSizeMarker(
+ frame: Uint8Array,
+ info: ZstdFrameInfo,
+ layout: ZstdBlockLayout,
+): Uint8Array {
+ // Magic, then a single-segment descriptor with an 8-byte content size. The
+ // checksum flag is dropped along with the trailing checksum bytes: nothing
+ // verifies them, and the marker has to be the frame's last block.
+ const headerSize = 4 + 1 + 8;
+ const markerSize = 3 + ZSTD_SIZE_MARKER_BYTES;
+ const blocks = frame.subarray(info.dataOffset, layout.blocksEnd);
+ const reframed = new Uint8Array(headerSize + blocks.byteLength + markerSize);
+ reframed.set(frame.subarray(0, 4));
+ reframed[4] = 0xe0;
+ let size = BigInt(
+ info.contentSize + ZSTD_SIZE_MARKER_BYTES + ZSTD_SIZE_SLACK_BYTES,
+ );
+ for (let index = 0; index < 8; index++) {
+ reframed[5 + index] = Number(size & 0xffn);
+ size >>= 8n;
+ }
+ reframed.set(blocks, headerSize);
+ // The marker block is the last one now, so the block that was carries the
+ // flag no longer.
+ reframed[headerSize + (layout.lastBlockOffset - info.dataOffset)] &= ~1;
+ const marker = headerSize + blocks.byteLength;
+ // Raw block, not RLE: the marker has to be eight chosen bytes, and an RLE
+ // block can only repeat one.
+ const header = 1 | (0 << 1) | (ZSTD_SIZE_MARKER_BYTES << 3);
+ reframed[marker] = header & 0xff;
+ reframed[marker + 1] = (header >>> 8) & 0xff;
+ reframed[marker + 2] = (header >>> 16) & 0xff;
+ reframed.set(ZSTD_SIZE_MARKER, marker + 3);
+ return reframed;
+}
+
+function hasSizeMarkerAt(output: Uint8Array, offset: number): boolean {
+ if (offset < 0 || offset + ZSTD_SIZE_MARKER_BYTES > output.byteLength) {
+ return false;
+ }
+ for (let index = 0; index < ZSTD_SIZE_MARKER_BYTES; index++) {
+ if (output[offset + index] !== ZSTD_SIZE_MARKER[index]) return false;
+ }
+ return true;
+}
+
+/**
+ * Where the marker landed, searched downwards from `from`, or -1.
+ *
+ * Only a diagnostic: whether the frame is well formed at all was already
+ * settled by testing the declared offset. fzstd stages a block's literals in
+ * the unwritten tail of the output buffer, so that tail is not reliably zero
+ * and the marker cannot be found by scanning back over zeros. The search is
+ * bounded because a hostile frame chooses how far off its output ends.
+ */
+function findSizeMarker(output: Uint8Array, from: number): number {
+ const start = Math.min(from, output.byteLength - ZSTD_SIZE_MARKER_BYTES);
+ const floor = Math.max(0, start - ZSTD_SIZE_SEARCH_BYTES);
+ for (let offset = start; offset >= floor; offset--) {
+ if (hasSizeMarkerAt(output, offset)) return offset;
+ }
+ return -1;
+}
+
+/** Rejects a frame whose output did not end where its header said it would. */
+function requireDeclaredSize(output: Uint8Array, contentSize: number): void {
+ // The marker lands exactly where the frame's own output ended, and no shift
+ // of it can spell itself, so this is the whole test: it holds for a frame
+ // that means what its header says and for no other. A frame that overshot by
+ // more than the slack could not land its marker inside the buffer at all,
+ // and fzstd has already rejected it by the time we get here.
+ if (hasSizeMarkerAt(output, contentSize)) return;
+ const decoded = findSizeMarker(output, contentSize + ZSTD_SIZE_SLACK_BYTES);
+ if (decoded >= 0 && decoded !== contentSize) {
+ throw new QwpProtocolError(
+ decoded < contentSize
+ ? `zstd decompressed size ${decoded} does not match frame content size ${contentSize}`
+ : `zstd output exceeds declared content size ${contentSize} by ${decoded - contentSize}`,
+ );
+ }
+ throw new QwpProtocolError(
+ `zstd output exceeds declared content size ${contentSize}`,
+ );
+}
+
+/** Decompresses the single bounded Zstd frame carried by a RESULT_BATCH. */
+export function decompressQwpZstdFrame(frame: Uint8Array): Uint8Array {
+ const info = inspectZstdFrame(frame);
+ const layout = validateSingleZstdFrame(frame, info);
+ let output: Uint8Array;
+ try {
+ // fzstd allocates the output itself, from the declared content size the
+ // marker is accounted for in. Handing it a buffer of our own instead costs
+ // more than the decompression does: it compares that argument against a
+ // sentinel with `!=`, and coercing a 64 MiB Uint8Array to a string for
+ // that comparison took 840 ms where the whole decode takes 8 ms.
+ output = decompress(frameWithSizeMarker(frame, info, layout));
+ } catch (error) {
+ if (error instanceof QwpProtocolError) throw error;
+ const detail = error instanceof Error ? `: ${error.message}` : "";
+ throw new QwpProtocolError(`zstd decompression failed${detail}`);
+ }
+ requireDeclaredSize(output, info.contentSize);
+ return output.subarray(0, info.contentSize);
+}
diff --git a/src/_qwp/_internal/async-queue.ts b/src/_qwp/_internal/async-queue.ts
new file mode 100644
index 0000000..42a4a28
--- /dev/null
+++ b/src/_qwp/_internal/async-queue.ts
@@ -0,0 +1,111 @@
+interface PendingNext {
+ resolve: (result: IteratorResult) => void;
+ reject: (error: unknown) => void;
+}
+
+interface QueueBarrier {
+ readonly kind: "barrier";
+ readonly resolve: () => void;
+ readonly reject: (error: unknown) => void;
+}
+
+interface QueueValue {
+ readonly kind: "value";
+ readonly value: T;
+}
+
+/** Single-consumer async queue used to preserve WebSocket message ordering. */
+export class QwpAsyncQueue implements AsyncIterable {
+ private readonly values: (QueueValue | QueueBarrier)[] = [];
+ private readonly pending: PendingNext[] = [];
+ private ended = false;
+ private failure: unknown;
+ private iteratorCreated = false;
+
+ push(value: T): void {
+ if (this.ended || this.failure !== undefined) return;
+ const pending = this.pending.shift();
+ if (pending) {
+ pending.resolve({ value, done: false });
+ } else {
+ this.values.push({ kind: "value", value });
+ }
+ }
+
+ end(): void {
+ if (this.ended || this.failure !== undefined) return;
+ this.ended = true;
+ this.settleBarriers();
+ for (const pending of this.pending.splice(0)) {
+ pending.resolve({ value: undefined, done: true });
+ }
+ }
+
+ fail(error: unknown): void {
+ if (this.ended || this.failure !== undefined) return;
+ this.failure = error;
+ this.settleBarriers(error);
+ for (const pending of this.pending.splice(0)) pending.reject(error);
+ }
+
+ /** Drops and returns values not yet handed to the single consumer. */
+ clear(): T[] {
+ const dropped: T[] = [];
+ for (const entry of this.values.splice(0)) {
+ if (entry.kind === "value") dropped.push(entry.value);
+ else entry.resolve();
+ }
+ return dropped;
+ }
+
+ /** Resolves once the consumer asks for the item after this queue position. */
+ barrier(): Promise {
+ if (this.ended || this.failure !== undefined || this.pending.length > 0) {
+ return Promise.resolve();
+ }
+ return new Promise((resolve, reject) => {
+ this.values.push({ kind: "barrier", resolve, reject });
+ });
+ }
+
+ [Symbol.asyncIterator](): AsyncIterator {
+ if (this.iteratorCreated) {
+ throw new Error("QWP message streams support only one consumer");
+ }
+ this.iteratorCreated = true;
+ return {
+ next: () => this.next(),
+ };
+ }
+
+ private next(): Promise> {
+ while (true) {
+ const entry = this.values.shift();
+ if (!entry) break;
+ if (entry.kind === "value") {
+ return Promise.resolve({ value: entry.value, done: false });
+ }
+ entry.resolve();
+ }
+ if (this.failure !== undefined) return Promise.reject(this.failure);
+ if (this.ended) {
+ return Promise.resolve({ value: undefined, done: true });
+ }
+ return new Promise((resolve, reject) => {
+ this.pending.push({ resolve, reject });
+ });
+ }
+
+ private settleBarriers(error?: unknown): void {
+ const entries = this.values.splice(0);
+ for (const entry of entries) {
+ if (entry.kind === "value") {
+ this.values.push(entry);
+ } else if (error === undefined) {
+ entry.resolve();
+ } else {
+ entry.reject(error);
+ }
+ }
+ }
+}
diff --git a/src/_qwp/_internal/egress-limits.ts b/src/_qwp/_internal/egress-limits.ts
new file mode 100644
index 0000000..1afc651
--- /dev/null
+++ b/src/_qwp/_internal/egress-limits.ts
@@ -0,0 +1,17 @@
+import { QWP_MAX_BATCH_ROWS_UPPER_BOUND } from "../_core";
+
+export function validateQwpMaxBatchRows(
+ value: number | undefined,
+): number | undefined {
+ if (value === undefined) return undefined;
+ if (
+ !Number.isSafeInteger(value) ||
+ value < 1 ||
+ value > QWP_MAX_BATCH_ROWS_UPPER_BOUND
+ ) {
+ throw new RangeError(
+ `maxBatchRows must be an integer between 1 and ${QWP_MAX_BATCH_ROWS_UPPER_BOUND}`,
+ );
+ }
+ return value;
+}
diff --git a/src/_qwp/_internal/egress-routing.ts b/src/_qwp/_internal/egress-routing.ts
new file mode 100644
index 0000000..5d15aec
--- /dev/null
+++ b/src/_qwp/_internal/egress-routing.ts
@@ -0,0 +1,140 @@
+import {
+ decodeQwpEgressMessage,
+ QWP_SERVER_ROLE,
+ QwpProtocolError,
+} from "../_core";
+import {
+ QwpBinaryConnection,
+ QwpConnectionFactory,
+ QwpSendClosedError,
+} from "../transport";
+import {
+ createQwpFailoverConnectionFactory,
+ QwpFailoverSelectionOptions,
+ QwpValidatedConnection,
+} from "./failover";
+
+/**
+ * Creates an egress endpoint walker that validates authoritative SERVER_INFO
+ * topology before exposing a connection. Reading the frame here works in both
+ * Node and browsers; the frame is replayed to the normal session consumer.
+ */
+export function createQwpEgressFailoverConnectionFactory(
+ preferredUrl: string | URL,
+ failoverUrls: readonly (string | URL)[] | undefined,
+ connect: (
+ endpoint: string | URL,
+ signal?: AbortSignal,
+ ) => Promise,
+ routing: QwpFailoverSelectionOptions,
+ serverInfoTimeoutMs: number,
+): QwpConnectionFactory {
+ return createQwpFailoverConnectionFactory(
+ preferredUrl,
+ failoverUrls,
+ connect,
+ {
+ ...routing,
+ validateConnection: (connection) =>
+ readAndReplayServerInfo(connection, serverInfoTimeoutMs),
+ },
+ );
+}
+
+async function readAndReplayServerInfo(
+ connection: QwpBinaryConnection,
+ timeoutMs: number,
+): Promise {
+ const iterator = connection.messages[Symbol.asyncIterator]();
+ let timer: ReturnType | undefined;
+ const timeout = new Promise((_resolve, reject) => {
+ timer = setTimeout(
+ () => reject(new Error("timed out waiting for QWP SERVER_INFO")),
+ timeoutMs,
+ );
+ });
+ try {
+ const result = await Promise.race([iterator.next(), timeout]);
+ if (result.done) {
+ throw new QwpSendClosedError(await connection.closed);
+ }
+ const serverInfo = decodeQwpEgressMessage(result.value);
+ if (serverInfo.kind !== "server-info") {
+ throw new QwpProtocolError(
+ "QWP egress connection did not begin with SERVER_INFO",
+ );
+ }
+ const serverRole = serverRoleName(serverInfo.role);
+ const serverZone =
+ serverInfo.zoneId ?? connection.handshake.serverZone ?? undefined;
+ return {
+ connection: prependMessage(connection, result.value, iterator, {
+ serverRole,
+ serverZone,
+ }),
+ serverRole,
+ serverZone,
+ };
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+}
+
+function serverRoleName(role: number): string {
+ switch (role) {
+ case QWP_SERVER_ROLE.STANDALONE:
+ return "STANDALONE";
+ case QWP_SERVER_ROLE.PRIMARY:
+ return "PRIMARY";
+ case QWP_SERVER_ROLE.REPLICA:
+ return "REPLICA";
+ case QWP_SERVER_ROLE.PRIMARY_CATCHUP:
+ return "PRIMARY_CATCHUP";
+ default:
+ return `UNKNOWN(${role})`;
+ }
+}
+
+function prependMessage(
+ connection: QwpBinaryConnection,
+ first: Uint8Array,
+ iterator: AsyncIterator,
+ topology: { readonly serverRole: string; readonly serverZone?: string },
+): QwpBinaryConnection {
+ let consumed = false;
+ const messages: AsyncIterable = {
+ async *[Symbol.asyncIterator]() {
+ if (consumed) {
+ throw new QwpProtocolError(
+ "QWP connection messages already have a consumer",
+ );
+ }
+ consumed = true;
+ yield first;
+ while (true) {
+ const result = await iterator.next();
+ if (result.done) return;
+ yield result.value;
+ }
+ },
+ };
+ const wrapped: QwpBinaryConnection = {
+ messages,
+ closed: connection.closed,
+ handshake: { ...connection.handshake, ...topology },
+ endpoint: connection.endpoint,
+ get ingressSymbolDictionary() {
+ return connection.ingressSymbolDictionary;
+ },
+ get ingressDeltaSymbolDictionaryEnabled() {
+ return connection.ingressDeltaSymbolDictionaryEnabled;
+ },
+ send: (payload) => connection.send(payload),
+ close: (code, reason) => connection.close(code, reason),
+ };
+ if (connection.ping) wrapped.ping = () => connection.ping!();
+ if (connection.getIngressMetrics) {
+ wrapped.getIngressMetrics = () => connection.getIngressMetrics!();
+ }
+ return wrapped;
+}
diff --git a/src/_qwp/_internal/failover.ts b/src/_qwp/_internal/failover.ts
new file mode 100644
index 0000000..c6d0ee9
--- /dev/null
+++ b/src/_qwp/_internal/failover.ts
@@ -0,0 +1,451 @@
+import {
+ QWP_TARGET,
+ QWP_UPGRADE_ERROR_KIND,
+ QwpBinaryConnection,
+ QwpConnectionFactory,
+ QwpEgressRoutingOptions,
+ QwpFailoverAttempt,
+ QwpFailoverError,
+ QwpRoleMismatchError,
+ QwpTarget,
+ QwpUpgradeError,
+} from "../transport";
+
+const HOST_STATE = {
+ HEALTHY: 0,
+ UNKNOWN: 1,
+ TRANSIENT_REJECT: 2,
+ TRANSPORT_ERROR: 3,
+ TOPOLOGY_REJECT: 4,
+} as const;
+
+type HostState = (typeof HOST_STATE)[keyof typeof HOST_STATE];
+
+const ZONE_TIER = {
+ SAME: 0,
+ UNKNOWN: 1,
+ OTHER: 2,
+} as const;
+
+type ZoneTier = (typeof ZONE_TIER)[keyof typeof ZONE_TIER];
+
+interface QwpEndpointHealth {
+ state: HostState;
+ zoneTier: ZoneTier;
+ lastSuccessEpoch: number;
+}
+
+/**
+ * Shared endpoint classifications used by independent connection walkers.
+ * Each factory keeps its own sweep cursor while publishing observations here,
+ * so concurrent pooled sessions and orphan drainers cannot steal attempts from
+ * one another but immediately benefit from one another's health discoveries.
+ */
+export class QwpFailoverHealthTracker {
+ private readonly endpointKeys: readonly string[];
+ private readonly health: QwpEndpointHealth[];
+ private successEpoch = 0;
+
+ constructor(
+ preferredUrl: string | URL,
+ failoverUrls: readonly (string | URL)[] | undefined,
+ private readonly target: QwpTarget,
+ private readonly configuredZone: string | undefined,
+ ) {
+ this.endpointKeys = endpointKeys(preferredUrl, failoverUrls);
+ const zoneBlind = this.zoneBlind;
+ this.health = this.endpointKeys.map(() => ({
+ state: HOST_STATE.UNKNOWN,
+ zoneTier: zoneBlind ? ZONE_TIER.SAME : ZONE_TIER.UNKNOWN,
+ lastSuccessEpoch: 0,
+ }));
+ }
+
+ assertCompatible(
+ preferredUrl: string | URL,
+ failoverUrls: readonly (string | URL)[] | undefined,
+ target: QwpTarget,
+ configuredZone: string | undefined,
+ ): void {
+ const keys = endpointKeys(preferredUrl, failoverUrls);
+ if (
+ target !== this.target ||
+ configuredZone !== this.configuredZone ||
+ keys.length !== this.endpointKeys.length ||
+ keys.some((key, index) => key !== this.endpointKeys[index])
+ ) {
+ throw new RangeError(
+ "QWP failover health tracker does not match the endpoint routing configuration",
+ );
+ }
+ }
+
+ newRoundCursor(deferredEndpoint?: number): QwpFailoverRoundCursor {
+ return new QwpFailoverRoundCursor(this.health, deferredEndpoint);
+ }
+
+ /**
+ * Starts a recovery round with stale classifications forgotten. The newest
+ * successful same-zone endpoint stays healthy, matching the Java client's
+ * locality-aware stickiness; learned zone tiers persist across rounds.
+ */
+ forgetClassifications(): void {
+ let stickyIndex = -1;
+ let newestSuccess = -1;
+ for (let index = 0; index < this.health.length; index++) {
+ const health = this.health[index];
+ if (
+ health.state === HOST_STATE.HEALTHY &&
+ health.zoneTier === ZONE_TIER.SAME &&
+ health.lastSuccessEpoch > newestSuccess
+ ) {
+ stickyIndex = index;
+ newestSuccess = health.lastSuccessEpoch;
+ }
+ }
+ for (let index = 0; index < this.health.length; index++) {
+ if (index !== stickyIndex) this.health[index].state = HOST_STATE.UNKNOWN;
+ }
+ }
+
+ recordFailure(index: number, error: unknown): void {
+ const health = this.health[index];
+ if (error instanceof QwpUpgradeError) {
+ this.recordZone(index, error.serverZone);
+ if (error.kind === QWP_UPGRADE_ERROR_KIND.ROLE_REJECTED) {
+ health.state =
+ normalizeRole(error.serverRole) === "PRIMARY_CATCHUP"
+ ? HOST_STATE.TRANSIENT_REJECT
+ : HOST_STATE.TOPOLOGY_REJECT;
+ return;
+ }
+ }
+ health.state = HOST_STATE.TRANSPORT_ERROR;
+ }
+
+ recordSuccess(index: number): void {
+ const health = this.health[index];
+ health.state = HOST_STATE.HEALTHY;
+ health.lastSuccessEpoch = ++this.successEpoch;
+ }
+
+ recordZone(index: number, serverZone: string | undefined): void {
+ const normalized = normalizeZone(serverZone);
+ if (!normalized) return;
+ this.health[index].zoneTier =
+ this.zoneBlind || normalized === this.configuredZone
+ ? ZONE_TIER.SAME
+ : ZONE_TIER.OTHER;
+ }
+
+ recordMidStreamFailure(index: number): void {
+ const health = this.health[index];
+ if (health.state === HOST_STATE.HEALTHY) {
+ health.state = HOST_STATE.TRANSPORT_ERROR;
+ }
+ }
+
+ recordTransientReject(index: number): void {
+ this.health[index].state = HOST_STATE.TRANSIENT_REJECT;
+ }
+
+ private get zoneBlind(): boolean {
+ return (
+ this.configuredZone === undefined || this.target === QWP_TARGET.PRIMARY
+ );
+ }
+}
+
+class QwpFailoverRoundCursor {
+ private readonly attempted = new Set();
+
+ constructor(
+ private readonly health: readonly QwpEndpointHealth[],
+ private readonly deferredEndpoint?: number,
+ ) {}
+
+ next(): number | undefined {
+ const selected = pickNextEndpoint(
+ this.health,
+ this.attempted,
+ this.deferredEndpoint,
+ );
+ if (selected === undefined) return undefined;
+ this.attempted.add(selected);
+ return selected;
+ }
+
+ get exhausted(): boolean {
+ return this.attempted.size === this.health.length;
+ }
+}
+
+export interface QwpValidatedConnection {
+ readonly connection: QwpBinaryConnection;
+ readonly serverRole?: string;
+ readonly serverZone?: string;
+}
+
+export interface QwpFailoverSelectionOptions extends QwpEgressRoutingOptions {
+ /** @internal Reads protocol-level topology metadata when headers are hidden. */
+ validateConnection?: (
+ connection: QwpBinaryConnection,
+ ) => Promise;
+ /** @internal Shares classifications without sharing a walker's cursor. */
+ healthTracker?: QwpFailoverHealthTracker;
+ /** @internal Background walkers must not reset shared classifications. */
+ resetClassificationsAfterExhaustion?: boolean;
+}
+
+/** Creates a health ledger that can be shared by independent walkers. */
+export function createQwpFailoverHealthTracker(
+ preferredUrl: string | URL,
+ failoverUrls: readonly (string | URL)[] | undefined,
+ options: QwpEgressRoutingOptions = {},
+): QwpFailoverHealthTracker {
+ return new QwpFailoverHealthTracker(
+ preferredUrl,
+ failoverUrls,
+ normalizeTarget(options.target),
+ normalizeZone(options.zone),
+ );
+}
+
+/**
+ * Creates a stateful endpoint walker ordered by health and then zone affinity.
+ * Every invocation still performs a complete sweep, so stale role/health data
+ * can never permanently exclude an endpoint whose state has changed.
+ */
+export function createQwpFailoverConnectionFactory(
+ preferredUrl: string | URL,
+ failoverUrls: readonly (string | URL)[] | undefined,
+ connect: (
+ endpoint: string | URL,
+ signal?: AbortSignal,
+ ) => Promise,
+ options: QwpFailoverSelectionOptions = {},
+): QwpConnectionFactory {
+ const endpoints = [preferredUrl, ...(failoverUrls ?? [])];
+ const target = normalizeTarget(options.target);
+ const configuredZone = normalizeZone(options.zone);
+ const healthTracker =
+ options.healthTracker ??
+ new QwpFailoverHealthTracker(
+ preferredUrl,
+ failoverUrls,
+ target,
+ configuredZone,
+ );
+ healthTracker.assertCompatible(
+ preferredUrl,
+ failoverUrls,
+ target,
+ configuredZone,
+ );
+ const resetClassificationsAfterExhaustion =
+ options.resetClassificationsAfterExhaustion !== false;
+ let deferredEndpoint: number | undefined;
+ let resetClassificationsBeforeSweep = false;
+
+ return async (signal?: AbortSignal): Promise => {
+ if (
+ resetClassificationsBeforeSweep &&
+ resetClassificationsAfterExhaustion
+ ) {
+ healthTracker.forgetClassifications();
+ }
+ resetClassificationsBeforeSweep = false;
+ const attempts: QwpFailoverAttempt[] = [];
+ const deferredForSweep = deferredEndpoint;
+ deferredEndpoint = undefined;
+ const cursor = healthTracker.newRoundCursor(deferredForSweep);
+
+ while (true) {
+ const index = cursor.next();
+ if (index === undefined) break;
+ const endpoint = endpoints[index];
+ let candidate: QwpBinaryConnection | undefined;
+ try {
+ candidate = await connect(endpoint, signal);
+ let validated: QwpValidatedConnection = {
+ connection: candidate,
+ serverRole: candidate.handshake.serverRole,
+ serverZone: candidate.handshake.serverZone,
+ };
+ if (options.validateConnection) {
+ const protocolValidated = await options.validateConnection(candidate);
+ validated = {
+ connection: protocolValidated.connection,
+ serverRole:
+ protocolValidated.serverRole ?? candidate.handshake.serverRole,
+ serverZone:
+ protocolValidated.serverZone ?? candidate.handshake.serverZone,
+ };
+ }
+ candidate = validated.connection;
+ healthTracker.recordZone(index, validated.serverZone);
+ if (!matchesTarget(validated.serverRole, target)) {
+ throw new QwpRoleMismatchError(
+ target,
+ validated.serverRole,
+ endpoint,
+ validated.serverZone,
+ );
+ }
+ healthTracker.recordSuccess(index);
+ resetClassificationsBeforeSweep = cursor.exhausted;
+ return observeConnectionHealth(
+ candidate,
+ () => {
+ healthTracker.recordMidStreamFailure(index);
+ },
+ () => {
+ healthTracker.recordTransientReject(index);
+ deferredEndpoint = index;
+ },
+ );
+ } catch (error) {
+ healthTracker.recordFailure(index, error);
+ attempts.push({ endpoint, error });
+ if (candidate) await candidate.close().catch(() => undefined);
+ // tryNextEndpoint is a tri-state: only an explicit false short-circuits
+ // the sweep. A browser cannot see the HTTP response, so every refused,
+ // reset, or non-101 upgrade it reports is `undefined`; treating that as
+ // "stop" would make failoverUrls unreachable in browsers. This matches
+ // isRetryableReconnectError(), which reads the sibling `retryable` flag
+ // of the same tri-state as `!== false`.
+ if (
+ error instanceof QwpUpgradeError &&
+ error.tryNextEndpoint === false
+ ) {
+ throw error;
+ }
+ }
+ }
+ resetClassificationsBeforeSweep = true;
+ if (attempts.length === 1) throw attempts[0].error;
+ throw new QwpFailoverError(attempts);
+ };
+}
+
+function normalizeTarget(target: QwpTarget | undefined): QwpTarget {
+ const effective = target ?? QWP_TARGET.ANY;
+ if (
+ effective !== QWP_TARGET.ANY &&
+ effective !== QWP_TARGET.PRIMARY &&
+ effective !== QWP_TARGET.REPLICA
+ ) {
+ throw new RangeError("target must be one of: any, primary, replica");
+ }
+ return effective;
+}
+
+function normalizeZone(zone: string | undefined): string | undefined {
+ const normalized = zone?.trim().toLowerCase();
+ return normalized || undefined;
+}
+
+function normalizeRole(role: string | undefined): string | undefined {
+ const normalized = role?.trim().toUpperCase().replace(/-/g, "_");
+ return normalized || undefined;
+}
+
+function matchesTarget(role: string | undefined, target: QwpTarget): boolean {
+ if (target === QWP_TARGET.ANY) return true;
+ const normalized = normalizeRole(role);
+ // An endpoint that declares no role is accepted whatever the target. Egress
+ // always learns one from SERVER_INFO, but ingress reads it from an upgrade
+ // response header that an older server may not send and a proxy may strip,
+ // and refusing to write to a node purely because it stayed silent would take
+ // a working deployment offline. A server that does know its role still
+ // rejects a misdirected write itself, with the 421 this client classifies as
+ // ROLE_REJECTED.
+ if (normalized === undefined) return true;
+ if (target === QWP_TARGET.REPLICA) return normalized === "REPLICA";
+ return (
+ normalized === "PRIMARY" ||
+ normalized === "PRIMARY_CATCHUP" ||
+ normalized === "STANDALONE"
+ );
+}
+
+function pickNextEndpoint(
+ health: readonly QwpEndpointHealth[],
+ attempted: ReadonlySet,
+ deferredEndpoint?: number,
+): number | undefined {
+ let selected = -1;
+ for (let index = 0; index < health.length; index++) {
+ if (attempted.has(index) || index === deferredEndpoint) continue;
+ if (selected < 0 || compareHealth(health[index], health[selected]) < 0) {
+ selected = index;
+ }
+ }
+ if (selected >= 0) return selected;
+ if (deferredEndpoint !== undefined && !attempted.has(deferredEndpoint)) {
+ return deferredEndpoint;
+ }
+ return undefined;
+}
+
+function compareHealth(
+ left: QwpEndpointHealth,
+ right: QwpEndpointHealth,
+): number {
+ if (left.state !== right.state) return left.state - right.state;
+ if (left.zoneTier !== right.zoneTier) return left.zoneTier - right.zoneTier;
+ return 0;
+}
+
+function observeConnectionHealth(
+ connection: QwpBinaryConnection,
+ demoteEndpoint: () => void,
+ deprioritizeEndpoint: () => void,
+): QwpBinaryConnection {
+ void connection.closed.then((info) => {
+ if (!info.wasClean) demoteEndpoint();
+ }, demoteEndpoint);
+ const observed: QwpBinaryConnection = {
+ messages: connection.messages,
+ closed: connection.closed,
+ handshake: connection.handshake,
+ endpoint: connection.endpoint,
+ get ingressSymbolDictionary() {
+ return connection.ingressSymbolDictionary;
+ },
+ get ingressDeltaSymbolDictionaryEnabled() {
+ return connection.ingressDeltaSymbolDictionaryEnabled;
+ },
+ deprioritizeEndpoint,
+ send: async (payload) => {
+ try {
+ await connection.send(payload);
+ } catch (error) {
+ demoteEndpoint();
+ throw error;
+ }
+ },
+ close: (code, reason) => connection.close(code, reason),
+ };
+ if (connection.ping) {
+ observed.ping = async () => {
+ try {
+ await connection.ping!();
+ } catch (error) {
+ demoteEndpoint();
+ throw error;
+ }
+ };
+ }
+ if (connection.getIngressMetrics) {
+ observed.getIngressMetrics = () => connection.getIngressMetrics!();
+ }
+ return observed;
+}
+
+function endpointKeys(
+ preferredUrl: string | URL,
+ failoverUrls: readonly (string | URL)[] | undefined,
+): readonly string[] {
+ return [preferredUrl, ...(failoverUrls ?? [])].map(String);
+}
diff --git a/src/_qwp/_internal/notification-dispatcher.ts b/src/_qwp/_internal/notification-dispatcher.ts
new file mode 100644
index 0000000..3eb1d7b
--- /dev/null
+++ b/src/_qwp/_internal/notification-dispatcher.ts
@@ -0,0 +1,143 @@
+import { isPromiseLike } from "./safe-callback";
+
+export interface QwpNotificationDispatcherMetrics {
+ readonly pending: number;
+ readonly delivered: number;
+ readonly dropped: number;
+ readonly closing: boolean;
+ readonly closed: boolean;
+}
+
+/**
+ * Browser-safe, bounded callback mailbox.
+ *
+ * One notification is delivered per event-loop turn so protocol work already
+ * queued by the WebSocket is not performed inside user callback stacks. When
+ * the inbox fills, the oldest pending notification is discarded and the most
+ * recent state is retained, matching the Java QWP dispatchers.
+ */
+export class QwpNotificationDispatcher {
+ private readonly queue: T[] = [];
+ private timer?: ReturnType;
+ private closeTimer?: ReturnType;
+ private closePromise?: Promise;
+ private resolveClose?: () => void;
+ private dispatching = false;
+ private closing = false;
+ private closed = false;
+ private delivered = 0;
+ private dropped = 0;
+
+ constructor(
+ private readonly handler: (notification: T) => unknown,
+ private readonly capacity: number,
+ ) {
+ if (!Number.isSafeInteger(capacity) || capacity < 1) {
+ throw new RangeError(
+ "QWP notification inbox capacity must be a positive safe integer",
+ );
+ }
+ }
+
+ get metrics(): QwpNotificationDispatcherMetrics {
+ return Object.freeze({
+ pending: this.queue.length,
+ delivered: this.delivered,
+ dropped: this.dropped,
+ closing: this.closing,
+ closed: this.closed,
+ });
+ }
+
+ /** Non-blocking enqueue with drop-oldest overflow. */
+ offer(notification: T): boolean {
+ if (this.closing || this.closed) return false;
+ if (this.queue.length >= this.capacity) {
+ this.queue.shift();
+ this.dropped++;
+ }
+ this.queue.push(notification);
+ this.schedule();
+ return true;
+ }
+
+ /**
+ * Stops accepting new notifications and best-effort drains the retained
+ * tail. Any entries still pending at the deadline are counted as dropped.
+ */
+ close(drainDeadlineMs = 100): Promise {
+ if (this.closePromise) return this.closePromise;
+ if (!Number.isFinite(drainDeadlineMs) || drainDeadlineMs < 0) {
+ return Promise.reject(
+ new RangeError(
+ "QWP notification drain deadline must be non-negative and finite",
+ ),
+ );
+ }
+ this.closing = true;
+ this.closePromise = new Promise((resolve) => {
+ this.resolveClose = resolve;
+ });
+ if (this.queue.length === 0 && !this.dispatching) {
+ this.finishClose();
+ return this.closePromise;
+ }
+ this.schedule();
+ this.closeTimer = setTimeout(() => {
+ this.closeTimer = undefined;
+ this.dropped += this.queue.length;
+ this.queue.length = 0;
+ if (!this.dispatching) this.finishClose();
+ }, drainDeadlineMs);
+ unrefTimer(this.closeTimer);
+ return this.closePromise;
+ }
+
+ private schedule(): void {
+ if (this.timer || this.dispatching || this.closed) return;
+ this.timer = setTimeout(() => {
+ this.timer = undefined;
+ this.dispatchOne();
+ }, 0);
+ unrefTimer(this.timer);
+ }
+
+ private dispatchOne(): void {
+ if (this.closed || this.dispatching) return;
+ const notification = this.queue.shift();
+ if (notification === undefined) {
+ if (this.closing) this.finishClose();
+ return;
+ }
+ this.dispatching = true;
+ this.delivered++;
+ try {
+ const result = this.handler(notification);
+ if (isPromiseLike(result)) void result.then(undefined, () => undefined);
+ } catch {
+ // Observability callbacks never participate in protocol progress.
+ } finally {
+ this.dispatching = false;
+ }
+ if (this.queue.length > 0) {
+ this.schedule();
+ } else if (this.closing) {
+ this.finishClose();
+ }
+ }
+
+ private finishClose(): void {
+ if (this.closed) return;
+ this.closed = true;
+ if (this.timer) clearTimeout(this.timer);
+ if (this.closeTimer) clearTimeout(this.closeTimer);
+ this.timer = undefined;
+ this.closeTimer = undefined;
+ this.resolveClose?.();
+ this.resolveClose = undefined;
+ }
+}
+
+function unrefTimer(timer: ReturnType): void {
+ (timer as ReturnType & { unref?: () => void }).unref?.();
+}
diff --git a/src/_qwp/_internal/reconnect-backoff.ts b/src/_qwp/_internal/reconnect-backoff.ts
new file mode 100644
index 0000000..2089829
--- /dev/null
+++ b/src/_qwp/_internal/reconnect-backoff.ts
@@ -0,0 +1,9 @@
+/**
+ * Applies full jitter to an exponential-backoff ceiling. Full jitter keeps the
+ * configured maximum a hard upper bound while spreading clients throughout
+ * every retry window after a shared outage.
+ */
+export function jitterReconnectDelayMs(ceilingMs: number): number {
+ if (ceilingMs <= 0) return 0;
+ return Math.floor(Math.random() * ceilingMs);
+}
diff --git a/src/_qwp/_internal/reconnecting-egress-connection.ts b/src/_qwp/_internal/reconnecting-egress-connection.ts
new file mode 100644
index 0000000..0131eac
--- /dev/null
+++ b/src/_qwp/_internal/reconnecting-egress-connection.ts
@@ -0,0 +1,729 @@
+import {
+ decodeQwpEgressMessage,
+ QWP_EGRESS_MESSAGE,
+ QwpProtocolError,
+ QwpServerInfoMessage,
+} from "../_core";
+import {
+ QWP_RECONNECT_EVENT_KIND,
+ QWP_UPGRADE_ERROR_KIND,
+ QwpBinaryConnection,
+ QwpConnectionCloseInfo,
+ QwpConnectionFactory,
+ QwpEgressReplayResetEvent,
+ QwpFailoverError,
+ QwpHandshakeMetadata,
+ QwpReconnectEvent,
+ QwpReconnectExhaustedError,
+ QwpReconnectOptions,
+ QwpSendClosedError,
+ QwpUpgradeError,
+} from "../transport";
+import { QwpAsyncQueue } from "./async-queue";
+import { jitterReconnectDelayMs } from "./reconnect-backoff";
+import { safelyInvoke } from "./safe-callback";
+
+type ReplayResetHandler = (
+ event: QwpEgressReplayResetEvent,
+) => void | Promise;
+type ConnectionResetHandler = (
+ serverInfo: QwpServerInfoMessage,
+) => void | Promise;
+type QueryRequestEncoder = (
+ serverInfo: QwpServerInfoMessage,
+ requestId: bigint,
+) => Uint8Array | Promise;
+
+class ReplayResetCallbackError extends Error {
+ readonly cause: unknown;
+
+ constructor(cause: unknown) {
+ super("QWP egress replay reset callback failed");
+ this.name = "ReplayResetCallbackError";
+ this.cause = cause;
+ }
+}
+
+class ReplayStateError extends QwpProtocolError {
+ readonly cause?: unknown;
+
+ constructor(message: string, cause?: unknown) {
+ super(message);
+ this.name = "ReplayStateError";
+ this.cause = cause;
+ }
+}
+
+/**
+ * Reconnects an egress wire and replays the in-flight request and its control
+ * messages. Statements may therefore be executed more than once when their
+ * outcome was lost with the connection.
+ */
+export class QwpReconnectingEgressConnection implements QwpBinaryConnection {
+ private readonly messagesQueue = new QwpAsyncQueue();
+ private readonly maxAttempts: number;
+ private readonly initialBackoffMs: number;
+ private readonly maxBackoffMs: number;
+ private readonly maxDurationMs: number;
+ private readonly resolveClosed: (info: QwpConnectionCloseInfo) => void;
+ private connection?: QwpBinaryConnection;
+ private connectingCandidate?: QwpBinaryConnection;
+ private connectAbort?: AbortController;
+ private lastHandshake?: QwpHandshakeMetadata;
+ private lastEndpoint?: string | URL;
+ private initialServerInfo?: QwpServerInfoMessage;
+ private currentServerInfo?: QwpServerInfoMessage;
+ private outboundReplay: Uint8Array[] = [];
+ private protocolRecoveries = 0;
+ private protocolRecoveryStartedAt = 0;
+ private generation = 0;
+ private sendTail: Promise = Promise.resolve();
+ private reconnectTask?: Promise;
+ private terminalError?: Error;
+ private cancelBackoff?: () => void;
+ private closing = false;
+ private closedSettled = false;
+ readonly messages: AsyncIterable = this.messagesQueue;
+ readonly closed: Promise;
+
+ private constructor(
+ private readonly factory: QwpConnectionFactory,
+ private readonly reconnectOptions: QwpReconnectOptions,
+ private readonly serverInfoTimeoutMs: number,
+ private readonly onConnectionReset: ConnectionResetHandler,
+ private readonly encodeQueryRequest: QueryRequestEncoder,
+ private readonly onReplayReset?: ReplayResetHandler,
+ private readonly retryInitialConnection = true,
+ ) {
+ this.maxAttempts = reconnectOptions.maxAttempts ?? 8;
+ this.initialBackoffMs = reconnectOptions.initialBackoffMs ?? 50;
+ this.maxBackoffMs = reconnectOptions.maxBackoffMs ?? 1_000;
+ this.maxDurationMs = reconnectOptions.maxDurationMs ?? 30_000;
+ validateReconnectPolicy(
+ this.maxAttempts,
+ this.initialBackoffMs,
+ this.maxBackoffMs,
+ this.maxDurationMs,
+ );
+ let resolveClosed!: (info: QwpConnectionCloseInfo) => void;
+ this.closed = new Promise((resolve) => {
+ resolveClosed = resolve;
+ });
+ this.resolveClosed = resolveClosed;
+ }
+
+ static async connect(
+ factory: QwpConnectionFactory,
+ reconnectOptions: QwpReconnectOptions,
+ serverInfoTimeoutMs: number,
+ onConnectionReset: ConnectionResetHandler,
+ encodeQueryRequest: QueryRequestEncoder,
+ onReplayReset?: ReplayResetHandler,
+ retryInitialConnection = true,
+ ): Promise {
+ const reconnecting = new QwpReconnectingEgressConnection(
+ factory,
+ reconnectOptions,
+ serverInfoTimeoutMs,
+ onConnectionReset,
+ encodeQueryRequest,
+ onReplayReset,
+ retryInitialConnection,
+ );
+ try {
+ await reconnecting.connectLoop(undefined, false);
+ return reconnecting;
+ } catch (error) {
+ await reconnecting.close().catch(() => undefined);
+ throw error;
+ }
+ }
+
+ get handshake(): QwpHandshakeMetadata {
+ if (!this.lastHandshake)
+ throw new Error("QWP connection is not established");
+ return this.lastHandshake;
+ }
+
+ get endpoint(): string | URL | undefined {
+ return this.lastEndpoint;
+ }
+
+ send(payload: Uint8Array): Promise {
+ if (this.terminalError) return Promise.reject(this.terminalError);
+ if (this.closing) return Promise.reject(new QwpSendClosedError());
+ const copy = payload.slice();
+ const sending = this.sendTail.then(async () => {
+ this.throwIfUnavailable();
+ const connection = await this.requireConnection();
+ const prepared = await this.prepareOutboundQuery(copy);
+ this.trackOutbound(prepared);
+ try {
+ await connection.send(prepared);
+ } catch (error) {
+ await this.requestReconnect(error, connection);
+ }
+ });
+ this.sendTail = sending.catch(() => undefined);
+ return sending;
+ }
+
+ async close(code = 1000, reason = ""): Promise {
+ if (this.closing) {
+ await this.closed;
+ return;
+ }
+ this.closing = true;
+ this.cancelBackoff?.();
+ this.messagesQueue.end();
+ const connection = this.connection;
+ // Tears down a connect that is still negotiating. Without this the socket
+ // and its deadline outlive close(), keeping the event loop open for up to
+ // connectTimeoutMs/authTimeoutMs after close() has already resolved.
+ this.connectAbort?.abort();
+ const connectingCandidate = this.connectingCandidate;
+ this.connection = undefined;
+ this.connectingCandidate = undefined;
+ let closeInfo: QwpConnectionCloseInfo = {
+ code,
+ reason,
+ wasClean: code === 1000,
+ };
+ if (connection) {
+ try {
+ await connection.close(code, reason);
+ closeInfo = await connection.closed;
+ } catch {
+ // Preserve the requested close result when transport shutdown races.
+ }
+ }
+ if (connectingCandidate && connectingCandidate !== connection) {
+ await connectingCandidate.close(code, reason).catch(() => undefined);
+ }
+ this.settleClosed(closeInfo);
+ }
+
+ private async connectLoop(
+ initialCause: unknown,
+ reconnecting: boolean,
+ skipQueueBarrier = false,
+ ): Promise {
+ const outageStarted = Date.now();
+ const previousEndpoint = this.lastEndpoint;
+ let attempt = 0;
+ let backoffMs = this.initialBackoffMs;
+ let lastError = initialCause;
+ if (reconnecting) {
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.RECONNECTING,
+ attempt: 0,
+ previousEndpoint,
+ cause: initialCause,
+ });
+ if (backoffMs > 0) {
+ await this.waitForBackoff(jitterReconnectDelayMs(backoffMs));
+ backoffMs = Math.min(Math.max(backoffMs * 2, 1), this.maxBackoffMs);
+ }
+ }
+
+ while (!this.closing) {
+ if (attempt > 0 && backoffMs > 0) {
+ await this.waitForBackoff(jitterReconnectDelayMs(backoffMs));
+ backoffMs = Math.min(Math.max(backoffMs * 2, 1), this.maxBackoffMs);
+ }
+ this.throwIfUnavailable();
+ attempt++;
+ let candidate: QwpBinaryConnection | undefined;
+ try {
+ const abort = new AbortController();
+ this.connectAbort = abort;
+ try {
+ candidate = await this.factory(abort.signal);
+ } finally {
+ if (this.connectAbort === abort) this.connectAbort = undefined;
+ }
+ this.connectingCandidate = candidate;
+ if (this.closing) {
+ await candidate.close().catch(() => undefined);
+ throw new QwpSendClosedError();
+ }
+ const iterator = candidate.messages[Symbol.asyncIterator]();
+ const serverInfoPayload = await this.readServerInfo(
+ iterator,
+ candidate,
+ );
+ const serverInfo = decodeQwpEgressMessage(serverInfoPayload);
+ if (serverInfo.kind !== "server-info") {
+ throw new QwpProtocolError(
+ "QWP egress connection did not begin with SERVER_INFO",
+ );
+ }
+ if (reconnecting) {
+ this.validateServerInfo(serverInfo, candidate);
+ await this.replayInto(
+ candidate,
+ serverInfo,
+ previousEndpoint,
+ initialCause,
+ skipQueueBarrier,
+ );
+ } else {
+ this.initialServerInfo = serverInfo;
+ this.currentServerInfo = serverInfo;
+ this.messagesQueue.push(serverInfoPayload);
+ }
+ if (this.closing) throw new QwpSendClosedError();
+ this.currentServerInfo = serverInfo;
+ this.install(candidate, iterator);
+ this.connectingCandidate = undefined;
+ if (reconnecting) {
+ this.emitEvent({
+ kind:
+ previousEndpoint !== undefined &&
+ String(previousEndpoint) !== String(candidate.endpoint)
+ ? QWP_RECONNECT_EVENT_KIND.FAILED_OVER
+ : QWP_RECONNECT_EVENT_KIND.RECONNECTED,
+ attempt,
+ endpoint: candidate.endpoint,
+ previousEndpoint,
+ });
+ } else {
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.CONNECTED,
+ attempt: 0,
+ endpoint: candidate.endpoint,
+ });
+ }
+ return;
+ } catch (error) {
+ lastError = error;
+ if (this.connectingCandidate === candidate) {
+ this.connectingCandidate = undefined;
+ }
+ if (candidate) await candidate.close().catch(() => undefined);
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.ATTEMPT_FAILED,
+ attempt,
+ endpoint: candidate?.endpoint,
+ previousEndpoint,
+ cause: error,
+ });
+ if (!isRetryableReconnectError(error)) throw error;
+ if (!reconnecting && !this.retryInitialConnection) throw error;
+ const attemptsExhausted =
+ this.maxAttempts > 0 && attempt >= this.maxAttempts;
+ const durationExhausted =
+ this.maxDurationMs > 0 &&
+ Date.now() - outageStarted >= this.maxDurationMs;
+ if (attemptsExhausted || durationExhausted) {
+ throw new QwpReconnectExhaustedError(attempt, lastError);
+ }
+ }
+ }
+ throw new QwpSendClosedError();
+ }
+
+ private install(
+ connection: QwpBinaryConnection,
+ iterator: AsyncIterator,
+ ): void {
+ this.connection = connection;
+ this.lastHandshake = connection.handshake;
+ this.lastEndpoint = connection.endpoint;
+ const generation = ++this.generation;
+ void this.pump(connection, iterator, generation);
+ }
+
+ private async pump(
+ connection: QwpBinaryConnection,
+ iterator: AsyncIterator,
+ generation: number,
+ ): Promise {
+ try {
+ while (true) {
+ const next = await iterator.next();
+ if (next.done) break;
+ if (this.closing || this.connection !== connection) return;
+ const message = decodeQwpEgressMessage(next.value);
+ if (message.kind === "server-info") {
+ throw new QwpProtocolError("received duplicate QWP SERVER_INFO");
+ } else if (
+ message.kind === "result-end" ||
+ message.kind === "exec-done" ||
+ message.kind === "query-error"
+ ) {
+ const activeRequestId = replayRequestId(this.outboundReplay);
+ if (activeRequestId === message.requestId) this.outboundReplay = [];
+ }
+ this.messagesQueue.push(next.value);
+ }
+ if (this.closing || this.connection !== connection) return;
+ await this.requestReconnect(
+ new QwpSendClosedError(await connection.closed),
+ connection,
+ ).catch((reconnectError) => this.failTerminal(reconnectError));
+ return;
+ } catch (error) {
+ if (
+ this.closing ||
+ this.connection !== connection ||
+ generation !== this.generation
+ ) {
+ return;
+ }
+ await this.requestReconnect(
+ error,
+ connection,
+ error instanceof QwpProtocolError ? 1002 : 1000,
+ error instanceof QwpProtocolError ? "invalid QWP egress message" : "",
+ ).catch((reconnectError) => this.failTerminal(reconnectError));
+ }
+ }
+
+ private async readServerInfo(
+ iterator: AsyncIterator,
+ connection: QwpBinaryConnection,
+ ): Promise {
+ let timer: ReturnType | undefined;
+ const timeout = new Promise((_resolve, reject) => {
+ timer = setTimeout(
+ () =>
+ reject(new Error("timed out waiting for QWP reconnect SERVER_INFO")),
+ this.serverInfoTimeoutMs,
+ );
+ });
+ try {
+ const result = await Promise.race([iterator.next(), timeout]);
+ if (result.done) {
+ throw new QwpSendClosedError(await connection.closed);
+ }
+ return result.value;
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+ }
+
+ private validateServerInfo(
+ serverInfo: QwpServerInfoMessage,
+ connection: QwpBinaryConnection,
+ ): void {
+ const initial = this.initialServerInfo;
+ if (!initial) {
+ throw new ReplayStateError(
+ "QWP reconnect started before the initial SERVER_INFO was received",
+ );
+ }
+ if (
+ initial.clusterId &&
+ serverInfo.clusterId &&
+ initial.clusterId !== serverInfo.clusterId
+ ) {
+ throw new QwpUpgradeError(
+ `QWP reconnect target belongs to a different cluster [expected=${initial.clusterId}, actual=${serverInfo.clusterId}]`,
+ {
+ kind: QWP_UPGRADE_ERROR_KIND.CAPABILITY_MISMATCH,
+ retryable: true,
+ tryNextEndpoint: true,
+ url: connection.endpoint,
+ },
+ );
+ }
+ }
+
+ private async replayInto(
+ connection: QwpBinaryConnection,
+ serverInfo: QwpServerInfoMessage,
+ previousEndpoint: string | URL | undefined,
+ cause: unknown,
+ skipQueueBarrier: boolean,
+ ): Promise {
+ if (this.outboundReplay.length === 0) {
+ // A terminal response may already be queued. Let the bounded session
+ // consume it before resetting connection-scoped decoder state.
+ if (skipQueueBarrier) this.messagesQueue.clear();
+ else await this.messagesQueue.barrier();
+ await this.onConnectionReset(serverInfo);
+ return;
+ }
+ // An active operation will be replayed from its request. Drop raw stale
+ // messages before resetting the decoded queue; waiting for a barrier here
+ // can deadlock when that queue is deliberately at its client-side bound.
+ this.messagesQueue.clear();
+ await this.onConnectionReset(serverInfo);
+ const requestId = replayRequestId(this.outboundReplay);
+ if (requestId === undefined) {
+ throw new ReplayStateError(
+ "QWP egress replay is missing its QUERY_REQUEST",
+ );
+ }
+ if (this.onReplayReset) {
+ try {
+ await this.onReplayReset({
+ requestId,
+ serverInfo,
+ previousEndpoint,
+ endpoint: connection.endpoint,
+ cause,
+ });
+ } catch (error) {
+ throw new ReplayResetCallbackError(error);
+ }
+ }
+ const request = await this.encodeReplayRequest(serverInfo, requestId);
+ validateEncodedRequest(request, requestId);
+ const preparedRequest = request.slice();
+ this.outboundReplay[0] = preparedRequest;
+ for (const payload of this.outboundReplay) await connection.send(payload);
+ }
+
+ private async prepareOutboundQuery(payload: Uint8Array): Promise {
+ if (payload[0] !== QWP_EGRESS_MESSAGE.QUERY_REQUEST) return payload;
+ const requestId = replayRequestId([payload]);
+ const serverInfo = this.currentServerInfo;
+ if (requestId === undefined || !serverInfo) {
+ throw new QwpProtocolError(
+ "QWP QUERY_REQUEST cannot be prepared before SERVER_INFO",
+ );
+ }
+ const encoded = await this.encodeReplayRequest(serverInfo, requestId);
+ validateEncodedRequest(encoded, requestId);
+ return encoded.slice();
+ }
+
+ private async encodeReplayRequest(
+ serverInfo: QwpServerInfoMessage,
+ requestId: bigint,
+ ): Promise {
+ try {
+ return await this.encodeQueryRequest(serverInfo, requestId);
+ } catch (error) {
+ throw new ReplayStateError(
+ `QWP egress could not reconstruct active request ID ${requestId}`,
+ error,
+ );
+ }
+ }
+
+ /** @internal Replaces a connection whose server response was invalid. */
+ async recoverProtocolFailure(error: QwpProtocolError): Promise {
+ this.throwIfUnavailable();
+ const connection = this.connection;
+ if (!connection) throw new QwpSendClosedError();
+ // Reconnecting replays the same QUERY_REQUEST, so a response this client
+ // cannot decode reproduces on the replacement connection. Each connect
+ // SUCCEEDS, so connectLoop's own budget is never consumed and the retry
+ // would otherwise run forever, rotating the whole cluster. Charge these
+ // recoveries to the same maxAttempts/maxDurationMs budget instead, the way
+ // the Java client counts every re-submission of one execute() against
+ // failover_max_attempts and failover_max_duration.
+ if (this.protocolRecoveries === 0) {
+ this.protocolRecoveryStartedAt = Date.now();
+ }
+ this.protocolRecoveries++;
+ // `>` not `>=`: maxAttempts counts reconnects here, as it does in
+ // connectLoop, so maxAttempts=1 still permits one recovery.
+ const attemptsExhausted =
+ this.maxAttempts > 0 && this.protocolRecoveries > this.maxAttempts;
+ const durationExhausted =
+ this.maxDurationMs > 0 &&
+ Date.now() - this.protocolRecoveryStartedAt >= this.maxDurationMs;
+ if (attemptsExhausted || durationExhausted) {
+ const exhausted = new QwpReconnectExhaustedError(
+ this.protocolRecoveries,
+ error,
+ );
+ this.failTerminal(exhausted);
+ throw exhausted;
+ }
+ try {
+ await this.requestReconnect(
+ error,
+ connection,
+ 1002,
+ "invalid QWP egress message",
+ true,
+ );
+ } catch (reconnectError) {
+ this.failTerminal(reconnectError);
+ throw reconnectError;
+ }
+ }
+
+ private trackOutbound(payload: Uint8Array): void {
+ switch (payload[0]) {
+ case QWP_EGRESS_MESSAGE.QUERY_REQUEST:
+ this.outboundReplay = [payload];
+ // A new application query is fresh progress, matching the Java
+ // client's per-execute() scoping. Replay does not come through here,
+ // so a request that keeps poisoning still exhausts its budget.
+ this.protocolRecoveries = 0;
+ break;
+ case QWP_EGRESS_MESSAGE.CREDIT:
+ case QWP_EGRESS_MESSAGE.CANCEL:
+ if (this.outboundReplay.length > 0) this.outboundReplay.push(payload);
+ break;
+ }
+ }
+
+ private async requireConnection(): Promise {
+ if (this.reconnectTask) await this.reconnectTask;
+ this.throwIfUnavailable();
+ if (!this.connection) throw new QwpSendClosedError();
+ return this.connection;
+ }
+
+ private async requestReconnect(
+ cause: unknown,
+ failedConnection: QwpBinaryConnection,
+ closeCode = 1000,
+ closeReason = "",
+ skipQueueBarrier = false,
+ ): Promise {
+ if (this.closing) throw new QwpSendClosedError();
+ if (this.connection && this.connection !== failedConnection) return;
+ if (this.reconnectTask) {
+ const activeReconnect = this.reconnectTask;
+ await activeReconnect;
+ if (this.connection === failedConnection && !this.closing) {
+ await this.requestReconnect(
+ cause,
+ failedConnection,
+ closeCode,
+ closeReason,
+ skipQueueBarrier,
+ );
+ }
+ return;
+ }
+
+ this.connection = undefined;
+ if (closeCode !== 1000) failedConnection.deprioritizeEndpoint?.();
+ void failedConnection.close(closeCode, closeReason).catch(() => undefined);
+ const reconnecting = this.connectLoop(cause, true, skipQueueBarrier);
+ this.reconnectTask = reconnecting;
+ try {
+ await reconnecting;
+ } finally {
+ if (this.reconnectTask === reconnecting) this.reconnectTask = undefined;
+ }
+ }
+
+ private async waitForBackoff(delayMs: number): Promise {
+ await new Promise((resolve) => {
+ const timer = setTimeout(() => {
+ if (this.cancelBackoff === cancel) this.cancelBackoff = undefined;
+ resolve();
+ }, delayMs);
+ const cancel = (): void => {
+ clearTimeout(timer);
+ if (this.cancelBackoff === cancel) this.cancelBackoff = undefined;
+ resolve();
+ };
+ this.cancelBackoff = cancel;
+ });
+ }
+
+ private emitEvent(event: Omit): void {
+ // Contain synchronous throws and rejected promises alike: a failing
+ // observer, sync or async, must never interfere with replay progress.
+ safelyInvoke(this.reconnectOptions.onEvent, {
+ ...event,
+ timestampMs: Date.now(),
+ });
+ }
+
+ private throwIfUnavailable(): void {
+ if (this.terminalError) throw this.terminalError;
+ if (this.closing) throw new QwpSendClosedError();
+ }
+
+ private failTerminal(error: unknown): void {
+ if (this.terminalError) return;
+ this.terminalError =
+ error instanceof Error
+ ? error
+ : new Error(`QWP reconnect failed: ${error}`);
+ this.cancelBackoff?.();
+ this.messagesQueue.fail(this.terminalError);
+ this.settleClosed({
+ code: 1011,
+ reason: this.terminalError.message,
+ wasClean: false,
+ });
+ void this.connection
+ ?.close(1011, "QWP reconnect failed")
+ .catch(() => undefined);
+ }
+
+ private settleClosed(info: QwpConnectionCloseInfo): void {
+ if (this.closedSettled) return;
+ this.closedSettled = true;
+ this.resolveClosed(info);
+ }
+}
+
+function replayRequestId(payloads: readonly Uint8Array[]): bigint | undefined {
+ const query = payloads.find(
+ (payload) => payload[0] === QWP_EGRESS_MESSAGE.QUERY_REQUEST,
+ );
+ if (!query || query.byteLength < 9) return undefined;
+ return new DataView(
+ query.buffer,
+ query.byteOffset,
+ query.byteLength,
+ ).getBigUint64(1, true);
+}
+
+function validateEncodedRequest(
+ payload: Uint8Array,
+ expectedRequestId: bigint,
+): void {
+ const requestId = replayRequestId([payload]);
+ if (requestId !== expectedRequestId) {
+ throw new ReplayStateError(
+ `QWP query encoder returned the wrong request [expected=${expectedRequestId}, actual=${requestId ?? "missing"}]`,
+ );
+ }
+}
+
+function validateReconnectPolicy(
+ maxAttempts: number,
+ initialBackoffMs: number,
+ maxBackoffMs: number,
+ maxDurationMs: number,
+): void {
+ if (!Number.isSafeInteger(maxAttempts) || maxAttempts < 0) {
+ throw new RangeError(
+ "reconnect maxAttempts must be a non-negative safe integer",
+ );
+ }
+ for (const [name, value] of [
+ ["initialBackoffMs", initialBackoffMs],
+ ["maxBackoffMs", maxBackoffMs],
+ ["maxDurationMs", maxDurationMs],
+ ] as const) {
+ if (!Number.isFinite(value) || value < 0) {
+ throw new RangeError(
+ `reconnect ${name} must be a non-negative finite number`,
+ );
+ }
+ }
+ if (maxBackoffMs < initialBackoffMs) {
+ throw new RangeError(
+ "reconnect maxBackoffMs must be greater than or equal to initialBackoffMs",
+ );
+ }
+}
+
+function isRetryableReconnectError(error: unknown): boolean {
+ if (error instanceof QwpUpgradeError) return error.retryable !== false;
+ if (error instanceof QwpFailoverError) {
+ return error.attempts.some((attempt) =>
+ isRetryableReconnectError(attempt.error),
+ );
+ }
+ return !(
+ error instanceof ReplayStateError ||
+ error instanceof ReplayResetCallbackError
+ );
+}
diff --git a/src/_qwp/_internal/reconnecting-ingress-connection.ts b/src/_qwp/_internal/reconnecting-ingress-connection.ts
new file mode 100644
index 0000000..2693807
--- /dev/null
+++ b/src/_qwp/_internal/reconnecting-ingress-connection.ts
@@ -0,0 +1,2299 @@
+import {
+ decodeQwpFrame,
+ decodeQwpIngressResponse,
+ decodeQwpIngressSymbolDictionaryDelta,
+ encodeQwpIngressSymbolDictionaryFrame,
+ QWP_FLAG_DEFER_COMMIT,
+ QWP_FLAG_DELTA_SYMBOL_DICTIONARY,
+ QWP_FLAG_DURABLE_ACK_POLL,
+ QWP_HEADER_SIZE,
+ QWP_STATUS,
+ QwpProtocolError,
+ qwpVarintSize,
+ utf8Length,
+} from "../_core";
+import {
+ QWP_INITIAL_CONNECT_MODE,
+ QWP_RECONNECT_EVENT_KIND,
+ QWP_UPGRADE_ERROR_KIND,
+ QwpBinaryConnection,
+ QwpConnectionCloseInfo,
+ QwpConnectionFactory,
+ QwpDurableAckUnavailableError,
+ QwpFailoverError,
+ QwpHandshakeMetadata,
+ QwpIngressReplayRecord,
+ QwpIngressReplayReference,
+ QwpIngressReplayStore,
+ QwpIngressTransportMetrics,
+ QwpInitialConnectMode,
+ QwpMemoryReplayAppendTimeoutError,
+ QwpMemoryReplayFrameTooLargeError,
+ QwpReconnectEvent,
+ QwpReconnectExhaustedError,
+ QwpReconnectOptions,
+ QwpReplayDictionaryError,
+ QwpReplayDictionaryPersistenceError,
+ QwpReplayRejectedError,
+ QwpSendClosedError,
+ QwpUnrecoverableReplayDictionaryError,
+ QwpUpgradeError,
+} from "../transport";
+import { QwpAsyncQueue } from "./async-queue";
+import { jitterReconnectDelayMs } from "./reconnect-backoff";
+import { QwpNotificationDispatcher } from "./notification-dispatcher";
+import {
+ createQwpProtocolViolationSenderError,
+ createQwpSenderError,
+ defaultQwpSenderErrorHandler,
+ type QwpSenderError,
+} from "../sender-error";
+
+const DEFAULT_CATCH_UP_CAP_GAP_MIN_ESCALATION_WINDOW_MS = 300_000;
+const MAX_CATCH_UP_CAP_GAP_ATTEMPTS = 16;
+const DEFAULT_ORPHAN_DURABLE_ACK_MISMATCH_MAX_DURATION_MS = 300_000;
+const MAX_ORPHAN_DURABLE_ACK_MISMATCH_ATTEMPTS = 16;
+const DEFAULT_MEMORY_REPLAY_MAX_BYTES = 128 * 1024 * 1024;
+const DEFAULT_MEMORY_REPLAY_APPEND_DEADLINE_MS = 30_000;
+// Charge a conservative fixed amount so even empty/very small opaque frames
+// cannot grow the replay Map without bound. Payload arrays are not copied by
+// the store, so the configured budget primarily tracks live frame storage.
+const MEMORY_REPLAY_RECORD_OVERHEAD_BYTES = 64;
+
+type ConnectAttemptPolicy = "single" | "configured" | "unbounded";
+
+export class QwpCatchUpCapGapError extends RangeError {
+ constructor(
+ readonly symbolId: number,
+ readonly frameLength: number,
+ readonly maxBatchSizeBytes: number,
+ details?: {
+ attempt: number;
+ episodeMs: number;
+ minEscalationWindowMs: number;
+ exhausted: boolean;
+ },
+ ) {
+ super(
+ `symbol dictionary entry exceeds reconnect target batch cap [id=${symbolId}, frameLength=${frameLength}, max=${maxBatchSizeBytes}` +
+ (details
+ ? `, attempt=${details.attempt}/${MAX_CATCH_UP_CAP_GAP_ATTEMPTS}, episodeMs=${details.episodeMs}/${details.minEscalationWindowMs}]${
+ details.exhausted
+ ? "; the data must be resent after the cap is raised"
+ : "; retrying because a larger-cap node may return"
+ }`
+ : "]"),
+ );
+ this.name = "QwpCatchUpCapGapError";
+ }
+}
+
+export class QwpDurableAckPersistentFailureError extends Error {
+ constructor(
+ readonly attempts: number,
+ readonly episodeMs: number,
+ readonly cause: QwpDurableAckUnavailableError,
+ ) {
+ super(
+ `QWP durable ACK remained unavailable for an orphan replay slot [attempts=${attempts}/${MAX_ORPHAN_DURABLE_ACK_MISMATCH_ATTEMPTS}, episodeMs=${episodeMs}]: ${cause.message}`,
+ );
+ this.name = "QwpDurableAckPersistentFailureError";
+ }
+}
+
+interface ReplayFrame extends Omit {
+ // Assigned inside send()'s serialized tail, immediately before the journal
+ // append, so a frame that never reaches the store consumes no sequence.
+ frameSequence: bigint;
+ payload?: Uint8Array;
+ readonly clientSequence?: bigint;
+ ackDelivered: boolean;
+ transmitted: boolean;
+ durableTargets?: Map;
+ dictionaryCatchup?: boolean;
+}
+
+type LoadedReplayRecord = QwpIngressReplayReference & {
+ readonly payload?: Uint8Array;
+};
+
+type LazyReplayStore = QwpIngressReplayStore &
+ Required>;
+
+interface RecoveredDiscardTail {
+ readonly startSequence: bigint;
+ readonly tipSequence: bigint;
+ readonly predecessorSequence?: bigint;
+}
+
+class RetriableIngressNackError extends Error {
+ constructor(
+ readonly frameSequence: bigint,
+ readonly status: number,
+ readonly retryDelayMs: number,
+ message?: string,
+ ) {
+ super(
+ `QuestDB temporarily rejected QWP frame [frameSequence=${frameSequence}, status=0x${status.toString(16)}]${
+ message ? `: ${message}` : ""
+ }`,
+ );
+ this.name = "RetriableIngressNackError";
+ }
+}
+
+class RetriableIngressConnectionError extends Error {
+ readonly cause: unknown;
+
+ constructor(
+ readonly retryDelayMs: number,
+ cause: unknown,
+ ) {
+ super(
+ cause instanceof Error
+ ? cause.message
+ : `QWP ingress connection was lost: ${cause}`,
+ );
+ this.name = "RetriableIngressConnectionError";
+ this.cause = cause;
+ }
+}
+
+class QwpMemoryReplayStore implements QwpIngressReplayStore {
+ private readonly records = new Map();
+ private readonly symbols: string[] = [];
+ private readonly capacityWaiters = new Set<{
+ resolve: () => void;
+ reject: (error: Error) => void;
+ timer: ReturnType;
+ }>();
+ private usedBytes = 0;
+ private closing = false;
+ private totalBackpressureStalls = 0;
+ private totalAppendTimeouts = 0;
+
+ constructor(
+ readonly maxBytes = DEFAULT_MEMORY_REPLAY_MAX_BYTES,
+ private readonly appendDeadlineMs = DEFAULT_MEMORY_REPLAY_APPEND_DEADLINE_MS,
+ ) {}
+
+ get metrics() {
+ return {
+ maxBytes: this.maxBytes,
+ usedBytes: this.usedBytes,
+ waitingAppends: this.capacityWaiters.size,
+ totalBackpressureStalls: this.totalBackpressureStalls,
+ totalAppendTimeouts: this.totalAppendTimeouts,
+ } as const;
+ }
+
+ async load(): Promise {
+ return Array.from(this.records, ([frameSequence, payload]) => ({
+ frameSequence,
+ payload: payload.slice(),
+ }));
+ }
+
+ async append(record: QwpIngressReplayRecord): Promise {
+ if (this.closing) throw new QwpSendClosedError();
+ if (this.records.has(record.frameSequence)) {
+ throw new Error(
+ `QWP memory replay sequence already exists [frameSequence=${record.frameSequence}]`,
+ );
+ }
+ const requiredBytes =
+ record.payload.byteLength + MEMORY_REPLAY_RECORD_OVERHEAD_BYTES;
+ if (requiredBytes > this.maxBytes) {
+ throw new QwpMemoryReplayFrameTooLargeError(
+ this.maxBytes,
+ record.payload.byteLength,
+ requiredBytes,
+ );
+ }
+ if (this.usedBytes + requiredBytes > this.maxBytes) {
+ this.totalBackpressureStalls++;
+ const deadline = Date.now() + this.appendDeadlineMs;
+ while (this.usedBytes + requiredBytes > this.maxBytes) {
+ const remainingMs = deadline - Date.now();
+ if (remainingMs <= 0) {
+ this.totalAppendTimeouts++;
+ throw new QwpMemoryReplayAppendTimeoutError(
+ this.maxBytes,
+ this.usedBytes,
+ requiredBytes,
+ this.appendDeadlineMs,
+ );
+ }
+ await this.waitForCapacity(remainingMs, requiredBytes);
+ if (this.closing) throw new QwpSendClosedError();
+ }
+ }
+ // send() already made the replay-owned payload copy. Sharing it between
+ // the connection and this accounting store avoids doubling the backlog.
+ this.records.set(record.frameSequence, record.payload);
+ this.usedBytes += requiredBytes;
+ }
+
+ async acknowledgeThrough(frameSequence: bigint): Promise {
+ for (const sequence of this.records.keys()) {
+ if (sequence > frameSequence) break;
+ const payload = this.records.get(sequence)!;
+ this.usedBytes -=
+ payload.byteLength + MEMORY_REPLAY_RECORD_OVERHEAD_BYTES;
+ this.records.delete(sequence);
+ }
+ this.releaseCapacityWaiters();
+ }
+
+ async loadSymbolDictionary(): Promise {
+ return this.symbols.slice();
+ }
+
+ async appendSymbolDictionary(
+ startId: number,
+ entries: readonly string[],
+ ): Promise {
+ if (startId !== this.symbols.length) {
+ throw new QwpReplayDictionaryError(
+ `memory replay dictionary is not dense [expected=${this.symbols.length}, received=${startId}]`,
+ );
+ }
+ this.symbols.push(...entries);
+ }
+
+ async close(): Promise {
+ if (this.closing) return;
+ this.closing = true;
+ const error = new QwpSendClosedError();
+ for (const waiter of this.capacityWaiters) {
+ clearTimeout(waiter.timer);
+ waiter.reject(error);
+ }
+ this.capacityWaiters.clear();
+ this.records.clear();
+ this.symbols.length = 0;
+ this.usedBytes = 0;
+ }
+
+ private waitForCapacity(
+ timeoutMs: number,
+ requiredBytes: number,
+ ): Promise {
+ return new Promise((resolve, reject) => {
+ const waiter = {
+ resolve: () => {
+ clearTimeout(waiter.timer);
+ this.capacityWaiters.delete(waiter);
+ resolve();
+ },
+ reject: (error: Error) => {
+ clearTimeout(waiter.timer);
+ this.capacityWaiters.delete(waiter);
+ reject(error);
+ },
+ timer: undefined as unknown as ReturnType,
+ };
+ waiter.timer = setTimeout(() => {
+ this.totalAppendTimeouts++;
+ waiter.reject(
+ new QwpMemoryReplayAppendTimeoutError(
+ this.maxBytes,
+ this.usedBytes,
+ requiredBytes,
+ this.appendDeadlineMs,
+ ),
+ );
+ }, timeoutMs);
+ this.capacityWaiters.add(waiter);
+ });
+ }
+
+ private releaseCapacityWaiters(): void {
+ for (const waiter of [...this.capacityWaiters]) waiter.resolve();
+ }
+}
+
+/**
+ * Reconnects an ingress wire and translates its per-connection ACK sequence
+ * back to stable replay records. Replay is deliberately at-least-once: a frame
+ * accepted by the server whose ACK was lost may be sent again.
+ */
+export class QwpReconnectingIngressConnection implements QwpBinaryConnection {
+ private readonly messagesQueue = new QwpAsyncQueue();
+ private readonly frames = new Map();
+ private readonly durableWatermarks = new Map();
+ private readonly symbolDictionary: string[];
+ private readonly store: QwpIngressReplayStore;
+ private readonly lazyReplayStore?: LazyReplayStore;
+ private readonly maxAttempts: number;
+ private readonly initialBackoffMs: number;
+ private readonly maxBackoffMs: number;
+ private readonly maxDurationMs: number;
+ private readonly maxFrameRejections: number;
+ private readonly poisonMinEscalationWindowMs: number;
+ private readonly catchUpCapGapMinEscalationWindowMs: number;
+ private readonly orphanDurableAckMismatchMaxDurationMs: number;
+ private readonly localMaxBatchSizeBytes?: number;
+ private readonly connectionDispatcher?: QwpNotificationDispatcher;
+ private readonly errorDispatcher?: QwpNotificationDispatcher;
+ private readonly resolveClosed: (info: QwpConnectionCloseInfo) => void;
+ private connection?: QwpBinaryConnection;
+ private connectingCandidate?: QwpBinaryConnection;
+ private connectAbort?: AbortController;
+ private lastHandshake?: QwpHandshakeMetadata;
+ private lastEndpoint?: string | URL;
+ // Wire log for the current connection, indexed by wire sequence minus
+ // wireFramesBase. Acknowledged frames are dropped and the base advances, so
+ // the log stays proportional to what is still unacknowledged rather than to
+ // everything ever sent on the connection.
+ private wireFrames: ReplayFrame[] = [];
+ private wireFramesBase = 0;
+ private nextFrameSequence = 0n;
+ private nextClientSequence = 0n;
+ private publishedFrameSequence = -1n;
+ private acknowledgedFrameSequence = -1n;
+ private highestOkFrameSequence = -1n;
+ private poisonFrameSequence?: bigint;
+ private poisonFirstStrikeMs = 0;
+ private poisonStrikes = 0;
+ private catchUpCapGapAttempts = 0;
+ private catchUpCapGapFirstMs = 0;
+ private durableAckMismatchAttempts = 0;
+ private durableAckMismatchFirstMs = 0;
+ private progressAtLastExemptRecycle = -1n;
+ private zeroProgressRecycles = 0;
+ private recoveredDiscardTail?: RecoveredDiscardTail;
+ private generation = 0;
+ private sendTail: Promise = Promise.resolve();
+ private drainTail: Promise = Promise.resolve();
+ private reconnectTask?: Promise;
+ private storeClosePromise?: Promise;
+ private terminalError?: Error;
+ private cancelBackoff?: () => void;
+ private closing = false;
+ private closedSettled = false;
+ private totalFramesSent = 0;
+ private totalBytesSent = 0;
+ private totalFramesReplayed = 0;
+ private totalBytesReplayed = 0;
+ private totalReconnectAttempts = 0;
+ private totalReconnectsSucceeded = 0;
+ private totalFailovers = 0;
+ private totalReconnectErrors = 0;
+ private totalServerNacks = 0;
+ private hasEverConnected = false;
+ private deltaSymbolDictionaryEnabled: boolean;
+ readonly messages: AsyncIterable = this.messagesQueue;
+ readonly closed: Promise;
+ readonly managesIngressSenderErrors = true;
+ ping?: () => Promise;
+
+ private constructor(
+ private readonly factory: QwpConnectionFactory,
+ private readonly reconnectOptions: QwpReconnectOptions,
+ store: QwpIngressReplayStore,
+ records: readonly LoadedReplayRecord[],
+ symbolDictionary: readonly string[],
+ recoveredDiscardTail: RecoveredDiscardTail | undefined,
+ localMaxBatchSizeBytes?: number,
+ private readonly backgroundStoreAndForward = false,
+ private readonly orphanStoreAndForward = false,
+ orphanDurableAckMismatchMaxDurationMs = DEFAULT_ORPHAN_DURABLE_ACK_MISMATCH_MAX_DURATION_MS,
+ catchUpCapGapMinEscalationWindowMs = DEFAULT_CATCH_UP_CAP_GAP_MIN_ESCALATION_WINDOW_MS,
+ connectionListenerInboxCapacity = 64,
+ errorInboxCapacity = 256,
+ onSenderError?: (error: QwpSenderError) => void,
+ ) {
+ this.store = store;
+ this.lazyReplayStore = isLazyReplayStore(store) ? store : undefined;
+ this.symbolDictionary = [...symbolDictionary];
+ this.deltaSymbolDictionaryEnabled =
+ store.loadSymbolDictionary !== undefined &&
+ store.appendSymbolDictionary !== undefined;
+ this.recoveredDiscardTail = recoveredDiscardTail;
+ this.localMaxBatchSizeBytes = localMaxBatchSizeBytes;
+ this.maxAttempts = reconnectOptions.maxAttempts ?? 3;
+ this.initialBackoffMs = reconnectOptions.initialBackoffMs ?? 100;
+ this.maxBackoffMs = reconnectOptions.maxBackoffMs ?? 5_000;
+ this.maxDurationMs = reconnectOptions.maxDurationMs ?? 30_000;
+ this.maxFrameRejections = reconnectOptions.maxFrameRejections ?? 4;
+ this.poisonMinEscalationWindowMs =
+ reconnectOptions.poisonMinEscalationWindowMs ?? 5_000;
+ this.catchUpCapGapMinEscalationWindowMs =
+ catchUpCapGapMinEscalationWindowMs;
+ this.orphanDurableAckMismatchMaxDurationMs =
+ orphanDurableAckMismatchMaxDurationMs;
+ if (reconnectOptions.onEvent) {
+ this.connectionDispatcher = new QwpNotificationDispatcher(
+ reconnectOptions.onEvent,
+ connectionListenerInboxCapacity,
+ );
+ }
+ this.errorDispatcher = new QwpNotificationDispatcher(
+ onSenderError ?? defaultQwpSenderErrorHandler,
+ errorInboxCapacity,
+ );
+ validateReconnectPolicy(
+ this.maxAttempts,
+ this.initialBackoffMs,
+ this.maxBackoffMs,
+ this.maxDurationMs,
+ this.maxFrameRejections,
+ this.poisonMinEscalationWindowMs,
+ this.catchUpCapGapMinEscalationWindowMs,
+ );
+ let resolveClosed!: (info: QwpConnectionCloseInfo) => void;
+ this.closed = new Promise((resolve) => {
+ resolveClosed = resolve;
+ });
+ this.resolveClosed = resolveClosed;
+
+ let previous = -1n;
+ for (const record of records) {
+ if (record.frameSequence < 0n || record.frameSequence <= previous) {
+ throw new Error(
+ "QWP replay store records must have strictly increasing non-negative sequences",
+ );
+ }
+ if (
+ !Number.isSafeInteger(record.payloadLength) ||
+ record.payloadLength < 0 ||
+ (record.payload !== undefined &&
+ record.payload.byteLength !== record.payloadLength)
+ ) {
+ throw new Error(
+ `QWP replay store returned an invalid payload length [frameSequence=${record.frameSequence}, payloadLength=${record.payloadLength}]`,
+ );
+ }
+ const frame: ReplayFrame = {
+ frameSequence: record.frameSequence,
+ payloadLength: record.payloadLength,
+ payload: record.payload?.slice(),
+ ackDelivered: true,
+ transmitted: true,
+ };
+ this.frames.set(frame.frameSequence, frame);
+ previous = frame.frameSequence;
+ }
+ if (records.length > 0) {
+ this.acknowledgedFrameSequence = records[0].frameSequence - 1n;
+ }
+ this.nextFrameSequence = previous + 1n;
+ this.publishedFrameSequence = previous;
+ }
+
+ static async connect(
+ factory: QwpConnectionFactory,
+ reconnectOptions: QwpReconnectOptions,
+ replayStore?: QwpIngressReplayStore,
+ localMaxBatchSizeBytes?: number,
+ memoryReplayMaxBytes = DEFAULT_MEMORY_REPLAY_MAX_BYTES,
+ memoryReplayAppendDeadlineMs = DEFAULT_MEMORY_REPLAY_APPEND_DEADLINE_MS,
+ backgroundStoreAndForward = false,
+ initialConnectMode: QwpInitialConnectMode = backgroundStoreAndForward
+ ? QWP_INITIAL_CONNECT_MODE.ASYNC
+ : QWP_INITIAL_CONNECT_MODE.SYNC,
+ orphanStoreAndForward = false,
+ orphanDurableAckMismatchMaxDurationMs = DEFAULT_ORPHAN_DURABLE_ACK_MISMATCH_MAX_DURATION_MS,
+ catchUpCapGapMinEscalationWindowMs = DEFAULT_CATCH_UP_CAP_GAP_MIN_ESCALATION_WINDOW_MS,
+ initialConnection?: Promise,
+ connectionListenerInboxCapacity = 64,
+ errorInboxCapacity = 256,
+ onSenderError?: (error: QwpSenderError) => void,
+ signal?: AbortSignal,
+ ): Promise {
+ const store: QwpIngressReplayStore =
+ replayStore ??
+ new QwpMemoryReplayStore(
+ memoryReplayMaxBytes,
+ memoryReplayAppendDeadlineMs,
+ );
+ let connection: QwpReconnectingIngressConnection | undefined;
+ // close() aborts this while a connect is still negotiating. Without it the
+ // caller returns from close() and this keeps going: a persistent store
+ // takes its slot lock after the sender is gone and holds it for the rest
+ // of the connect budget, and the abandoned session goes on to send frames
+ // and even quarantine directories.
+ const abortError = () =>
+ signal?.reason ?? new Error("QWP connect was aborted");
+ if (signal?.aborted) throw abortError();
+ try {
+ const lazyStore = isLazyReplayStore(store) ? store : undefined;
+ const records: readonly LoadedReplayRecord[] = lazyStore
+ ? await lazyStore.loadReferences()
+ : (await store.load()).map((record) => ({
+ ...record,
+ payloadLength: record.payload.byteLength,
+ }));
+ const sortedRecords = [...records].sort((a, b) =>
+ a.frameSequence < b.frameSequence
+ ? -1
+ : a.frameSequence > b.frameSequence
+ ? 1
+ : 0,
+ );
+ let persistedSymbolDictionary: readonly string[] = [];
+ let persistedSymbolDictionaryFailure: unknown;
+ if (store.loadSymbolDictionary) {
+ try {
+ persistedSymbolDictionary = await store.loadSymbolDictionary();
+ } catch (error) {
+ if (!store.replaceSymbolDictionary) throw error;
+ persistedSymbolDictionaryFailure = error;
+ }
+ }
+ const loadPayload = (record: LoadedReplayRecord) =>
+ record.payload
+ ? Promise.resolve(record.payload)
+ : lazyStore!.readPayload(record.frameSequence);
+ const recoveredDiscardTail = await analyzeRecoveredDiscardTail(
+ sortedRecords,
+ loadPayload,
+ );
+ const symbolDictionary = await recoverSymbolDictionary(
+ sortedRecords,
+ loadPayload,
+ persistedSymbolDictionary,
+ recoveredDiscardTail,
+ store,
+ persistedSymbolDictionaryFailure,
+ );
+ connection = new QwpReconnectingIngressConnection(
+ factory,
+ reconnectOptions,
+ store,
+ sortedRecords,
+ symbolDictionary,
+ recoveredDiscardTail,
+ localMaxBatchSizeBytes,
+ backgroundStoreAndForward,
+ orphanStoreAndForward,
+ orphanDurableAckMismatchMaxDurationMs,
+ catchUpCapGapMinEscalationWindowMs,
+ connectionListenerInboxCapacity,
+ errorInboxCapacity,
+ onSenderError,
+ );
+ // The store's lock is held from here on, so an abort has something to
+ // release and must reach the connect that is about to run.
+ if (signal?.aborted) throw abortError();
+ const onAbort = () => {
+ void connection?.close().catch(() => undefined);
+ };
+ signal?.addEventListener("abort", onAbort, { once: true });
+ try {
+ await connection.retireRecoveredDiscardTailIfReady();
+ if (
+ backgroundStoreAndForward &&
+ initialConnectMode === QWP_INITIAL_CONNECT_MODE.ASYNC
+ ) {
+ connection.startBackgroundConnect();
+ } else {
+ await connection.connectLoopOrCatchUp(
+ initialConnectMode,
+ initialConnection,
+ backgroundStoreAndForward,
+ orphanStoreAndForward,
+ );
+ }
+ } finally {
+ signal?.removeEventListener("abort", onAbort);
+ }
+ if (signal?.aborted) throw abortError();
+ return connection;
+ } catch (error) {
+ await connection?.close().catch(() => undefined);
+ if (!connection) {
+ const opened = await initialConnection?.catch(() => undefined);
+ await opened?.close().catch(() => undefined);
+ await store.close().catch(() => undefined);
+ }
+ throw error;
+ }
+ }
+
+ /** The foreground connect, with Java's catch-up fallback around it. */
+ private async connectLoopOrCatchUp(
+ initialConnectMode: QwpInitialConnectMode,
+ initialConnection: Promise | undefined,
+ backgroundStoreAndForward: boolean,
+ orphanStoreAndForward: boolean,
+ ): Promise {
+ try {
+ await this.connectLoop(
+ undefined,
+ false,
+ initialConnectMode === QWP_INITIAL_CONNECT_MODE.OFF
+ ? "single"
+ : "configured",
+ initialConnection,
+ );
+ } catch (error) {
+ if (
+ backgroundStoreAndForward &&
+ !orphanStoreAndForward &&
+ error instanceof QwpCatchUpCapGapError
+ ) {
+ // Java returns the foreground sender once the wire has connected,
+ // then moves recovered-dictionary catch-up to its unbounded I/O
+ // loop. Do the same instead of making OFF/SYNC construction wait
+ // forever for a larger-cap node.
+ this.startBackgroundConnect();
+ } else {
+ throw error;
+ }
+ }
+ }
+
+ get handshake(): QwpHandshakeMetadata {
+ if (!this.lastHandshake) {
+ if (this.backgroundStoreAndForward) return { qwpVersion: 1 };
+ throw new Error("QWP connection is not established");
+ }
+ return this.lastHandshake;
+ }
+
+ get endpoint(): string | URL | undefined {
+ return this.lastEndpoint;
+ }
+
+ get ingressSymbolDictionary(): readonly string[] {
+ return this.symbolDictionary.slice();
+ }
+
+ get ingressDeltaSymbolDictionaryEnabled(): boolean {
+ return this.deltaSymbolDictionaryEnabled;
+ }
+
+ getIngressMetrics(): QwpIngressTransportMetrics {
+ let pendingReplayBytes = 0;
+ for (const frame of this.frames.values()) {
+ pendingReplayBytes += frame.payloadLength;
+ }
+ const memoryMetrics =
+ this.store instanceof QwpMemoryReplayStore
+ ? this.store.metrics
+ : undefined;
+ return Object.freeze({
+ publishedFrameSequence: this.publishedFrameSequence,
+ acknowledgedFrameSequence: this.acknowledgedFrameSequence,
+ pendingReplayFrames: this.frames.size,
+ pendingReplayBytes,
+ memoryReplayMaxBytes: memoryMetrics?.maxBytes,
+ memoryReplayUsedBytes: memoryMetrics?.usedBytes,
+ waitingMemoryReplayAppends: memoryMetrics?.waitingAppends ?? 0,
+ totalMemoryReplayBackpressureStalls:
+ memoryMetrics?.totalBackpressureStalls ?? 0,
+ totalMemoryReplayAppendTimeouts: memoryMetrics?.totalAppendTimeouts ?? 0,
+ totalFramesSent: this.totalFramesSent,
+ totalBytesSent: this.totalBytesSent,
+ totalFramesReplayed: this.totalFramesReplayed,
+ totalBytesReplayed: this.totalBytesReplayed,
+ totalReconnectAttempts: this.totalReconnectAttempts,
+ totalReconnectsSucceeded: this.totalReconnectsSucceeded,
+ totalFailovers: this.totalFailovers,
+ totalReconnectErrors: this.totalReconnectErrors,
+ totalServerNacks: this.totalServerNacks,
+ deliveredConnectionNotifications:
+ this.connectionDispatcher?.metrics.delivered ?? 0,
+ droppedConnectionNotifications:
+ this.connectionDispatcher?.metrics.dropped ?? 0,
+ deliveredErrorNotifications: this.errorDispatcher?.metrics.delivered ?? 0,
+ droppedErrorNotifications: this.errorDispatcher?.metrics.dropped ?? 0,
+ });
+ }
+
+ getIngressFrameSequence(clientSequence: bigint): bigint | undefined {
+ for (const frame of this.frames.values()) {
+ if (frame.clientSequence === clientSequence) return frame.frameSequence;
+ }
+ return undefined;
+ }
+
+ skipIngressClientSequence(): void {
+ // Only the client sequence is reserved. The skipped frame never reaches
+ // the journal, so consuming a frame sequence here would leave a hole that
+ // makes every later append non-contiguous.
+ this.nextClientSequence++;
+ }
+
+ send(payload: Uint8Array): Promise {
+ if (this.terminalError) return Promise.reject(this.terminalError);
+ if (this.closing) return Promise.reject(new QwpSendClosedError());
+ const frame: ReplayFrame = {
+ // Placeholder; the real sequence is allocated in the tail below, once
+ // the journal has accepted the frame.
+ frameSequence: -1n,
+ clientSequence: this.nextClientSequence++,
+ payload: payload.slice(),
+ payloadLength: payload.byteLength,
+ ackDelivered: false,
+ transmitted: false,
+ };
+ const publishing = this.sendTail.then(async () => {
+ this.throwIfUnavailable();
+ const delta = readSymbolDictionaryDelta(frame.payload!);
+ if (delta) {
+ if (!this.deltaSymbolDictionaryEnabled) {
+ throw new QwpReplayDictionaryError(
+ "QWP delta symbol dictionaries are disabled because replay dictionary persistence is unavailable; encode symbols with full inline dictionaries",
+ );
+ }
+ await this.persistSymbolDictionaryDelta(delta);
+ }
+ // Sends are serialized on sendTail, so allocating here rather than at
+ // call time keeps frame sequences dense and in append order. A rejected
+ // append -- an exhausted journal, a missed append deadline -- must not
+ // consume one: the store enforces contiguity, so a hole would make every
+ // later append fail until the journal drained completely.
+ const frameSequence = this.nextFrameSequence;
+ await this.store.append({
+ frameSequence,
+ payload: frame.payload!,
+ });
+ this.nextFrameSequence = frameSequence + 1n;
+ frame.frameSequence = frameSequence;
+ this.frames.set(frame.frameSequence, frame);
+ this.publishedFrameSequence = frame.frameSequence;
+ if (this.backgroundStoreAndForward) {
+ if (this.lazyReplayStore) frame.payload = undefined;
+ this.enqueueDrain(frame);
+ return;
+ }
+ try {
+ await this.transmit(frame);
+ } catch (error) {
+ this.failTerminal(error);
+ throw error;
+ }
+ });
+ this.sendTail = publishing.catch(() => undefined);
+ return publishing;
+ }
+
+ private enqueueDrain(frame: ReplayFrame): void {
+ const draining = this.drainTail.then(async () => {
+ if (this.closing) return;
+ await this.transmit(frame);
+ });
+ this.drainTail = draining.catch((error: unknown) => {
+ if (!this.closing) this.failTerminal(error);
+ });
+ }
+
+ private startBackgroundConnect(): void {
+ const connecting = this.connectLoop(undefined, false, "unbounded");
+ this.reconnectTask = connecting;
+ void connecting
+ .catch((error: unknown) => {
+ if (!this.closing) this.failTerminal(error);
+ })
+ .finally(() => {
+ if (this.reconnectTask === connecting) this.reconnectTask = undefined;
+ });
+ }
+
+ async close(code = 1000, reason = ""): Promise {
+ if (this.closing) {
+ await this.closed;
+ return;
+ }
+ this.closing = true;
+ this.cancelBackoff?.();
+ this.messagesQueue.end();
+ const connection = this.connection;
+ // Tears down a connect that is still negotiating. Without this the socket
+ // and its deadline outlive close(), keeping the event loop open for up to
+ // connectTimeoutMs/authTimeoutMs after close() has already resolved.
+ this.connectAbort?.abort();
+ const connectingCandidate = this.connectingCandidate;
+ this.connection = undefined;
+ this.connectingCandidate = undefined;
+ let closeInfo: QwpConnectionCloseInfo = {
+ code,
+ reason,
+ wasClean: code === 1000,
+ };
+ if (connection) {
+ try {
+ await connection.close(code, reason);
+ closeInfo = await connection.closed;
+ } catch {
+ // The persistent store still has to close after a transport close race.
+ }
+ }
+ if (connectingCandidate && connectingCandidate !== connection) {
+ await connectingCandidate.close(code, reason).catch(() => undefined);
+ }
+ try {
+ await this.closeStore();
+ } finally {
+ this.releaseMemoryReplayReferences();
+ await Promise.all([
+ this.connectionDispatcher?.close(),
+ this.errorDispatcher?.close(),
+ ]);
+ this.settleClosed(closeInfo);
+ }
+ }
+
+ private async connectLoop(
+ initialCause: unknown,
+ reconnecting: boolean,
+ attemptPolicy: ConnectAttemptPolicy = this.backgroundStoreAndForward
+ ? "unbounded"
+ : "configured",
+ initialConnection?: Promise,
+ ): Promise {
+ const outageStarted = Date.now();
+ const previousEndpoint = this.lastEndpoint;
+ let attempt = 0;
+ let backoffMs = this.initialBackoffMs;
+ let lastError = initialCause;
+ let primaryUnavailableAttempts = 0;
+ if (reconnecting) {
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.RECONNECTING,
+ attempt: 0,
+ previousEndpoint,
+ cause: initialCause,
+ });
+ }
+
+ const initialRetryDelayMs = reconnectDelayMs(initialCause);
+ if (initialRetryDelayMs > 0) {
+ await this.waitForBackoff(jitterReconnectDelayMs(initialRetryDelayMs));
+ } else if (reconnecting && backoffMs > 0) {
+ await this.waitForBackoff(jitterReconnectDelayMs(backoffMs));
+ backoffMs = Math.min(Math.max(backoffMs * 2, 1), this.maxBackoffMs);
+ }
+
+ while (!this.closing) {
+ if (attempt > 0 && backoffMs > 0) {
+ await this.waitForBackoff(jitterReconnectDelayMs(backoffMs));
+ backoffMs = Math.min(Math.max(backoffMs * 2, 1), this.maxBackoffMs);
+ }
+ this.throwIfUnavailable();
+ attempt++;
+ if (reconnecting) this.totalReconnectAttempts++;
+ let candidate: QwpBinaryConnection | undefined;
+ try {
+ if (attempt === 1 && initialConnection) {
+ candidate = await initialConnection;
+ } else {
+ const abort = new AbortController();
+ this.connectAbort = abort;
+ try {
+ candidate = await this.factory(abort.signal);
+ } finally {
+ if (this.connectAbort === abort) this.connectAbort = undefined;
+ }
+ }
+ this.hasEverConnected = true;
+ this.connectingCandidate = candidate;
+ if (this.closing) {
+ await candidate.close().catch(() => undefined);
+ throw new QwpSendClosedError();
+ }
+ const replayed = await this.replayInto(candidate);
+ if (this.closing) throw new QwpSendClosedError();
+ this.install(candidate, replayed);
+ this.resetCatchUpCapGapEpisode();
+ this.resetDurableAckMismatchEpisode();
+ this.connectingCandidate = undefined;
+ if (reconnecting) {
+ this.totalReconnectsSucceeded++;
+ const failedOver =
+ previousEndpoint !== undefined &&
+ String(previousEndpoint) !== String(candidate.endpoint);
+ if (failedOver) this.totalFailovers++;
+ this.emitEvent({
+ kind: failedOver
+ ? QWP_RECONNECT_EVENT_KIND.FAILED_OVER
+ : QWP_RECONNECT_EVENT_KIND.RECONNECTED,
+ attempt,
+ endpoint: candidate.endpoint,
+ previousEndpoint,
+ });
+ } else {
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.CONNECTED,
+ attempt: 0,
+ endpoint: candidate.endpoint,
+ });
+ }
+ return;
+ } catch (error) {
+ if (reconnecting) this.totalReconnectErrors++;
+ lastError = error;
+ if (this.connectingCandidate === candidate) {
+ this.connectingCandidate = undefined;
+ }
+ if (candidate) await candidate.close().catch(() => undefined);
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.ATTEMPT_FAILED,
+ attempt,
+ endpoint: candidate?.endpoint,
+ previousEndpoint,
+ cause: error,
+ });
+ const capGapError =
+ error instanceof QwpCatchUpCapGapError
+ ? this.applyCatchUpCapGapPolicy(error)
+ : undefined;
+ if (!capGapError) this.resetCatchUpCapGapEpisode();
+ if (capGapError?.exhausted) throw capGapError.error;
+ if (
+ capGapError &&
+ !this.orphanStoreAndForward &&
+ attemptPolicy !== "unbounded"
+ ) {
+ throw capGapError.error;
+ }
+ const durableAckMismatch = durableAckUnavailableCause(error);
+ if (
+ durableAckMismatch &&
+ (!this.backgroundStoreAndForward || attemptPolicy !== "unbounded")
+ ) {
+ this.resetDurableAckMismatchEpisode();
+ throw durableAckMismatch;
+ }
+ const durableAckPolicy =
+ durableAckMismatch &&
+ this.backgroundStoreAndForward &&
+ attemptPolicy === "unbounded"
+ ? this.applyDurableAckMismatchPolicy(durableAckMismatch)
+ : undefined;
+ if (!durableAckPolicy) this.resetDurableAckMismatchEpisode();
+ if (durableAckPolicy?.exhausted) throw durableAckPolicy.error;
+ if (
+ this.orphanStoreAndForward &&
+ attemptPolicy === "unbounded" &&
+ isPrimaryUnavailableError(error)
+ ) {
+ primaryUnavailableAttempts++;
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.PRIMARY_UNAVAILABLE,
+ attempt: primaryUnavailableAttempts,
+ previousEndpoint,
+ cause: error,
+ });
+ }
+ if (!durableAckPolicy && !this.isRetryableReconnectError(error)) {
+ throw error;
+ }
+ const attemptsExhausted =
+ attemptPolicy === "single" ||
+ (attemptPolicy === "configured" &&
+ this.maxAttempts > 0 &&
+ attempt >= this.maxAttempts);
+ const durationExhausted =
+ attemptPolicy === "configured" &&
+ this.maxDurationMs > 0 &&
+ Date.now() - outageStarted >= this.maxDurationMs;
+ if (attemptsExhausted || durationExhausted) {
+ if (attemptPolicy === "single") throw error;
+ throw new QwpReconnectExhaustedError(attempt, lastError);
+ }
+ }
+ }
+ throw new QwpSendClosedError();
+ }
+
+ private applyCatchUpCapGapPolicy(error: QwpCatchUpCapGapError): {
+ exhausted: boolean;
+ error: QwpCatchUpCapGapError;
+ } {
+ // Foreground SF owns producer data and must wait for a larger-cap node.
+ if (!this.orphanStoreAndForward) {
+ return { exhausted: false, error };
+ }
+ const now = monotonicNowMs();
+ if (this.catchUpCapGapAttempts === 0) {
+ this.catchUpCapGapFirstMs = now;
+ }
+ this.catchUpCapGapAttempts++;
+ const episodeMs = Math.max(0, now - this.catchUpCapGapFirstMs);
+ const exhausted =
+ this.catchUpCapGapAttempts >= MAX_CATCH_UP_CAP_GAP_ATTEMPTS &&
+ episodeMs >= this.catchUpCapGapMinEscalationWindowMs;
+ return {
+ exhausted,
+ error: new QwpCatchUpCapGapError(
+ error.symbolId,
+ error.frameLength,
+ error.maxBatchSizeBytes,
+ {
+ attempt: this.catchUpCapGapAttempts,
+ episodeMs,
+ minEscalationWindowMs: this.catchUpCapGapMinEscalationWindowMs,
+ exhausted,
+ },
+ ),
+ };
+ }
+
+ private resetCatchUpCapGapEpisode(): void {
+ this.catchUpCapGapAttempts = 0;
+ this.catchUpCapGapFirstMs = 0;
+ }
+
+ private applyDurableAckMismatchPolicy(error: QwpDurableAckUnavailableError): {
+ exhausted: boolean;
+ error: QwpDurableAckUnavailableError | QwpDurableAckPersistentFailureError;
+ } {
+ const now = monotonicNowMs();
+ if (this.durableAckMismatchAttempts === 0) {
+ this.durableAckMismatchFirstMs = now;
+ }
+ this.durableAckMismatchAttempts++;
+ const episodeMs = Math.max(0, now - this.durableAckMismatchFirstMs);
+ const durationExhausted =
+ this.orphanDurableAckMismatchMaxDurationMs > 0 &&
+ episodeMs >= this.orphanDurableAckMismatchMaxDurationMs;
+ const exhausted =
+ this.orphanStoreAndForward &&
+ (this.durableAckMismatchAttempts >=
+ MAX_ORPHAN_DURABLE_ACK_MISMATCH_ATTEMPTS ||
+ durationExhausted);
+ if (exhausted) {
+ const persistent = new QwpDurableAckPersistentFailureError(
+ this.durableAckMismatchAttempts,
+ episodeMs,
+ error,
+ );
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.DURABLE_ACK_PERSISTENT_FAILURE,
+ attempt: this.durableAckMismatchAttempts,
+ previousEndpoint: this.lastEndpoint,
+ cause: persistent,
+ episodeMs,
+ });
+ return { exhausted: true, error: persistent };
+ }
+ this.emitEvent({
+ kind: QWP_RECONNECT_EVENT_KIND.DURABLE_ACK_UNAVAILABLE,
+ attempt: this.durableAckMismatchAttempts,
+ previousEndpoint: this.lastEndpoint,
+ cause: error,
+ episodeMs,
+ });
+ return { exhausted: false, error };
+ }
+
+ private resetDurableAckMismatchEpisode(): void {
+ this.durableAckMismatchAttempts = 0;
+ this.durableAckMismatchFirstMs = 0;
+ }
+
+ private isRetryableReconnectError(error: unknown): boolean {
+ if (
+ this.backgroundStoreAndForward &&
+ !this.orphanStoreAndForward &&
+ this.hasEverConnected &&
+ isEndpointPolicyFailure(error)
+ ) {
+ return true;
+ }
+ return isRetryableReconnectError(error);
+ }
+
+ private async replayInto(
+ connection: QwpBinaryConnection,
+ ): Promise {
+ const replayed: ReplayFrame[] = [];
+ const cap = minimumDefined(
+ connection.handshake.maxBatchSizeBytes,
+ this.localMaxBatchSizeBytes,
+ );
+ this.durableWatermarks.clear();
+ for (const payload of dictionaryCatchupFrames(this.symbolDictionary, cap)) {
+ const frame: ReplayFrame = {
+ frameSequence: -1n,
+ payload,
+ payloadLength: payload.byteLength,
+ ackDelivered: true,
+ transmitted: true,
+ dictionaryCatchup: true,
+ };
+ replayed.push(frame);
+ await this.sendPhysical(connection, payload, false);
+ }
+ for (const frame of this.frames.values()) {
+ if (!frame.transmitted) continue;
+ if (this.isRecoveredDiscardFrame(frame.frameSequence)) continue;
+ frame.durableTargets = undefined;
+ if (cap !== undefined && frame.payloadLength > cap) {
+ throw new RangeError(
+ `persisted QWP frame exceeds reconnect target batch cap [size=${frame.payloadLength}, max=${cap}]`,
+ );
+ }
+ const payload = await this.readFramePayload(frame);
+ replayed.push(frame);
+ await this.sendPhysical(connection, payload, true);
+ }
+ return replayed;
+ }
+
+ private install(
+ connection: QwpBinaryConnection,
+ wireFrames: ReplayFrame[],
+ ): void {
+ this.hasEverConnected = true;
+ this.connection = connection;
+ this.lastHandshake = connection.handshake;
+ this.lastEndpoint = connection.endpoint;
+ this.wireFrames = wireFrames;
+ this.wireFramesBase = 0;
+ if (connection.ping && !this.ping) {
+ // Assigned only when the initial transport supports PING so browser
+ // connections keep the optional capability genuinely absent.
+ this.ping = () => this.pingWithReconnect();
+ }
+ const generation = ++this.generation;
+ void this.pump(connection, generation);
+ }
+
+ private async pump(
+ connection: QwpBinaryConnection,
+ generation: number,
+ ): Promise {
+ try {
+ for await (const payload of connection.messages) {
+ if (
+ this.closing ||
+ this.connection !== connection ||
+ generation !== this.generation
+ ) {
+ return;
+ }
+ let translated: Uint8Array | undefined;
+ try {
+ translated = await this.translateResponse(payload);
+ } catch (error) {
+ if (
+ error instanceof RetriableIngressNackError ||
+ error instanceof QwpProtocolError ||
+ error instanceof QwpReplayRejectedError
+ ) {
+ throw error;
+ }
+ // The wire payload decoded successfully. Failures from this point
+ // are local replay-store/bookkeeping failures, not evidence that
+ // the server rejected the head frame.
+ if (isRetryableReconnectError(error)) {
+ // A journal fault here is usually transient: a briefly full or
+ // read-only filesystem parks maintenanceFailure for about a
+ // second and the store clears it on the next successful batch.
+ // failTerminal() is permanent, so latching would brick a running
+ // producer for the rest of the process lifetime -- the outcome
+ // the store-level retry exists to prevent. transmitOnce() routes
+ // the identical class to requestReconnect() for that reason and
+ // this path has to agree. acknowledgeThrough() persists its
+ // cursor before it mutates anything, so a failure here leaves
+ // exactly the state a crash at this instant would leave, and
+ // replay resumes from the persisted watermark.
+ await this.requestReconnect(error, connection).catch(
+ (reconnectError) => this.failTerminal(reconnectError),
+ );
+ return;
+ }
+ this.failTerminal(error);
+ await connection
+ .close(1011, "QWP ingress response processing failed")
+ .catch(() => undefined);
+ return;
+ }
+ if (translated) this.messagesQueue.push(translated);
+ if (this.terminalError) return;
+ }
+ if (
+ this.closing ||
+ this.connection !== connection ||
+ generation !== this.generation
+ ) {
+ return;
+ }
+ const info = await connection.closed;
+ const cause = this.classifyConnectionLoss(
+ new QwpSendClosedError(info),
+ info,
+ );
+ if (cause instanceof QwpProtocolError) {
+ this.failTerminal(cause);
+ await connection
+ .close(1002, "poisoned QWP ingress frame")
+ .catch(() => undefined);
+ return;
+ }
+ await this.requestReconnect(cause, connection).catch((reconnectError) =>
+ this.failTerminal(reconnectError),
+ );
+ return;
+ } catch (error) {
+ if (
+ this.closing ||
+ this.connection !== connection ||
+ generation !== this.generation
+ ) {
+ return;
+ }
+ if (
+ error instanceof QwpProtocolError ||
+ error instanceof QwpReplayRejectedError
+ ) {
+ this.failTerminal(error);
+ await connection
+ .close(1002, "terminal QWP response")
+ .catch(() => undefined);
+ return;
+ }
+ const cause =
+ error instanceof RetriableIngressNackError
+ ? error
+ : this.classifyConnectionLoss(
+ error,
+ error instanceof QwpSendClosedError ? error.closeInfo : undefined,
+ );
+ if (cause instanceof QwpProtocolError) {
+ this.failTerminal(cause);
+ await connection
+ .close(1002, "poisoned QWP ingress frame")
+ .catch(() => undefined);
+ return;
+ }
+ await this.requestReconnect(cause, connection).catch((reconnectError) => {
+ this.failTerminal(reconnectError);
+ });
+ }
+ }
+
+ private async translateResponse(
+ payload: Uint8Array,
+ ): Promise {
+ const response = decodeQwpIngressResponse(payload);
+ if (response.status === QWP_STATUS.DURABLE_ACK) {
+ for (const table of response.tables) {
+ const current = this.durableWatermarks.get(table.name);
+ if (current === undefined || table.sequenceTransaction > current) {
+ this.durableWatermarks.set(table.name, table.sequenceTransaction);
+ }
+ }
+ await this.trimDurablePrefix();
+ return payload;
+ }
+ if (response.sequence === null) {
+ throw new QwpProtocolError("QWP response is missing its wire sequence");
+ }
+ if (response.sequence < 0n) {
+ throw new QwpProtocolError(
+ `QWP response sequence is negative: ${response.sequence}`,
+ );
+ }
+ const highestWireIndex = this.wireFramesBase + this.wireFrames.length - 1;
+ if (response.sequence > BigInt(highestWireIndex)) {
+ // Reject an over-range sequence rather than clamping it, matching the null
+ // and negative guards above. A frame is logged here before it is sent, so
+ // a conforming server can only acknowledge a sequence it has received,
+ // never one beyond the last frame sent. Clamping a bogus over-range value
+ // onto the newest in-flight frame would retire every unacknowledged frame
+ // below it and delete journal records the server never confirmed -- the
+ // watermark must never advance past an unacknowledged frame.
+ throw new QwpProtocolError(
+ `QWP response sequence is beyond the last frame sent: ${response.sequence} > ${highestWireIndex}`,
+ );
+ }
+ const wireIndex = Number(response.sequence);
+ const localIndex = wireIndex - this.wireFramesBase;
+ const frame = localIndex >= 0 ? this.wireFrames[localIndex] : undefined;
+ if (!frame) {
+ // Either nothing has been sent on this connection yet, or this sequence
+ // was covered by an earlier cumulative ACK and trimmed. A duplicate OK
+ // has already been delivered; a NACK still has to be reported.
+ if (response.status === QWP_STATUS.OK) return undefined;
+ this.totalServerNacks++;
+ const pending = this.pendingFsnRange();
+ this.emitSenderError(
+ createQwpSenderError(response, {
+ messageSequence: response.sequence ?? undefined,
+ fromFsn: pending?.from,
+ toFsn: pending?.to,
+ }),
+ );
+ if (isRetriableIngressStatus(response.status)) {
+ throw new RetriableIngressNackError(
+ -1n,
+ response.status,
+ this.nextExemptRecycleDelay(),
+ response.errorMessage,
+ );
+ }
+ throw new QwpProtocolError(
+ `QuestDB rejected ingress before any frame was sent [status=0x${response.status.toString(16)}]${
+ response.errorMessage ? `: ${response.errorMessage}` : ""
+ }`,
+ );
+ }
+
+ if (response.status === QWP_STATUS.OK) {
+ if (frame.dictionaryCatchup) return undefined;
+ const covered = this.wireFrames.slice(0, localIndex + 1);
+ const clientTarget = findLastClientFrame(covered);
+ const shouldDeliver = covered.some(
+ (candidate) =>
+ candidate.clientSequence !== undefined && !candidate.ackDelivered,
+ );
+ for (const candidate of covered) candidate.ackDelivered = true;
+ if (frame.frameSequence > this.highestOkFrameSequence) {
+ this.highestOkFrameSequence = frame.frameSequence;
+ }
+ this.clearPoisonThrough(frame.frameSequence);
+ if (this.handshake.durableAckEnabled) {
+ frame.durableTargets = new Map(
+ response.tables.map((table) => [
+ table.name,
+ table.sequenceTransaction,
+ ]),
+ );
+ await this.trimDurablePrefix();
+ } else {
+ await this.acknowledgeThrough(frame.frameSequence);
+ }
+ // ACKs are cumulative, so nothing reads the covered prefix again.
+ // Dropping it keeps both the log and the payloads it pins bounded, and
+ // keeps each ACK proportional to the frames it actually covers.
+ this.wireFrames.splice(0, localIndex + 1);
+ this.wireFramesBase += localIndex + 1;
+ if (!shouldDeliver || clientTarget?.clientSequence === undefined) {
+ return undefined;
+ }
+ return rewriteResponseSequence(payload, clientTarget.clientSequence);
+ }
+
+ this.totalServerNacks++;
+ const pending = frame.dictionaryCatchup
+ ? this.pendingFsnRange()
+ : undefined;
+ this.emitSenderError(
+ createQwpSenderError(response, {
+ messageSequence: response.sequence,
+ fromFsn: pending?.from ?? frame.frameSequence,
+ toFsn: pending?.to ?? frame.frameSequence,
+ }),
+ );
+
+ if (isRetriableIngressStatus(response.status)) {
+ const exempt =
+ frame.dictionaryCatchup || response.status === QWP_STATUS.NOT_WRITABLE;
+ if (exempt) {
+ throw new RetriableIngressNackError(
+ frame.frameSequence,
+ response.status,
+ this.nextExemptRecycleDelay(),
+ response.errorMessage,
+ );
+ }
+ if (this.recordPoisonStrike(frame.frameSequence)) {
+ this.emitSenderError(
+ createQwpProtocolViolationSenderError(
+ `frame remained rejected after ${this.poisonStrikes} attempts${
+ response.errorMessage ? `: ${response.errorMessage}` : ""
+ }`,
+ frame.frameSequence,
+ ),
+ );
+ throw new QwpReplayRejectedError(
+ frame.frameSequence,
+ response.status,
+ `frame remained rejected after ${this.poisonStrikes} attempts${
+ response.errorMessage ? `: ${response.errorMessage}` : ""
+ }`,
+ );
+ }
+ throw new RetriableIngressNackError(
+ frame.frameSequence,
+ response.status,
+ cappedExponentialBackoff(
+ this.initialBackoffMs,
+ this.maxBackoffMs,
+ this.poisonStrikes - 1,
+ ),
+ response.errorMessage,
+ );
+ }
+
+ if (frame.dictionaryCatchup) {
+ const error = new QwpProtocolError(
+ `QuestDB rejected QWP symbol dictionary catch-up [status=0x${response.status.toString(16)}]${
+ response.errorMessage ? `: ${response.errorMessage}` : ""
+ }`,
+ );
+ this.failTerminal(error);
+ return undefined;
+ }
+
+ const replayError = new QwpReplayRejectedError(
+ frame.frameSequence,
+ response.status,
+ response.errorMessage,
+ );
+ if (frame.clientSequence === undefined) {
+ this.failTerminal(replayError);
+ return undefined;
+ }
+ const translated = rewriteResponseSequence(payload, frame.clientSequence);
+ this.messagesQueue.push(translated);
+ this.failTerminal(replayError);
+ return undefined;
+ }
+
+ private async trimDurablePrefix(): Promise {
+ let lastCovered: bigint | undefined;
+ for (const frame of this.frames.values()) {
+ // Successful ingress ACKs are cumulative. Deferred frames therefore
+ // have no checkpoint of their own; a later commit-bearing ACK covers
+ // them and its durable targets retire the whole preceding range.
+ if (!frame.durableTargets) continue;
+ if (!areTargetsCovered(frame.durableTargets, this.durableWatermarks)) {
+ break;
+ }
+ lastCovered = frame.frameSequence;
+ }
+ if (lastCovered !== undefined) await this.acknowledgeThrough(lastCovered);
+ }
+
+ private clearPoisonThrough(frameSequence: bigint): void {
+ if (
+ this.poisonFrameSequence === undefined ||
+ frameSequence < this.poisonFrameSequence
+ ) {
+ return;
+ }
+ this.poisonFrameSequence = undefined;
+ this.poisonFirstStrikeMs = 0;
+ this.poisonStrikes = 0;
+ }
+
+ private recordPoisonStrike(frameSequence: bigint): boolean {
+ const now = Date.now();
+ if (this.poisonFrameSequence === frameSequence) {
+ this.poisonStrikes++;
+ } else {
+ this.poisonFrameSequence = frameSequence;
+ this.poisonStrikes = 1;
+ this.poisonFirstStrikeMs = now;
+ }
+ return (
+ this.poisonStrikes >= this.maxFrameRejections &&
+ now - this.poisonFirstStrikeMs >= this.poisonMinEscalationWindowMs
+ );
+ }
+
+ private classifyConnectionLoss(
+ cause: unknown,
+ closeInfo?: QwpConnectionCloseInfo,
+ ): Error {
+ const orderly = closeInfo?.code === 1000 || closeInfo?.code === 1001;
+ const head = orderly ? undefined : this.currentPoisonHead();
+ if (!head) {
+ return new RetriableIngressConnectionError(
+ this.nextExemptRecycleDelay(),
+ cause,
+ );
+ }
+ if (this.recordPoisonStrike(head.frameSequence)) {
+ const closeDetail = closeInfo
+ ? `code=${closeInfo.code}, reason=${closeInfo.reason}`
+ : "transport ended without an orderly close";
+ const message = `QWP ingress frame repeatedly caused a non-orderly connection loss [frameSequence=${head.frameSequence}, strikes=${this.poisonStrikes}, ${closeDetail}]`;
+ this.emitSenderError(
+ createQwpProtocolViolationSenderError(
+ message,
+ head.frameSequence,
+ this.nextFrameSequence - 1n,
+ ),
+ );
+ return new QwpProtocolError(message);
+ }
+ return new RetriableIngressConnectionError(
+ cappedExponentialBackoff(
+ this.initialBackoffMs,
+ this.maxBackoffMs,
+ this.poisonStrikes - 1,
+ ),
+ cause,
+ );
+ }
+
+ private currentPoisonHead(): ReplayFrame | undefined {
+ const progress =
+ this.highestOkFrameSequence > this.acknowledgedFrameSequence
+ ? this.highestOkFrameSequence
+ : this.acknowledgedFrameSequence;
+ return this.wireFrames.find(
+ (frame) => !frame.dictionaryCatchup && frame.frameSequence > progress,
+ );
+ }
+
+ private nextExemptRecycleDelay(): number {
+ const progress =
+ this.highestOkFrameSequence > this.acknowledgedFrameSequence
+ ? this.highestOkFrameSequence
+ : this.acknowledgedFrameSequence;
+ if (progress > this.progressAtLastExemptRecycle) {
+ this.zeroProgressRecycles = 0;
+ }
+ this.progressAtLastExemptRecycle = progress;
+ const level = this.zeroProgressRecycles++;
+ if (level === 0) return 0;
+ return cappedExponentialBackoff(
+ this.initialBackoffMs,
+ this.maxBackoffMs,
+ level - 1,
+ );
+ }
+
+ private async acknowledgeThrough(frameSequence: bigint): Promise {
+ await this.acknowledgeStoredFramesThrough(frameSequence);
+ await this.retireRecoveredDiscardTailIfReady();
+ }
+
+ private async acknowledgeStoredFramesThrough(
+ frameSequence: bigint,
+ ): Promise {
+ await this.store.acknowledgeThrough(frameSequence);
+ for (const sequence of this.frames.keys()) {
+ if (sequence > frameSequence) break;
+ this.frames.delete(sequence);
+ }
+ if (frameSequence > this.acknowledgedFrameSequence) {
+ this.acknowledgedFrameSequence = frameSequence;
+ }
+ }
+
+ private isRecoveredDiscardFrame(frameSequence: bigint): boolean {
+ const tail = this.recoveredDiscardTail;
+ return (
+ tail !== undefined &&
+ frameSequence >= tail.startSequence &&
+ frameSequence <= tail.tipSequence
+ );
+ }
+
+ private async retireRecoveredDiscardTailIfReady(): Promise {
+ const tail = this.recoveredDiscardTail;
+ if (!tail) return;
+ if (
+ tail.predecessorSequence !== undefined &&
+ this.frames.has(tail.predecessorSequence)
+ ) {
+ return;
+ }
+ await this.acknowledgeStoredFramesThrough(tail.tipSequence);
+ this.recoveredDiscardTail = undefined;
+ }
+
+ private async persistSymbolDictionaryDelta(
+ delta: NonNullable>,
+ ): Promise {
+ if (
+ !this.store.loadSymbolDictionary ||
+ !this.store.appendSymbolDictionary
+ ) {
+ throw new QwpReplayDictionaryError(
+ "QWP delta symbol dictionaries require a replay store with dictionary persistence",
+ );
+ }
+ if (delta.startId > this.symbolDictionary.length) {
+ throw new QwpReplayDictionaryError(
+ `QWP symbol dictionary has a gap [expectedAtMost=${this.symbolDictionary.length}, received=${delta.startId}]`,
+ );
+ }
+ const overlap = Math.min(
+ this.symbolDictionary.length - delta.startId,
+ delta.entries.length,
+ );
+ for (let index = 0; index < overlap; index++) {
+ const id = delta.startId + index;
+ if (this.symbolDictionary[id] !== delta.entries[index]) {
+ throw new QwpReplayDictionaryError(
+ `QWP symbol dictionary conflicts at ID ${id}`,
+ );
+ }
+ }
+ const firstNewEntry = Math.max(
+ this.symbolDictionary.length - delta.startId,
+ 0,
+ );
+ const newEntries = delta.entries.slice(firstNewEntry);
+ if (newEntries.length === 0) return;
+ const startId = this.symbolDictionary.length;
+ try {
+ await this.store.appendSymbolDictionary(startId, newEntries);
+ } catch (error) {
+ this.deltaSymbolDictionaryEnabled = false;
+ throw new QwpReplayDictionaryPersistenceError(error);
+ }
+ this.symbolDictionary.push(...newEntries);
+ }
+
+ private async transmit(frame: ReplayFrame): Promise {
+ // Loops when a reconnect completes while this frame's payload is being
+ // read; see the currency check below.
+ for (;;) {
+ if (await this.transmitOnce(frame)) return;
+ }
+ }
+
+ /** Returns false when a reconnect invalidated the captured connection. */
+ private async transmitOnce(frame: ReplayFrame): Promise {
+ const connection = await this.requireConnection();
+ const generation = this.generation;
+ const cap = minimumDefined(
+ connection.handshake.maxBatchSizeBytes,
+ this.localMaxBatchSizeBytes,
+ );
+ if (cap !== undefined && frame.payloadLength > cap) {
+ // Data the producer already handed over is never reclassified as
+ // unsendable because a failover landed on a smaller-cap node: that would
+ // invent a terminal for a frame an earlier node would have taken. Treat
+ // it as a connection-level failure, exactly as replayInto() does with the
+ // identical check, so the reconnect loop keeps looking for a node that
+ // can take it. Marking it transmitted is what puts it in replayInto()'s
+ // resend set; it is deliberately not pushed onto the wire log, because
+ // nothing reached the wire and the log is indexed by wire sequence.
+ frame.transmitted = true;
+ await this.requestReconnect(
+ new RangeError(
+ `QWP frame exceeds reconnect target batch cap [size=${frame.payloadLength}, max=${cap}]`,
+ ),
+ connection,
+ );
+ return true;
+ }
+ let payload: Uint8Array;
+ try {
+ payload = await this.readFramePayload(frame);
+ } catch (error) {
+ // A journal read can fail transiently: a briefly full or read-only
+ // filesystem parks maintenanceFailure for about a second, and the store
+ // clears it on the next successful batch. enqueueDrain's only handler is
+ // failTerminal, so letting this escape would brick a running producer for
+ // the rest of the process lifetime -- the very outcome the store-level
+ // retry was added to prevent. replayInto() makes the identical read and
+ // connectLoop retries its failures, so route this one the same way.
+ // Deterministic corruption still escapes and stays terminal.
+ if (!isRetryableReconnectError(error)) throw error;
+ // Nothing reached the wire, so the frame is deliberately kept off the
+ // wire log; marking it transmitted is what puts it in replayInto()'s
+ // resend set, exactly as the batch-cap branch above does.
+ frame.transmitted = true;
+ await this.requestReconnect(error, connection);
+ return true;
+ }
+ // The journal read above yields, and with a lazy store it can park behind
+ // an fsyncing append for longer than a jittered reconnect takes. install()
+ // swaps this.wireFrames wholesale and resets wireFramesBase, while
+ // replayInto() skipped this frame because it was not transmitted yet.
+ // Pushing it now would log it against the replacement connection's wire
+ // sequence while sending it on the dead one, so the replacement's next
+ // cumulative ACK would retire a frame no server ever received and delete
+ // its journal record. Retry against the current connection instead.
+ if (this.connection !== connection || this.generation !== generation) {
+ return false;
+ }
+ frame.transmitted = true;
+ this.wireFrames.push(frame);
+ try {
+ await this.sendPhysical(connection, payload, false);
+ if (this.lazyReplayStore) frame.payload = undefined;
+ } catch (error) {
+ await this.requestReconnect(error, connection);
+ if (this.lazyReplayStore) frame.payload = undefined;
+ }
+ return true;
+ }
+
+ private async readFramePayload(frame: ReplayFrame): Promise {
+ let payload = frame.payload;
+ if (!payload) {
+ if (!this.lazyReplayStore) {
+ throw new QwpProtocolError(
+ `QWP replay payload is unavailable [frameSequence=${frame.frameSequence}]`,
+ );
+ }
+ payload = await this.lazyReplayStore.readPayload(frame.frameSequence);
+ }
+ if (payload.byteLength !== frame.payloadLength) {
+ throw new QwpProtocolError(
+ `persisted QWP frame length changed [frameSequence=${frame.frameSequence}, expected=${frame.payloadLength}, received=${payload.byteLength}]`,
+ );
+ }
+ return payload;
+ }
+
+ private async sendPhysical(
+ connection: QwpBinaryConnection,
+ payload: Uint8Array,
+ replayed: boolean,
+ ): Promise {
+ this.totalFramesSent++;
+ this.totalBytesSent += payload.byteLength;
+ if (replayed) {
+ this.totalFramesReplayed++;
+ this.totalBytesReplayed += payload.byteLength;
+ }
+ await connection.send(payload);
+ }
+
+ private async requireConnection(): Promise {
+ if (this.reconnectTask) await this.reconnectTask;
+ this.throwIfUnavailable();
+ if (!this.connection) throw new QwpSendClosedError();
+ return this.connection;
+ }
+
+ private async requestReconnect(
+ cause: unknown,
+ failedConnection: QwpBinaryConnection,
+ ): Promise {
+ if (this.closing) throw new QwpSendClosedError();
+ if (this.connection && this.connection !== failedConnection) return;
+ if (this.reconnectTask) {
+ const activeReconnect = this.reconnectTask;
+ await activeReconnect;
+ if (this.connection === failedConnection && !this.closing) {
+ await this.requestReconnect(cause, failedConnection);
+ }
+ return;
+ }
+
+ if (
+ cause instanceof RetriableIngressNackError &&
+ cause.status === QWP_STATUS.NOT_WRITABLE
+ ) {
+ // NOT_WRITABLE describes this node, not the replayed frame. Preserve the
+ // frame and make the next factory sweep start at another endpoint.
+ failedConnection.deprioritizeEndpoint?.();
+ }
+ this.connection = undefined;
+ void failedConnection.close().catch(() => undefined);
+ const reconnecting = this.connectLoop(
+ cause,
+ true,
+ this.backgroundStoreAndForward ? "unbounded" : "configured",
+ );
+ this.reconnectTask = reconnecting;
+ try {
+ await reconnecting;
+ } finally {
+ if (this.reconnectTask === reconnecting) this.reconnectTask = undefined;
+ }
+ }
+
+ private async pingWithReconnect(): Promise {
+ const connection = await this.requireConnection();
+ if (!connection.ping) {
+ throw new Error("QWP reconnect target does not support WebSocket PING");
+ }
+ try {
+ await connection.ping();
+ } catch (error) {
+ await this.requestReconnect(error, connection);
+ const replacement = await this.requireConnection();
+ if (!replacement.ping) {
+ throw new Error("QWP reconnect target does not support WebSocket PING");
+ }
+ await replacement.ping();
+ }
+ }
+
+ private async waitForBackoff(delayMs: number): Promise {
+ await new Promise((resolve) => {
+ const timer = setTimeout(() => {
+ if (this.cancelBackoff === cancel) this.cancelBackoff = undefined;
+ resolve();
+ }, delayMs);
+ const cancel = (): void => {
+ clearTimeout(timer);
+ if (this.cancelBackoff === cancel) this.cancelBackoff = undefined;
+ resolve();
+ };
+ this.cancelBackoff = cancel;
+ });
+ }
+
+ private emitEvent(event: Omit): void {
+ this.connectionDispatcher?.offer({
+ ...event,
+ timestampMs: Date.now(),
+ });
+ }
+
+ private emitSenderError(error: QwpSenderError): void {
+ this.errorDispatcher?.offer(error);
+ }
+
+ private pendingFsnRange(): { from: bigint; to: bigint } | undefined {
+ const iterator = this.frames.keys();
+ const first = iterator.next();
+ if (first.done) return undefined;
+ let to = first.value;
+ for (const frameSequence of iterator) to = frameSequence;
+ return { from: first.value, to };
+ }
+
+ private throwIfUnavailable(): void {
+ if (this.terminalError) throw this.terminalError;
+ if (this.closing) throw new QwpSendClosedError();
+ }
+
+ private failTerminal(error: unknown): void {
+ if (this.terminalError) return;
+ this.terminalError =
+ error instanceof Error
+ ? error
+ : new Error(`QWP reconnect failed: ${error}`);
+ this.cancelBackoff?.();
+ this.messagesQueue.fail(this.terminalError);
+ this.settleClosed({
+ code: 1011,
+ reason: this.terminalError.message,
+ wasClean: false,
+ });
+ void this.closeStore()
+ .catch(() => undefined)
+ .finally(() => this.releaseMemoryReplayReferences());
+ void this.connection
+ ?.close(1011, "QWP reconnect failed")
+ .catch(() => undefined);
+ }
+
+ private settleClosed(info: QwpConnectionCloseInfo): void {
+ if (this.closedSettled) return;
+ this.closedSettled = true;
+ this.resolveClosed(info);
+ }
+
+ private closeStore(): Promise {
+ if (!this.storeClosePromise) {
+ this.storeClosePromise = Promise.resolve().then(() => this.store.close());
+ }
+ return this.storeClosePromise;
+ }
+
+ private releaseMemoryReplayReferences(): void {
+ if (!(this.store instanceof QwpMemoryReplayStore)) return;
+ this.frames.clear();
+ this.wireFrames = [];
+ this.wireFramesBase = 0;
+ this.symbolDictionary.length = 0;
+ this.durableWatermarks.clear();
+ }
+}
+
+function isLazyReplayStore(
+ store: QwpIngressReplayStore,
+): store is LazyReplayStore {
+ return (
+ typeof store.loadReferences === "function" &&
+ typeof store.readPayload === "function"
+ );
+}
+
+function readSymbolDictionaryDelta(payload: Uint8Array) {
+ // Preserve support for opaque/custom payloads used with the low-level API.
+ if (
+ payload.byteLength < QWP_HEADER_SIZE ||
+ (payload[5] & QWP_FLAG_DELTA_SYMBOL_DICTIONARY) === 0
+ ) {
+ return undefined;
+ }
+ return decodeQwpIngressSymbolDictionaryDelta(payload);
+}
+
+async function analyzeRecoveredDiscardTail(
+ records: readonly LoadedReplayRecord[],
+ loadPayload: (record: LoadedReplayRecord) => Promise,
+): Promise {
+ let boundaryIndex = -1;
+ for (let index = 0; index < records.length; index++) {
+ if (isRecoveredCommitBarrier(await loadPayload(records[index]))) {
+ boundaryIndex = index;
+ }
+ }
+ if (boundaryIndex === records.length - 1) return undefined;
+ return {
+ startSequence: records[boundaryIndex + 1].frameSequence,
+ tipSequence: records[records.length - 1].frameSequence,
+ predecessorSequence:
+ boundaryIndex < 0 ? undefined : records[boundaryIndex].frameSequence,
+ };
+}
+
+function isRecoveredCommitBarrier(payload: Uint8Array): boolean {
+ try {
+ const frame = decodeQwpFrame(payload);
+ if ((frame.flags & QWP_FLAG_DEFER_COMMIT) !== 0) return false;
+ // A durable-ACK poll is side-effect-free and cannot cover deferred data
+ // before it. Treat an exact poll as transparent during the recovery scan.
+ if (
+ frame.flags === QWP_FLAG_DURABLE_ACK_POLL &&
+ frame.tableCount === 0 &&
+ frame.payloadLength === 0
+ ) {
+ return false;
+ }
+ return true;
+ } catch {
+ // Opaque low-level payloads and malformed QWP records are never silently
+ // retired. They remain replay barriers and preserve the existing behavior.
+ return true;
+ }
+}
+
+async function recoverSymbolDictionary(
+ records: readonly LoadedReplayRecord[],
+ loadPayload: (record: LoadedReplayRecord) => Promise,
+ persistedDictionary: readonly string[],
+ discardTail: RecoveredDiscardTail | undefined,
+ store: QwpIngressReplayStore,
+ persistedDictionaryFailure?: unknown,
+): Promise {
+ const hasDictionaryPersistence =
+ store.loadSymbolDictionary !== undefined &&
+ store.appendSymbolDictionary !== undefined;
+ let recoveredFromPersisted = true;
+ let dictionary: string[];
+ try {
+ dictionary = await reconstructSymbolDictionary(
+ records,
+ loadPayload,
+ persistedDictionary,
+ discardTail,
+ hasDictionaryPersistence,
+ persistedDictionaryFailure,
+ );
+ } catch (error) {
+ if (!store.replaceSymbolDictionary || persistedDictionary.length === 0) {
+ throw error;
+ }
+ // A structurally valid sidecar can still belong to an older dictionary
+ // generation. Only discard it when the committed frames independently
+ // reconstruct a complete dense dictionary from ID zero.
+ dictionary = await reconstructSymbolDictionary(
+ records,
+ loadPayload,
+ [],
+ discardTail,
+ hasDictionaryPersistence,
+ error,
+ );
+ recoveredFromPersisted = false;
+ }
+ const replacePersistedDictionary =
+ persistedDictionaryFailure !== undefined || !recoveredFromPersisted;
+ if (replacePersistedDictionary) {
+ try {
+ await store.replaceSymbolDictionary!(dictionary);
+ } catch (error) {
+ throw new QwpReplayDictionaryError(
+ "could not replace the unusable QWP symbol dictionary from surviving frame deltas",
+ error,
+ );
+ }
+ } else if (dictionary.length > persistedDictionary.length) {
+ try {
+ await store.appendSymbolDictionary!(
+ persistedDictionary.length,
+ dictionary.slice(persistedDictionary.length),
+ );
+ } catch (error) {
+ throw new QwpReplayDictionaryError(
+ "could not heal the recovered QWP symbol dictionary from surviving frame deltas",
+ error,
+ );
+ }
+ }
+ return dictionary;
+}
+
+async function reconstructSymbolDictionary(
+ records: readonly LoadedReplayRecord[],
+ loadPayload: (record: LoadedReplayRecord) => Promise,
+ baseline: readonly string[],
+ discardTail: RecoveredDiscardTail | undefined,
+ hasDictionaryPersistence: boolean,
+ recoveryCause?: unknown,
+): Promise {
+ const dictionary = [...baseline];
+ const dictionaryIds = new Map(dictionary.map((entry, id) => [entry, id]));
+ for (const record of records) {
+ // A wholly deferred recovery tail is retired locally and never replayed.
+ // Its dictionary additions therefore cannot make a committed prefix safe.
+ if (
+ discardTail !== undefined &&
+ record.frameSequence >= discardTail.startSequence
+ ) {
+ break;
+ }
+ let delta: ReturnType;
+ try {
+ delta = readSymbolDictionaryDelta(await loadPayload(record));
+ } catch (error) {
+ throw new QwpUnrecoverableReplayDictionaryError(
+ `persisted QWP frame contains an invalid symbol dictionary delta [sequence=${record.frameSequence}]`,
+ recoveryCause ?? error,
+ );
+ }
+ if (!delta) continue;
+ if (!hasDictionaryPersistence) {
+ throw new QwpUnrecoverableReplayDictionaryError(
+ "persisted QWP delta frames require a replay store with dictionary persistence",
+ );
+ }
+ if (delta.startId > dictionary.length) {
+ throw new QwpUnrecoverableReplayDictionaryError(
+ `persisted QWP frame references a symbol dictionary gap that cannot be reconstructed [startId=${delta.startId}, dictionarySize=${dictionary.length}]`,
+ recoveryCause,
+ );
+ }
+ delta.entries.forEach((entry, index) => {
+ const id = delta.startId + index;
+ const existing = dictionary[id];
+ if (existing !== undefined && existing !== entry) {
+ throw new QwpUnrecoverableReplayDictionaryError(
+ `persisted QWP frame conflicts with symbol dictionary at ID ${id}`,
+ recoveryCause,
+ );
+ }
+ if (id === dictionary.length) {
+ const duplicateId = dictionaryIds.get(entry);
+ if (duplicateId !== undefined) {
+ throw new QwpUnrecoverableReplayDictionaryError(
+ `persisted QWP frame assigns symbol dictionary value ${JSON.stringify(entry)} to both ID ${duplicateId} and ID ${id}`,
+ recoveryCause,
+ );
+ }
+ dictionary.push(entry);
+ dictionaryIds.set(entry, id);
+ }
+ });
+ }
+ return dictionary;
+}
+
+function dictionaryCatchupFrames(
+ entries: readonly string[],
+ maxBatchSizeBytes?: number,
+): Uint8Array[] {
+ if (entries.length === 0) return [];
+ if (maxBatchSizeBytes === undefined) {
+ return [encodeQwpIngressSymbolDictionaryFrame(0, entries)];
+ }
+ const result: Uint8Array[] = [];
+ let startId = 0;
+ while (startId < entries.length) {
+ let count = 0;
+ let entriesSize = 0;
+ while (startId + count < entries.length) {
+ const entryLength = utf8Length(entries[startId + count]);
+ const nextEntriesSize =
+ entriesSize + qwpVarintSize(entryLength) + entryLength;
+ const nextCount = count + 1;
+ const size =
+ QWP_HEADER_SIZE +
+ qwpVarintSize(startId) +
+ qwpVarintSize(nextCount) +
+ nextEntriesSize;
+ if (size > maxBatchSizeBytes) break;
+ count = nextCount;
+ entriesSize = nextEntriesSize;
+ }
+ if (count === 0) {
+ const entryLength = utf8Length(entries[startId]);
+ const frameLength =
+ QWP_HEADER_SIZE +
+ qwpVarintSize(startId) +
+ qwpVarintSize(1) +
+ qwpVarintSize(entryLength) +
+ entryLength;
+ throw new QwpCatchUpCapGapError(startId, frameLength, maxBatchSizeBytes);
+ }
+ result.push(
+ encodeQwpIngressSymbolDictionaryFrame(
+ startId,
+ entries.slice(startId, startId + count),
+ ),
+ );
+ startId += count;
+ }
+ return result;
+}
+
+function minimumDefined(
+ first: number | undefined,
+ second: number | undefined,
+): number | undefined {
+ return first === undefined
+ ? second
+ : second === undefined
+ ? first
+ : Math.min(first, second);
+}
+
+function monotonicNowMs(): number {
+ return typeof performance === "undefined" ? Date.now() : performance.now();
+}
+
+function validateReconnectPolicy(
+ maxAttempts: number,
+ initialBackoffMs: number,
+ maxBackoffMs: number,
+ maxDurationMs: number,
+ maxFrameRejections: number,
+ poisonMinEscalationWindowMs: number,
+ catchUpCapGapMinEscalationWindowMs: number,
+): void {
+ if (!Number.isSafeInteger(maxAttempts) || maxAttempts < 0) {
+ throw new RangeError(
+ "reconnect maxAttempts must be a non-negative safe integer",
+ );
+ }
+ for (const [name, value] of [
+ ["initialBackoffMs", initialBackoffMs],
+ ["maxBackoffMs", maxBackoffMs],
+ ["maxDurationMs", maxDurationMs],
+ ["poisonMinEscalationWindowMs", poisonMinEscalationWindowMs],
+ ["catchUpCapGapMinEscalationWindowMs", catchUpCapGapMinEscalationWindowMs],
+ ] as const) {
+ if (!Number.isFinite(value) || value < 0) {
+ throw new RangeError(
+ `reconnect ${name} must be a non-negative finite number`,
+ );
+ }
+ }
+ if (maxBackoffMs < initialBackoffMs) {
+ throw new RangeError(
+ "reconnect maxBackoffMs must be greater than or equal to initialBackoffMs",
+ );
+ }
+ if (!Number.isSafeInteger(maxFrameRejections) || maxFrameRejections < 1) {
+ throw new RangeError(
+ "reconnect maxFrameRejections must be a positive safe integer",
+ );
+ }
+}
+
+function isRetryableReconnectError(error: unknown): boolean {
+ if (error instanceof QwpUpgradeError) return error.retryable !== false;
+ if (error instanceof QwpFailoverError) {
+ return error.attempts.some((attempt) =>
+ isRetryableReconnectError(attempt.error),
+ );
+ }
+ if (
+ error instanceof QwpReplayRejectedError ||
+ error instanceof QwpProtocolError
+ ) {
+ return false;
+ }
+ // Replay-store errors are declared in the Node-only layer, so the journal's
+ // own verdict -- structural corruption, or a slot lock another process took
+ // over -- is read structurally through the `retryable` flag those classes
+ // carry. Every reconnect/replay path must honour the same verdict.
+ return (
+ (error as { retryable?: unknown } | null | undefined)?.retryable !== false
+ );
+}
+
+function isEndpointPolicyFailure(error: unknown): boolean {
+ if (error instanceof QwpUpgradeError) return true;
+ return (
+ error instanceof QwpFailoverError &&
+ error.attempts.some((attempt) => isEndpointPolicyFailure(attempt.error))
+ );
+}
+
+/** Returns the typed capability gap retained anywhere in a failed endpoint sweep. */
+function durableAckUnavailableCause(
+ error: unknown,
+): QwpDurableAckUnavailableError | undefined {
+ if (error instanceof QwpDurableAckUnavailableError) return error;
+ if (!(error instanceof QwpFailoverError) || error.attempts.length === 0) {
+ return undefined;
+ }
+ for (const attempt of error.attempts) {
+ const cause = durableAckUnavailableCause(attempt.error);
+ if (cause) return cause;
+ }
+ return undefined;
+}
+
+function isPrimaryUnavailableError(error: unknown): boolean {
+ if (error instanceof QwpUpgradeError) {
+ return error.kind === QWP_UPGRADE_ERROR_KIND.ROLE_REJECTED;
+ }
+ return (
+ error instanceof QwpFailoverError &&
+ error.attempts.length > 0 &&
+ error.attempts.every((attempt) => isPrimaryUnavailableError(attempt.error))
+ );
+}
+
+function reconnectDelayMs(error: unknown): number {
+ return error instanceof RetriableIngressNackError ||
+ error instanceof RetriableIngressConnectionError
+ ? error.retryDelayMs
+ : 0;
+}
+
+function isRetriableIngressStatus(status: number): boolean {
+ return (
+ status !== QWP_STATUS.SCHEMA_MISMATCH &&
+ status !== QWP_STATUS.PARSE_ERROR &&
+ status !== QWP_STATUS.SECURITY_ERROR
+ );
+}
+
+function cappedExponentialBackoff(
+ initialMs: number,
+ maximumMs: number,
+ exponent: number,
+): number {
+ if (initialMs === 0 || maximumMs === 0) return 0;
+ return Math.min(initialMs * 2 ** Math.min(exponent, 52), maximumMs);
+}
+
+function findLastClientFrame(
+ frames: readonly ReplayFrame[],
+): ReplayFrame | undefined {
+ for (let index = frames.length - 1; index >= 0; index--) {
+ if (frames[index].clientSequence !== undefined) return frames[index];
+ }
+ return undefined;
+}
+
+function rewriteResponseSequence(
+ payload: Uint8Array,
+ sequence: bigint,
+): Uint8Array {
+ const translated = payload.slice();
+ new DataView(
+ translated.buffer,
+ translated.byteOffset,
+ translated.byteLength,
+ ).setBigUint64(1, sequence, true);
+ return translated;
+}
+
+function areTargetsCovered(
+ targets: ReadonlyMap,
+ watermarks: ReadonlyMap,
+): boolean {
+ for (const [table, target] of targets) {
+ const watermark = watermarks.get(table);
+ if (watermark === undefined || watermark < target) return false;
+ }
+ return true;
+}
diff --git a/src/_qwp/_internal/safe-callback.ts b/src/_qwp/_internal/safe-callback.ts
new file mode 100644
index 0000000..69f4eda
--- /dev/null
+++ b/src/_qwp/_internal/safe-callback.ts
@@ -0,0 +1,60 @@
+/**
+ * Containment for user-supplied observability callbacks.
+ *
+ * Notification callbacks (reconnect events, sender errors, recovery reports)
+ * run purely for their side effects and must never interfere with protocol
+ * progress. A synchronous throw is easy to contain with try/catch, but an
+ * `async` callback returns a promise: if it rejects, the rejection escapes the
+ * surrounding try/catch and Node treats it as an unhandled rejection, which
+ * terminates the host process by default (Node >= 15). This helper contains
+ * both failure modes so a broken callback can never crash the client's host or
+ * stall protocol work.
+ */
+
+/**
+ * Invokes an observability callback without letting a synchronous throw or a
+ * rejected promise (from an `async` callback) escape. On either failure the
+ * optional {@link onFailure} handler runs; it is itself guarded so it can never
+ * re-escape the containment it backs.
+ */
+export function safelyInvoke(
+ callback: ((event: T) => unknown) | undefined,
+ event: T,
+ onFailure?: (error: unknown) => void,
+): void {
+ if (!callback) return;
+ try {
+ const result = callback(event);
+ if (isPromiseLike(result)) {
+ void result.then(undefined, (error) => reportFailure(onFailure, error));
+ }
+ } catch (error) {
+ reportFailure(onFailure, error);
+ }
+}
+
+function reportFailure(
+ onFailure: ((error: unknown) => void) | undefined,
+ error: unknown,
+): void {
+ if (!onFailure) return;
+ try {
+ onFailure(error);
+ } catch {
+ // A failing fallback must not re-escape the containment it backs.
+ }
+}
+
+/**
+ * Minimal Promises/A+ thenable test. A genuine thenable only guarantees a
+ * `then` method, so `then(undefined, onRejected)` — not `catch` — is the
+ * portable way to attach a rejection handler.
+ */
+export function isPromiseLike(value: unknown): value is PromiseLike {
+ return (
+ value !== null &&
+ (typeof value === "object" || typeof value === "function") &&
+ "then" in value &&
+ typeof value.then === "function"
+ );
+}
diff --git a/src/_qwp/_internal/websocket-connection.ts b/src/_qwp/_internal/websocket-connection.ts
new file mode 100644
index 0000000..206b6f8
--- /dev/null
+++ b/src/_qwp/_internal/websocket-connection.ts
@@ -0,0 +1,623 @@
+import { QwpProtocolError } from "../_core";
+import {
+ QWP_UPGRADE_ERROR_KIND,
+ QWP_UPGRADE_TIMEOUT_PHASE,
+ QwpBinaryConnection,
+ QwpConnectionCloseInfo,
+ QwpHandshakeMetadata,
+ QwpSendClosedError,
+ QwpSendError,
+ QwpSendTimeoutError,
+ QwpUpgradeError,
+} from "../transport";
+import { QwpAsyncQueue } from "./async-queue";
+
+interface QwpWebSocketMessageEvent {
+ data: unknown;
+}
+
+interface QwpWebSocketCloseEvent {
+ code?: number;
+ reason?: string;
+ wasClean?: boolean;
+}
+
+export interface QwpWebSocketLike {
+ binaryType: string;
+ readonly readyState: number;
+ /** WebSocket subprotocol selected by the server, or an empty string. */
+ readonly protocol?: string;
+ /** Number of application bytes queued by WHATWG-compatible WebSockets. */
+ readonly bufferedAmount?: number;
+ send(data: Uint8Array): void;
+ /** Node adapter hook for the `ws.send(data, callback)` completion signal. */
+ sendWithCallback?(data: Uint8Array, callback: (error?: Error) => void): void;
+ /** Node WebSocket implementations may expose control-frame PING. */
+ ping?(): void;
+ /** Node WebSocket implementations may support immediate termination. */
+ terminate?(): void;
+ close(code?: number, reason?: string): void;
+ addEventListener(
+ type: "open",
+ listener: (event: unknown) => void,
+ options?: { once?: boolean },
+ ): void;
+ addEventListener(
+ type: "message",
+ listener: (event: QwpWebSocketMessageEvent) => void,
+ ): void;
+ addEventListener(
+ type: "error",
+ listener: (event: unknown) => void,
+ options?: { once?: boolean },
+ ): void;
+ addEventListener(
+ type: "close",
+ listener: (event: QwpWebSocketCloseEvent) => void,
+ options?: { once?: boolean },
+ ): void;
+ /** Optional cleanup hook implemented by browser WebSocket and Node `ws`. */
+ removeEventListener?(type: "open", listener: (event: unknown) => void): void;
+ removeEventListener?(
+ type: "message",
+ listener: (event: QwpWebSocketMessageEvent) => void,
+ ): void;
+ removeEventListener?(type: "error", listener: (event: unknown) => void): void;
+ removeEventListener?(
+ type: "close",
+ listener: (event: QwpWebSocketCloseEvent) => void,
+ ): void;
+}
+
+export interface QwpWebSocketOpenOptions {
+ url: string | URL;
+ connectTimeoutMs?: number;
+ /** Node-only HTTP authentication and WebSocket upgrade deadline. */
+ authTimeoutMs?: number;
+ /** Resolves after the Node TCP/TLS transport has connected. */
+ transportConnected?: Promise;
+ sendTimeoutMs?: number;
+ closeTimeoutMs?: number;
+ completeHandshake: () => QwpHandshakeMetadata;
+ /** Node adapters use this to surface non-101 HTTP responses from `ws`. */
+ openingFailure?: Promise;
+ /** Browsers hide the HTTP response behind a generic WebSocket error event. */
+ opaqueErrors?: boolean;
+ /**
+ * Tears the pending upgrade down immediately. Without it a close() issued
+ * while the peer has accepted the TCP connection but not answered the
+ * upgrade leaves the socket and its deadline alive until that deadline
+ * fires, which keeps the Node event loop open long after close() resolved.
+ */
+ signal?: AbortSignal;
+}
+
+const WEBSOCKET_OPEN = 1;
+const WEBSOCKET_CLOSED = 3;
+const BUFFERED_AMOUNT_POLL_MS = 4;
+const DEFAULT_TIMEOUT_MS = 15_000;
+
+export function validateQwpWebSocketTimeouts(options: {
+ connectTimeoutMs?: number;
+ authTimeoutMs?: number;
+ sendTimeoutMs?: number;
+ closeTimeoutMs?: number;
+}): void {
+ for (const [name, value] of [
+ ["connectTimeoutMs", options.connectTimeoutMs],
+ ["authTimeoutMs", options.authTimeoutMs],
+ ["sendTimeoutMs", options.sendTimeoutMs],
+ ["closeTimeoutMs", options.closeTimeoutMs],
+ ] as const) {
+ if (value !== undefined && (!Number.isFinite(value) || value <= 0)) {
+ throw new RangeError(`${name} must be a positive finite number`);
+ }
+ }
+}
+
+async function normalizeBinaryMessage(data: unknown): Promise {
+ if (data instanceof ArrayBuffer) return new Uint8Array(data);
+ if (ArrayBuffer.isView(data)) {
+ return new Uint8Array(
+ data.buffer,
+ data.byteOffset,
+ data.byteLength,
+ ).slice();
+ }
+ if (typeof Blob !== "undefined" && data instanceof Blob) {
+ return new Uint8Array(await data.arrayBuffer());
+ }
+ throw new QwpProtocolError("QWP WebSocket received a non-binary message");
+}
+
+/** Wraps a WHATWG-style WebSocket and resolves once its opening handshake succeeds. */
+/** Absorbs a socket `error` raised before the real listeners are attached. */
+const ignoreSocketError = (): void => undefined;
+
+export function openQwpWebSocket(
+ socket: QwpWebSocketLike,
+ options: QwpWebSocketOpenOptions,
+): Promise {
+ try {
+ validateQwpWebSocketTimeouts(options);
+ } catch (error) {
+ try {
+ // Tearing down a CONNECTING socket makes `ws` emit `error`, and nothing
+ // has subscribed to this one yet. Absorb it rather than let an
+ // EventEmitter with no listener rethrow it into the process.
+ socket.addEventListener("error", ignoreSocketError);
+ if (socket.terminate) socket.terminate();
+ else if (socket.readyState !== WEBSOCKET_CLOSED) socket.close();
+ } catch {
+ // Configuration validation remains authoritative.
+ }
+ return Promise.reject(error);
+ }
+ const connectTimeoutMs = options.connectTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+ // Opening a connection is two deadlines: connectTimeoutMs covers the TCP/TLS
+ // transport, and authTimeoutMs takes over for the upgrade and authentication
+ // exchange the moment transportConnected resolves. A caller who narrows only
+ // the first is bounding how long establishing one connection may take, and
+ // the upgrade is part of that -- inheriting keeps an explicit 200 ms from
+ // being exceeded 75x by a default nobody chose, which is what a peer that
+ // accepts TCP and never answers the upgrade used to cost. Setting
+ // authTimeoutMs restores an independent budget for the slower phase.
+ const authTimeoutMs =
+ options.authTimeoutMs ?? options.connectTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+ const sendTimeoutMs = options.sendTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+ const closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+
+ const messages = new QwpAsyncQueue();
+ let resolveClosed!: (info: QwpConnectionCloseInfo) => void;
+ const closed = new Promise((resolve) => {
+ resolveClosed = resolve;
+ });
+ let opened = false;
+ let openingSettled = false;
+ let messageTail: Promise = Promise.resolve();
+ let sendTail: Promise = Promise.resolve();
+ let terminalSendError: QwpSendError | undefined;
+ let rejectActiveSend: ((error: QwpSendError) => void) | undefined;
+ let closeSettled = false;
+ let closeTask: Promise | undefined;
+ let cleanupTask: Promise = Promise.resolve();
+ let removeSocketListeners = (): void => undefined;
+
+ const failSends = (error: QwpSendError): QwpSendError => {
+ terminalSendError ??= error;
+ rejectActiveSend?.(terminalSendError);
+ return terminalSendError;
+ };
+
+ const settleClosed = (info: QwpConnectionCloseInfo): void => {
+ if (closeSettled) return;
+ closeSettled = true;
+ resolveClosed(info);
+ if (opened) failSends(new QwpSendClosedError(info));
+ removeSocketListeners();
+ cleanupTask = (async () => {
+ let timer: ReturnType | undefined;
+ const timedOut = await Promise.race([
+ messageTail.then(
+ () => false,
+ () => false,
+ ),
+ new Promise((resolve) => {
+ timer = setTimeout(() => resolve(true), closeTimeoutMs);
+ }),
+ ]);
+ if (timer) clearTimeout(timer);
+ if (timedOut) messageTail = Promise.resolve();
+ messages.end();
+ })();
+ };
+
+ const closeSocket = (code = 1000, reason = ""): Promise => {
+ if (closeTask) return closeTask;
+ closeTask = (async () => {
+ const requestedInfo: QwpConnectionCloseInfo = {
+ code,
+ reason,
+ wasClean: code === 1000,
+ };
+ if (opened) failSends(new QwpSendClosedError(requestedInfo));
+ if (socket.readyState === WEBSOCKET_CLOSED) {
+ settleClosed(requestedInfo);
+ } else {
+ try {
+ socket.close(code, reason);
+ } catch {
+ try {
+ socket.terminate?.();
+ } catch {
+ // The synthetic close below still releases local resources.
+ }
+ settleClosed({
+ code: 1006,
+ reason: "QWP WebSocket close failed",
+ wasClean: false,
+ });
+ }
+ }
+ if (!closeSettled) {
+ let timer: ReturnType | undefined;
+ const timedOut = await Promise.race([
+ closed.then(() => false),
+ new Promise((resolve) => {
+ timer = setTimeout(() => resolve(true), closeTimeoutMs);
+ }),
+ ]);
+ if (timer) clearTimeout(timer);
+ if (timedOut && !closeSettled) {
+ try {
+ socket.terminate?.();
+ } catch {
+ // Local state must still settle when forced termination throws.
+ }
+ settleClosed({
+ code: 1006,
+ reason: `QWP WebSocket close timed out after ${closeTimeoutMs}ms`,
+ wasClean: false,
+ });
+ }
+ }
+ await cleanupTask;
+ })();
+ return closeTask;
+ };
+
+ const abortAfterSendFailure = (): void => {
+ void closeSocket(1011, "QWP send failed");
+ };
+
+ const sendWithBackpressure = (payload: Uint8Array): Promise => {
+ if (terminalSendError) return Promise.reject(terminalSendError);
+ if (socket.readyState !== WEBSOCKET_OPEN) {
+ return Promise.reject(failSends(new QwpSendClosedError()));
+ }
+
+ return new Promise((resolveSend, rejectSend) => {
+ let settled = false;
+ let drainPoll: ReturnType | undefined;
+
+ const settle = (error?: QwpSendError): void => {
+ if (settled) return;
+ settled = true;
+ if (drainPoll) clearTimeout(drainPoll);
+ clearTimeout(sendTimeout);
+ if (rejectActiveSend === rejectPending) rejectActiveSend = undefined;
+ if (error) rejectSend(error);
+ else resolveSend();
+ };
+ const rejectPending = (error: QwpSendError): void => settle(error);
+ const failSend = (error: QwpSendError): void => {
+ settle(failSends(error));
+ abortAfterSendFailure();
+ };
+
+ rejectActiveSend = rejectPending;
+ const sendTimeout = setTimeout(() => {
+ const bufferedAmount = socket.bufferedAmount;
+ failSend(
+ new QwpSendTimeoutError(
+ sendTimeoutMs,
+ typeof bufferedAmount === "number" ? bufferedAmount : undefined,
+ ),
+ );
+ }, sendTimeoutMs);
+
+ if (socket.sendWithCallback) {
+ try {
+ socket.sendWithCallback(payload, (error) => {
+ if (error) {
+ failSend(
+ new QwpSendError(
+ "QWP WebSocket send failed; delivery outcome is unknown",
+ error,
+ ),
+ );
+ } else {
+ settle();
+ }
+ });
+ } catch (error) {
+ failSend(
+ new QwpSendError(
+ "QWP WebSocket send failed before it could be queued",
+ error,
+ ),
+ );
+ }
+ return;
+ }
+
+ const initialBufferedAmount = socket.bufferedAmount;
+ try {
+ socket.send(payload);
+ } catch (error) {
+ failSend(
+ new QwpSendError(
+ "QWP WebSocket send failed before it could be queued",
+ error,
+ ),
+ );
+ return;
+ }
+
+ if (typeof initialBufferedAmount !== "number") {
+ // Backwards compatibility for custom adapters without a drain signal.
+ settle();
+ return;
+ }
+
+ const waitForDrain = (): void => {
+ if (socket.readyState !== WEBSOCKET_OPEN) {
+ settle(failSends(new QwpSendClosedError()));
+ return;
+ }
+ if (
+ typeof socket.bufferedAmount !== "number" ||
+ socket.bufferedAmount <= initialBufferedAmount
+ ) {
+ settle();
+ return;
+ }
+ drainPoll = setTimeout(waitForDrain, BUFFERED_AMOUNT_POLL_MS);
+ };
+ waitForDrain();
+ });
+ };
+
+ return new Promise((resolve, reject) => {
+ let timeout: ReturnType | undefined;
+
+ const armOpeningTimeout = (
+ timeoutMs: number,
+ phase?: "connect" | "authentication",
+ ): void => {
+ if (timeout) clearTimeout(timeout);
+ timeout = setTimeout(() => {
+ const message =
+ phase === QWP_UPGRADE_TIMEOUT_PHASE.CONNECT
+ ? `QWP TCP/TLS connection timed out after ${timeoutMs}ms`
+ : phase === QWP_UPGRADE_TIMEOUT_PHASE.AUTHENTICATION
+ ? `QWP authentication/WebSocket upgrade timed out after ${timeoutMs}ms`
+ : `QWP WebSocket connection timed out after ${timeoutMs}ms`;
+ failOpening(
+ new QwpUpgradeError(message, {
+ kind: QWP_UPGRADE_ERROR_KIND.TIMEOUT,
+ retryable: true,
+ tryNextEndpoint: true,
+ url: options.url,
+ timeoutPhase: phase,
+ }),
+ 1000,
+ "QWP connection timeout",
+ );
+ }, timeoutMs);
+ };
+
+ const failOpening = (
+ error: Error,
+ closeCode = 1000,
+ closeReason = "QWP upgrade failed",
+ ): void => {
+ if (openingSettled) return;
+ openingSettled = true;
+ if (timeout) clearTimeout(timeout);
+ options.signal?.removeEventListener("abort", abortOpening);
+ void closeSocket(closeCode, closeReason);
+ reject(error);
+ };
+
+ const abortOpening = (): void => {
+ failOpening(
+ new QwpSendClosedError(),
+ 1000,
+ "QWP connection closed while connecting",
+ );
+ };
+ // Aborting closes the socket, and closing a CONNECTING `ws` socket makes it
+ // emit `error` on the next tick. This executor attaches the socket's
+ // listeners last, so acting on an already-aborted signal here would leave
+ // that event unhandled and terminate the process. A failover sweep hands
+ // the same signal to every remaining endpoint after close() aborts it, so
+ // this is the ordinary shape for a multi-address client, not a rare race.
+ // Record the abort and apply it once the listeners are in place.
+ let abortedBeforeListening = false;
+ if (options.signal) {
+ if (options.signal.aborted) {
+ abortedBeforeListening = true;
+ } else {
+ options.signal.addEventListener("abort", abortOpening, { once: true });
+ }
+ }
+
+ armOpeningTimeout(
+ connectTimeoutMs,
+ options.transportConnected
+ ? QWP_UPGRADE_TIMEOUT_PHASE.CONNECT
+ : undefined,
+ );
+ void options.transportConnected?.then(
+ () => {
+ if (openingSettled) return;
+ armOpeningTimeout(
+ authTimeoutMs,
+ QWP_UPGRADE_TIMEOUT_PHASE.AUTHENTICATION,
+ );
+ },
+ (error: unknown) => {
+ failOpening(
+ new QwpUpgradeError(
+ "QWP TCP/TLS transport failed while establishing a connection",
+ {
+ kind: QWP_UPGRADE_ERROR_KIND.TRANSPORT,
+ retryable: true,
+ tryNextEndpoint: true,
+ url: options.url,
+ cause: error,
+ },
+ ),
+ );
+ },
+ );
+
+ const onOpen = (): void => {
+ if (openingSettled) return;
+ let handshake: QwpHandshakeMetadata;
+ try {
+ handshake = Object.freeze({ ...options.completeHandshake() });
+ } catch (error) {
+ failOpening(
+ error instanceof Error
+ ? error
+ : new Error("QWP WebSocket upgrade validation failed"),
+ 1000,
+ "QWP upgrade validation failed",
+ );
+ return;
+ }
+ openingSettled = true;
+ opened = true;
+ if (timeout) clearTimeout(timeout);
+ options.signal?.removeEventListener("abort", abortOpening);
+ const connection: QwpBinaryConnection = {
+ messages,
+ closed,
+ handshake,
+ endpoint: options.url,
+ send(payload: Uint8Array): Promise {
+ const sending = sendTail.then(() => sendWithBackpressure(payload));
+ sendTail = sending.catch(() => undefined);
+ return sending;
+ },
+ async close(code = 1000, reason = ""): Promise {
+ await closeSocket(code, reason);
+ },
+ };
+ if (socket.ping) {
+ connection.ping = async (): Promise => {
+ if (socket.readyState !== WEBSOCKET_OPEN) {
+ throw new Error("QWP WebSocket is not open");
+ }
+ socket.ping!();
+ };
+ }
+ resolve(connection);
+ };
+
+ const onMessage = (event: QwpWebSocketMessageEvent): void => {
+ if (openingSettled && !opened) return;
+ messageTail = messageTail
+ .then(async () =>
+ messages.push(await normalizeBinaryMessage(event.data)),
+ )
+ .catch((error: unknown) => {
+ messages.fail(error);
+ void closeSocket(1002, "invalid QWP payload");
+ });
+ };
+
+ options.openingFailure?.catch((error: unknown) => {
+ failOpening(
+ error instanceof Error
+ ? error
+ : new QwpUpgradeError("QWP WebSocket upgrade failed", {
+ kind: QWP_UPGRADE_ERROR_KIND.TRANSPORT,
+ retryable: true,
+ tryNextEndpoint: true,
+ url: options.url,
+ cause: error,
+ }),
+ );
+ });
+
+ const onError = (event: unknown): void => {
+ if (opened) {
+ const eventError = (event as { error?: unknown }).error;
+ failSends(
+ new QwpSendError(
+ "QWP WebSocket transport error while sending",
+ eventError ?? event,
+ ),
+ );
+ messages.fail(new Error("QWP WebSocket transport error"));
+ abortAfterSendFailure();
+ return;
+ }
+ const opaque = options.opaqueErrors === true;
+ const eventError = (event as { error?: unknown }).error;
+ const error = new QwpUpgradeError(
+ opaque
+ ? "QWP WebSocket upgrade failed; the browser did not expose the HTTP response"
+ : "QWP WebSocket transport error during upgrade",
+ {
+ kind: opaque
+ ? QWP_UPGRADE_ERROR_KIND.OPAQUE
+ : QWP_UPGRADE_ERROR_KIND.TRANSPORT,
+ retryable: opaque ? undefined : true,
+ tryNextEndpoint: opaque ? undefined : true,
+ url: options.url,
+ cause: eventError ?? event,
+ },
+ );
+ failOpening(error);
+ };
+
+ const onClose = (event: QwpWebSocketCloseEvent): void => {
+ clearTimeout(timeout);
+ const info = {
+ code: event.code ?? 1006,
+ reason: event.reason ?? "",
+ wasClean: event.wasClean ?? false,
+ };
+ settleClosed(info);
+ if (!opened) {
+ failOpening(
+ new QwpUpgradeError(
+ `QWP WebSocket closed during handshake [code=${info.code}, reason=${info.reason}]`,
+ {
+ kind: options.opaqueErrors
+ ? QWP_UPGRADE_ERROR_KIND.OPAQUE
+ : QWP_UPGRADE_ERROR_KIND.TRANSPORT,
+ retryable: options.opaqueErrors ? undefined : true,
+ tryNextEndpoint: options.opaqueErrors ? undefined : true,
+ url: options.url,
+ closeCode: info.code,
+ },
+ ),
+ );
+ return;
+ }
+ };
+
+ removeSocketListeners = (): void => {
+ try {
+ socket.removeEventListener?.("open", onOpen);
+ socket.removeEventListener?.("message", onMessage);
+ socket.removeEventListener?.("error", onError);
+ socket.removeEventListener?.("close", onClose);
+ } catch {
+ // Transport cleanup must not make connection close reject.
+ }
+ };
+ try {
+ socket.binaryType = "arraybuffer";
+ socket.addEventListener("open", onOpen, { once: true });
+ socket.addEventListener("message", onMessage);
+ socket.addEventListener("error", onError);
+ socket.addEventListener("close", onClose, { once: true });
+ } catch (error) {
+ failOpening(
+ error instanceof Error
+ ? error
+ : new Error("failed to configure QWP WebSocket listeners"),
+ );
+ }
+ // Safe now: `onError` is attached, so the close this triggers has a
+ // subscriber. A failed attachment above already settled the opening, and
+ // failOpening() is idempotent, so this is a no-op in that case.
+ if (abortedBeforeListening) abortOpening();
+ });
+}
diff --git a/src/_qwp/client.ts b/src/_qwp/client.ts
new file mode 100644
index 0000000..fa220e3
--- /dev/null
+++ b/src/_qwp/client.ts
@@ -0,0 +1,898 @@
+import {
+ QwpEgressQuery,
+ QwpEgressQueryOptions,
+ QwpEgressSession,
+ QwpEgressViewQuery,
+ QwpResultBatchViewHandler,
+} from "./egress-session";
+import { QwpSender } from "./sender";
+import { QwpHandshakeMetadata } from "./transport";
+import type {
+ QwpNegotiatedEgressCompression,
+ QwpServerInfoMessage,
+} from "./_core";
+
+const DEFAULT_POOL_MIN = 1;
+const DEFAULT_POOL_MAX = 4;
+const DEFAULT_ACQUIRE_TIMEOUT_MS = 5_000;
+const DEFAULT_IDLE_TIMEOUT_MS = 60_000;
+const DEFAULT_MAX_LIFETIME_MS = 30 * 60_000;
+const DEFAULT_HOUSEKEEPING_INTERVAL_MS = 5_000;
+const MIN_HOUSEKEEPING_INTERVAL_MS = 100;
+const MAX_CLOSE_CREATION_WAIT_MS = 5_000;
+const MAX_CLOSE_LEASE_WAIT_MS = 5_000;
+
+export interface QwpClientPoolOptions {
+ /** Warm ingress connections created by connect(). Defaults to 1. */
+ senderPoolMin?: number;
+ /** Maximum concurrently borrowed ingress senders. Defaults to 4. */
+ senderPoolMax?: number;
+ /** Warm egress connections created by connect(). Defaults to 1. */
+ queryPoolMin?: number;
+ /** Maximum concurrently borrowed query connections. Defaults to 4. */
+ queryPoolMax?: number;
+ /** Idle time before an excess pooled connection is closed. Defaults to 60s; zero disables. */
+ idleTimeoutMs?: number;
+ /** Maximum pooled connection age before recycling it while idle. Defaults to 30m; zero disables. */
+ maxLifetimeMs?: number;
+ /** Idle/lifetime sweep interval. Defaults to 5s and must be at least 100ms. */
+ housekeepingIntervalMs?: number;
+ /**
+ * Maximum wait for a returned pool slot and for leases during shutdown.
+ * The shutdown wait is capped at 5 seconds. Defaults to 5 seconds.
+ */
+ acquireTimeoutMs?: number;
+}
+
+export interface QwpClientFactories {
+ createSender(slot: number): Promise;
+ createQuerySession(slot: number): Promise;
+ /** @internal Coordinates stable persistent sender slots with recovery. */
+ senderSlotReservation?: QwpPoolSlotReservation;
+ /** @internal Starts runtime-specific background services on first use. */
+ start?(): void | Promise;
+ /** @internal Stops runtime-specific background services during close. */
+ close?(): void | Promise;
+}
+
+/** @internal Cross-owner reservation for stable pooled sender slot indexes. */
+export interface QwpPoolSlotReservation {
+ tryReserve(slot: number): boolean;
+ release(slot: number): void;
+ onAvailable(listener: () => void): () => void;
+}
+
+export interface QwpResourcePoolMetrics {
+ readonly minimum: number;
+ readonly maximum: number;
+ readonly total: number;
+ readonly available: number;
+ readonly leased: number;
+ readonly creating: number;
+ readonly waiting: number;
+}
+
+export interface QwpClientMetrics {
+ readonly senders: QwpResourcePoolMetrics;
+ readonly queries: QwpResourcePoolMetrics;
+ readonly closing: boolean;
+ readonly closed: boolean;
+}
+
+/** A bounded QWP pool could not provide a connection before its deadline. */
+export class QwpPoolAcquireTimeoutError extends Error {
+ constructor(
+ readonly resource: "sender" | "query",
+ readonly timeoutMs: number,
+ ) {
+ super(
+ `timed out waiting for a QWP ${resource} from the pool after ${timeoutMs}ms`,
+ );
+ this.name = "QwpPoolAcquireTimeoutError";
+ }
+}
+
+/** A pooled resource failed while a new slot was being connected. */
+export class QwpPoolResourceError extends Error {
+ readonly cause: unknown;
+
+ constructor(
+ readonly resource: "sender" | "query",
+ cause: unknown,
+ ) {
+ super(
+ `failed to create pooled QWP ${resource}${
+ cause instanceof Error ? `: ${cause.message}` : ""
+ }`,
+ );
+ this.name = "QwpPoolResourceError";
+ this.cause = cause;
+ }
+}
+
+/** The owning QWP client, or one of its returned lease handles, is closed. */
+export class QwpClientClosedError extends Error {
+ constructor(message = "QWP client is closed") {
+ super(message);
+ this.name = "QwpClientClosedError";
+ }
+}
+
+interface ValidatedPoolOptions {
+ readonly senderPoolMin: number;
+ readonly senderPoolMax: number;
+ readonly queryPoolMin: number;
+ readonly queryPoolMax: number;
+ readonly acquireTimeoutMs: number;
+ readonly idleTimeoutMs: number;
+ readonly maxLifetimeMs: number;
+ readonly housekeepingIntervalMs: number;
+}
+
+interface PoolEntry {
+ readonly slot: number;
+ readonly value: T;
+ readonly createdAtMs: number;
+ idleSinceMs: number;
+ leased: boolean;
+ destroyPromise?: Promise;
+}
+
+interface PoolWaiter {
+ readonly resolve: () => void;
+ readonly reject: (error: unknown) => void;
+ readonly timer?: ReturnType;
+}
+
+interface PoolCloseWaiter {
+ readonly resolve: () => void;
+ readonly timer: ReturnType;
+}
+
+class QwpResourcePool {
+ private readonly all = new Map>();
+ private readonly available: PoolEntry[] = [];
+ private readonly creatingSlots = new Set();
+ private readonly destroyingSlots = new Set();
+ private readonly creationOperations = new Set>();
+ private readonly waiters = new Set();
+ private readonly closeWaiters = new Set();
+ private readonly reservedSlots = new Set();
+ private readonly unsubscribeSlotAvailability?: () => void;
+ private closePromise?: Promise;
+ private pendingLeaseTeardowns = 0;
+ private closed = false;
+
+ constructor(
+ private readonly resource: "sender" | "query",
+ private readonly minimum: number,
+ private readonly maximum: number,
+ private readonly acquireTimeoutMs: number,
+ private readonly idleTimeoutMs: number,
+ private readonly maxLifetimeMs: number,
+ private readonly createResource: (slot: number) => Promise,
+ private readonly destroyResource: (resource: T) => Promise,
+ private readonly closeLeasedOnShutdown = false,
+ private readonly slotReservation?: QwpPoolSlotReservation,
+ ) {
+ this.unsubscribeSlotAvailability = slotReservation?.onAvailable(() =>
+ this.wakeWaiters(),
+ );
+ }
+
+ get metrics(): QwpResourcePoolMetrics {
+ return Object.freeze({
+ minimum: this.minimum,
+ maximum: this.maximum,
+ total: this.all.size,
+ available: this.available.length,
+ leased: Array.from(this.all.values()).filter((entry) => entry.leased)
+ .length,
+ creating: this.creatingSlots.size,
+ waiting: this.waiters.size,
+ });
+ }
+
+ async prewarm(): Promise {
+ const needed = Math.max(
+ 0,
+ this.minimum - this.all.size - this.creatingSlots.size,
+ );
+ const acquired = await Promise.allSettled(
+ Array.from({ length: needed }, () => this.acquire()),
+ );
+ await Promise.all(
+ acquired.map((result) =>
+ result.status === "fulfilled"
+ ? this.release(result.value, true)
+ : Promise.resolve(),
+ ),
+ );
+ const failure = acquired.find(
+ (result): result is PromiseRejectedResult => result.status === "rejected",
+ );
+ if (failure) throw failure.reason;
+ }
+
+ async acquire(): Promise> {
+ const deadline = Date.now() + this.acquireTimeoutMs;
+ while (true) {
+ this.throwIfClosed();
+ const available = this.available.shift();
+ if (available) {
+ available.leased = true;
+ return available;
+ }
+ const slot = this.reserveSlot();
+ if (slot !== undefined) return this.createLeased(slot);
+ const remaining = deadline - Date.now();
+ if (remaining <= 0) {
+ throw new QwpPoolAcquireTimeoutError(
+ this.resource,
+ this.acquireTimeoutMs,
+ );
+ }
+ await this.waitForChange(remaining);
+ }
+ }
+
+ async release(entry: PoolEntry, reusable: boolean): Promise {
+ if (!entry.leased) return;
+ entry.leased = false;
+ if (this.closed || !reusable || this.all.get(entry.slot) !== entry) {
+ if (this.all.get(entry.slot) === entry) this.all.delete(entry.slot);
+ this.pendingLeaseTeardowns++;
+ try {
+ await this.destroyRetired(entry);
+ } finally {
+ this.pendingLeaseTeardowns--;
+ this.wakeWaiters();
+ this.wakeCloseWaiters();
+ }
+ return;
+ }
+ entry.idleSinceMs = Date.now();
+ this.available.push(entry);
+ this.wakeWaiters();
+ this.wakeCloseWaiters();
+ }
+
+ close(): Promise {
+ if (!this.closePromise) this.closePromise = this.closeNow();
+ return this.closePromise;
+ }
+
+ async reapIdle(nowMs = Date.now()): Promise {
+ if (this.closed || this.all.size <= this.minimum) return;
+ const reaped: PoolEntry[] = [];
+ let index = 0;
+ while (index < this.available.length && this.all.size > this.minimum) {
+ const entry = this.available[index];
+ const idleExpired =
+ this.idleTimeoutMs > 0 &&
+ nowMs - entry.idleSinceMs >= this.idleTimeoutMs;
+ const lifetimeExpired =
+ this.maxLifetimeMs > 0 &&
+ nowMs - entry.createdAtMs >= this.maxLifetimeMs;
+ if (!idleExpired && !lifetimeExpired) {
+ index++;
+ continue;
+ }
+ this.available.splice(index, 1);
+ this.all.delete(entry.slot);
+ reaped.push(entry);
+ }
+ if (reaped.length === 0) return;
+ await Promise.all(reaped.map((entry) => this.destroyRetired(entry)));
+ }
+
+ private async closeNow(): Promise {
+ if (this.closed) return;
+ this.closed = true;
+ this.unsubscribeSlotAvailability?.();
+ for (const waiter of this.waiters) {
+ if (waiter.timer) clearTimeout(waiter.timer);
+ waiter.reject(new QwpClientClosedError());
+ }
+ this.waiters.clear();
+ // Idle entries always belong to the closing thread. Borrowed senders remain
+ // owner-managed, while borrowed query sessions are retired with them below.
+ const entries = this.available.splice(0);
+ for (const entry of entries) this.all.delete(entry.slot);
+ const idleTeardown = Promise.all(
+ entries.map((entry) => this.destroy(entry)),
+ );
+ let leasedTeardown: Promise | undefined;
+ if (this.closeLeasedOnShutdown) {
+ const leased = Array.from(this.all.values()).filter(
+ (entry) => entry.leased,
+ );
+ for (const entry of leased) this.all.delete(entry.slot);
+ // Invoke every query teardown before awaiting any one WebSocket's bounded
+ // close handshake, so one slow idle socket cannot delay active-query
+ // cancellation on the other pool entries.
+ leasedTeardown = Promise.all(leased.map((entry) => this.destroy(entry)));
+ }
+ await idleTeardown;
+ const creations = Array.from(this.creationOperations);
+ if (creations.length > 0) {
+ const waitMs = Math.min(
+ this.acquireTimeoutMs,
+ MAX_CLOSE_CREATION_WAIT_MS,
+ );
+ let timer: ReturnType | undefined;
+ try {
+ await Promise.race([
+ Promise.allSettled(creations),
+ new Promise((resolve) => {
+ timer = setTimeout(resolve, waitMs);
+ }),
+ ]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+ }
+ if (leasedTeardown) {
+ await leasedTeardown;
+ this.wakeCloseWaiters();
+ return;
+ }
+ await this.waitForLeases(
+ Math.min(this.acquireTimeoutMs, MAX_CLOSE_LEASE_WAIT_MS),
+ );
+ }
+
+ private reserveSlot(): number | undefined {
+ if (
+ this.all.size + this.creatingSlots.size + this.destroyingSlots.size >=
+ this.maximum
+ ) {
+ return undefined;
+ }
+ for (let slot = 0; slot < this.maximum; slot++) {
+ if (
+ !this.all.has(slot) &&
+ !this.creatingSlots.has(slot) &&
+ !this.destroyingSlots.has(slot)
+ ) {
+ if (this.slotReservation && !this.slotReservation.tryReserve(slot)) {
+ continue;
+ }
+ if (this.slotReservation) this.reservedSlots.add(slot);
+ this.creatingSlots.add(slot);
+ return slot;
+ }
+ }
+ return undefined;
+ }
+
+ private async createLeased(slot: number): Promise> {
+ let finishCreation!: () => void;
+ const operation = new Promise((resolve) => {
+ finishCreation = resolve;
+ });
+ this.creationOperations.add(operation);
+ let retained = false;
+ try {
+ let value: T;
+ try {
+ value = await this.createResource(slot);
+ } catch (error) {
+ throw new QwpPoolResourceError(this.resource, error);
+ }
+ if (this.closed) {
+ await this.destroyResource(value).catch(() => undefined);
+ throw new QwpClientClosedError();
+ }
+ const nowMs = Date.now();
+ const entry: PoolEntry = {
+ slot,
+ value,
+ createdAtMs: nowMs,
+ idleSinceMs: nowMs,
+ leased: true,
+ };
+ this.all.set(slot, entry);
+ retained = true;
+ return entry;
+ } finally {
+ this.creatingSlots.delete(slot);
+ if (!retained) this.releaseSlotReservation(slot);
+ finishCreation();
+ this.creationOperations.delete(operation);
+ this.wakeWaiters();
+ }
+ }
+
+ private waitForChange(timeoutMs: number): Promise {
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ this.waiters.delete(waiter);
+ reject(
+ new QwpPoolAcquireTimeoutError(this.resource, this.acquireTimeoutMs),
+ );
+ }, timeoutMs);
+ const waiter: PoolWaiter = { resolve, reject, timer };
+ this.waiters.add(waiter);
+ });
+ }
+
+ private wakeWaiters(): void {
+ for (const waiter of this.waiters) {
+ this.waiters.delete(waiter);
+ if (waiter.timer) clearTimeout(waiter.timer);
+ waiter.resolve();
+ }
+ }
+
+ private wakeCloseWaiters(): void {
+ for (const waiter of this.closeWaiters) {
+ this.closeWaiters.delete(waiter);
+ clearTimeout(waiter.timer);
+ waiter.resolve();
+ }
+ }
+
+ private outstandingLeases(): number {
+ let count = this.pendingLeaseTeardowns;
+ for (const entry of this.all.values()) {
+ if (entry.leased) count++;
+ }
+ return count;
+ }
+
+ private async waitForLeases(timeoutMs: number): Promise {
+ const deadline = Date.now() + timeoutMs;
+ while (this.outstandingLeases() > 0) {
+ const remaining = deadline - Date.now();
+ if (remaining <= 0) return;
+ await new Promise((resolve) => {
+ const waiter: PoolCloseWaiter = {
+ resolve,
+ timer: setTimeout(() => {
+ this.closeWaiters.delete(waiter);
+ resolve();
+ }, remaining),
+ };
+ this.closeWaiters.add(waiter);
+ });
+ }
+ }
+
+ private destroy(entry: PoolEntry): Promise {
+ if (!entry.destroyPromise) {
+ entry.destroyPromise = this.destroyResource(entry.value)
+ .catch(() => undefined)
+ .finally(() => this.releaseSlotReservation(entry.slot));
+ }
+ return entry.destroyPromise;
+ }
+
+ private releaseSlotReservation(slot: number): void {
+ if (!this.reservedSlots.delete(slot)) return;
+ this.slotReservation?.release(slot);
+ }
+
+ private async destroyRetired(entry: PoolEntry): Promise {
+ this.destroyingSlots.add(entry.slot);
+ try {
+ await this.destroy(entry);
+ } finally {
+ this.destroyingSlots.delete(entry.slot);
+ this.wakeWaiters();
+ this.wakeCloseWaiters();
+ }
+ }
+
+ private throwIfClosed(): void {
+ if (this.closed) throw new QwpClientClosedError();
+ }
+}
+
+/** One exclusively borrowed egress session from a QwpClient query pool. */
+export class QwpQueryLease {
+ private closePromise?: Promise;
+ private released = false;
+
+ /** Initial SERVER_INFO; use serverInfo for the current post-failover snapshot. */
+ readonly ready: Promise;
+
+ /** @internal */
+ constructor(
+ private readonly session: QwpEgressSession,
+ private readonly releaseSession: (reusable: boolean) => Promise,
+ ) {
+ this.ready = session.ready;
+ }
+
+ get handshake(): QwpHandshakeMetadata {
+ this.throwIfReleased();
+ return this.session.handshake;
+ }
+
+ /**
+ * Cached immutable SERVER_INFO for this lease's currently bound endpoint.
+ * Reading it does not drive failover; a successful query replay refreshes it.
+ */
+ get serverInfo(): QwpServerInfoMessage | undefined {
+ this.throwIfReleased();
+ return this.session.serverInfo;
+ }
+
+ get negotiatedCompression(): QwpNegotiatedEgressCompression | undefined {
+ this.throwIfReleased();
+ return this.session.negotiatedCompression;
+ }
+
+ get negotiatedZstdLevel(): number {
+ this.throwIfReleased();
+ return this.session.negotiatedZstdLevel;
+ }
+
+ query(
+ sql: string,
+ options: QwpEgressQueryOptions = {},
+ ): Promise {
+ this.throwIfReleased();
+ return this.session.query(sql, options);
+ }
+
+ queryViews(
+ sql: string,
+ onBatch: QwpResultBatchViewHandler,
+ options: QwpEgressQueryOptions = {},
+ ): Promise {
+ this.throwIfReleased();
+ return this.session.queryViews(sql, onBatch, options);
+ }
+
+ close(): Promise {
+ if (!this.closePromise) this.closePromise = this.closeNow();
+ return this.closePromise;
+ }
+
+ private async closeNow(): Promise {
+ if (this.released) return;
+ this.released = true;
+ let reusable = false;
+ try {
+ reusable = await this.session.prepareForPoolRelease();
+ } finally {
+ await this.releaseSession(reusable);
+ }
+ }
+
+ private throwIfReleased(): void {
+ if (this.released) {
+ throw new QwpClientClosedError("QWP query lease is closed");
+ }
+ }
+}
+
+/**
+ * Browser-safe facade owning bounded ingress and egress connection pools.
+ * Borrowed handles are exclusive; separate query leases execute concurrently.
+ */
+export class QwpClient {
+ private readonly senderPool: QwpResourcePool;
+ private readonly queryPool: QwpResourcePool;
+ private connectPromise?: Promise;
+ private startPromise?: Promise;
+ private closePromise?: Promise;
+ private readonly startFactories?: () => void | Promise;
+ private readonly closeFactories?: () => void | Promise;
+ private readonly housekeepingIntervalMs: number;
+ private housekeepingTask: Promise = Promise.resolve();
+ private housekeeperTimer?: ReturnType;
+ private closing = false;
+ private closed = false;
+
+ constructor(
+ factories: QwpClientFactories,
+ options: QwpClientPoolOptions = {},
+ ) {
+ const validated = validatePoolOptions(options);
+ this.senderPool = new QwpResourcePool(
+ "sender",
+ validated.senderPoolMin,
+ validated.senderPoolMax,
+ validated.acquireTimeoutMs,
+ validated.idleTimeoutMs,
+ validated.maxLifetimeMs,
+ factories.createSender,
+ (sender) => sender.close(),
+ false,
+ factories.senderSlotReservation,
+ );
+ this.queryPool = new QwpResourcePool(
+ "query",
+ validated.queryPoolMin,
+ validated.queryPoolMax,
+ validated.acquireTimeoutMs,
+ validated.idleTimeoutMs,
+ validated.maxLifetimeMs,
+ factories.createQuerySession,
+ (session) => session.shutdownForClientClose(),
+ true,
+ );
+ this.startFactories = factories.start;
+ this.closeFactories = factories.close;
+ this.housekeepingIntervalMs = validated.housekeepingIntervalMs;
+ }
+
+ /** Pre-connects the configured minimum sender and query pool sizes. */
+ connect(): Promise {
+ if (!this.connectPromise) this.connectPromise = this.connectNow();
+ return this.connectPromise;
+ }
+
+ get metrics(): QwpClientMetrics {
+ return Object.freeze({
+ senders: this.senderPool.metrics,
+ queries: this.queryPool.metrics,
+ closing: this.closing,
+ closed: this.closed,
+ });
+ }
+
+ /** Borrows an exclusive fluent sender; close() flushes and returns its slot. */
+ async borrowSender(): Promise {
+ this.throwIfUnavailable();
+ await this.ensureStarted();
+ this.throwIfUnavailable();
+ const entry = await this.senderPool.acquire();
+ return createSenderLease(entry.value, async (reusable) => {
+ await this.senderPool.release(entry, reusable);
+ });
+ }
+
+ /** Borrows one exclusive egress connection for one or more serial queries. */
+ async borrowQuery(): Promise {
+ this.throwIfUnavailable();
+ await this.ensureStarted();
+ this.throwIfUnavailable();
+ const entry = await this.queryPool.acquire();
+ return new QwpQueryLease(entry.value, async (reusable) => {
+ await this.queryPool.release(entry, reusable);
+ });
+ }
+
+ /**
+ * Rejects new borrows and closes idle resources. Borrowed query sessions are
+ * cancelled and closed; borrowed senders retain ownership during a bounded
+ * drain and own their teardown if they outlive it.
+ */
+ close(): Promise {
+ if (!this.closePromise) this.closePromise = this.closeNow();
+ return this.closePromise;
+ }
+
+ private async connectNow(): Promise {
+ this.throwIfUnavailable();
+ try {
+ await this.ensureStarted();
+ this.throwIfUnavailable();
+ await Promise.all([this.senderPool.prewarm(), this.queryPool.prewarm()]);
+ return this;
+ } catch (error) {
+ await this.close();
+ throw error;
+ }
+ }
+
+ private async closeNow(): Promise {
+ if (this.closed) return;
+ this.closing = true;
+ this.stopHousekeeper();
+ await this.startPromise?.catch(() => undefined);
+ await this.housekeepingTask;
+ try {
+ let runtimeClose: Promise;
+ try {
+ runtimeClose = Promise.resolve(this.closeFactories?.()).catch(
+ () => undefined,
+ );
+ } catch {
+ runtimeClose = Promise.resolve();
+ }
+ // Stop runtime scanners and reject pool waiters in the same phase. This
+ // prevents a recovery-slot release during shutdown from waking an older
+ // borrow into a newly created foreground connection.
+ await Promise.all([
+ runtimeClose,
+ this.queryPool.close(),
+ this.senderPool.close(),
+ ]);
+ } finally {
+ this.closed = true;
+ }
+ }
+
+ private ensureStarted(): Promise {
+ if (!this.startPromise) {
+ this.startPromise = Promise.resolve()
+ .then(() => this.startFactories?.())
+ .then(() => {
+ if (!this.closing && !this.closed) this.startHousekeeper();
+ });
+ }
+ return this.startPromise;
+ }
+
+ private startHousekeeper(): void {
+ if (this.housekeeperTimer) return;
+ this.housekeeperTimer = setInterval(() => {
+ this.housekeepingTask = this.housekeepingTask
+ .then(async () => {
+ if (this.closing || this.closed) return;
+ const nowMs = Date.now();
+ await Promise.all([
+ this.senderPool.reapIdle(nowMs),
+ this.queryPool.reapIdle(nowMs),
+ ]);
+ })
+ .catch(() => undefined);
+ }, this.housekeepingIntervalMs);
+ const timer = this.housekeeperTimer as unknown as { unref?: () => void };
+ timer.unref?.();
+ }
+
+ private stopHousekeeper(): void {
+ if (!this.housekeeperTimer) return;
+ clearInterval(this.housekeeperTimer);
+ this.housekeeperTimer = undefined;
+ }
+
+ private throwIfUnavailable(): void {
+ if (this.closing || this.closed) throw new QwpClientClosedError();
+ }
+}
+
+function createSenderLease(
+ sender: QwpSender,
+ releaseSender: (reusable: boolean) => Promise,
+): QwpSender {
+ let released = false;
+ let closePromise: Promise | undefined;
+ const methods = new Map