From b5bbba89fd219f3936518c66822917dcf2a0f2ed Mon Sep 17 00:00:00 2001 From: lizhixuan Date: Sat, 18 Jul 2026 15:02:14 +0800 Subject: [PATCH] =?UTF-8?q?feat(core,mcp):=20binary=20cursor=20v2=20?= =?UTF-8?q?=E2=80=94=20full-load=20~2.7k=20chars=20(was=20~5k),=20maxCurso?= =?UTF-8?q?rSeen=20knob?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor wire format v2: base64url of [magic Rk + version][page u32 LE][seen u32 LE × N] instead of v1's JSON array of base36 hash strings — the seen keys are packed as the raw fnv1a words (hash.ts gains fnv1a32; public fnv1a unchanged). Cursors ride inside LLM tool outputs downstream, where hosts may clamp long strings (4096-char defenses) and every char is replayed through conversation history. Contract preserved and hardened over v1: - opaque, self-contained, no instance state; decode throws "invalid cursor" on anything not produced by meta.nextCursor (magic + version + length + page checks), including legacy v1 JSON cursors — no migration, cursors are short-lived load-more state - non-canonical base64url (tampered trailing bits) is rejected instead of silently aliasing a valid cursor - an out-of-uint32-range caller-supplied controls.page (negative, fractional, NaN, ≥ 2^32) encodes as a poison cursor that fails loudly on the next call — v1's zod schema did; naive binary packing would have silently wrapped New createRefkit({ maxCursorSeen }) caps the remembered seen keys (default still 500). Infinity disables the cap; effective floor is the batch just returned, so a too-small cap can never make load-more repeat the batch it just handed back (it would never converge). The zero-config MCP CLI reads REFKIT_MAX_CURSOR_SEEN (strict decimal; invalid values warn on stderr and fall back). --- .changeset/cursor-v2-binary-encoding.md | 30 +++++ .changeset/mcp-max-cursor-seen-env.md | 9 ++ README.md | 2 +- packages/core/src/__tests__/client.test.ts | 48 ++++++++ packages/core/src/__tests__/cursor.test.ts | 81 +++++++++++++ packages/core/src/client.ts | 23 +++- packages/core/src/cursor.ts | 127 ++++++++++++++++----- packages/core/src/hash.ts | 13 ++- packages/mcp/README.md | 2 + packages/mcp/src/__tests__/mcp.test.ts | 30 ++++- packages/mcp/src/cli.ts | 23 +++- 11 files changed, 347 insertions(+), 41 deletions(-) create mode 100644 .changeset/cursor-v2-binary-encoding.md create mode 100644 .changeset/mcp-max-cursor-seen-env.md create mode 100644 packages/core/src/__tests__/cursor.test.ts diff --git a/.changeset/cursor-v2-binary-encoding.md b/.changeset/cursor-v2-binary-encoding.md new file mode 100644 index 0000000..48036d8 --- /dev/null +++ b/.changeset/cursor-v2-binary-encoding.md @@ -0,0 +1,30 @@ +--- +"@refkit/core": minor +--- + +Shrink the load-more cursor to roughly half its size: `meta.nextCursor` is now +a binary-packed base64url string (magic + version + page + raw fnv1a uint32 +seen-keys) instead of v1's JSON array of base36 hash strings — a full 500-entry +cursor drops from ~5k to ~2.7k chars. Cursors ride inside LLM tool outputs +downstream (and get replayed through conversation history), so every char +counts; ~2.7k also clears consumers that clamp tool-output strings at 4k. + +The cursor stays opaque and self-contained: pass back `meta.nextCursor`, get +the next deduped batch, no caller-side bookkeeping, no client instance state. +Anything else — including a v1 JSON cursor from a previous release — still +fails loudly with "invalid cursor" rather than quietly restarting from page 1 +(cursors are short-lived load-more state, not durable ids; there is no v1 +migration). + +New `createRefkit({ maxCursorSeen })` caps how many already-returned keys the +cursor remembers (default unchanged at 500, most recent kept, ~5.4 chars each) +for callers who want an even tighter cursor and can accept re-showing +long-evicted results sooner. `Infinity` disables the cap; the effective floor +is the batch just returned, so a too-small cap can never make load-more repeat +the batch it just handed back. + +Hardening over v1, both restoring guarantees the removed zod schema provided: +an out-of-uint32-range `controls.page` (negative, fractional, `NaN`, ≥ 2^32) +encodes as a poison cursor that fails loudly on the next call instead of +silently wrapping to a different page, and non-canonical base64url (tampered +trailing bits) is rejected rather than silently aliased to a valid cursor. diff --git a/.changeset/mcp-max-cursor-seen-env.md b/.changeset/mcp-max-cursor-seen-env.md new file mode 100644 index 0000000..f214403 --- /dev/null +++ b/.changeset/mcp-max-cursor-seen-env.md @@ -0,0 +1,9 @@ +--- +"@refkit/mcp": minor +--- + +`REFKIT_MAX_CURSOR_SEEN` env var for the zero-config CLI: caps how many +already-returned keys the load-more cursor remembers (core's `maxCursorSeen`), +for hosts that clamp tool-output strings — the default 500-key cursor is ~2.7k +chars; `REFKIT_MAX_CURSOR_SEEN=200` brings it near ~1.1k. Invalid values warn +on stderr and fall back to the core default. diff --git a/README.md b/README.md index 44407da..ff17cb7 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,7 @@ Agents can use refkit in two ways: npx -y @refkit/mcp ``` -It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results; `rerank: true` for query-aware re-ranking (term coverage incl. CJK, resolution, source diversity); `cursor` (from the previous result's top-level `nextCursor` — always returned, no `explain` needed) to page through results without repeats. BYOK provider packages are `optionalDependencies` of `@refkit/mcp` — installed by default (zero-config `npx` keeps working), but an install with `--omit=optional` skips them, and a key whose package is missing just logs a stderr warning instead of crashing the server. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp). +It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results; `rerank: true` for query-aware re-ranking (term coverage incl. CJK, resolution, source diversity); `cursor` (from the previous result's top-level `nextCursor` — always returned, no `explain` needed) to page through results without repeats; `REFKIT_MAX_CURSOR_SEEN` shrinks the cursor for hosts that clamp tool-output strings. BYOK provider packages are `optionalDependencies` of `@refkit/mcp` — installed by default (zero-config `npx` keeps working), but an install with `--omit=optional` skips them, and a key whose package is missing just logs a stderr warning instead of crashing the server. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp). ## Not legal advice diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index 5ee4b0d..2575800 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { createRefkit } from '../client' +import { cursorSeenKey, decodeCursor } from '../cursor' import { defineProvider, type ReferenceProvider } from '../provider' import { lexicalReranker } from '../rerank' import type { Reference } from '../reference' @@ -122,6 +123,53 @@ describe('createRefkit', () => { const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')])] }) await expect(rk.search({ query: 'x', modalities: ['image'], cursor: 'not-a-cursor' })).rejects.toThrow(/invalid cursor/) await expect(rk.search({ query: 'x', modalities: ['image'], cursor: '{"v":9}' })).rejects.toThrow(/invalid cursor/) + // Legacy v1 JSON cursors are short-lived load-more state, not durable ids — + // they fail like any other foreign string instead of being migrated. + await expect(rk.search({ query: 'x', modalities: ['image'], cursor: '{"v":1,"page":1,"seen":["abc"]}' })).rejects.toThrow(/invalid cursor/) + }) + + it('cursor: a bad caller-supplied controls.page fails loudly on the next call, not silently', async () => { + // v1's zod decode rejected out-of-range pages when the cursor came back; + // v2 must preserve that instead of wrapping to some other uint32 page. + const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')])] }) + const out = await rk.searchWithMeta({ query: 'x', modalities: ['image'], controls: { page: -1 } }) + expect(out.references).toHaveLength(1) + await expect(rk.search({ query: 'x', modalities: ['image'], cursor: out.meta.nextCursor })).rejects.toThrow(/invalid cursor/) + }) + + it('cursor: seen never evicts the batch just returned, even when maxCursorSeen is smaller', async () => { + // A cap below the batch size would re-show this batch on the very next + // call and pagination would never converge. + const refs = [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3'), ref('a-4', 'https://a/4')] + const rk = createRefkit({ providers: [provider('a', refs)], maxCursorSeen: 1 }) + const batch1 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2 }) + expect(batch1.references.map(r => r.canonicalUrl)).toEqual(['https://a/1', 'https://a/2']) + const batch2 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: batch1.meta.nextCursor }) + expect(batch2.references.map(r => r.canonicalUrl)).toEqual(['https://a/3', 'https://a/4']) + }) + + it('cursor: maxCursorSeen Infinity disables the cap instead of falling back to the default', async () => { + // 501 results in one batch: the default cap would trim seen to 500. + const many = Array.from({ length: 501 }, (_, i) => ref(`a-${i}`, `https://a/${i}`)) + const rk = createRefkit({ providers: [provider('a', many)], maxCursorSeen: Infinity }) + const batch = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 501 }) + expect(batch.references).toHaveLength(501) + expect(decodeCursor(batch.meta.nextCursor!).seen).toHaveLength(501) + }) + + it('cursor: maxCursorSeen caps remembered keys (oldest evicted first)', async () => { + const refs = [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3'), ref('a-4', 'https://a/4')] + const rk = createRefkit({ providers: [provider('a', refs)], maxCursorSeen: 2 }) + const search = (cursor?: string) => rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 1, cursor }) + + const batch1 = await search() + const batch2 = await search(batch1.meta.nextCursor) + const batch3 = await search(batch2.meta.nextCursor) + expect(batch3.references.map(r => r.canonicalUrl)).toEqual(['https://a/3']) + // Only the 2 most recent keys survive; batch1's key was evicted. + expect(decodeCursor(batch3.meta.nextCursor!).seen).toEqual( + [cursorSeenKey('https://a/2'), cursorSeenKey('https://a/3')], + ) }) it('rejects a Promise passed as providers (un-awaited async factory)', () => { diff --git a/packages/core/src/__tests__/cursor.test.ts b/packages/core/src/__tests__/cursor.test.ts new file mode 100644 index 0000000..39ee8d2 --- /dev/null +++ b/packages/core/src/__tests__/cursor.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest' +import { cursorSeenKey, decodeCursor, encodeCursor } from '../cursor' + +const INVALID = /invalid cursor/ + +describe('cursor v2 encoding', () => { + it('roundtrips page + seen through an opaque base64url string', () => { + const state = { page: 7, seen: [0, 1, 0x7fffffff, 0xffffffff, 123456789] } + const encoded = encodeCursor(state) + expect(encoded).toMatch(/^[A-Za-z0-9_-]+$/) + expect(decodeCursor(encoded)).toEqual(state) + }) + + it('roundtrips an empty seen list', () => { + expect(decodeCursor(encodeCursor({ page: 1, seen: [] }))).toEqual({ page: 1, seen: [] }) + }) + + it('cursorSeenKey is a deterministic uint32', () => { + const k = cursorSeenKey('https://a/1') + expect(Number.isInteger(k)).toBe(true) + expect(k).toBeGreaterThanOrEqual(0) + expect(k).toBeLessThan(2 ** 32) + expect(cursorSeenKey('https://a/1')).toBe(k) + }) + + it('stays under 2.8k chars at the 500-entry seen cap (v1 JSON was ~5k)', () => { + const seen = Array.from({ length: 500 }, (_, i) => (i * 2654435761) >>> 0) + const encoded = encodeCursor({ page: 9, seen }) + expect(encoded.length).toBeLessThanOrEqual(2700) + expect(decodeCursor(encoded).seen).toEqual(seen) + }) + + it('throws "invalid cursor" on strings not produced by encodeCursor', () => { + for (const bad of [ + '', + 'not a cursor!', // characters outside the base64url alphabet + '{"v":1,"page":1,"seen":["abc"]}', // legacy v1 JSON cursor — no longer accepted + 'not-a-cursor', // base64url alphabet but not our layout + 'AAAAAAAA', // well-formed base64url, decodes to 6 bytes — shorter than the header + 'abc', // too short to hold a header + ]) { + expect(() => decodeCursor(bad), JSON.stringify(bad)).toThrow(INVALID) + } + }) + + it('throws on wrong magic and wrong version (payload long enough to reach those checks)', () => { + const v2 = (bytes: number[]) => Buffer.from(bytes).toString('base64url') + // Sanity: Buffer's base64url of a genuine header matches our encoder. + expect(v2([0x52, 0x6b, 0x02, 1, 0, 0, 0])).toBe(encodeCursor({ page: 1, seen: [] })) + expect(() => decodeCursor(v2([0x00, 0x6b, 0x02, 1, 0, 0, 0]))).toThrow(INVALID) // bad magic byte 0 + expect(() => decodeCursor(v2([0x52, 0x00, 0x02, 1, 0, 0, 0]))).toThrow(INVALID) // bad magic byte 1 + expect(() => decodeCursor(v2([0x52, 0x6b, 0x01, 1, 0, 0, 0]))).toThrow(INVALID) // version 1 + expect(() => decodeCursor(v2([0x52, 0x6b, 0x03, 1, 0, 0, 0]))).toThrow(INVALID) // version 3 + }) + + it('throws on non-canonical trailing bits (single-char tamper the encoder could never emit)', () => { + // Both base64url remainder classes: rem-2 (4 unused bits) and rem-3 (2 unused bits). + for (const state of [{ page: 1, seen: [] }, { page: 3, seen: [42] }]) { + const encoded = encodeCursor(state) + expect(encoded.endsWith('A')).toBe(true) // unused trailing bits are zero + expect(() => decodeCursor(encoded.slice(0, -1) + 'B'), encoded).toThrow(INVALID) + } + }) + + it('throws on a truncated but otherwise genuine cursor', () => { + const encoded = encodeCursor({ page: 1, seen: [1, 2, 3] }) + expect(() => decodeCursor(encoded.slice(0, -2))).toThrow(INVALID) + }) + + it('throws on a cursor carrying page 0', () => { + expect(() => decodeCursor(encodeCursor({ page: 0, seen: [] }))).toThrow(INVALID) + }) + + it('encodes an out-of-uint32-range page as a poison cursor that fails loudly on decode', () => { + // A bad caller-supplied controls.page must NOT silently wrap to a different + // page (v1 failed loudly on the next call; v2 must too). + for (const page of [-1, 2.5, Number.NaN, 2 ** 32, 2 ** 32 + 5]) { + expect(() => decodeCursor(encodeCursor({ page, seen: [7] })), String(page)).toThrow(INVALID) + } + }) +}) diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index b3e40a1..64afe95 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -45,6 +45,15 @@ export interface RefkitOptions { * sources at once — a provider's timeout only starts when its slot starts, so * queueing never burns a queued provider's deadline. */ concurrency?: number + /** Cap on already-returned keys remembered inside the load-more cursor (most + * recent kept). Each key costs ~5.4 chars of cursor, so this bounds + * `meta.nextCursor` length (~2.7k chars at the default 500). Lower it when + * the cursor travels a size-sensitive channel (e.g. LLM tool output); + * overflowing just risks re-showing results evicted long ago. `Infinity` + * disables the cap. Effective floor is the batch just returned — evicting + * keys the same call produced would repeat them immediately and load-more + * would never converge. */ + maxCursorSeen?: number } export interface ProviderError { @@ -146,9 +155,10 @@ const DEFAULT_CACHE_TTL_MS = 300_000 // Cursor: how many further provider pages one load-more call may try when the // current page's pool is fully consumed, before reporting an empty batch. const MAX_CURSOR_ADVANCES = 3 -// Cursor: cap on remembered already-returned keys (most recent kept). Bounds -// cursor size (~7 bytes/key); overflowing just risks re-showing very old results. -const MAX_CURSOR_SEEN = 500 +// Cursor: default cap on remembered already-returned keys (most recent kept; +// see RefkitOptions.maxCursorSeen). Bounds cursor size (~5.4 chars/key packed); +// overflowing just risks re-showing very old results. +const DEFAULT_MAX_CURSOR_SEEN = 500 function errorSummary(error: unknown): string { if (error instanceof Error) return error.message @@ -336,13 +346,16 @@ export function createRefkit(options: RefkitOptions): RefkitClient { } const references = pass.refs.slice(0, limit) + // Never below this batch's size (evicting keys just returned would repeat + // them on the very next call); Infinity = uncapped, NaN falls back. + const rawMaxSeen = options.maxCursorSeen ?? DEFAULT_MAX_CURSOR_SEEN + const maxCursorSeen = Math.max(Number.isNaN(rawMaxSeen) ? DEFAULT_MAX_CURSOR_SEEN : rawMaxSeen, references.length) const nextCursor = references.length > 0 ? encodeCursor({ - v: 1, // Same page on purpose — its overfetched pool may still hold // unreturned results; the next call advances internally if not. page: page ?? 1, - seen: [...(cursorState?.seen ?? []), ...references.map(r => cursorSeenKey(r.canonicalUrl))].slice(-MAX_CURSOR_SEEN), + seen: [...(cursorState?.seen ?? []), ...references.map(r => cursorSeenKey(r.canonicalUrl))].slice(-maxCursorSeen), }) : undefined const warnings: string[] = [] diff --git a/packages/core/src/cursor.ts b/packages/core/src/cursor.ts index aff274f..20c4962 100644 --- a/packages/core/src/cursor.ts +++ b/packages/core/src/cursor.ts @@ -1,16 +1,19 @@ -// Unified "load more" cursor (v1). Providers fetch an overfetched pool per page -// (fetchLimit ≥ limit) while each call returns only `limit`, and RRF fusion makes -// raw provider pages overlap — so the cursor carries the CURRENT provider-local -// page plus compact hashes of every already-returned result. The client filters -// repeats out and advances the page internally only once a page's pool is -// exhausted. The string is an implementation detail — treat it as opaque; only -// `meta.nextCursor` from a previous search is a valid input. -import { z } from 'zod' -import { fnv1a } from './hash' +// Unified "load more" cursor (v2, binary). Providers fetch an overfetched pool +// per page (fetchLimit ≥ limit) while each call returns only `limit`, and RRF +// fusion makes raw provider pages overlap — so the cursor carries the CURRENT +// provider-local page plus compact hashes of every already-returned result. The +// client filters repeats out and advances the page internally only once a +// page's pool is exhausted. The string is an implementation detail — treat it +// as opaque; only `meta.nextCursor` from a previous search is a valid input. +// +// Wire format: base64url of [magic 'R' 'k', version 0x02][page uint32 LE] +// [seen uint32 LE × N]. Packing the raw fnv1a words (instead of the v1 JSON +// array of base36 strings) keeps a full 500-entry cursor at ~2.7k chars vs ~5k +// — cursors ride inside LLM tool outputs downstream, where every char counts. +import { fnv1a32 } from './hash' import { canonicalizeUrl } from './dedup-key' export interface SearchCursorState { - v: 1 /** Provider-local page the current pool comes from (routed as controls.page; * 1-based). Advanced by the client, not per call. */ page: number @@ -18,37 +21,103 @@ export interface SearchCursorState { * by the client (most recent kept) so cursor size stays bounded; a 32-bit * hash keeps entries compact — the worst case of a collision or an evicted * entry is one result suppressed or repeated. */ - seen: string[] + seen: number[] } -const cursorSchema = z.object({ - v: z.literal(1), - page: z.number().int().min(1), - seen: z.array(z.string()), -}) +const MAGIC_0 = 0x52 // 'R' +const MAGIC_1 = 0x6b // 'k' +const VERSION = 0x02 +const HEADER_BYTES = 7 // magic (2) + version (1) + page uint32 (4) -/** Compact already-seen key for a result — same URL canonicalization as merge/dedup. */ -export function cursorSeenKey(canonicalUrl: string): string { - return fnv1a(canonicalizeUrl(canonicalUrl)) +const B64URL = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_' +const B64URL_INDEX: Record = Object.fromEntries([...B64URL].map((c, i) => [c, i])) + +function toBase64url(bytes: Uint8Array): string { + let out = '' + let i = 0 + for (; i + 2 < bytes.length; i += 3) { + const n = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2] + out += B64URL[n >> 18] + B64URL[(n >> 12) & 63] + B64URL[(n >> 6) & 63] + B64URL[n & 63] + } + const rest = bytes.length - i + if (rest === 1) { + out += B64URL[bytes[i] >> 2] + B64URL[(bytes[i] & 3) << 4] + } else if (rest === 2) { + const n = (bytes[i] << 8) | bytes[i + 1] + out += B64URL[n >> 10] + B64URL[(n >> 4) & 63] + B64URL[(n & 15) << 2] + } + return out +} + +/** Decode unpadded base64url; returns undefined on any string a + * {@link toBase64url} call could not have produced — including non-canonical + * encodings whose unused trailing bits are non-zero. */ +function fromBase64url(s: string): Uint8Array | undefined { + const rem = s.length % 4 + if (rem === 1) return undefined + const bytes = new Uint8Array((s.length >> 2) * 3 + (rem === 2 ? 1 : rem === 3 ? 2 : 0)) + let bi = 0 + let acc = 0 + let accBits = 0 + for (const c of s) { + const v = B64URL_INDEX[c] + if (v === undefined) return undefined + acc = (acc << 6) | v + accBits += 6 + if (accBits >= 8) { + accBits -= 8 + bytes[bi++] = (acc >> accBits) & 0xff + } + } + // Unused trailing bits must be zero, or up to 16 tampered strings would + // silently alias one cursor. + if ((acc & ((1 << accBits) - 1)) !== 0) return undefined + return bytes +} + +/** Compact already-seen key for a result (raw fnv1a uint32) — same URL + * canonicalization as merge/dedup. */ +export function cursorSeenKey(canonicalUrl: string): number { + return fnv1a32(canonicalizeUrl(canonicalUrl)) } export function encodeCursor(state: SearchCursorState): string { - return JSON.stringify(state) + const bytes = new Uint8Array(HEADER_BYTES + state.seen.length * 4) + const view = new DataView(bytes.buffer) + bytes[0] = MAGIC_0 + bytes[1] = MAGIC_1 + bytes[2] = VERSION + // A page uint32 can't represent (bad caller-supplied controls.page: negative, + // fractional, NaN, ≥ 2^32) encodes as 0 — a poison value decodeCursor rejects + // — so it fails loudly on the next call like v1's schema did, instead of + // silently wrapping to some other page. + const page = Number.isInteger(state.page) && state.page >= 1 && state.page <= 0xffffffff ? state.page : 0 + view.setUint32(3, page, true) + state.seen.forEach((key, i) => view.setUint32(HEADER_BYTES + i * 4, key, true)) + return toBase64url(bytes) } +const invalidCursor = () => new Error('refkit.search: invalid cursor (not produced by meta.nextCursor)') + /** Parse and validate a cursor string. Throws on anything that is not a cursor * this library produced — a corrupted cursor must fail loudly, not quietly * restart from page 1. */ export function decodeCursor(cursor: string): SearchCursorState { - let parsed: unknown - try { - parsed = JSON.parse(cursor) - } catch { - throw new Error('refkit.search: invalid cursor (not produced by meta.nextCursor)') + const bytes = fromBase64url(cursor) + if ( + bytes === undefined || + bytes.length < HEADER_BYTES || + (bytes.length - HEADER_BYTES) % 4 !== 0 || + bytes[0] !== MAGIC_0 || bytes[1] !== MAGIC_1 || bytes[2] !== VERSION + ) { + throw invalidCursor() } - const result = cursorSchema.safeParse(parsed) - if (!result.success) { - throw new Error('refkit.search: invalid cursor (not produced by meta.nextCursor)') + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const page = view.getUint32(3, true) + if (page < 1) throw invalidCursor() + const seen: number[] = [] + for (let offset = HEADER_BYTES; offset < bytes.length; offset += 4) { + seen.push(view.getUint32(offset, true)) } - return result.data + return { page, seen } } diff --git a/packages/core/src/hash.ts b/packages/core/src/hash.ts index 3d437dc..fa5c4ba 100644 --- a/packages/core/src/hash.ts +++ b/packages/core/src/hash.ts @@ -1,10 +1,17 @@ -// FNV-1a 32-bit, base36. Deterministic, dependency-free, runtime-agnostic. +// FNV-1a 32-bit. Deterministic, dependency-free, runtime-agnostic. // Used only for content-addressed ids / dedup keys — not for security. -export function fnv1a(str: string): string { + +/** Raw uint32 FNV-1a — for callers that pack the hash into binary (cursor). */ +export function fnv1a32(str: string): number { let h = 0x811c9dc5 for (let i = 0; i < str.length; i++) { h ^= str.charCodeAt(i) h = Math.imul(h, 0x01000193) } - return (h >>> 0).toString(36) + return h >>> 0 +} + +/** Base36 rendering of {@link fnv1a32} — for string ids / dedup keys. */ +export function fnv1a(str: string): string { + return fnv1a32(str).toString(36) } diff --git a/packages/mcp/README.md b/packages/mcp/README.md index bd968e5..88d1f1b 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -19,6 +19,8 @@ REFKIT_UNSPLASH_KEY=… REFKIT_PEXELS_KEY=… REFKIT_PIXABAY_KEY=… REFKIT_FLIC UNSPLASH_KEY=… PEXELS_KEY=… PIXABAY_KEY=… FLICKR_KEY=… SI_KEY=… BRAVE_TOKEN=… npx -y @refkit/mcp ``` +If your MCP client clamps tool-output strings, `REFKIT_MAX_CURSOR_SEEN` shrinks the load-more cursor: it caps how many already-returned keys `nextCursor` remembers (default 500 ≈ 2.7k chars; `REFKIT_MAX_CURSOR_SEEN=200` ≈ 1.1k). A lower cap only risks re-showing results from that many batches ago on very deep pagination. + Example MCP client config: ```json diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index 7a82f59..d9f9714 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -1,11 +1,11 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js' import { createRefkit, defineProvider } from '@refkit/core' import { openverse } from '@refkit/provider-openverse' import { readFileSync } from 'node:fs' import { createRefkitMcpServer } from '../index' -import { defaultProviders, BYOK_SOURCES } from '../cli' +import { defaultProviders, maxCursorSeenFromEnv, BYOK_SOURCES } from '../cli' const OPENVERSE = { results: [ { id: 'aaa', title: 'cc0 sky', creator: 'Alice', foreign_landing_url: 'https://ov/aaa', url: 'https://cdn/aaa.jpg', thumbnail: 'https://ov/aaa/thumb', width: 10, height: 10, license: 'cc0', license_version: '1.0', license_url: 'https://cc/cc0' }, @@ -396,6 +396,32 @@ describe('build_attribution tool', () => { }) }) +describe('maxCursorSeenFromEnv (cursor size knob for size-clamped tool outputs)', () => { + it('parses a positive integer REFKIT_MAX_CURSOR_SEEN', () => { + expect(maxCursorSeenFromEnv({})).toBeUndefined() + expect(maxCursorSeenFromEnv({ REFKIT_MAX_CURSOR_SEEN: '200' })).toBe(200) + }) + + it('warns on stderr and falls back to the core default on garbage values', () => { + const spy = vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + // Strict decimal digits only — Number()'s hex/exponent/whitespace/sign + // leniency must not slip through ('1e100' would defeat the cap entirely, + // and a beyond-safe-integer literal can't be represented faithfully). + const garbage = ['0', '-5', '2.5', 'abc', '0x10', '1e3', ' 5 ', '+7', '1e100', '9007199254740993'] + for (const bad of garbage) { + expect(maxCursorSeenFromEnv({ REFKIT_MAX_CURSOR_SEEN: bad }), bad).toBeUndefined() + } + expect(spy).toHaveBeenCalledTimes(garbage.length) + // '' means unset: no warning. + expect(maxCursorSeenFromEnv({ REFKIT_MAX_CURSOR_SEEN: '' })).toBeUndefined() + expect(spy).toHaveBeenCalledTimes(garbage.length) + } finally { + spy.mockRestore() + } + }) +}) + describe('defaultProviders (zero-config CLI wiring)', () => { it('includes every keyless provider by default', async () => { const ids = (await defaultProviders({})).map(p => p.id) diff --git a/packages/mcp/src/cli.ts b/packages/mcp/src/cli.ts index 49c2619..0e434b2 100644 --- a/packages/mcp/src/cli.ts +++ b/packages/mcp/src/cli.ts @@ -130,6 +130,27 @@ const isEntry = (() => { } })() +/** Cursor size knob for size-clamped tool-output channels: REFKIT_MAX_CURSOR_SEEN + * caps how many already-returned keys the load-more cursor remembers (see + * RefkitOptions.maxCursorSeen in @refkit/core — default 500 ≈ 2.7k chars of + * nextCursor). Invalid values warn on stderr and fall back to that default. */ +export function maxCursorSeenFromEnv(env: NodeJS.ProcessEnv = process.env): number | undefined { + const raw = env.REFKIT_MAX_CURSOR_SEEN + if (raw === undefined || raw === '') return undefined + // Strict decimal digits only — Number()'s hex/exponent/whitespace leniency + // would silently accept values like '1e100' that defeat the cap entirely. + const n = /^\d+$/.test(raw) ? Number(raw) : Number.NaN + if (!Number.isSafeInteger(n) || n < 1) { + console.error(`[refkit-mcp] ignoring invalid REFKIT_MAX_CURSOR_SEEN=${JSON.stringify(raw)} — expected a positive integer.`) + return undefined + } + return n +} + if (isEntry) { - await serveStdio(createRefkit({ providers: await defaultProviders() })) + const maxCursorSeen = maxCursorSeenFromEnv() + await serveStdio(createRefkit({ + providers: await defaultProviders(), + ...(maxCursorSeen !== undefined ? { maxCursorSeen } : {}), + })) }