From b34d840d330185811e0f364f4ce4ff9fd0a56916 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Sat, 23 May 2026 01:59:55 +0900 Subject: [PATCH 01/11] Add @fedify/backfill --- deno.json | 1 + deno.lock | 13 ------ packages/backfill/README.md | 58 +++++++++++++++++++++++ packages/backfill/deno.json | 22 +++++++++ packages/backfill/package.json | 66 ++++++++++++++++++++++++++ packages/backfill/src/mod.ts | 8 ++++ packages/backfill/tsdown.config.ts | 14 ++++++ packages/fedify/README.md | 3 ++ pnpm-lock.yaml | 75 +++++++++++++++++------------- pnpm-workspace.yaml | 1 + 10 files changed, 215 insertions(+), 46 deletions(-) create mode 100644 packages/backfill/README.md create mode 100644 packages/backfill/deno.json create mode 100644 packages/backfill/package.json create mode 100644 packages/backfill/src/mod.ts create mode 100644 packages/backfill/tsdown.config.ts diff --git a/deno.json b/deno.json index 2740fe03e..92d23ca59 100644 --- a/deno.json +++ b/deno.json @@ -2,6 +2,7 @@ "workspace": [ "./packages/amqp", "./packages/astro", + "./packages/backfill", "./packages/cfworkers", "./packages/cli", "./packages/debugger", diff --git a/deno.lock b/deno.lock index bef73340b..f7cd75f23 100644 --- a/deno.lock +++ b/deno.lock @@ -84,7 +84,6 @@ "npm:@jimp/core@^1.6.1": "1.6.1", "npm:@jimp/wasm-webp@^1.6.1": "1.6.1", "npm:@js-temporal/polyfill@~0.5.1": "0.5.1", - "npm:@jsr/std__assert@0.226": "0.226.0", "npm:@multiformats/base-x@^4.0.1": "4.0.1", "npm:@nestjs/common@^11.0.1": "11.1.19_reflect-metadata@0.2.2_rxjs@7.8.2", "npm:@nurodev/astro-bun@^2.1.2": "2.1.2_astro@5.18.1__@types+node@24.12.2__mysql2@3.22.3___@types+node@24.12.2_@types+node@24.12.2_mysql2@3.22.3__@types+node@24.12.2", @@ -2382,17 +2381,6 @@ "wasm-feature-detect" ] }, - "@jsr/std__assert@0.226.0": { - "integrity": "sha512-xCuUFDfHkIZd96glKgjZbnYFqu6blu8Y53SyvDMlFDJm1Y/j+/FcW6xq7TzGFIaF5B9QecIlDfamfhzA8ZdVbg==", - "dependencies": [ - "@jsr/std__internal" - ], - "tarball": "https://npm.jsr.io/~/11/@jsr/std__assert/0.226.0.tgz" - }, - "@jsr/std__internal@1.0.12": { - "integrity": "sha512-6xReMW9p+paJgqoFRpOE2nogJFvzPfaLHLIlyADYjKMUcwDyjKZxryIbgcU+gxiTygn8yCjld1HoI0ET4/iZeA==", - "tarball": "https://npm.jsr.io/~/11/@jsr/std__internal/1.0.12.tgz" - }, "@logtape/logtape@1.3.7": { "integrity": "sha512-YgF+q9op97oLLPwc7TcTNIllTArVtTwkwyKky6XVzAXQcBrvFXXtMuwJSryONAyOUSItrx994O/HABOrszZyFg==" }, @@ -9472,7 +9460,6 @@ "packageJson": { "dependencies": [ "npm:@js-temporal/polyfill@~0.5.1", - "npm:@jsr/std__assert@0.226", "npm:@types/node@^24.2.1", "npm:json-canon@^1.0.1", "npm:jsonld@9", diff --git a/packages/backfill/README.md b/packages/backfill/README.md new file mode 100644 index 000000000..ff7f95f60 --- /dev/null +++ b/packages/backfill/README.md @@ -0,0 +1,58 @@ + + +@fedify/backfill: ActivityPub backfill for Fedify +================================================= + +[![JSR][JSR badge]][JSR] +[![npm][npm badge]][npm] +[![Follow @fedify@hollo.social][@fedify@hollo.social badge]][@fedify@hollo.social] + +*This package is available since Fedify 2.3.0.* + +This package provides the scaffold for ActivityPub backfill support in the +[Fedify] ecosystem. It is intended to host APIs for retrieving and processing +historical federated content, but the implementation has not been added yet. + +[JSR badge]: https://jsr.io/badges/@fedify/backfill +[JSR]: https://jsr.io/@fedify/backfill +[npm badge]: https://img.shields.io/npm/v/@fedify/backfill?logo=npm +[npm]: https://www.npmjs.com/package/@fedify/backfill +[@fedify@hollo.social badge]: https://fedi-badge.deno.dev/@fedify@hollo.social/followers.svg +[@fedify@hollo.social]: https://hollo.social/@fedify +[Fedify]: https://fedify.dev/ + + +Installation +------------ + +::: code-group + +~~~~ sh [Deno] +deno add jsr:@fedify/backfill +~~~~ + +~~~~ sh [npm] +npm add @fedify/backfill +~~~~ + +~~~~ sh [pnpm] +pnpm add @fedify/backfill +~~~~ + +~~~~ sh [Yarn] +yarn add @fedify/backfill +~~~~ + +~~~~ sh [Bun] +bun add @fedify/backfill +~~~~ + +::: + + +Status +------ + +The package structure and publishing metadata are in place. Public runtime +APIs will be added in follow-up changes once the backfill workflow and data +model are finalized. diff --git a/packages/backfill/deno.json b/packages/backfill/deno.json new file mode 100644 index 000000000..cd151e493 --- /dev/null +++ b/packages/backfill/deno.json @@ -0,0 +1,22 @@ +{ + "name": "@fedify/backfill", + "version": "2.3.0", + "license": "MIT", + "exports": { + ".": "./src/mod.ts" + }, + "exclude": [ + "dist/", + "node_modules/" + ], + "publish": { + "exclude": [ + "**/*.test.ts", + "tsdown.config.ts" + ] + }, + "tasks": { + "check": "deno fmt --check && deno lint && deno check src/**/*.ts", + "test": "deno test" + } +} diff --git a/packages/backfill/package.json b/packages/backfill/package.json new file mode 100644 index 000000000..3266f1a3b --- /dev/null +++ b/packages/backfill/package.json @@ -0,0 +1,66 @@ +{ + "name": "@fedify/backfill", + "version": "2.3.0", + "description": "ActivityPub backfill support for Fedify", + "keywords": [ + "Fedify", + "ActivityPub", + "Fediverse", + "Backfill" + ], + "author": { + "name": "Jiwon Kwon", + "email": "work@kwonjiwon.org" + }, + "homepage": "https://fedify.dev/", + "repository": { + "type": "git", + "url": "git+https://github.com/fedify-dev/fedify.git", + "directory": "packages/backfill" + }, + "license": "MIT", + "bugs": { + "url": "https://github.com/fedify-dev/fedify/issues" + }, + "funding": [ + "https://opencollective.com/fedify", + "https://github.com/sponsors/dahlia" + ], + "type": "module", + "main": "./dist/mod.cjs", + "module": "./dist/mod.js", + "types": "./dist/mod.d.ts", + "exports": { + ".": { + "types": { + "import": "./dist/mod.d.ts", + "require": "./dist/mod.d.cts", + "default": "./dist/mod.d.ts" + }, + "import": "./dist/mod.js", + "require": "./dist/mod.cjs", + "default": "./dist/mod.js" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist/", + "package.json", + "README.md" + ], + "dependencies": {}, + "devDependencies": { + "tsdown": "catalog:", + "typescript": "catalog:" + }, + "scripts": { + "build:self": "tsdown", + "build": "pnpm --filter @fedify/backfill... run build:self", + "prepack": "pnpm build", + "prepublish": "pnpm build", + "pretest": "pnpm build", + "test": "cd dist/ && node --test", + "pretest:bun": "pnpm build", + "test:bun": "cd dist/ && bun test --timeout 60000" + } +} diff --git a/packages/backfill/src/mod.ts b/packages/backfill/src/mod.ts new file mode 100644 index 000000000..b98af542e --- /dev/null +++ b/packages/backfill/src/mod.ts @@ -0,0 +1,8 @@ +/** + * ActivityPub backfill support for Fedify. + * + * This package is currently a scaffold for upcoming backfill features. + * + * @module + */ +export {}; diff --git a/packages/backfill/tsdown.config.ts b/packages/backfill/tsdown.config.ts new file mode 100644 index 000000000..bf33f512d --- /dev/null +++ b/packages/backfill/tsdown.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: ["src/mod.ts"], + dts: true, + format: ["esm", "cjs"], + platform: "node", + outExtensions({ format }) { + return { + js: format === "cjs" ? ".cjs" : ".js", + dts: format === "cjs" ? ".d.cts" : ".d.ts", + }; + }, +}); diff --git a/packages/fedify/README.md b/packages/fedify/README.md index 8c17b11a4..c08313151 100644 --- a/packages/fedify/README.md +++ b/packages/fedify/README.md @@ -100,6 +100,7 @@ Here is the list of packages: | [@fedify/create](/packages/create/) | | [npm][npm:@fedify/create] | Create a new Fedify project | | [@fedify/amqp](/packages/amqp/) | [JSR][jsr:@fedify/amqp] | [npm][npm:@fedify/amqp] | AMQP/RabbitMQ driver | | [@fedify/astro](/packages/astro/) | [JSR][jsr:@fedify/astro] | [npm][npm:@fedify/astro] | Astro integration | +| [@fedify/backfill](/packages/backfill/) | [JSR][jsr:@fedify/backfill] | [npm][npm:@fedify/backfill] | ActivityPub backfill support | | [@fedify/cfworkers](/packages/cfworkers/) | [JSR][jsr:@fedify/cfworkers] | [npm][npm:@fedify/cfworkers] | Cloudflare Workers integration | | [@fedify/debugger](/packages/debugger/) | [JSR][jsr:@fedify/debugger] | [npm][npm:@fedify/debugger] | Embedded ActivityPub debug dashboard | | [@fedify/denokv](/packages/denokv/) | [JSR][jsr:@fedify/denokv] | | Deno KV integration | @@ -136,6 +137,8 @@ Here is the list of packages: [npm:@fedify/amqp]: https://www.npmjs.com/package/@fedify/amqp [jsr:@fedify/astro]: https://jsr.io/@fedify/astro [npm:@fedify/astro]: https://www.npmjs.com/package/@fedify/astro +[jsr:@fedify/backfill]: https://jsr.io/@fedify/backfill +[npm:@fedify/backfill]: https://www.npmjs.com/package/@fedify/backfill [jsr:@fedify/cfworkers]: https://jsr.io/@fedify/cfworkers [npm:@fedify/cfworkers]: https://www.npmjs.com/package/@fedify/cfworkers [jsr:@fedify/debugger]: https://jsr.io/@fedify/debugger diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f3eba0e1b..a15948ba8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -849,7 +849,7 @@ importers: version: 0.10.8 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -865,7 +865,16 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + typescript: + specifier: 'catalog:' + version: 6.0.3 + + packages/backfill: + devDependencies: + tsdown: + specifier: 'catalog:' + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -884,7 +893,7 @@ importers: version: 0.8.71(@cloudflare/workers-types@4.20260511.1)(@vitest/runner@3.2.4)(@vitest/snapshot@3.2.4)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.3.0)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.46.1)(tsx@4.21.0)(yaml@2.9.0)) tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1020,7 +1029,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1045,7 +1054,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1079,7 +1088,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1098,7 +1107,7 @@ importers: version: 1.2.19(@types/react@19.1.8) tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1120,7 +1129,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1142,7 +1151,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1224,7 +1233,7 @@ importers: version: 4.20250617.4 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) tsx: specifier: ^4.21.0 version: 4.21.0 @@ -1258,7 +1267,7 @@ importers: version: 0.5.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1277,7 +1286,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1293,7 +1302,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1333,7 +1342,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1355,7 +1364,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1386,7 +1395,7 @@ importers: version: 9.32.0(jiti@2.6.1) tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1417,7 +1426,7 @@ importers: version: link:../testing tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1442,7 +1451,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1458,7 +1467,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1489,7 +1498,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1520,7 +1529,7 @@ importers: version: '@jsr/std__async@1.0.13' tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1554,7 +1563,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1579,7 +1588,7 @@ importers: version: link:../vocab-runtime tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1598,7 +1607,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1626,7 +1635,7 @@ importers: version: '@jsr/std__async@1.0.13' tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1642,7 +1651,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1670,7 +1679,7 @@ importers: version: '@jsr/std__async@1.0.13' tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1685,7 +1694,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1740,7 +1749,7 @@ importers: version: 12.6.0 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1780,7 +1789,7 @@ importers: version: 12.6.0 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1805,7 +1814,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1836,7 +1845,7 @@ importers: version: 12.6.0 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) typescript: specifier: 'catalog:' version: 6.0.3 @@ -25494,7 +25503,7 @@ snapshots: minimist: 1.2.8 strip-bom: 3.0.0 - tsdown@0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34): + tsdown@0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)): dependencies: ansis: 4.3.0 cac: 7.0.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 896419acb..9f8bf38b5 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ packages: - packages/amqp - packages/astro +- packages/backfill - packages/cfworkers - packages/cli - packages/debugger From 1e7bd343a1aadb81779c128d098c16b9e5df1f7a Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 26 May 2026 15:31:24 +0900 Subject: [PATCH 02/11] Add backfill API surface Define the initial @fedify/backfill async generator API around a typed BackfillContext, note seed object, traversal options, and BackfillItem wrappers. The generator remains a stub so tests and traversal logic can be added in follow-up commits. Assisted-by: Codex:gpt-5 --- packages/backfill/package.json | 4 +- packages/backfill/src/backfill.ts | 27 ++++++++ packages/backfill/src/mod.ts | 14 +++- packages/backfill/src/types.ts | 111 ++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 packages/backfill/src/backfill.ts create mode 100644 packages/backfill/src/types.ts diff --git a/packages/backfill/package.json b/packages/backfill/package.json index 3266f1a3b..1e80dd9d8 100644 --- a/packages/backfill/package.json +++ b/packages/backfill/package.json @@ -48,7 +48,9 @@ "package.json", "README.md" ], - "dependencies": {}, + "dependencies": { + "@fedify/vocab": "workspace:*" + }, "devDependencies": { "tsdown": "catalog:", "typescript": "catalog:" diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts new file mode 100644 index 000000000..e82c9f952 --- /dev/null +++ b/packages/backfill/src/backfill.ts @@ -0,0 +1,27 @@ +import type * as vocab from "@fedify/vocab"; + +import type { + BackfillContext, + BackfillItem, + BackfillOptions, +} from "./types.ts"; + +/** + * Backfills post-like objects related to a seed object. + * + * The seed object is not yielded by default, but its ID is treated as already + * seen so it will not be yielded again if the collection contains it. + */ +export async function* backfill< + TObject extends vocab.Object = vocab.Object, +>( + context: BackfillContext, + note: TObject, + options: BackfillOptions = {}, +): AsyncGenerator, void, void> { + void context; + void note; + void options; + + yield* [] satisfies BackfillItem[]; +} diff --git a/packages/backfill/src/mod.ts b/packages/backfill/src/mod.ts index b98af542e..6ceb69ac2 100644 --- a/packages/backfill/src/mod.ts +++ b/packages/backfill/src/mod.ts @@ -1,8 +1,18 @@ /** * ActivityPub backfill support for Fedify. * - * This package is currently a scaffold for upcoming backfill features. + * This package provides async generator APIs for collecting historical + * ActivityPub objects related to a seed object. * * @module */ -export {}; +export { backfill } from "./backfill.ts"; +export type { + BackfillContext, + BackfillDocumentLoader, + BackfillDocumentLoaderOptions, + BackfillItem, + BackfillOptions, + BackfillOrigin, + BackfillStrategy, +} from "./types.ts"; diff --git a/packages/backfill/src/types.ts b/packages/backfill/src/types.ts new file mode 100644 index 000000000..c0f50a2c2 --- /dev/null +++ b/packages/backfill/src/types.ts @@ -0,0 +1,111 @@ +import type * as vocab from "@fedify/vocab"; + +/** + * Backfill traversal strategy used to discover the returned object. + */ +export type BackfillStrategy = "context-posts"; + +/** + * Source relation that produced a backfilled object. + */ +export type BackfillOrigin = "context" | "collection"; + +/** + * Options passed to {@link BackfillDocumentLoader}. + */ +export interface BackfillDocumentLoaderOptions { + /** + * Cancellation signal for the current dereference operation. + */ + signal?: AbortSignal; +} + +/** + * Dereferences an ActivityPub object or collection IRI. + */ +export type BackfillDocumentLoader = ( + iri: URL, + options?: BackfillDocumentLoaderOptions, +) => Promise; + +/** + * Dependencies used by backfill traversal. + */ +export interface BackfillContext { + /** + * Dereferences context collections and collection item IRIs. + */ + documentLoader: BackfillDocumentLoader; +} + +/** + * Controls direct context collection backfill traversal. + */ +export interface BackfillOptions< + TObject extends vocab.Object = vocab.Object, +> { + /** + * Maximum number of items to yield. Skipped duplicates do not count. + */ + maxItems?: number; + + /** + * Maximum traversal depth. This is reserved for future reply-tree traversal; + */ + maxDepth?: number; + + /** + * Maximum number of calls to {@link BackfillContext.documentLoader}. + * + * Dereferencing the note context, collection item IRIs, and future page IRIs + * all count as requests. Embedded collection items do not count. + */ + maxRequests?: number; + + /** + * Delay between `documentLoader` requests. + * + * When a callback is provided, `iteration` is the zero-based request index. + */ + interval?: + | Temporal.DurationLike + | ((iteration: number) => Temporal.DurationLike); + + /** + * Cancels traversal before requests and before yields. + */ + signal?: AbortSignal; +} + +/** + * A single object discovered by backfill traversal. + */ +export interface BackfillItem< + TObject extends vocab.Object = vocab.Object, +> { + /** + * The discovered ActivityPub object. + */ + object: TObject; + + /** + * The object's ActivityPub ID, when present. + */ + id?: URL; + + /** + * The traversal strategy that produced this item. + */ + strategy: BackfillStrategy; + + /** + * The source relation that produced this item. + */ + origin: BackfillOrigin; + + /** + * Traversal depth. Direct context collection items are depth 0; deeper + * values are reserved for future reply-tree traversal. + */ + depth?: number; +} From 6970c156e582ac19c6b2993521c51a56240c85d0 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Wed, 27 May 2026 16:11:51 +0900 Subject: [PATCH 03/11] Implement context collection backfill Add the initial context-posts traversal for @fedify/backfill. The implementation dereferences the seed object's context, accepts direct ActivityStreams collections and collection pages, yields post-like objects, and enforces request, item, interval, abort, and duplicate-id handling. Add tests for the PR 1 behavior across Deno, Node.js, and Bun. Assisted-by: Codex:gpt-5 --- packages/backfill/src/backfill.test.ts | 339 +++++++++++++++++++++++++ packages/backfill/src/backfill.ts | 166 +++++++++++- packages/backfill/src/types.ts | 8 +- packages/backfill/tsdown.config.ts | 41 ++- pnpm-lock.yaml | 4 + 5 files changed, 536 insertions(+), 22 deletions(-) create mode 100644 packages/backfill/src/backfill.test.ts diff --git a/packages/backfill/src/backfill.test.ts b/packages/backfill/src/backfill.test.ts new file mode 100644 index 000000000..f36b46f74 --- /dev/null +++ b/packages/backfill/src/backfill.test.ts @@ -0,0 +1,339 @@ +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert"; +import test, { describe } from "node:test"; +import { backfill, type BackfillContext } from "./mod.ts"; +import { Collection, Create, Note } from "@fedify/vocab"; + +async function collect( + context: BackfillContext, + note: Note, + options: Parameters[2] = {}, +) { + return await Array.fromAsync(backfill(context, note, options)); +} + +describe("backfill", () => { + test("package exports backfill", () => { + strictEqual(typeof backfill, "function"); + }); + + test("context missing yields nothing", async () => { + const note = new Note({ + id: new URL("https://example.com/notes/1"), + }); + const context: BackfillContext = { + documentLoader: () => { + throw new Error("documentLoader should not be called"); + }, + }; + + deepStrictEqual(await collect(context, note), []); + }); + + test("context resolves to non-collection yields nothing", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Note({ + id: new URL("https://example.com/notes/2"), + }), + ), + }; + + deepStrictEqual(await collect(context, note), []); + }); + + test("context collection with embedded objects yields items", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const item = new Note({ + id: new URL("https://example.com/notes/2"), + content: "hello", + }); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Collection({ + id: contextId, + items: [item], + }), + ), + }; + + const items = await collect(context, note); + + strictEqual(items.length, 1); + strictEqual(items[0].object, item); + deepStrictEqual(items[0].id, item.id); + strictEqual(items[0].strategy, "context-posts"); + strictEqual(items[0].origin, "collection"); + }); + + test("embedded object without id is yielded without id", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const item = new Note({ content: "anonymous" }); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Collection({ + id: contextId, + items: [item], + }), + ), + }; + + const items = await collect(context, note); + + strictEqual(items.length, 1); + strictEqual(items[0].object, item); + strictEqual(items[0].id, undefined); + }); + + test("activity objects in collection are skipped", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const activity = new Create({ + id: new URL("https://example.com/activities/1"), + object: new Note({ id: new URL("https://example.com/notes/2") }), + }); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Collection({ + id: contextId, + items: [activity], + }), + ), + }; + + deepStrictEqual(await collect(context, note), []); + }); + + test("context collection with URL items loads and yields objects", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const itemId = new URL("https://example.com/notes/2"); + const item = new Note({ + id: itemId, + content: "hello", + }); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const requests: URL[] = []; + const context: BackfillContext = { + documentLoader: (iri) => { + requests.push(iri); + if (iri.href === contextId.href) { + return Promise.resolve( + new Collection({ + id: contextId, + items: [itemId], + }), + ); + } + if (iri.href === itemId.href) return Promise.resolve(item); + return Promise.resolve(null); + }, + }; + + const items = await collect(context, note); + + strictEqual(items.length, 1); + ok(items[0].id instanceof URL); + strictEqual(items[0].id.href, itemId.href); + deepStrictEqual(requests.map((url) => url.href), [ + contextId.href, + itemId.href, + ]); + }); + + test("seed is not yielded again when present in collection", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const other = new Note({ + id: new URL("https://example.com/notes/2"), + }); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Collection({ + id: contextId, + items: [note, other], + }), + ), + }; + + const items = await collect(context, note); + + strictEqual(items.length, 1); + strictEqual(items[0].object, other); + }); + + test("duplicate object IDs are skipped", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const duplicateId = new URL("https://example.com/notes/2"); + const first = new Note({ id: duplicateId, content: "first" }); + const second = new Note({ id: duplicateId, content: "second" }); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Collection({ + id: contextId, + items: [first, second], + }), + ), + }; + + const items = await collect(context, note); + + strictEqual(items.length, 1); + strictEqual(items[0].object, first); + }); + + test("maxItems limits yielded items", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Collection({ + id: contextId, + items: [ + new Note({ id: new URL("https://example.com/notes/2") }), + new Note({ id: new URL("https://example.com/notes/3") }), + ], + }), + ), + }; + + const items = await collect(context, note, { maxItems: 1 }); + + strictEqual(items.length, 1); + strictEqual(items[0].id?.href, "https://example.com/notes/2"); + }); + + test("maxRequests limits dereferencing", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const itemId = new URL("https://example.com/notes/2"); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: (iri) => { + if (iri.href === contextId.href) { + return Promise.resolve( + new Collection({ + id: contextId, + items: [itemId], + }), + ); + } + return Promise.resolve(new Note({ id: iri })); + }, + }; + + deepStrictEqual(await collect(context, note, { maxRequests: 1 }), []); + }); + + test("AbortSignal stops traversal", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const controller = new AbortController(); + controller.abort(); + const context: BackfillContext = { + documentLoader: () => + Promise.resolve( + new Collection({ + id: contextId, + items: [new Note({ id: new URL("https://example.com/notes/2") })], + }), + ), + }; + + await rejects( + collect(context, note, { signal: controller.signal }), + { name: "AbortError" }, + ); + }); + + test("documentLoader receives AbortSignal", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const controller = new AbortController(); + let receivedSignal: AbortSignal | undefined; + const context: BackfillContext = { + documentLoader: (_iri, options) => { + receivedSignal = options?.signal; + return Promise.resolve(new Collection({ id: contextId, items: [] })); + }, + }; + + await collect(context, note, { signal: controller.signal }); + + strictEqual(receivedSignal, controller.signal); + }); + + test("interval callback receives zero-based request index", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const itemId = new URL("https://example.com/notes/2"); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const iterations: number[] = []; + const context: BackfillContext = { + documentLoader: (iri) => { + if (iri.href === contextId.href) { + return Promise.resolve( + new Collection({ + id: contextId, + items: [itemId], + }), + ); + } + return Promise.resolve(new Note({ id: iri })); + }, + }; + + await collect(context, note, { + interval: (iteration) => { + iterations.push(iteration); + return { milliseconds: 0 }; + }, + }); + + deepStrictEqual(iterations, [0, 1]); + }); +}); diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts index e82c9f952..fcb27f903 100644 --- a/packages/backfill/src/backfill.ts +++ b/packages/backfill/src/backfill.ts @@ -1,4 +1,12 @@ -import type * as vocab from "@fedify/vocab"; +import { + Activity, + Collection, + CollectionPage, + type Link, + Object as APObject, + OrderedCollection, + OrderedCollectionPage, +} from "@fedify/vocab"; import type { BackfillContext, @@ -6,6 +14,13 @@ import type { BackfillOptions, } from "./types.ts"; +class MaxRequestsExceeded extends Error {} + +interface RequestBudget { + readonly signal?: AbortSignal; + requestCount: number; +} + /** * Backfills post-like objects related to a seed object. * @@ -13,15 +28,154 @@ import type { * seen so it will not be yielded again if the collection contains it. */ export async function* backfill< - TObject extends vocab.Object = vocab.Object, + TObject extends APObject = APObject, >( context: BackfillContext, note: TObject, options: BackfillOptions = {}, ): AsyncGenerator, void, void> { - void context; - void note; - void options; + if (options.maxItems != null && options.maxItems <= 0) return; + + const contextId = note.contextIds[0]; + if (contextId == null) return; + + const budget: RequestBudget = { + signal: options.signal, + requestCount: 0, + }; + const seenIds = new Set(); + if (note.id != null) seenIds.add(note.id.href); + + const collection = await loadObject(context, contextId, options, budget); + if (!isCollection(collection)) return; + + let yielded = 0; + try { + for await ( + const object of getCollectionItems(context, collection, options, budget) + ) { + if (!isContextPostObject(object)) continue; + const id = object.id ?? undefined; + if (id != null) { + if (seenIds.has(id.href)) continue; + seenIds.add(id.href); + } + + options.signal?.throwIfAborted(); + yield { + object: object as TObject, + id, + strategy: "context-posts", + origin: "collection", + depth: 0, + }; + + yielded++; + if (options.maxItems != null && yielded >= options.maxItems) return; + } + } catch (error) { + if (error instanceof MaxRequestsExceeded) return; + throw error; + } +} + +async function* getCollectionItems( + context: BackfillContext, + collection: BackfillCollection, + options: BackfillOptions, + budget: RequestBudget, +): AsyncIterable { + yield* collection.getItems({ + documentLoader: async (url) => { + const object = await loadObject( + context, + new URL(url), + options, + budget, + true, + ); + if (object == null) throw new MaxRequestsExceeded(); + return { + contextUrl: null, + documentUrl: url, + document: await object.toJsonLd(), + }; + }, + crossOrigin: "trust", + }); +} + +async function loadObject( + context: BackfillContext, + iri: URL, + options: BackfillOptions, + budget: RequestBudget, + throwOnBudgetExceeded = false, +): Promise { + budget.signal?.throwIfAborted(); + if ( + options.maxRequests != null && + budget.requestCount >= options.maxRequests + ) { + if (throwOnBudgetExceeded) throw new MaxRequestsExceeded(); + return null; + } + + await waitForInterval(options, budget); + budget.signal?.throwIfAborted(); + + budget.requestCount++; + return await context.documentLoader(iri, { signal: budget.signal }); +} + +async function waitForInterval( + options: BackfillOptions, + budget: RequestBudget, +): Promise { + if (options.interval == null) return; + const duration = typeof options.interval === "function" + ? options.interval(budget.requestCount) + : options.interval; + const milliseconds = durationToMilliseconds(duration); + if (milliseconds <= 0) return; + await new Promise((resolve, reject) => { + const timeout = setTimeout(resolve, milliseconds); + budget.signal?.addEventListener("abort", () => { + clearTimeout(timeout); + reject(budget.signal?.reason); + }, { once: true }); + }); +} + +function durationToMilliseconds(duration: Temporal.DurationLike): number { + return ( + (duration.milliseconds ?? 0) + + (duration.seconds ?? 0) * 1000 + + (duration.minutes ?? 0) * 60 * 1000 + + (duration.hours ?? 0) * 60 * 60 * 1000 + + (duration.days ?? 0) * 24 * 60 * 60 * 1000 + ); +} + +type BackfillCollection = + | Collection + | OrderedCollection + | CollectionPage + | OrderedCollectionPage; + +function isCollection( + object: APObject | null, +): object is BackfillCollection { + return object instanceof Collection || + object instanceof OrderedCollection || + object instanceof CollectionPage || + object instanceof OrderedCollectionPage; +} - yield* [] satisfies BackfillItem[]; +function isContextPostObject( + object: APObject | Link, +): object is APObject { + return object instanceof APObject && + !(object instanceof Activity) && + !isCollection(object); } diff --git a/packages/backfill/src/types.ts b/packages/backfill/src/types.ts index c0f50a2c2..bcf82f004 100644 --- a/packages/backfill/src/types.ts +++ b/packages/backfill/src/types.ts @@ -1,4 +1,4 @@ -import type * as vocab from "@fedify/vocab"; +import type { Object as APObject } from "@fedify/vocab"; /** * Backfill traversal strategy used to discover the returned object. @@ -26,7 +26,7 @@ export interface BackfillDocumentLoaderOptions { export type BackfillDocumentLoader = ( iri: URL, options?: BackfillDocumentLoaderOptions, -) => Promise; +) => Promise; /** * Dependencies used by backfill traversal. @@ -42,7 +42,7 @@ export interface BackfillContext { * Controls direct context collection backfill traversal. */ export interface BackfillOptions< - TObject extends vocab.Object = vocab.Object, + TObject extends APObject = APObject, > { /** * Maximum number of items to yield. Skipped duplicates do not count. @@ -81,7 +81,7 @@ export interface BackfillOptions< * A single object discovered by backfill traversal. */ export interface BackfillItem< - TObject extends vocab.Object = vocab.Object, + TObject extends APObject = APObject, > { /** * The discovered ActivityPub object. diff --git a/packages/backfill/tsdown.config.ts b/packages/backfill/tsdown.config.ts index bf33f512d..9f43a88ac 100644 --- a/packages/backfill/tsdown.config.ts +++ b/packages/backfill/tsdown.config.ts @@ -1,14 +1,31 @@ +import { glob } from "node:fs/promises"; +import { sep } from "node:path"; import { defineConfig } from "tsdown"; -export default defineConfig({ - entry: ["src/mod.ts"], - dts: true, - format: ["esm", "cjs"], - platform: "node", - outExtensions({ format }) { - return { - js: format === "cjs" ? ".cjs" : ".js", - dts: format === "cjs" ? ".d.cts" : ".d.ts", - }; - }, -}); +export default [ + defineConfig({ + entry: ["src/mod.ts"], + dts: true, + format: ["esm", "cjs"], + platform: "node", + outExtensions({ format }) { + return { + js: format === "cjs" ? ".cjs" : ".js", + dts: format === "cjs" ? ".d.cts" : ".d.ts", + }; + }, + }), + defineConfig({ + entry: (await Array.fromAsync(glob(`src/**/*.test.ts`))) + .map((f) => f.replace(sep, "/")), + format: ["esm", "cjs"], + platform: "node", + outExtensions({ format }) { + return { + js: format === "cjs" ? ".cjs" : ".js", + dts: format === "cjs" ? ".d.cts" : ".d.ts", + }; + }, + deps: { neverBundle: [/^node:/] }, + }), +]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a15948ba8..0dcee0d3e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -871,6 +871,10 @@ importers: version: 6.0.3 packages/backfill: + dependencies: + '@fedify/vocab': + specifier: workspace:* + version: link:../vocab devDependencies: tsdown: specifier: 'catalog:' From 392730b89211293dea8c263e4b061d92bfcb9511 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Wed, 27 May 2026 16:27:08 +0900 Subject: [PATCH 04/11] Document backfill API usage Replace the scaffold status text with a short description of the initial context collection backfill behavior and a minimal usage example. Assisted-by: Codex:gpt-5 --- packages/backfill/README.md | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/backfill/README.md b/packages/backfill/README.md index ff7f95f60..3d38a762c 100644 --- a/packages/backfill/README.md +++ b/packages/backfill/README.md @@ -9,9 +9,11 @@ *This package is available since Fedify 2.3.0.* -This package provides the scaffold for ActivityPub backfill support in the -[Fedify] ecosystem. It is intended to host APIs for retrieving and processing -historical federated content, but the implementation has not been added yet. +This package provides ActivityPub conversation backfill support for the +[Fedify] ecosystem. It can retrieve post-like objects from a seed object's +context collection, following the direct FEP-f228-style path where the +context dereferences to a `Collection`, `OrderedCollection`, `CollectionPage`, +or `OrderedCollectionPage`. [JSR badge]: https://jsr.io/badges/@fedify/backfill [JSR]: https://jsr.io/@fedify/backfill @@ -50,9 +52,28 @@ bun add @fedify/backfill ::: -Status ------- +Usage +----- -The package structure and publishing metadata are in place. Public runtime -APIs will be added in follow-up changes once the backfill workflow and data -model are finalized. +The `backfill()` function accepts a backfill context, a seed object, and +traversal options: + +~~~~ typescript +import { backfill } from "@fedify/backfill"; +import { lookupObject } from "@fedify/vocab"; + +const documentLoader = (iri: URL, options?: { signal?: AbortSignal }) => + lookupObject(iri, { signal: options?.signal }); + +for await ( + const item of backfill({ documentLoader }, note, { + maxItems: 20, + maxRequests: 50, + }) +) { + console.log(item.id?.href); +} +~~~~ + +The seed object itself is not yielded. If it appears in the discovered +collection, it is skipped by ID. From ff8f75ffb05783b1f714820c8108df2640eb2dd0 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Wed, 27 May 2026 16:35:58 +0900 Subject: [PATCH 05/11] Clean up backfill lockfile changes Remove unrelated lockfile churn from the backfill branch, keeping only the new package importer required for @fedify/backfill. Assisted-by: Codex:gpt-5 --- deno.lock | 13 ++++++++++ pnpm-lock.yaml | 68 +++++++++++++++++++++++++------------------------- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/deno.lock b/deno.lock index f7cd75f23..bef73340b 100644 --- a/deno.lock +++ b/deno.lock @@ -84,6 +84,7 @@ "npm:@jimp/core@^1.6.1": "1.6.1", "npm:@jimp/wasm-webp@^1.6.1": "1.6.1", "npm:@js-temporal/polyfill@~0.5.1": "0.5.1", + "npm:@jsr/std__assert@0.226": "0.226.0", "npm:@multiformats/base-x@^4.0.1": "4.0.1", "npm:@nestjs/common@^11.0.1": "11.1.19_reflect-metadata@0.2.2_rxjs@7.8.2", "npm:@nurodev/astro-bun@^2.1.2": "2.1.2_astro@5.18.1__@types+node@24.12.2__mysql2@3.22.3___@types+node@24.12.2_@types+node@24.12.2_mysql2@3.22.3__@types+node@24.12.2", @@ -2381,6 +2382,17 @@ "wasm-feature-detect" ] }, + "@jsr/std__assert@0.226.0": { + "integrity": "sha512-xCuUFDfHkIZd96glKgjZbnYFqu6blu8Y53SyvDMlFDJm1Y/j+/FcW6xq7TzGFIaF5B9QecIlDfamfhzA8ZdVbg==", + "dependencies": [ + "@jsr/std__internal" + ], + "tarball": "https://npm.jsr.io/~/11/@jsr/std__assert/0.226.0.tgz" + }, + "@jsr/std__internal@1.0.12": { + "integrity": "sha512-6xReMW9p+paJgqoFRpOE2nogJFvzPfaLHLIlyADYjKMUcwDyjKZxryIbgcU+gxiTygn8yCjld1HoI0ET4/iZeA==", + "tarball": "https://npm.jsr.io/~/11/@jsr/std__internal/1.0.12.tgz" + }, "@logtape/logtape@1.3.7": { "integrity": "sha512-YgF+q9op97oLLPwc7TcTNIllTArVtTwkwyKky6XVzAXQcBrvFXXtMuwJSryONAyOUSItrx994O/HABOrszZyFg==" }, @@ -9460,6 +9472,7 @@ "packageJson": { "dependencies": [ "npm:@js-temporal/polyfill@~0.5.1", + "npm:@jsr/std__assert@0.226", "npm:@types/node@^24.2.1", "npm:json-canon@^1.0.1", "npm:jsonld@9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0dcee0d3e..df3689951 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -849,7 +849,7 @@ importers: version: 0.10.8 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -865,7 +865,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -878,7 +878,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -897,7 +897,7 @@ importers: version: 0.8.71(@cloudflare/workers-types@4.20260511.1)(@vitest/runner@3.2.4)(@vitest/snapshot@3.2.4)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.3.0)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.46.1)(tsx@4.21.0)(yaml@2.9.0)) tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1033,7 +1033,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1058,7 +1058,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1092,7 +1092,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1111,7 +1111,7 @@ importers: version: 1.2.19(@types/react@19.1.8) tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1133,7 +1133,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1155,7 +1155,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1237,7 +1237,7 @@ importers: version: 4.20250617.4 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) tsx: specifier: ^4.21.0 version: 4.21.0 @@ -1271,7 +1271,7 @@ importers: version: 0.5.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1290,7 +1290,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1306,7 +1306,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1346,7 +1346,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1368,7 +1368,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1399,7 +1399,7 @@ importers: version: 9.32.0(jiti@2.6.1) tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1430,7 +1430,7 @@ importers: version: link:../testing tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1455,7 +1455,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1471,7 +1471,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1502,7 +1502,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1533,7 +1533,7 @@ importers: version: '@jsr/std__async@1.0.13' tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1567,7 +1567,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1592,7 +1592,7 @@ importers: version: link:../vocab-runtime tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1611,7 +1611,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1639,7 +1639,7 @@ importers: version: '@jsr/std__async@1.0.13' tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1655,7 +1655,7 @@ importers: devDependencies: tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1683,7 +1683,7 @@ importers: version: '@jsr/std__async@1.0.13' tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1698,7 +1698,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1753,7 +1753,7 @@ importers: version: 12.6.0 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1793,7 +1793,7 @@ importers: version: 12.6.0 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1818,7 +1818,7 @@ importers: version: 22.19.1 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -1849,7 +1849,7 @@ importers: version: 12.6.0 tsdown: specifier: 'catalog:' - version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)) + version: 0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34) typescript: specifier: 'catalog:' version: 6.0.3 @@ -25507,7 +25507,7 @@ snapshots: minimist: 1.2.8 strip-bom: 3.0.0 - tsdown@0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)): + tsdown@0.22.0(tsx@4.21.0)(typescript@6.0.3)(unrun@0.2.34): dependencies: ansis: 4.3.0 cac: 7.0.0 From a9cb0db721803ae5f66a8e3dec79007853104c14 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Wed, 27 May 2026 17:14:31 +0900 Subject: [PATCH 06/11] Address backfill review feedback Skip collection URL items that fail to dereference instead of terminating the whole traversal, while still stopping when the request budget is exhausted. Also clean up interval abort listeners after successful waits. Assisted-by: Codex:gpt-5 --- packages/backfill/src/backfill.test.ts | 38 +++++++++++++++++++ packages/backfill/src/backfill.ts | 52 ++++++++++++++++++++------ 2 files changed, 79 insertions(+), 11 deletions(-) diff --git a/packages/backfill/src/backfill.test.ts b/packages/backfill/src/backfill.test.ts index f36b46f74..4170f34be 100644 --- a/packages/backfill/src/backfill.test.ts +++ b/packages/backfill/src/backfill.test.ts @@ -162,6 +162,44 @@ describe("backfill", () => { ]); }); + test("failed URL collection items are skipped", async () => { + const contextId = new URL("https://example.com/contexts/1"); + const missingItemId = new URL("https://example.com/notes/missing"); + const failedItemId = new URL("https://example.com/notes/failed"); + const itemId = new URL("https://example.com/notes/2"); + const item = new Note({ + id: itemId, + content: "hello", + }); + const note = new Note({ + id: new URL("https://example.com/notes/1"), + contexts: [contextId], + }); + const context: BackfillContext = { + documentLoader: (iri) => { + if (iri.href === contextId.href) { + return Promise.resolve( + new Collection({ + id: contextId, + items: [missingItemId, failedItemId, itemId], + }), + ); + } + if (iri.href === missingItemId.href) return Promise.resolve(null); + if (iri.href === failedItemId.href) { + return Promise.reject(new Error("failed to load")); + } + if (iri.href === itemId.href) return Promise.resolve(item); + return Promise.resolve(null); + }, + }; + + const items = await collect(context, note); + + strictEqual(items.length, 1); + strictEqual(items[0].id?.href, itemId.href); + }); + test("seed is not yielded again when present in collection", async () => { const contextId = new URL("https://example.com/contexts/1"); const note = new Note({ diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts index fcb27f903..c91f63e15 100644 --- a/packages/backfill/src/backfill.ts +++ b/packages/backfill/src/backfill.ts @@ -87,14 +87,21 @@ async function* getCollectionItems( ): AsyncIterable { yield* collection.getItems({ documentLoader: async (url) => { - const object = await loadObject( - context, - new URL(url), - options, - budget, - true, - ); - if (object == null) throw new MaxRequestsExceeded(); + let object: APObject | null; + try { + object = await loadObject( + context, + new URL(url), + options, + budget, + true, + ); + } catch (error) { + if (error instanceof MaxRequestsExceeded) throw error; + budget.signal?.throwIfAborted(); + return skippedCollectionItemDocument(url); + } + if (object == null) return skippedCollectionItemDocument(url); return { contextUrl: null, documentUrl: url, @@ -105,6 +112,17 @@ async function* getCollectionItems( }); } +function skippedCollectionItemDocument(url: string) { + return { + contextUrl: null, + documentUrl: url, + document: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Activity", + }, + }; +} + async function loadObject( context: BackfillContext, iri: URL, @@ -139,15 +157,27 @@ async function waitForInterval( const milliseconds = durationToMilliseconds(duration); if (milliseconds <= 0) return; await new Promise((resolve, reject) => { - const timeout = setTimeout(resolve, milliseconds); - budget.signal?.addEventListener("abort", () => { + if (budget.signal?.aborted) { + reject(budget.signal.reason); + return; + } + const timeout = setTimeout(() => { + budget.signal?.removeEventListener("abort", onAbort); + resolve(); + }, milliseconds); + const onAbort = () => { clearTimeout(timeout); reject(budget.signal?.reason); - }, { once: true }); + }; + budget.signal?.addEventListener("abort", onAbort, { once: true }); }); } function durationToMilliseconds(duration: Temporal.DurationLike): number { + if (typeof duration === "string") { + return Temporal.Duration.from(duration).total({ unit: "milliseconds" }); + } + return ( (duration.milliseconds ?? 0) + (duration.seconds ?? 0) * 1000 + From cdcb0660b65187b88dd363b1c4262d2831f7e723 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Wed, 27 May 2026 20:09:39 +0900 Subject: [PATCH 07/11] Normalize backfill test entry paths Normalize every platform path separator in globbed test entries before passing those paths to tsdown. Assisted-by: Codex:gpt-5 --- packages/backfill/tsdown.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backfill/tsdown.config.ts b/packages/backfill/tsdown.config.ts index 9f43a88ac..c9ddde870 100644 --- a/packages/backfill/tsdown.config.ts +++ b/packages/backfill/tsdown.config.ts @@ -17,7 +17,7 @@ export default [ }), defineConfig({ entry: (await Array.fromAsync(glob(`src/**/*.test.ts`))) - .map((f) => f.replace(sep, "/")), + .map((f) => f.replaceAll(sep, "/")), format: ["esm", "cjs"], platform: "node", outExtensions({ format }) { From 3f87eaabfda208ad92bac36f55c17630b89eade7 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Mon, 1 Jun 2026 18:32:03 +0900 Subject: [PATCH 08/11] Guard string interval without Temporal When callers pass a string interval, report a clear error if Temporal is not available globally instead of failing with an unhelpful ReferenceError. Assisted-by: Codex:gpt-5 --- packages/backfill/src/backfill.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts index c91f63e15..cbc7164b0 100644 --- a/packages/backfill/src/backfill.ts +++ b/packages/backfill/src/backfill.ts @@ -175,6 +175,13 @@ async function waitForInterval( function durationToMilliseconds(duration: Temporal.DurationLike): number { if (typeof duration === "string") { + if (typeof Temporal === "undefined") { + throw new TypeError( + "Temporal is not globally available; pass interval as a " + + "Temporal.DurationLike object instead of a string, or provide a " + + "Temporal polyfill.", + ); + } return Temporal.Duration.from(duration).total({ unit: "milliseconds" }); } From 0ad8d5d5399126e65a3e140ce57bbba29527d491 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 2 Jun 2026 16:47:38 +0900 Subject: [PATCH 09/11] Address backfill review details Make the interval option explicitly accept string durations, keep the internal duration helper aligned with that public type, and use strict Node assertions in the backfill tests. Explicitly externalize @fedify/vocab in the backfill tsdown build so vocabulary class identity remains stable for instanceof checks. Assisted-by: Codex:gpt-5 --- packages/backfill/src/backfill.test.ts | 2 +- packages/backfill/src/backfill.ts | 4 +++- packages/backfill/src/types.ts | 3 ++- packages/backfill/tsdown.config.ts | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/backfill/src/backfill.test.ts b/packages/backfill/src/backfill.test.ts index 4170f34be..9c827ee92 100644 --- a/packages/backfill/src/backfill.test.ts +++ b/packages/backfill/src/backfill.test.ts @@ -1,4 +1,4 @@ -import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert"; +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; import test, { describe } from "node:test"; import { backfill, type BackfillContext } from "./mod.ts"; import { Collection, Create, Note } from "@fedify/vocab"; diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts index cbc7164b0..b0bf00cea 100644 --- a/packages/backfill/src/backfill.ts +++ b/packages/backfill/src/backfill.ts @@ -173,7 +173,9 @@ async function waitForInterval( }); } -function durationToMilliseconds(duration: Temporal.DurationLike): number { +function durationToMilliseconds( + duration: Temporal.DurationLike | string, +): number { if (typeof duration === "string") { if (typeof Temporal === "undefined") { throw new TypeError( diff --git a/packages/backfill/src/types.ts b/packages/backfill/src/types.ts index bcf82f004..b6db32be8 100644 --- a/packages/backfill/src/types.ts +++ b/packages/backfill/src/types.ts @@ -69,7 +69,8 @@ export interface BackfillOptions< */ interval?: | Temporal.DurationLike - | ((iteration: number) => Temporal.DurationLike); + | string + | ((iteration: number) => Temporal.DurationLike | string); /** * Cancels traversal before requests and before yields. diff --git a/packages/backfill/tsdown.config.ts b/packages/backfill/tsdown.config.ts index c9ddde870..bd0f4b7a0 100644 --- a/packages/backfill/tsdown.config.ts +++ b/packages/backfill/tsdown.config.ts @@ -14,6 +14,7 @@ export default [ dts: format === "cjs" ? ".d.cts" : ".d.ts", }; }, + deps: { neverBundle: ["@fedify/vocab"] }, }), defineConfig({ entry: (await Array.fromAsync(glob(`src/**/*.test.ts`))) @@ -26,6 +27,6 @@ export default [ dts: format === "cjs" ? ".d.cts" : ".d.ts", }; }, - deps: { neverBundle: [/^node:/] }, + deps: { neverBundle: [/^node:/, "@fedify/vocab"] }, }), ]; From bdf65df6d3b7cef4513dbf727ad05ee9772a5d75 Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 2 Jun 2026 21:43:02 +0900 Subject: [PATCH 10/11] Polish backfill public API docs Replace the package README's VitePress-only installation block with plain Markdown, document the new public API surface with since tags, and mark exported option and result properties as readonly. Export the request-budget error class so callers can identify it when it escapes traversal internals. Assisted-by: Codex:gpt-5 --- packages/backfill/README.md | 26 ++++-------------- packages/backfill/src/backfill.test.ts | 3 +- packages/backfill/src/backfill.ts | 9 +++++- packages/backfill/src/mod.ts | 2 +- packages/backfill/src/types.ts | 38 ++++++++++++++++++-------- 5 files changed, 42 insertions(+), 36 deletions(-) diff --git a/packages/backfill/README.md b/packages/backfill/README.md index 3d38a762c..7c8a4ceab 100644 --- a/packages/backfill/README.md +++ b/packages/backfill/README.md @@ -27,30 +27,14 @@ or `OrderedCollectionPage`. Installation ------------ -::: code-group - -~~~~ sh [Deno] +~~~~ sh deno add jsr:@fedify/backfill +npm add @fedify/backfill +pnpm add @fedify/backfill +yarn add @fedify/backfill +bun add @fedify/backfill ~~~~ -~~~~ sh [npm] -npm add @fedify/backfill -~~~~ - -~~~~ sh [pnpm] -pnpm add @fedify/backfill -~~~~ - -~~~~ sh [Yarn] -yarn add @fedify/backfill -~~~~ - -~~~~ sh [Bun] -bun add @fedify/backfill -~~~~ - -::: - Usage ----- diff --git a/packages/backfill/src/backfill.test.ts b/packages/backfill/src/backfill.test.ts index 9c827ee92..fe957187d 100644 --- a/packages/backfill/src/backfill.test.ts +++ b/packages/backfill/src/backfill.test.ts @@ -1,6 +1,6 @@ import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert/strict"; import test, { describe } from "node:test"; -import { backfill, type BackfillContext } from "./mod.ts"; +import { backfill, type BackfillContext, MaxRequestsExceeded } from "./mod.ts"; import { Collection, Create, Note } from "@fedify/vocab"; async function collect( @@ -14,6 +14,7 @@ async function collect( describe("backfill", () => { test("package exports backfill", () => { strictEqual(typeof backfill, "function"); + strictEqual(typeof MaxRequestsExceeded, "function"); }); test("context missing yields nothing", async () => { diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts index b0bf00cea..ffee632ae 100644 --- a/packages/backfill/src/backfill.ts +++ b/packages/backfill/src/backfill.ts @@ -14,7 +14,12 @@ import type { BackfillOptions, } from "./types.ts"; -class MaxRequestsExceeded extends Error {} +/** + * Thrown when backfill traversal exceeds the configured request budget. + * + * @since 2.3.0 + */ +export class MaxRequestsExceeded extends Error {} interface RequestBudget { readonly signal?: AbortSignal; @@ -26,6 +31,8 @@ interface RequestBudget { * * The seed object is not yielded by default, but its ID is treated as already * seen so it will not be yielded again if the collection contains it. + * + * @since 2.3.0 */ export async function* backfill< TObject extends APObject = APObject, diff --git a/packages/backfill/src/mod.ts b/packages/backfill/src/mod.ts index 6ceb69ac2..d72c3c339 100644 --- a/packages/backfill/src/mod.ts +++ b/packages/backfill/src/mod.ts @@ -6,7 +6,7 @@ * * @module */ -export { backfill } from "./backfill.ts"; +export { backfill, MaxRequestsExceeded } from "./backfill.ts"; export type { BackfillContext, BackfillDocumentLoader, diff --git a/packages/backfill/src/types.ts b/packages/backfill/src/types.ts index b6db32be8..03d39872f 100644 --- a/packages/backfill/src/types.ts +++ b/packages/backfill/src/types.ts @@ -2,26 +2,34 @@ import type { Object as APObject } from "@fedify/vocab"; /** * Backfill traversal strategy used to discover the returned object. + * + * @since 2.3.0 */ export type BackfillStrategy = "context-posts"; /** * Source relation that produced a backfilled object. + * + * @since 2.3.0 */ export type BackfillOrigin = "context" | "collection"; /** * Options passed to {@link BackfillDocumentLoader}. + * + * @since 2.3.0 */ export interface BackfillDocumentLoaderOptions { /** * Cancellation signal for the current dereference operation. */ - signal?: AbortSignal; + readonly signal?: AbortSignal; } /** * Dereferences an ActivityPub object or collection IRI. + * + * @since 2.3.0 */ export type BackfillDocumentLoader = ( iri: URL, @@ -30,16 +38,20 @@ export type BackfillDocumentLoader = ( /** * Dependencies used by backfill traversal. + * + * @since 2.3.0 */ export interface BackfillContext { /** * Dereferences context collections and collection item IRIs. */ - documentLoader: BackfillDocumentLoader; + readonly documentLoader: BackfillDocumentLoader; } /** * Controls direct context collection backfill traversal. + * + * @since 2.3.0 */ export interface BackfillOptions< TObject extends APObject = APObject, @@ -47,12 +59,12 @@ export interface BackfillOptions< /** * Maximum number of items to yield. Skipped duplicates do not count. */ - maxItems?: number; + readonly maxItems?: number; /** * Maximum traversal depth. This is reserved for future reply-tree traversal; */ - maxDepth?: number; + readonly maxDepth?: number; /** * Maximum number of calls to {@link BackfillContext.documentLoader}. @@ -60,14 +72,14 @@ export interface BackfillOptions< * Dereferencing the note context, collection item IRIs, and future page IRIs * all count as requests. Embedded collection items do not count. */ - maxRequests?: number; + readonly maxRequests?: number; /** * Delay between `documentLoader` requests. * * When a callback is provided, `iteration` is the zero-based request index. */ - interval?: + readonly interval?: | Temporal.DurationLike | string | ((iteration: number) => Temporal.DurationLike | string); @@ -75,11 +87,13 @@ export interface BackfillOptions< /** * Cancels traversal before requests and before yields. */ - signal?: AbortSignal; + readonly signal?: AbortSignal; } /** * A single object discovered by backfill traversal. + * + * @since 2.3.0 */ export interface BackfillItem< TObject extends APObject = APObject, @@ -87,26 +101,26 @@ export interface BackfillItem< /** * The discovered ActivityPub object. */ - object: TObject; + readonly object: TObject; /** * The object's ActivityPub ID, when present. */ - id?: URL; + readonly id?: URL; /** * The traversal strategy that produced this item. */ - strategy: BackfillStrategy; + readonly strategy: BackfillStrategy; /** * The source relation that produced this item. */ - origin: BackfillOrigin; + readonly origin: BackfillOrigin; /** * Traversal depth. Direct context collection items are depth 0; deeper * values are reserved for future reply-tree traversal. */ - depth?: number; + readonly depth?: number; } From 5d2a8ed19be739dfa58320530553cdb54b0798ae Mon Sep 17 00:00:00 2001 From: Jiwon Kwon Date: Tue, 2 Jun 2026 22:43:22 +0900 Subject: [PATCH 11/11] Use provisional backfill since tags Keep the new backfill API documentation on provisional 2.x.0 since tags until the feature branch is ready to merge and the target release version is known. Assisted-by: Codex:gpt-5 --- packages/backfill/src/backfill.ts | 4 ++-- packages/backfill/src/types.ts | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts index ffee632ae..5e3968b82 100644 --- a/packages/backfill/src/backfill.ts +++ b/packages/backfill/src/backfill.ts @@ -17,7 +17,7 @@ import type { /** * Thrown when backfill traversal exceeds the configured request budget. * - * @since 2.3.0 + * @since 2.x.0 */ export class MaxRequestsExceeded extends Error {} @@ -32,7 +32,7 @@ interface RequestBudget { * The seed object is not yielded by default, but its ID is treated as already * seen so it will not be yielded again if the collection contains it. * - * @since 2.3.0 + * @since 2.x.0 */ export async function* backfill< TObject extends APObject = APObject, diff --git a/packages/backfill/src/types.ts b/packages/backfill/src/types.ts index 03d39872f..25555f30e 100644 --- a/packages/backfill/src/types.ts +++ b/packages/backfill/src/types.ts @@ -3,21 +3,21 @@ import type { Object as APObject } from "@fedify/vocab"; /** * Backfill traversal strategy used to discover the returned object. * - * @since 2.3.0 + * @since 2.x.0 */ export type BackfillStrategy = "context-posts"; /** * Source relation that produced a backfilled object. * - * @since 2.3.0 + * @since 2.x.0 */ export type BackfillOrigin = "context" | "collection"; /** * Options passed to {@link BackfillDocumentLoader}. * - * @since 2.3.0 + * @since 2.x.0 */ export interface BackfillDocumentLoaderOptions { /** @@ -29,7 +29,7 @@ export interface BackfillDocumentLoaderOptions { /** * Dereferences an ActivityPub object or collection IRI. * - * @since 2.3.0 + * @since 2.x.0 */ export type BackfillDocumentLoader = ( iri: URL, @@ -39,7 +39,7 @@ export type BackfillDocumentLoader = ( /** * Dependencies used by backfill traversal. * - * @since 2.3.0 + * @since 2.x.0 */ export interface BackfillContext { /** @@ -51,7 +51,7 @@ export interface BackfillContext { /** * Controls direct context collection backfill traversal. * - * @since 2.3.0 + * @since 2.x.0 */ export interface BackfillOptions< TObject extends APObject = APObject, @@ -93,7 +93,7 @@ export interface BackfillOptions< /** * A single object discovered by backfill traversal. * - * @since 2.3.0 + * @since 2.x.0 */ export interface BackfillItem< TObject extends APObject = APObject,