From e16ee98a7a795d52e1d06e75f100ee747a72544b Mon Sep 17 00:00:00 2001 From: Shlok Date: Fri, 24 Apr 2026 22:42:07 -0700 Subject: [PATCH 1/3] Weekly AI digest pipeline - scripts/generate-digests.mjs: groups events.csv by ISO week and asks Claude (Sonnet) for a one-paragraph briefing per week. Backfill mode generates every historical week; default mode does only the latest completed week. Writes digests.json incrementally. - .github/workflows/weekly-digest.yml: runs every Monday 09:00 UTC and on manual dispatch (with optional --backfill / --force inputs). Commits digests.json back to the repo. - package.json: add @anthropic-ai/sdk; expose npm run digest scripts. - digests.json: empty placeholder; the first run will populate it. Requires ANTHROPIC_API_KEY repository secret to be set in GitHub Actions. --- .github/workflows/weekly-digest.yml | 56 ++++ digests.json | 1 + package-lock.json | 421 ++++++++++++++++++++++++++++ package.json | 6 + scripts/generate-digests.mjs | 231 +++++++++++++++ 5 files changed, 715 insertions(+) create mode 100644 .github/workflows/weekly-digest.yml create mode 100644 digests.json create mode 100644 scripts/generate-digests.mjs diff --git a/.github/workflows/weekly-digest.yml b/.github/workflows/weekly-digest.yml new file mode 100644 index 0000000..3db37a9 --- /dev/null +++ b/.github/workflows/weekly-digest.yml @@ -0,0 +1,56 @@ +name: Weekly AV digest + +# Runs every Monday at 09:00 UTC. Generates a digest for the most recent +# completed ISO week and commits digests.json. Can also be triggered manually. + +on: + schedule: + - cron: "0 9 * * 1" + workflow_dispatch: + inputs: + backfill: + description: "Backfill all weeks (otherwise only the latest)" + type: boolean + default: false + force: + description: "Re-generate even if cached" + type: boolean + default: false + +permissions: + contents: write + +jobs: + generate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + + - name: Install deps + run: npm ci || npm install + + - name: Generate digest(s) + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + ARGS="" + if [ "${{ inputs.backfill }}" = "true" ]; then ARGS="$ARGS --backfill"; fi + if [ "${{ inputs.force }}" = "true" ]; then ARGS="$ARGS --force"; fi + node scripts/generate-digests.mjs $ARGS + + - name: Commit digests.json + run: | + if [ -n "$(git status --porcelain digests.json)" ]; then + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add digests.json + git commit -m "chore: update weekly digests" + git push + else + echo "No digest changes to commit" + fi diff --git a/digests.json b/digests.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/digests.json @@ -0,0 +1 @@ +[] diff --git a/package-lock.json b/package-lock.json index 6feee38..8dc99b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,14 +5,435 @@ "packages": { "": { "dependencies": { + "@anthropic-ai/sdk": "^0.39.0", "csv-parse": "^6.1.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.39.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.39.0.tgz", + "integrity": "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==", + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/csv-parse": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz", "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==", "license": "MIT" + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } } } } diff --git a/package.json b/package.json index d44ebb1..02bee28 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,11 @@ { + "type": "module", + "scripts": { + "digest": "node scripts/generate-digests.mjs", + "digest:backfill": "node scripts/generate-digests.mjs --backfill" + }, "dependencies": { + "@anthropic-ai/sdk": "^0.39.0", "csv-parse": "^6.1.0" } } diff --git a/scripts/generate-digests.mjs b/scripts/generate-digests.mjs new file mode 100644 index 0000000..82504fa --- /dev/null +++ b/scripts/generate-digests.mjs @@ -0,0 +1,231 @@ +#!/usr/bin/env node +// Generate weekly AI digests of AV-Map events using Anthropic's Claude API. +// +// Usage: +// node scripts/generate-digests.mjs # Generate the most recent complete week +// node scripts/generate-digests.mjs --backfill # Generate every week back to the dataset start +// node scripts/generate-digests.mjs --force # Re-generate even if already in digests.json +// +// Requires: +// ANTHROPIC_API_KEY in env +// @anthropic-ai/sdk + csv-parse installed + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parse } from 'csv-parse/sync'; +import Anthropic from '@anthropic-ai/sdk'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..'); +const EVENTS_CSV = path.join(REPO_ROOT, 'events.csv'); +const DIGESTS_JSON = path.join(REPO_ROOT, 'digests.json'); + +const MODEL = 'claude-sonnet-4-5-20250929'; + +const args = new Set(process.argv.slice(2)); +const BACKFILL = args.has('--backfill'); +const FORCE = args.has('--force'); + +if (!process.env.ANTHROPIC_API_KEY) { + console.error('ANTHROPIC_API_KEY env var is required'); + process.exit(1); +} + +const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); + +// ── ISO week helpers ──────────────────────────────────────────────── +// ISO week: weeks start on Monday; week 1 contains the year's first Thursday. + +function toUtcMidnight(d) { + return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); +} + +function isoWeekStart(d) { + const u = toUtcMidnight(d); + const day = u.getUTCDay() || 7; // 1=Mon..7=Sun + if (day !== 1) u.setUTCDate(u.getUTCDate() - (day - 1)); + return u; +} + +function isoWeekKey(d) { + // Returns "YYYY-Www" string per ISO 8601. + const u = toUtcMidnight(d); + u.setUTCDate(u.getUTCDate() + 4 - (u.getUTCDay() || 7)); // Thu of this week + const yearStart = new Date(Date.UTC(u.getUTCFullYear(), 0, 1)); + const week = Math.ceil(((u - yearStart) / 86400000 + 1) / 7); + return `${u.getUTCFullYear()}-W${String(week).padStart(2, '0')}`; +} + +function addDays(d, n) { + const x = new Date(d); + x.setUTCDate(x.getUTCDate() + n); + return x; +} + +function fmtDate(d) { + return d.toISOString().slice(0, 10); +} + +// ── Load events ───────────────────────────────────────────────────── + +function loadEvents() { + const raw = fs.readFileSync(EVENTS_CSV, 'utf8'); + const rows = parse(raw, { columns: true, skip_empty_lines: true, trim: true }); + const events = rows + .filter((r) => r.date && r.event_type && r.company) + .map((r) => ({ + ...r, + _date: new Date(`${r.date}T00:00:00Z`), + })) + .filter((r) => !isNaN(r._date.getTime())); + events.sort((a, b) => a._date - b._date); + return events; +} + +function groupByWeek(events) { + const weeks = new Map(); + for (const e of events) { + const key = isoWeekKey(e._date); + if (!weeks.has(key)) { + const start = isoWeekStart(e._date); + weeks.set(key, { week: key, start, end: addDays(start, 6), events: [] }); + } + weeks.get(key).events.push(e); + } + return [...weeks.values()].sort((a, b) => a.start - b.start); +} + +// ── Prompt construction ───────────────────────────────────────────── + +const SYSTEM_PROMPT = `You are an analyst writing concise weekly briefings for the AV Map (avmap.io), which tracks the global rollout of autonomous vehicle (AV) services. + +You will receive a list of events that occurred during a single week. Each event describes a real-world change to an AV deployment (a launch, a policy change, an area expansion, etc.). + +Write ONE plain-prose paragraph (3-5 sentences, ~80 words) capturing the most important narrative threads of the week. Lead with what changed strategically, not a list. Mention specific companies and cities. Be terse. No hedging. No "in conclusion." Plain English. + +Also produce a 6-12 word headline that captures the dominant story. + +Output STRICT JSON only, no prose around it: +{ "headline": "...", "summary": "..." } + +If the week has only one or two minor events, say so briefly — do not pad.`; + +function eventLine(e) { + const parts = [ + e.date, + e.event_type, + e.company, + e.city || '', + e.access || '', + e.supervision || '', + e.fares ? `fares=${e.fares}` : '', + e.notes ? `(${e.notes})` : '', + ].filter(Boolean); + return parts.join(' | '); +} + +async function generateDigest(weekBucket) { + const eventsBlock = weekBucket.events.map(eventLine).join('\n'); + const userPrompt = `Week: ${fmtDate(weekBucket.start)} to ${fmtDate(weekBucket.end)}\nEvents:\n${eventsBlock}`; + + const resp = await client.messages.create({ + model: MODEL, + max_tokens: 400, + system: [ + { type: 'text', text: SYSTEM_PROMPT, cache_control: { type: 'ephemeral' } }, + ], + messages: [{ role: 'user', content: userPrompt }], + }); + + const text = resp.content.map((b) => (b.type === 'text' ? b.text : '')).join('').trim(); + // Strip code fences if present. + const cleaned = text.replace(/^```json\s*/i, '').replace(/^```\s*/, '').replace(/```\s*$/, '').trim(); + + let parsed; + try { + parsed = JSON.parse(cleaned); + } catch (err) { + throw new Error(`Failed to parse model JSON: ${cleaned.slice(0, 200)}`); + } + if (typeof parsed.summary !== 'string' || typeof parsed.headline !== 'string') { + throw new Error(`Model returned unexpected shape: ${JSON.stringify(parsed).slice(0, 200)}`); + } + return parsed; +} + +// ── Main ──────────────────────────────────────────────────────────── + +function loadExistingDigests() { + if (!fs.existsSync(DIGESTS_JSON)) return []; + try { + return JSON.parse(fs.readFileSync(DIGESTS_JSON, 'utf8')); + } catch { + return []; + } +} + +function saveDigests(digests) { + digests.sort((a, b) => (a.week < b.week ? 1 : -1)); // newest first + fs.writeFileSync(DIGESTS_JSON, JSON.stringify(digests, null, 2) + '\n'); +} + +async function main() { + const events = loadEvents(); + const weeks = groupByWeek(events); + + // Exclude the current (incomplete) week — only generate completed weeks. + const today = toUtcMidnight(new Date()); + const currentWeekStart = isoWeekStart(today); + const completed = weeks.filter((w) => w.start < currentWeekStart); + + const targets = BACKFILL ? completed : completed.slice(-1); + + const existing = loadExistingDigests(); + const existingByWeek = new Map(existing.map((d) => [d.week, d])); + + console.log(`Total weeks with events: ${completed.length}`); + console.log(`Targets this run: ${targets.length} (${BACKFILL ? 'backfill' : 'latest only'})`); + console.log(`Already cached: ${existing.length}`); + + let generated = 0; + let skipped = 0; + let failed = 0; + + for (const w of targets) { + if (!FORCE && existingByWeek.has(w.week)) { + skipped++; + continue; + } + try { + console.log(`→ ${w.week} (${w.events.length} events)`); + const { headline, summary } = await generateDigest(w); + const entry = { + week: w.week, + start: fmtDate(w.start), + end: fmtDate(w.end), + headline, + summary, + event_count: w.events.length, + event_types: [...new Set(w.events.map((e) => e.event_type))].sort(), + companies: [...new Set(w.events.map((e) => e.company))].sort(), + generated_at: new Date().toISOString(), + }; + existingByWeek.set(w.week, entry); + generated++; + // Persist incrementally so a crash mid-backfill doesn't lose progress. + saveDigests([...existingByWeek.values()]); + } catch (err) { + console.error(` ✗ ${w.week} failed: ${err.message}`); + failed++; + } + } + + console.log(`\nDone. generated=${generated} skipped=${skipped} failed=${failed}`); + if (failed > 0) process.exit(1); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); From e22976363d59fcdc5f144de0f17c24bfafa39cec Mon Sep 17 00:00:00 2001 From: Shlok Date: Sat, 25 Apr 2026 00:07:24 -0700 Subject: [PATCH 2/3] Remove AI digest pipeline (script, workflow, sdk dep) --- .github/workflows/weekly-digest.yml | 56 ---- digests.json | 1 - package-lock.json | 427 +--------------------------- package.json | 6 - scripts/generate-digests.mjs | 231 --------------- 5 files changed, 3 insertions(+), 718 deletions(-) delete mode 100644 .github/workflows/weekly-digest.yml delete mode 100644 digests.json delete mode 100644 scripts/generate-digests.mjs diff --git a/.github/workflows/weekly-digest.yml b/.github/workflows/weekly-digest.yml deleted file mode 100644 index 3db37a9..0000000 --- a/.github/workflows/weekly-digest.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Weekly AV digest - -# Runs every Monday at 09:00 UTC. Generates a digest for the most recent -# completed ISO week and commits digests.json. Can also be triggered manually. - -on: - schedule: - - cron: "0 9 * * 1" - workflow_dispatch: - inputs: - backfill: - description: "Backfill all weeks (otherwise only the latest)" - type: boolean - default: false - force: - description: "Re-generate even if cached" - type: boolean - default: false - -permissions: - contents: write - -jobs: - generate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: "20" - cache: "npm" - - - name: Install deps - run: npm ci || npm install - - - name: Generate digest(s) - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - ARGS="" - if [ "${{ inputs.backfill }}" = "true" ]; then ARGS="$ARGS --backfill"; fi - if [ "${{ inputs.force }}" = "true" ]; then ARGS="$ARGS --force"; fi - node scripts/generate-digests.mjs $ARGS - - - name: Commit digests.json - run: | - if [ -n "$(git status --porcelain digests.json)" ]; then - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add digests.json - git commit -m "chore: update weekly digests" - git push - else - echo "No digest changes to commit" - fi diff --git a/digests.json b/digests.json deleted file mode 100644 index fe51488..0000000 --- a/digests.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/package-lock.json b/package-lock.json index 8dc99b4..9a5f860 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,435 +5,14 @@ "packages": { "": { "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", "csv-parse": "^6.1.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.39.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.39.0.tgz", - "integrity": "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==", - "license": "MIT", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/csv-parse": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz", - "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.2.1.tgz", + "integrity": "sha512-LRLMV+UCyfMokp8Wb411duBf1gaBKJfOfBWU9eHMJ+b+cJYZsNu3AFmjJf3+yPGd59Exz1TsMjaSFyxnYB9+IQ==", "license": "MIT" - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } } } } diff --git a/package.json b/package.json index 02bee28..d44ebb1 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,5 @@ { - "type": "module", - "scripts": { - "digest": "node scripts/generate-digests.mjs", - "digest:backfill": "node scripts/generate-digests.mjs --backfill" - }, "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", "csv-parse": "^6.1.0" } } diff --git a/scripts/generate-digests.mjs b/scripts/generate-digests.mjs deleted file mode 100644 index 82504fa..0000000 --- a/scripts/generate-digests.mjs +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env node -// Generate weekly AI digests of AV-Map events using Anthropic's Claude API. -// -// Usage: -// node scripts/generate-digests.mjs # Generate the most recent complete week -// node scripts/generate-digests.mjs --backfill # Generate every week back to the dataset start -// node scripts/generate-digests.mjs --force # Re-generate even if already in digests.json -// -// Requires: -// ANTHROPIC_API_KEY in env -// @anthropic-ai/sdk + csv-parse installed - -import fs from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { parse } from 'csv-parse/sync'; -import Anthropic from '@anthropic-ai/sdk'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const REPO_ROOT = path.resolve(__dirname, '..'); -const EVENTS_CSV = path.join(REPO_ROOT, 'events.csv'); -const DIGESTS_JSON = path.join(REPO_ROOT, 'digests.json'); - -const MODEL = 'claude-sonnet-4-5-20250929'; - -const args = new Set(process.argv.slice(2)); -const BACKFILL = args.has('--backfill'); -const FORCE = args.has('--force'); - -if (!process.env.ANTHROPIC_API_KEY) { - console.error('ANTHROPIC_API_KEY env var is required'); - process.exit(1); -} - -const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); - -// ── ISO week helpers ──────────────────────────────────────────────── -// ISO week: weeks start on Monday; week 1 contains the year's first Thursday. - -function toUtcMidnight(d) { - return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); -} - -function isoWeekStart(d) { - const u = toUtcMidnight(d); - const day = u.getUTCDay() || 7; // 1=Mon..7=Sun - if (day !== 1) u.setUTCDate(u.getUTCDate() - (day - 1)); - return u; -} - -function isoWeekKey(d) { - // Returns "YYYY-Www" string per ISO 8601. - const u = toUtcMidnight(d); - u.setUTCDate(u.getUTCDate() + 4 - (u.getUTCDay() || 7)); // Thu of this week - const yearStart = new Date(Date.UTC(u.getUTCFullYear(), 0, 1)); - const week = Math.ceil(((u - yearStart) / 86400000 + 1) / 7); - return `${u.getUTCFullYear()}-W${String(week).padStart(2, '0')}`; -} - -function addDays(d, n) { - const x = new Date(d); - x.setUTCDate(x.getUTCDate() + n); - return x; -} - -function fmtDate(d) { - return d.toISOString().slice(0, 10); -} - -// ── Load events ───────────────────────────────────────────────────── - -function loadEvents() { - const raw = fs.readFileSync(EVENTS_CSV, 'utf8'); - const rows = parse(raw, { columns: true, skip_empty_lines: true, trim: true }); - const events = rows - .filter((r) => r.date && r.event_type && r.company) - .map((r) => ({ - ...r, - _date: new Date(`${r.date}T00:00:00Z`), - })) - .filter((r) => !isNaN(r._date.getTime())); - events.sort((a, b) => a._date - b._date); - return events; -} - -function groupByWeek(events) { - const weeks = new Map(); - for (const e of events) { - const key = isoWeekKey(e._date); - if (!weeks.has(key)) { - const start = isoWeekStart(e._date); - weeks.set(key, { week: key, start, end: addDays(start, 6), events: [] }); - } - weeks.get(key).events.push(e); - } - return [...weeks.values()].sort((a, b) => a.start - b.start); -} - -// ── Prompt construction ───────────────────────────────────────────── - -const SYSTEM_PROMPT = `You are an analyst writing concise weekly briefings for the AV Map (avmap.io), which tracks the global rollout of autonomous vehicle (AV) services. - -You will receive a list of events that occurred during a single week. Each event describes a real-world change to an AV deployment (a launch, a policy change, an area expansion, etc.). - -Write ONE plain-prose paragraph (3-5 sentences, ~80 words) capturing the most important narrative threads of the week. Lead with what changed strategically, not a list. Mention specific companies and cities. Be terse. No hedging. No "in conclusion." Plain English. - -Also produce a 6-12 word headline that captures the dominant story. - -Output STRICT JSON only, no prose around it: -{ "headline": "...", "summary": "..." } - -If the week has only one or two minor events, say so briefly — do not pad.`; - -function eventLine(e) { - const parts = [ - e.date, - e.event_type, - e.company, - e.city || '', - e.access || '', - e.supervision || '', - e.fares ? `fares=${e.fares}` : '', - e.notes ? `(${e.notes})` : '', - ].filter(Boolean); - return parts.join(' | '); -} - -async function generateDigest(weekBucket) { - const eventsBlock = weekBucket.events.map(eventLine).join('\n'); - const userPrompt = `Week: ${fmtDate(weekBucket.start)} to ${fmtDate(weekBucket.end)}\nEvents:\n${eventsBlock}`; - - const resp = await client.messages.create({ - model: MODEL, - max_tokens: 400, - system: [ - { type: 'text', text: SYSTEM_PROMPT, cache_control: { type: 'ephemeral' } }, - ], - messages: [{ role: 'user', content: userPrompt }], - }); - - const text = resp.content.map((b) => (b.type === 'text' ? b.text : '')).join('').trim(); - // Strip code fences if present. - const cleaned = text.replace(/^```json\s*/i, '').replace(/^```\s*/, '').replace(/```\s*$/, '').trim(); - - let parsed; - try { - parsed = JSON.parse(cleaned); - } catch (err) { - throw new Error(`Failed to parse model JSON: ${cleaned.slice(0, 200)}`); - } - if (typeof parsed.summary !== 'string' || typeof parsed.headline !== 'string') { - throw new Error(`Model returned unexpected shape: ${JSON.stringify(parsed).slice(0, 200)}`); - } - return parsed; -} - -// ── Main ──────────────────────────────────────────────────────────── - -function loadExistingDigests() { - if (!fs.existsSync(DIGESTS_JSON)) return []; - try { - return JSON.parse(fs.readFileSync(DIGESTS_JSON, 'utf8')); - } catch { - return []; - } -} - -function saveDigests(digests) { - digests.sort((a, b) => (a.week < b.week ? 1 : -1)); // newest first - fs.writeFileSync(DIGESTS_JSON, JSON.stringify(digests, null, 2) + '\n'); -} - -async function main() { - const events = loadEvents(); - const weeks = groupByWeek(events); - - // Exclude the current (incomplete) week — only generate completed weeks. - const today = toUtcMidnight(new Date()); - const currentWeekStart = isoWeekStart(today); - const completed = weeks.filter((w) => w.start < currentWeekStart); - - const targets = BACKFILL ? completed : completed.slice(-1); - - const existing = loadExistingDigests(); - const existingByWeek = new Map(existing.map((d) => [d.week, d])); - - console.log(`Total weeks with events: ${completed.length}`); - console.log(`Targets this run: ${targets.length} (${BACKFILL ? 'backfill' : 'latest only'})`); - console.log(`Already cached: ${existing.length}`); - - let generated = 0; - let skipped = 0; - let failed = 0; - - for (const w of targets) { - if (!FORCE && existingByWeek.has(w.week)) { - skipped++; - continue; - } - try { - console.log(`→ ${w.week} (${w.events.length} events)`); - const { headline, summary } = await generateDigest(w); - const entry = { - week: w.week, - start: fmtDate(w.start), - end: fmtDate(w.end), - headline, - summary, - event_count: w.events.length, - event_types: [...new Set(w.events.map((e) => e.event_type))].sort(), - companies: [...new Set(w.events.map((e) => e.company))].sort(), - generated_at: new Date().toISOString(), - }; - existingByWeek.set(w.week, entry); - generated++; - // Persist incrementally so a crash mid-backfill doesn't lose progress. - saveDigests([...existingByWeek.values()]); - } catch (err) { - console.error(` ✗ ${w.week} failed: ${err.message}`); - failed++; - } - } - - console.log(`\nDone. generated=${generated} skipped=${skipped} failed=${failed}`); - if (failed > 0) process.exit(1); -} - -main().catch((e) => { - console.error(e); - process.exit(1); -}); From 019c0f27e14d2b8fd61863e3acbe4bd677083d8d Mon Sep 17 00:00:00 2001 From: Shlok Date: Sat, 25 Apr 2026 10:22:49 -0700 Subject: [PATCH 3/3] Stavanger row: company is ADASTEC (L4 software), Vy/Kolumbus are operators --- events.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/events.csv b/events.csv index 78ab553..9d358a7 100644 --- a/events.csv +++ b/events.csv @@ -220,7 +220,7 @@ date,event_type,company,city,geometry_file,vehicles,platform,fares,direct_bookin "2026-04-01","service_created","WeRide","Singapore","103.9072,1.4043","WeRide Robotaxi GXR","Grab","Yes","No","Flexible","Autonomous","Public","","","https://www.weride.ai/","https://www.grab.com","https://www.grab.com/sg/press/others/grab-in-partnership-with-weride-officially-launch-singapores-first-autonomous-public-ride-service-in-punggol/","Ai.R autonomous ride service in Punggol; 11 vehicles; SE Asia first" "2026-04-01","supervision_updated","Tesla","Austin","","","","","","","Autonomous","","","","","","https://www.basenor.com/blogs/news/tesla-austin-robotaxi-fleet-hits-10-unsupervised-model-ys","10 unsupervised Model Ys confirmed in fleet" "2026-04-06","service_testing","Tesla","Phoenix","-112.074,33.4484","Tesla Model Y","","","","","","","","","https://www.tesla.com/robotaxi","","https://www.basenor.com/blogs/news/tesla-robotaxi-expansion-60-model-ys-spotted-in-phoenix","~60 Robotaxi-configured Model Ys spotted at staging lot" -"2026-04-07","service_created","Waymo","Nashville","waymo-nashville-april-7-2026-boundary.geojson","Jaguar I-Pace","Waymo","Yes","Yes","Flexible","Autonomous","Waitlist","Flexdrive","","https://waymo.com/blog/2026/04/your-waymo-ride-now-arriving-in-nashville/","","https://techcrunch.com/2026/04/07/waymo-opens-robotaxi-service-in-nashville-partners-with-lyft/","11th US city; Lyft/Flexdrive fleet management; 60 sq mi initial area" +"2026-04-07","service_created","Waymo","Nashville","waymo-nashville-april-7-2026-boundary.geojson","Jaguar I-Pace","Waymo","Yes","Yes","Flexible","Autonomous","Waitlist","Flexdrive","","https://waymo.com/blog/2026/04/your-waymo-ride-now-arriving-in-nashville/","","https://techcrunch.com/2026/04/07/waymo-opens-robotaxi-service-in-nashville-partners-with-lyft/","11th US city; Lyft/Flexdrive fleet management; 60 sq mi initial area" "2026-04-08","service_created","Pony.ai","Zagreb","15.9819,45.8150","BAIC Arcfox Alpha T5","Verne;Uber","Yes","Yes","Flexible","Safety Driver","Public","Verne","","https://pony.ai/","https://www.letsverne.com","https://www.prnewswire.com/news-releases/ponyai-advances-global-deployment-with-launch-of-europes-first-commercial-robotaxi-service-in-zagreb-302736854.html","Europe's first commercial robotaxi; ~90 km² area; 7am-9pm daily" "2026-04-08","service_testing","VW ADMT","Los Angeles","-118.2437,34.0522","VW ID. Buzz AD","Uber","","","","Safety Driver","","","","https://www.moia.io/","https://www.uber.com","https://techcrunch.com/2026/04/08/volkswagen-moia-uber-los-angeles-testing-self-driving-microbuses-id-buzz/","MOIA America testing begins; ~10 vehicles scaling to 100+; commercial launch late 2026" "2026-04-13","service_announced","Didi","Dubai","55.2708,25.2048","","","","","","","","","2026","https://www.didiglobal.com/","","https://www.cnbc.com/2026/04/15/china-robotaxi-didi-weride-baidu-dubai-uae-expansion.html","UAE confirmed as Didi's first overseas robotaxi market" @@ -231,6 +231,6 @@ date,event_type,company,city,geometry_file,vehicles,platform,fares,direct_bookin "2026-04-14","service_announced","Glydways","Dubai","55.1810,25.0772","","","","","","","","","2026","https://www.glydways.com/","","https://www.globenewswire.com/news-release/2026/04/14/3273722/0/en/Glydways-Raises-Oversubscribed-170M-Series-C-as-First-Autonomous-Vehicle-Networks-Enter-Public-Operation-in-2026.html","Bluewaters area ATN pilot with Dubai RTA; MOU to expand across 4 transit corridors" "2026-04-14","service_announced","Glydways","New York","-74.0060,40.7128","","","","","","","","","2026","https://www.glydways.com/","","https://www.globenewswire.com/news-release/2026/04/14/3273722/0/en/Glydways-Raises-Oversubscribed-170M-Series-C-as-First-Autonomous-Vehicle-Networks-Enter-Public-Operation-in-2026.html","Greater NYC autonomous transit network pilot planned for 2026" "2026-04-14","service_announced","Glydways","Atlanta","-84.4469,33.6391","","","","","","","","","December 2026","https://www.glydways.com/","","https://www.globenewswire.com/news-release/2026/04/14/3273722/0/en/Glydways-Raises-Oversubscribed-170M-Series-C-as-First-Autonomous-Vehicle-Networks-Enter-Public-Operation-in-2026.html","South Metro Atlanta 0.5-mile guideway connecting ATL SkyTrain to Gateway Center Arena; free on-demand service December 2026" -"2026-04-17","service_created","Vy;Kolumbus","Stavanger","5.7331,58.9700","Karsan e-ATAK","","No","No","Fixed Route","Autonomous","Public","","","https://www.vy.no/en/the-vy-group/autonomous-vehicles","","https://www.sustainable-bus.com/autonomous-driving/stavanger-autonomous-bus-without-safety-driver-approved/","Norway's first fully driverless bus authorization without safety driver; ADASTEC autonomous software; Applied Autonomy xFlow fleet management" +"2026-04-17","service_created","ADASTEC","Stavanger","5.7331,58.9700","Karsan e-ATAK","","No","No","Fixed Route","Autonomous","Public","Vy; Kolumbus; Applied Autonomy","","https://www.adastec.com/","","https://www.sustainable-bus.com/autonomous-driving/stavanger-autonomous-bus-without-safety-driver-approved/","Norway's first fully driverless bus authorization without safety driver; ADASTEC L4 autonomous software running on Karsan e-ATAK; operated by Vy and Kolumbus with Applied Autonomy xFlow fleet management" "2026-04-18","service_created","Tesla","Dallas","-96.7970,32.7767","Tesla Model Y","Robotaxi","Yes","Yes","Flexible","Autonomous","Public","","","https://www.tesla.com/robotaxi","","https://techcrunch.com/2026/04/18/tesla-brings-its-robotaxi-service-to-dallas-and-houston/","Fully unsupervised launch skipping safety monitor phase; ~30-35 sq mi geofence covering urban core and Park Cities" "2026-04-18","service_created","Tesla","Houston","-95.3698,29.7604","Tesla Model Y","Robotaxi","Yes","Yes","Flexible","Autonomous","Public","","","https://www.tesla.com/robotaxi","","https://techcrunch.com/2026/04/18/tesla-brings-its-robotaxi-service-to-dallas-and-houston/","Fully unsupervised launch; ~12-15 sq mi geofence focused on Northwest region covering Jersey Village and Willowbrook areas" \ No newline at end of file