src, scripts: use kv to cache directories

.env.example

@@ -0,0 +1,30 @@
+# Your Cloudflare account tag.
+# 
+# Needed for:
+#  - Directory cache scripts
+CLOUDFLARE_ACCOUNT_ID=
+
+# Cloudflare V4 API token.
+# 
+# Needed for:
+#  - Directory cache scripts
+# 
+# Required permissions:
+#  - `Workers KV Storage`: Edit
+#  - `Workers R2 Storage`: Read
+# 
+# See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/
+CLOUDFLARE_API_TOKEN=
+
+# S3 credentials for your R2 bucket.
+# 
+# Needed for:
+#  - Directory listings in the worker.
+#  - Directory cache scripts
+# 
+# Required permissions:
+#  - `Object Read Only`
+# 
+# See https://dash.cloudflare.com/?account=/r2/api-tokens
+S3_ACCESS_KEY_ID=
+S3_ACCESS_KEY_SECRET=

.github/workflows/build-directory-cache.yaml

@@ -0,0 +1,77 @@
+name: Build Directory Cache
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-directory-cache:
+    name: Build Directory Cache
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
+        with:
+          egress-policy: audit
+
+      - name: Git Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Cache Dependencies
+        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: |
+            ~/.npm
+            node_modules/.cache
+          key: ${{ runner.os }}-npm-${{ hashFiles('**/workflows/format.yml') }}
+          restore-keys: ${{ runner.os }}-npm-
+
+      - name: Setup Node
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version: lts/*
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci && npm update nodejs-latest-linker --save
+
+      - name: Build Directory Cache
+        run: node scripts/build-directory-cache.mjs && node --run format
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+          S3_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
+          S3_ACCESS_KEY_SECRET: ${{ secrets.CF_SECRET_ACCESS_KEY }}
+
+      - name: Commit Changes
+        id: git_auto_commit
+        uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0
+        with:
+          commit_options: '--no-verify --no-signoff'
+          commit_message: 'chore: update redirect links'
+          branch: build-directory-cache
+          create_branch: true
+
+      - name: Open and Merge Pull Request
+        if: steps.git_auto_commit.outputs.changes_detected == 'true'
+        run: |
+          gh pr create --fill
+          gh pr merge --squash --delete-branch --admin
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
+
+      - name: Deploy to Production
+        if: steps.git_auto_commit.outputs.changes_detected == 'true'
+        run: |
+          gh workflow run deploy.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
+
+      - name: Alert on Failure
+        if: failure() && github.repository == 'nodejs/release-cloudflare-worker'
+        uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # 2.3.3
+        env:
+          SLACK_COLOR: '#DE512A'
+          SLACK_ICON: https://github.com/nodejs.png?size=48
+          SLACK_TITLE: Build Directory Cache failed (${{ github.ref }})
+          SLACK_MESSAGE: The `build-directory-cache.yaml` action has failed.
+          SLACK_USERNAME: nodejs-bot
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/update-links.yml

@@ -6,6 +6,11 @@ permissions:
 on:
   # Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml
   workflow_dispatch:
+    inputs:
+      version:
+        description: 'Node.js version (ex/ `v20.0.0`)'
+        required: true
+        type: string
 
 concurrency:
   group: update-redirect-links
@@ -22,6 +27,7 @@ jobs:
           egress-policy: block
           allowed-endpoints: >
             api.github.com:443
+            api.cloudflare.com:443
             dist-prod.07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com:443
             github.com:443
             hooks.slack.com:443
@@ -49,11 +55,13 @@ jobs:
       - name: Install dependencies
         run: npm ci && npm update nodejs-latest-linker --save
 
-      - name: Update Redirect Links
-        run: node scripts/build-r2-symlinks.mjs && node --run format
+      - name: Update Directory Cache
+        run: node scripts/update-directory-cache.mjs "$VERSION_INPUT" && node --run format
         env:
-          CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
-          CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }}
+          VERSION_INPUT: '${{ inputs.version }}'
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+          S3_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
+          S3_ACCESS_KEY_SECRET: ${{ secrets.CF_SECRET_ACCESS_KEY }}
 
       - name: Commit Changes
         id: git_auto_commit

.gitignore

@@ -3,3 +3,4 @@ node_modules/
 dist/
 .dev.vars
 .sentryclirc
+.env

e2e-tests/directory.kv.test.ts

@@ -0,0 +1,89 @@
+import { env, createExecutionContext } from 'cloudflare:test';
+import { test, beforeAll, expect, vi } from 'vitest';
+import {
+  populateDirectoryCacheWithDevBucket,
+  populateR2WithDevBucket,
+} from './util';
+import worker from '../src/worker';
+import type { Env } from '../src/env';
+import { CACHE_HEADERS } from '../src/constants/cache';
+
+const mockedEnv: Env = {
+  ...env,
+  ENVIRONMENT: 'e2e-tests',
+  CACHING: false,
+  LOG_ERRORS: true,
+  USE_KV: true,
+};
+
+beforeAll(async () => {
+  await populateR2WithDevBucket();
+  await populateDirectoryCacheWithDevBucket();
+
+  vi.mock(
+    import('../src/constants/latestVersions.json'),
+    async importOriginal => {
+      const original = await importOriginal();
+
+      // Point all `latest-` directories to one that exists in the dev bucket
+      Object.keys(original.default).forEach(branch => {
+        let updatedValue: string;
+        if (branch === 'node-latest.tar.gz') {
+          updatedValue = 'latest/node-v20.0.0.tar.gz';
+        } else {
+          updatedValue = 'v20.0.0';
+        }
+
+        // @ts-expect-error
+        original.default[branch] = updatedValue;
+      });
+
+      return original;
+    }
+  );
+});
+
+// Ensure essential endpoints are routable
+for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
+  test(`GET \`${path}\` returns 200`, async () => {
+    const ctx = createExecutionContext();
+
+    const res = await worker.fetch(
+      new Request(`https://localhost${path}`),
+      mockedEnv,
+      ctx
+    );
+
+    // Consume body promise
+    await res.text();
+
+    expect(res.status).toBe(200);
+  });
+}
+
+test('GET `/dist/unknown-directory/` returns 404', async () => {
+  const ctx = createExecutionContext();
+
+  const res = await worker.fetch(
+    new Request('https://localhost/dist/unknown-directory/'),
+    mockedEnv,
+    ctx
+  );
+
+  expect(res.status).toBe(404);
+  expect(res.headers.get('cache-control')).toStrictEqual(CACHE_HEADERS.failure);
+  expect(await res.text()).toStrictEqual('Directory not found');
+});
+
+test('GET `/dist` redirects to `/dist/`', async () => {
+  const ctx = createExecutionContext();
+
+  const res = await worker.fetch(
+    new Request('https://localhost/dist'),
+    mockedEnv,
+    ctx
+  );
+
+  expect(res.status).toBe(301);
+  expect(res.headers.get('location')).toStrictEqual('https://localhost/dist/');
+});

e2e-tests/util.ts

@@ -1,7 +1,9 @@
 import { env } from 'cloudflare:test';
+import { join } from 'node:path';
 import { inject } from 'vitest';
-import type { Env } from '../env';
-import type { Directory } from '../../vitest-setup';
+import type { Env } from '../src/env';
+import type { Directory } from '../vitest-setup';
+import type { ReadDirectoryResult, File } from '../src/providers/provider';
 
 async function populateR2BucketDirectory(directory: Directory): Promise<void> {
   const promises: Array<Promise<unknown>> = [];
@@ -10,7 +12,7 @@ async function populateR2BucketDirectory(directory: Directory): Promise<void> {
     const file = directory.files[path];
 
     promises.push(
-      env.R2_BUCKET.put(path, file.contents, {
+      env.R2_BUCKET.put(join(directory.name, path), file.contents, {
         customMetadata: {
           // This is added by rclone when copying the release assets to the
           //  bucket.
@@ -27,6 +29,41 @@ async function populateR2BucketDirectory(directory: Directory): Promise<void> {
   await Promise.all(promises);
 }
 
+async function populateDirectoryCache(directory: Directory): Promise<void> {
+  let hasIndexHtmlFile = false;
+
+  const files: File[] = Object.keys(directory.files).map(name => {
+    const file = directory.files[name];
+
+    if (!hasIndexHtmlFile && name.match(/index.htm(?:l)$/)) {
+      hasIndexHtmlFile = true;
+    }
+
+    return {
+      name,
+      lastModified: new Date(file.lastModified),
+      size: file.size,
+    };
+  });
+
+  const cachedDirectory: ReadDirectoryResult = {
+    subdirectories: Object.keys(directory.subdirectories),
+    files,
+    hasIndexHtmlFile,
+    lastModified: new Date(),
+  };
+
+  const promises: Array<Promise<void>> = [
+    env.DIRECTORY_CACHE.put(
+      `${directory.name}/`,
+      JSON.stringify(cachedDirectory)
+    ),
+    ...Object.values(directory.subdirectories).map(populateDirectoryCache),
+  ];
+
+  await Promise.all(promises);
+}
+
 /**
  * Writes the contents of the dev bucket into the R2 bucket given in {@link env}
  */
@@ -38,6 +75,14 @@ export async function populateR2WithDevBucket(): Promise<void> {
   await populateR2BucketDirectory(devBucket);
 }
 
+export async function populateDirectoryCacheWithDevBucket(): Promise<void> {
+  // Grab the contents of the dev bucket
+  const devBucket = inject('devBucket');
+
+  // Write it to KV
+  await populateDirectoryCache(devBucket);
+}
+
 declare module 'cloudflare:test' {
   // eslint-disable-next-line @typescript-eslint/no-empty-object-type
   interface ProvidedEnv extends Env {}

lib/README.md

@@ -0,0 +1,3 @@
+# `lib/`
+
+Utilities used in local scripts and in the deployed worker.

lib/limits.mjs

@@ -0,0 +1,19 @@
+/**
+ * Max amount of retries for requests to R2
+ */
+export const R2_RETRY_LIMIT = 5;
+
+/**
+ * Max amount of retries for requests to KV
+ */
+export const KV_RETRY_LIMIT = 5;
+
+/**
+ * Max amount of keys to be returned in a S3 request
+ */
+export const S3_MAX_KEYS = 1000;
+
+/**
+ * Max amount of keys we can have in a KV request
+ */
+export const KV_MAX_KEYS = 10_000;