Auto-discover and verify new gov dependencie

.github/workflows/update-gov-dependencies.yml

@@ -0,0 +1,86 @@
+name: Update Gov Dependencies
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * 1"
+
+jobs:
+  update-gov-dependencies:
+    runs-on: ubuntu-latest
+    # The first run has no cache and sweeps every allowlisted org, so it takes a
+    # while. Later runs only revisit repos that changed and finish much faster.
+    timeout-minutes: 360
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm install
+
+      # Discovery results record each repo's last push time, so a restored cache
+      # lets the run skip every repo that has not changed. The run id in the key
+      # means each run saves a fresh copy for the next one.
+      - name: Restore discovery cache
+        uses: actions/cache@v4
+        with:
+          path: .gov-update-cache
+          key: gov-update-cache-${{ github.run_id }}
+          restore-keys: gov-update-cache-
+
+      - name: Discover and verify packages
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GOV_UPDATE_CACHE: .gov-update-cache/repos.json
+          GOV_UPDATE_REPORT: gov-update-report.md
+        run: npm run update-gov-dependencies
+
+      - name: Check for changes
+        id: changes
+        run: |
+          if git diff --quiet -- src/gov-dependencies.data.ts; then
+            echo "changed=false" >> $GITHUB_OUTPUT
+          else
+            echo "changed=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Read report
+        if: steps.changes.outputs.changed == 'true'
+        id: report
+        run: |
+          {
+            echo "text<<GOV_UPDATE_REPORT_EOF"
+            cat gov-update-report.md
+            echo "GOV_UPDATE_REPORT_EOF"
+          } >> $GITHUB_OUTPUT
+
+      - name: Create Pull Request
+        if: steps.changes.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: "chore: update federal dependency list"
+          title: "chore: update federal dependency list"
+          body: |
+            ${{ steps.report.outputs.text }}
+
+            Each package above passed verification: its registry metadata points
+            at a repository in an allowlisted federal organization, and that
+            repository declares the package name itself.
+
+            Anything listed as flagged was **not** added and needs a maintainer
+            to decide. Entries are only ever added, never removed automatically.
+          branch: automated/update-gov-dependencies
+          delete-branch: true
+          # The report and cache are build artifacts, so only the data file is
+          # allowed into the commit.
+          add-paths: src/gov-dependencies.data.ts

.gitignore

@@ -103,3 +103,7 @@ __tests__/runner/*
 # IDE files
 .idea
 *.code-workspace
+
+# Gov dependency updater artifacts
+.gov-update-cache/
+gov-update-report.md

README.md

@@ -229,6 +229,46 @@ The automated code.json generator calculates specific fields by analyzing your r
 
 **feedbackMechanism**: The repository's issues URL in the format of {repositoryURL}/issues. If you already have a code.json file with existing feedback mechanisms, the generator preserves those values. No configuration needed.
 
+**reusedCode**: The generator scans your `package.json` and `requirements.txt` for dependencies published by federal agencies and lists them here, each linked to the agency repository it comes from. It matches against a curated list of federal packages (see below). Entries already in your code.json are preserved. No configuration needed.
+
+## Federal Dependency List
+
+The `reusedCode` field is matched against a curated list of federal npm and PyPI packages in `src/gov-dependencies.data.ts`, each mapped to the agency and repository it comes from.
+
+That list is kept current by an automated job under `src/gov-update/`. It crawls federal GitHub and npm organizations and verifies every package before adding it: the package's registry metadata must point at a repository in a known federal organization, and that repository must itself declare the package. Each addition goes through a pull request for a maintainer to review.
+
+### Running the job
+
+The job runs every Monday through [`update-gov-dependencies.yml`](.github/workflows/update-gov-dependencies.yml), and can also be started by hand from the Actions tab with **Run workflow**. When it finds something new it opens a pull request against `src/gov-dependencies.data.ts` with the verification report as the description. If nothing new turns up, it opens nothing.
+
+To run it locally:
+
+```bash
+npm install
+npm run update-gov-dependencies
+```
+
+That rewrites `src/gov-dependencies.data.ts` in place, so check `git diff` afterwards. Three optional environment variables:
+
+| Variable | Purpose |
+| --- | --- |
+| `GITHUB_TOKEN` | Raises the GitHub API rate limit from 60 to 5,000 requests an hour. A full run needs it. |
+| `GOV_UPDATE_CACHE` | Path to a cache file. Repos that have not been pushed to since the last run are skipped, which is the difference between a long run and a short one. |
+| `GOV_UPDATE_REPORT` | Path to write the markdown report of what was added and what was flagged. |
+
+```bash
+GITHUB_TOKEN=<your token> \
+GOV_UPDATE_CACHE=.gov-update-cache/repos.json \
+GOV_UPDATE_REPORT=gov-update-report.md \
+npm run update-gov-dependencies
+```
+
+The first run has nothing cached and visits every repository in every allowlisted organization, so expect it to take a while. Later runs reuse the cache and are much quicker.
+
+The organizations it trusts live in `src/gov-update/allowlist.json`, mapping each GitHub organization name to its agency. Most are confirmed federal against CISA's official `.gov` domain registry ([cisagov/dotgov-data](https://github.com/cisagov/dotgov-data)). The registry only covers `.gov`, so military organizations and a handful of others are manually verified instead.
+
+To add an organization, add its GitHub organization name and agency name to `allowlist.json` and open a pull request. New organizations should always be reviewed, never added automatically. GitHub does not verify organization ownership, so this stays human-vetted rather than cryptographic proof.
+
 ## Project Vision
 
 To streamline federal agencies' compliance with open source requirements by automating the maintenance of code.json files, reducing manual effort and improving accuracy of repository metadata.

package.json

@@ -34,7 +34,8 @@
     "package:watch": "npm run package -- --watch",
     "test": "NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 npx jest",
     "all": "npm run format:write && npm run lint && npm run test && npm run coverage && npm run package",
-    "generate-schema": "npx tsx src/scripts/generate-schema.ts"
+    "generate-schema": "npx tsx src/scripts/generate-schema.ts",
+    "update-gov-dependencies": "npx tsx src/gov-update/run.ts"
   },
   "license": "MIT",
   "dependencies": {

src/__tests__/unit/gov-update-codegen.test.ts

@@ -0,0 +1,135 @@
+import { describe, it, expect } from "@jest/globals";
+import {
+  addEntries,
+  existingKeys,
+  DataFileEntry,
+} from "../../gov-update/codegen.js";
+
+const FIXTURE = `export interface ReusedCodeEntry {
+  name: string;
+  URL: string;
+}
+
+const DESIGN_SYSTEM: ReusedCodeEntry = {
+  name: "design-system (CMS)",
+  URL: "https://github.com/cmsgov/design-system",
+};
+
+const NGX_TOOL: ReusedCodeEntry = {
+  name: "ngx-tool (GSA)",
+  URL: "https://github.com/gsa/ngx-tool",
+};
+
+export const GOV_DEPENDENCIES: Record<string, ReusedCodeEntry> = {
+  "@gsa/ngx-tool": NGX_TOOL,
+  "design-system": DESIGN_SYSTEM,
+};
+
+export const GOV_DEPENDENCIES_PYPI: Record<string, ReusedCodeEntry> = {
+  "design-system": DESIGN_SYSTEM,
+};
+`;
+
+describe("existingKeys", () => {
+  it("reads keys from both maps", () => {
+    expect([...existingKeys(FIXTURE, "npm")].sort()).toEqual([
+      "@gsa/ngx-tool",
+      "design-system",
+    ]);
+    expect([...existingKeys(FIXTURE, "pypi")]).toEqual(["design-system"]);
+  });
+});
+
+describe("addEntries", () => {
+  it("inserts an unscoped npm key alphabetically with a new const", () => {
+    const entries: DataFileEntry[] = [
+      {
+        eco: "npm",
+        key: "amortize",
+        displayName: "amortize (CFPB)",
+        url: "https://github.com/cfpb/amortize",
+      },
+    ];
+    const out = addEntries(FIXTURE, entries);
+    expect(out).toContain("const AMORTIZE: ReusedCodeEntry = {");
+    expect(out).toContain('  name: "amortize (CFPB)",');
+    const npmBlock = out.slice(out.indexOf("GOV_DEPENDENCIES:"));
+    expect(npmBlock.indexOf("amortize: AMORTIZE")).toBeLessThan(
+      npmBlock.indexOf('"design-system": DESIGN_SYSTEM'),
+    );
+  });
+
+  it("groups scoped npm keys among the existing scoped block", () => {
+    const entries: DataFileEntry[] = [
+      {
+        eco: "npm",
+        key: "@cfpb/analytics",
+        displayName: "analytics (CFPB)",
+        url: "https://github.com/cfpb/analytics",
+      },
+    ];
+    const out = addEntries(FIXTURE, entries);
+    const npmBlock = out.slice(
+      out.indexOf("GOV_DEPENDENCIES:"),
+      out.indexOf("GOV_DEPENDENCIES_PYPI:"),
+    );
+    expect(npmBlock).toContain('"@cfpb/analytics": CFPB_ANALYTICS');
+    expect(npmBlock.indexOf("@cfpb/analytics")).toBeLessThan(
+      npmBlock.indexOf('"design-system": DESIGN_SYSTEM'),
+    );
+  });
+
+  it("reuses an existing const when the repo URL already appears", () => {
+    const entries: DataFileEntry[] = [
+      {
+        eco: "npm",
+        key: "@cmsgov/design-system",
+        displayName: "design-system (CMS)",
+        url: "https://github.com/cmsgov/design-system",
+      },
+    ];
+    const out = addEntries(FIXTURE, entries);
+    expect(out).toContain('"@cmsgov/design-system": DESIGN_SYSTEM');
+    expect(out.match(/const DESIGN_SYSTEM:/g)).toHaveLength(1);
+  });
+
+  it("quotes keys that are not valid identifiers", () => {
+    const entries: DataFileEntry[] = [
+      {
+        eco: "pypi",
+        key: "scikit-learn-gov",
+        displayName: "scikit-learn-gov (NIST)",
+        url: "https://github.com/usnistgov/scikit-learn-gov",
+      },
+    ];
+    const out = addEntries(FIXTURE, entries);
+    expect(out).toContain('"scikit-learn-gov": SCIKIT_LEARN_GOV');
+  });
+
+  it("prefixes const names that would start with a digit", () => {
+    const entries: DataFileEntry[] = [
+      {
+        eco: "pypi",
+        key: "3d-viz",
+        displayName: "3d-viz (CMS)",
+        url: "https://github.com/cmsgov/3d-viz",
+      },
+    ];
+    const out = addEntries(FIXTURE, entries);
+    expect(out).toContain("const PKG_3D_VIZ: ReusedCodeEntry = {");
+    expect(out).toContain('"3d-viz": PKG_3D_VIZ');
+  });
+
+  it("is idempotent-safe for keys already present", () => {
+    const before = existingKeys(FIXTURE, "npm").size;
+    const out = addEntries(FIXTURE, [
+      {
+        eco: "npm",
+        key: "amortize",
+        displayName: "amortize (CFPB)",
+        url: "https://github.com/cfpb/amortize",
+      },
+    ]);
+    expect(existingKeys(out, "npm").size).toBe(before + 1);
+  });
+});