diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index b263f7bff..733fbdeb8 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -178,7 +178,7 @@ modules: - cp -r ../../hugo/assets ./hugo/assets - cp -r ../../hugo/data ./hugo/data - cp -r ../../hugo/i18n ./hugo/i18n - - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" + - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json ../../srv/lib/provenance-handlers.js ../../srv/lib/provenance-envelope.js ../../srv/lib/provenance-data.js ../../srv/lib/provenance-keys.js ../../srv/lib/provenance-freshness.js srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" - bash -c "node -e \"const p=require('./package.json'); p.dependencies=Object.assign(p.dependencies||{},{cheerio:'^1.2.0','@sap-ai-sdk/foundation-models':'^2.10.0'}); require('fs').writeFileSync('./package.json', JSON.stringify(p,null,2));\"" properties: EXPOSE_CAP_UI: false diff --git a/CLAUDE.md b/CLAUDE.md index f9281be8e..9ca222ea0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -134,3 +134,4 @@ The load-bearing few. **Full detail for every relocated item → [tutorials-ims- - **`test:e2e` is post-deploy only, not on PRs** — self-skips without `SMOKE_BASE_URL`. Served tutorials render `
`+`

`, NOT `
`. Runbook: `test/e2e/README.md`. - **Freshness detector grounding needs the corpus-embedding backfill** — until `srv/jobs/freshness-corpus-embedding-job.js` runs, every API-obsolescence claim degrades to `confidence: Low`. Tutorial source from `ContentFiles.sourceContent` via `getTutorialSource(slug)`, NOT `Steps.description`. → gotchas.md "Freshness detector". - **External channels subsystem** — `Channels` entity (`db/channels.cds`) is the source of truth; re-ingest via `npm run seed-channels`; directory at `/channels` (baked by `fetch-channels` in `build:all`); verb-lane fill via `npm run promote-channels`; community items never land in `START_HERE`. → [channels.md](docs/developers/reference/channels.md). +- **Signed provenance envelope (#2245)** — `PROVENANCE_ENVELOPE_ENABLED` (DB config, default OFF, DEV-first); two anonymous Express endpoints (`/content/tutorials/:slug/provenance` and `/.well-known/tutorial-provenance/jwks.json`); signing key = `PROVENANCE_SIGNING_KEY` credstore secret (Ed25519 PKCS8 PEM); rotate via `/admin-ui/#secrets`. Fail-open. → gotchas.md "Signed provenance envelope". diff --git a/db/_content-shape.cds b/db/_content-shape.cds index 35c9bd076..26b0a60db 100644 --- a/db/_content-shape.cds +++ b/db/_content-shape.cds @@ -39,6 +39,7 @@ aspect ContentFilesAspect : managed { // skips slugs whose `sourceHash` is null on the server side. sourceContent : LargeBinary; sourceHash : Sha256; + sourceCommit : String(64); // git commit SHA of source .md at publish time (#2245); null for pre-2245 rows } aspect ContentManifestAspect : managed { @@ -78,6 +79,7 @@ aspect ContentCurrentAspect : managed { mimeType : String(100) default 'text/html'; sourceContent : LargeBinary; sourceHash : Sha256; + sourceCommit : String(64); // git commit SHA of source .md at publish time (#2245); null for pre-2245 rows sourceVersion : Integer; } @@ -97,6 +99,7 @@ aspect ContentHistoryAspect : managed { mimeType : String(100) default 'text/html'; sourceContent : LargeBinary; sourceHash : Sha256; + sourceCommit : String(64); // git commit SHA of source .md at publish time (#2245); null for pre-2245 rows } aspect TutorialBodyTextAspect : managed { diff --git a/db/last-dev/csn.json b/db/last-dev/csn.json index 19c8dcaa8..67e455ab4 100644 --- a/db/last-dev/csn.json +++ b/db/last-dev/csn.json @@ -4104,6 +4104,11 @@ "type": "cds.String", "length": 64, "@cds.persistence.name": "SOURCEHASH" + }, + "sourceCommit": { + "type": "cds.String", + "length": 64, + "@cds.persistence.name": "SOURCECOMMIT" } }, "@cds.persistence.name": "COM_SAP_DEVELOPERS_IMS_CONTENTFILES" diff --git a/db/src/com.sap.developers.ims.ContentFiles.hdbmigrationtable b/db/src/com.sap.developers.ims.ContentFiles.hdbmigrationtable index 8409a66d8..f13d38dbe 100644 --- a/db/src/com.sap.developers.ims.ContentFiles.hdbmigrationtable +++ b/db/src/com.sap.developers.ims.ContentFiles.hdbmigrationtable @@ -1,4 +1,4 @@ -== version=2 +== version=3 COLUMN TABLE com_sap_developers_ims_ContentFiles ( createdAt TIMESTAMP, createdBy NVARCHAR(255), @@ -13,9 +13,14 @@ COLUMN TABLE com_sap_developers_ims_ContentFiles ( "MIMETYPE" NVARCHAR(100) DEFAULT 'text/html', sourceContent BLOB, sourceHash NVARCHAR(64), + sourceCommit NVARCHAR(64), PRIMARY KEY(slug, version) ) +== migration=3 +-- generated by cds-compiler version 7.0.1 +ALTER TABLE com_sap_developers_ims_ContentFiles ADD (sourceCommit NVARCHAR(64)); + == migration=2 -- generated by cds-compiler version 6.9.0 ALTER TABLE com_sap_developers_ims_ContentFiles ADD (sourceContent BLOB, sourceHash NVARCHAR(64)); diff --git a/docs/developers/operations/testing-endpoints.md b/docs/developers/operations/testing-endpoints.md index 637c6958a..9f8f1ea25 100644 --- a/docs/developers/operations/testing-endpoints.md +++ b/docs/developers/operations/testing-endpoints.md @@ -183,6 +183,7 @@ When `EXPOSE_CAP_UI=true` is set on the CAP srv app, these are accessible throug | `/a2a` | POST | A2A JSON-RPC 2.0 endpoint (`message/send`, `message/stream`, `tasks/get`, `tasks/cancel`) for central Joule consumption (#1220). Skill via `metadata.skillId`; defaults to conversational `tutorial-chat`. Enable/config via `/admin-ui/#joule` (ChatSettings `a2aEnabled`). | XSUAA + `Tutorial.MCP` | | `/.well-known/agent-card.json` | GET | A2A Agent Card — public discovery document (5 skills, streaming, xsuaa security scheme). Base URL + token URL from `/admin-ui/#joule` (ChatSettings). | None | | `/.well-known/a2a-instructions.md` | GET | A2A consumption guide (how to authenticate + call) | None | +| `/.well-known/tutorial-provenance/jwks.json` | GET | Ed25519 JWKS for verifying tutorial provenance JWS tokens (`{ keys: [...] }`). Returns 404 when `PROVENANCE_ENVELOPE_ENABLED` flag is OFF. Anonymous Express route — `@requires` not applicable. Pair: `/content/tutorials/:slug/provenance`. | None | | `/api/codecheck` | POST | AI code-check spike (issue #171, gated on `ChatSettings.codeCheckEnabled`). Body: `{ tutorialSlug, stepNumber, submittedCode, language? }`. Returns `{ verdict: 'pass'\|'partial'\|'fail', summary, suggestions[], correctAspects[] }`. 503 when flag off; 429 with `Retry-After` on per-user 30/hr or per-(user,slug,step) 5/5min cap. | XSUAA | | `/author/generateOsVariants` | POST | AI-assisted OS variant generation for the VS Code authoring plugin (issue #173). Body: `{ sourceMarkdown, sourceOS, targetOSes[], context? }`. Returns `{ variants[], model, tokensUsed, requestId }`. 60/hr per author. See spec [#173](../../superpowers/specs/2026-06-09-173-os-conditional-content-design.md) §5. | XSUAA + `Tutorial.Author` | | `/admin/embeddings/stats` | GET | Tutorial embedding coverage / drift statistics | XSUAA + `Admin` | @@ -210,6 +211,7 @@ When `EXPOSE_CAP_UI=true` is set on the CAP srv app, these are accessible throug | URL | Method | Description | Auth | |-----|--------|-------------|------| | `/content/tutorials/{slug}` | GET | Serve tutorial HTML from HANA (ETag, Cache-Control) | None | +| `/content/tutorials/{slug}/provenance` | GET | Signed provenance JWS for a tutorial (`{ jws, jwks_url }`). `jws` is a compact Ed25519-signed JWS attesting `contentHash`, `sourceCommit`, `builtAt`, and freshness (`confidence`: `high`/`medium`/`low`/`unknown`). Returns 404 when `PROVENANCE_ENVELOPE_ENABLED` flag is OFF or slug unknown. Anonymous Express route — `@requires` not applicable. Pair: `/.well-known/tutorial-provenance/jwks.json`. Requires `PROVENANCE_SIGNING_KEY` credstore secret (Ed25519 PKCS8 PEM). | None | | `/content/hashes` | GET | SHA-256 map of active content (`{ slug: hash }`) | None | | `/content/nav` | GET | Navigation metadata for published tutorials | None | | `/content/publish` | POST | **Deprecated** — single-shot publish (base64-gzipped files). Kept for one release cycle; new clients use the chunked protocol below. | Bearer (`CONTENT_API_KEY`) | diff --git a/docs/developers/reference/tutorials-ims-gotchas.md b/docs/developers/reference/tutorials-ims-gotchas.md index bb4a08286..e672b5c78 100644 --- a/docs/developers/reference/tutorials-ims-gotchas.md +++ b/docs/developers/reference/tutorials-ims-gotchas.md @@ -147,3 +147,25 @@ All default OFF and DEV-only unless noted. Toggles fail-open on every fault path ## Freshness detector - **Freshness detector grounding needs the corpus-embedding backfill** — the `checkFreshness`/`freshness-scan` engine cosine-searches `ApiDocs`/`Samples` embeddings. Those columns are populated by `srv/jobs/freshness-corpus-embedding-job.js` (nightly `17 3` + on-demand `runJob`). Until it runs in an env, grounding returns nothing and every API-obsolescence claim degrades to `confidence: Low` (fail-open, by design). LLM calls use the SAP AI SDK directly (`@sap-ai-sdk/orchestration`, forced tool-call), NOT `@cap-js/ai`; unit tests inject `globalThis.__FRESHNESS_TEST_IMPL__`. Bulk scan gated by `FRESHNESS_SCAN_ENABLED` (default OFF). **Tutorial markdown is sourced from `ContentFiles.sourceContent` via `getTutorialSource(slug)` in `srv/lib/content-store.js` — NOT from `Steps.description`** (Steps are never populated with step markdown; reading Steps would yield nothing). Findings carry a **global `codeBlockIndex`** across the whole-tutorial markdown — per-step attribution is deferred because the persisted source is not split per step. + +## Signed provenance envelope (issue #2245) + +- **`PROVENANCE_ENVELOPE_ENABLED` is a DB config flag (ImsConfig), default OFF, DEV-first** — controlled via `ImsConfig` key `flag.provenance.envelope` (registered in `srv/lib/feature-flags/registry.js`). When OFF, `GET /content/tutorials/:slug/provenance` returns 404 and `GET /.well-known/tutorial-provenance/jwks.json` returns 404. No env var alternative; never store the signing key as an env var directly (use the credstore — see below). Flip via `/admin-ui/#featureFlags` (DEV); confirm PROD behaviour before enabling there. + +- **Two anonymous public endpoints** — both are plain Express routes registered in `srv/server.js`, intentionally outside any CAP service. `@requires` / `@restrict` do not apply. Both are read-only content-distribution endpoints, safe to serve unauthenticated: + - `GET /content/tutorials/:slug/provenance` — returns `{ jws, jwks_url }`. `jws` is a compact Ed25519-signed JWS (JWT serialisation via `jose`'s `SignJWT`) whose payload attests `{ sub, contentHash, sourceCommit, builtAt, freshness: { confidence, runAt, openHighCount, openMediumCount } }`. Returns 404 when the flag is OFF, 404 when the slug is unknown, or 503 on signing failure (fail-open: content still serves normally). + - `GET /.well-known/tutorial-provenance/jwks.json` — returns the Ed25519 JWKS `{ keys: [...] }` for out-of-band JWS verification. Returns 404 when the flag is OFF. Key ID (`kid`) in the JWKS matches the `kid` header in every issued JWS, enabling key rotation without re-verifying old tokens. + +- **`PROVENANCE_SIGNING_KEY` credstore secret** — the Ed25519 private key (PKCS8 PEM format) is stored in the target environment's BTP Credential Store as `PROVENANCE_SIGNING_KEY` and surfaced as an env var at runtime by the credstore binding. **Never commit a key or paste one into `.mtaext`, env files, or source.** Rotation: generate a new key (see below), store it via `/admin-ui/#secrets` (the per-env credstore rotation flow), then `cf restart tutorials-srv` — the new `kid` propagates to the JWKS automatically on next request. Old JWS tokens signed with the retired key will fail verification once the key is removed from the JWKS; that is expected. + +- **Generating a DEV signing key** — run locally and copy the PEM output, then paste it into `/admin-ui/#secrets` as `PROVENANCE_SIGNING_KEY` on the target env. Never write the output to a file you might commit. + + ```bash + node -e "import('jose').then(async j=>{const {privateKey}=await j.generateKeyPair('EdDSA',{crv:'Ed25519',extractable:true});console.log(await j.exportPKCS8(privateKey))})" + ``` + +- **Freshness confidence values** — derived by `deriveConfidence` in `srv/lib/provenance-freshness.js`. Possible values: `high` (freshness report DONE within 30 days, no open findings), `medium` (DONE but 30–90 days old or has open medium-severity findings), `low` (DONE but > 90 days old or any open high-severity finding), `unknown` (no freshness report or report status not DONE). Corpus-embedding backfill must have run before any value other than `unknown` can be returned — see "Freshness detector" above. + +- **Advisory headers on the HTML serve path** — when the flag is ON, `GET /content/tutorials/:slug` also sets `X-Freshness-Confidence: ` and `X-Content-Provenance: ` on the HTML response. These are advisory only; caching behaviour is unchanged. Any error deriving the advisory is swallowed silently (fail-open). + +- **`sourceCommit` plumbing** — `ContentCurrent.sourceCommit` (String(64)) is populated by the publish pipeline via the `source_commit` field in the publish payload; `scripts/fetch-tutorials.ts` threads the HEAD commit SHA through to the publish client. On older published content the column is NULL; the provenance JWS payload will carry `sourceCommit: null` in that case, which is valid. diff --git a/docs/superpowers/plans/2026-09-11-signed-provenance-envelope.md b/docs/superpowers/plans/2026-09-11-signed-provenance-envelope.md new file mode 100644 index 000000000..5a2fc559c --- /dev/null +++ b/docs/superpowers/plans/2026-09-11-signed-provenance-envelope.md @@ -0,0 +1,959 @@ +# Signed Tutorial Provenance & Freshness Attestation — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Serve a per-tutorial cryptographically signed envelope (compact JWS / EdDSA) carrying separate provenance and freshness claims, plus advisory headers on the HTML serve path, so any third-party AI agent can verify and down-weight stale SAP tutorial content. + +**Architecture:** New `srv/lib/provenance-*.js` modules build claims from the existing `FreshnessReport` + `ContentCurrent` rows, sign them with `jose` using an Ed25519 key loaded from the BTP Credential Store, and cache the JWS by `(contentHash, runAt)`. A sibling `/content/tutorials/:slug/provenance` route serves the JWS; a `/.well-known/tutorial-provenance/jwks.json` route publishes the public key. Advisory (unsigned) headers are added inside `serveStoredSlug`. Everything is DB-flag-gated and fail-open. + +**Tech Stack:** CAP Node.js (`@sap/cds` 10), Express (bootstrap block in `srv/server.js`), `jose` 6.2.3 (EdDSA / JWS / JWKS), `node:crypto`, HANA/SQLite via CDS, vitest 4 + `@cap-js/cds-test`. + +**Spec:** `docs/superpowers/specs/2026-09-11-2245-signed-provenance-envelope-design.md` + +## Global Constraints + +- **Node floor `>=22.12`; ESM only** (`"type":"module"`) — use `import`. +- **Feature flags are DB config, never env** — register in `srv/lib/feature-flags/registry.js` (`kind:'db'`), read via synchronous `isFlagEnabled(key)` from `srv/lib/feature-flags/db-flags.js`. A drift test fails the build if a flag is used without a registry entry. +- **Fail-open everywhere** — any signing/key/lookup error serves content normally; the provenance endpoint returns 503; nothing throws into the content path. +- **No new runtime dependency** — `jose` is already in `dependencies`; do not add crypto libs. +- **Secrets never in source/env-committed** — the Ed25519 private key comes from the BTP Credential Store, surfaced at runtime as env `PROVENANCE_SIGNING_KEY` (PKCS8 PEM). +- **Slugs are lowercase-canonical** — `.toLowerCase()` before any slug comparison/lookup. +- **Thresholds are module constants for v1:** `FRESH_MAX_AGE_DAYS = 30`, `STALE_AGE_DAYS = 90`, `ATTESTATION_TTL_SECONDS = 86400`. +- **`srv/lib/` change → re-audit `srv-qa` `cp` list** in `.deploy/mta.yaml` for any new transitive `./` import reachable from `content-store.js`. + +--- + +### Task 1: Register the `PROVENANCE_ENVELOPE_ENABLED` feature flag + +**Files:** +- Modify: `srv/lib/feature-flags/registry.js` (add entry near `FRESHNESS_SCAN_ENABLED`) +- Test: `test/unit/feature-flags-registry.test.js` (existing drift test — must stay green) + +**Interfaces:** +- Produces: registry key `'PROVENANCE_ENVELOPE_ENABLED'` (ImsConfig key `flag.provenance.envelope`), read via `isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED')`. + +- [ ] **Step 1: Run the existing registry drift test to confirm baseline green** + +Run: `npx vitest run --project unit test/unit/feature-flags-registry.test.js` +Expected: PASS. + +- [ ] **Step 2: Add the flag entry** + +In `srv/lib/feature-flags/registry.js`, next to the `FRESHNESS_SCAN_ENABLED` entry, add: + +```js +{ + key: 'PROVENANCE_ENVELOPE_ENABLED', label: 'Signed provenance & freshness envelope', category: 'Content', + kind: 'db', imsConfigKey: 'flag.provenance.envelope', + valueType: 'boolean', default: false, status: 'dev-only', + description: 'When true, serves the signed provenance JWS at /content/tutorials/:slug/provenance, publishes the JWKS at /.well-known/tutorial-provenance/jwks.json, and emits advisory X-Freshness-Confidence / X-Content-Provenance headers. DB-driven config (ImsConfig key flag.provenance.envelope); no env var. Default OFF.', + howToChange: featureFlagUpsert('PROVENANCE_ENVELOPE_ENABLED', 'flag.provenance.envelope'), +}, +``` + +- [ ] **Step 3: Run the drift test again** + +Run: `npx vitest run --project unit test/unit/feature-flags-registry.test.js` +Expected: PASS (new flag recognized; no "unregistered flag" failure). + +- [ ] **Step 4: Commit** + +```bash +git add srv/lib/feature-flags/registry.js +git commit -m "feat(2245): register PROVENANCE_ENVELOPE_ENABLED db feature flag" +``` + +--- + +### Task 2: Ed25519 key module — load signer + expose JWKS + +**Files:** +- Create: `srv/lib/provenance-keys.js` +- Test: `test/unit/provenance-keys.test.js` + +**Interfaces:** +- Consumes: env `PROVENANCE_SIGNING_KEY` (PKCS8 PEM, Ed25519). +- Produces: + - `async getSigningKey()` → `{ key: KeyLike, kid: string } | null` (null when key absent/invalid — fail-open signal) + - `async getJwks()` → `{ keys: JWK[] }` (empty `keys: []` when no key) + - test helper `__setKeyForTest(pkcs8Pem | null)` and `__resetKeysForTest()` + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/provenance-keys.test.js +import { describe, it, expect, beforeAll, afterEach } from 'vitest'; +import { generateKeyPair, exportPKCS8, jwtVerify, importJWK } from 'jose'; +import { getSigningKey, getJwks, __setKeyForTest, __resetKeysForTest } from '../../srv/lib/provenance-keys.js'; + +let pem; +beforeAll(async () => { + const { privateKey } = await generateKeyPair('EdDSA', { crv: 'Ed25519', extractable: true }); + pem = await exportPKCS8(privateKey); +}); +afterEach(() => __resetKeysForTest()); + +describe('provenance-keys', () => { + it('returns null signer when no key configured', async () => { + __setKeyForTest(null); + expect(await getSigningKey()).toBeNull(); + expect((await getJwks()).keys).toEqual([]); + }); + + it('loads an Ed25519 signer and publishes a matching public JWK', async () => { + __setKeyForTest(pem); + const signer = await getSigningKey(); + expect(signer).not.toBeNull(); + expect(signer.kid).toMatch(/.+/); + const jwks = await getJwks(); + expect(jwks.keys).toHaveLength(1); + expect(jwks.keys[0]).toMatchObject({ kty: 'OKP', crv: 'Ed25519', use: 'sig', alg: 'EdDSA', kid: signer.kid }); + expect(jwks.keys[0].d).toBeUndefined(); // never leak the private scalar + // round-trip: sign with signer, verify with the published public JWK + const { SignJWT } = await import('jose'); + const jws = await new SignJWT({ t: 1 }).setProtectedHeader({ alg: 'EdDSA', kid: signer.kid }).sign(signer.key); + const pub = await importJWK(jwks.keys[0], 'EdDSA'); + const { payload } = await jwtVerify(jws, pub); + expect(payload.t).toBe(1); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/provenance-keys.test.js` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement `srv/lib/provenance-keys.js`** + +```js +import { importPKCS8, exportJWK, calculateJwkThumbprint } from 'jose'; + +let _testPem; // when set (incl. null), overrides env — for tests only +let _cache; // memoized { key, kid, jwk } | null + +function readPem() { + if (_testPem !== undefined) return _testPem; + return process.env.PROVENANCE_SIGNING_KEY || null; +} + +async function load() { + if (_cache !== undefined) return _cache; + const pem = readPem(); + if (!pem) { _cache = null; return _cache; } + try { + const key = await importPKCS8(pem, 'EdDSA', { extractable: true }); + const priv = await exportJWK(key); + const jwk = { kty: priv.kty, crv: priv.crv, x: priv.x }; // public-only + const kid = await calculateJwkThumbprint(jwk); + _cache = { key, kid, jwk: { ...jwk, use: 'sig', alg: 'EdDSA', kid } }; + } catch (e) { + console.warn('[provenance-keys] failed to load signing key, disabling:', e.message); + _cache = null; + } + return _cache; +} + +export async function getSigningKey() { + const c = await load(); + return c ? { key: c.key, kid: c.kid } : null; +} + +export async function getJwks() { + const c = await load(); + return { keys: c ? [c.jwk] : [] }; +} + +export function __setKeyForTest(pem) { _testPem = pem; _cache = undefined; } +export function __resetKeysForTest() { _testPem = undefined; _cache = undefined; } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/provenance-keys.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/provenance-keys.js test/unit/provenance-keys.test.js +git commit -m "feat(2245): Ed25519 provenance key loader + JWKS export" +``` + +--- + +### Task 3: Freshness confidence derivation (pure function) + +**Files:** +- Create: `srv/lib/provenance-freshness.js` +- Test: `test/unit/provenance-freshness.test.js` + +**Interfaces:** +- Produces: `deriveConfidence({ report, now = Date.now() })` where `report` is `{ status, openHighCount, openMediumCount, runAt } | null`. Returns `'high' | 'medium' | 'low' | 'unknown'`. Also exports constants `FRESH_MAX_AGE_DAYS = 30`, `STALE_AGE_DAYS = 90`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/provenance-freshness.test.js +import { describe, it, expect } from 'vitest'; +import { deriveConfidence } from '../../srv/lib/provenance-freshness.js'; + +const DAY = 86400000; +const now = Date.UTC(2026, 8, 11); +const ago = d => new Date(now - d * DAY).toISOString(); + +describe('deriveConfidence', () => { + it('unknown when no report', () => { + expect(deriveConfidence({ report: null, now })).toBe('unknown'); + }); + it('unknown when report not DONE', () => { + expect(deriveConfidence({ report: { status: 'FAILED', openHighCount: 0, openMediumCount: 0, runAt: ago(1) }, now })).toBe('unknown'); + }); + it('high: fresh scan, no high, no medium', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: ago(10) }, now })).toBe('high'); + }); + it('medium: clean but aging (30-90d)', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: ago(45) }, now })).toBe('medium'); + }); + it('medium: fresh scan but only medium findings', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 2, runAt: ago(5) }, now })).toBe('medium'); + }); + it('low: any open high finding', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 1, openMediumCount: 0, runAt: ago(1) }, now })).toBe('low'); + }); + it('low: scan older than 90d even if clean', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: ago(120) }, now })).toBe('low'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/provenance-freshness.test.js` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement `srv/lib/provenance-freshness.js`** + +```js +export const FRESH_MAX_AGE_DAYS = 30; +export const STALE_AGE_DAYS = 90; +const DAY = 86400000; + +export function deriveConfidence({ report, now = Date.now() }) { + if (!report || report.status !== 'DONE' || !report.runAt) return 'unknown'; + const ageDays = (now - new Date(report.runAt).getTime()) / DAY; + const high = report.openHighCount || 0; + const medium = report.openMediumCount || 0; + if (high > 0) return 'low'; + if (ageDays > STALE_AGE_DAYS) return 'low'; + if (ageDays > FRESH_MAX_AGE_DAYS || medium > 0) return 'medium'; + return 'high'; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/provenance-freshness.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/provenance-freshness.js test/unit/provenance-freshness.test.js +git commit -m "feat(2245): freshness confidence derivation" +``` + +> **Note for Task 6:** `FreshnessReport` has `openHighCount` but NOT `openMediumCount` today. Task 6 computes `openMediumCount` from `FreshnessFinding` rows (disposition OPEN, severity Medium) at read time, since it is not stored. The derivation treats a missing `openMediumCount` as `0`. + +--- + +### Task 4: `sourceCommit` schema column + publish append plumbing + +**Files:** +- Modify: `db/_content-shape.cds` (add `sourceCommit` to `ContentFilesAspect`, `ContentCurrentAspect`, `ContentHistoryAspect`) +- Modify: `srv/lib/content-publish-session.js` (accept `sourceCommits`, stamp onto ContentFiles row + carry to ContentCurrent) +- Modify: `srv/lib/content-store.js` (`appendHandler` — destructure + forward `sourceCommits`) +- Test: `test/lib/content-store.test.js` (extend publish/append test) + +**Interfaces:** +- Consumes: append payload gains optional `sourceCommits: Record`. +- Produces: `ContentCurrent.sourceCommit` (String(64)) populated per slug; read by Task 6. + +- [ ] **Step 1: Add the schema column** + +In `db/_content-shape.cds`, add to `ContentFilesAspect`, `ContentCurrentAspect`, and `ContentHistoryAspect` (after `sourceHash`): + +```cds + sourceCommit : String(64); // git commit SHA of source .md at publish time (#2245); null for pre-2245 rows +``` + +- [ ] **Step 2: Verify the model still compiles** + +Run: `npx cds compile db/ srv/ > /dev/null && echo OK` +Expected: `OK` (no compile error). + +- [ ] **Step 3: Write the failing test** + +Add to `test/lib/content-store.test.js` inside the `POST /content/publish` describe (mirror the existing append flow — this repo uses the session begin/append/commit endpoints; follow the existing append test in this file for the exact begin/commit calls): + +```js +it('persists sourceCommit onto ContentCurrent when supplied', async () => { + const { ContentCurrent } = cds.entities('com.sap.developers.ims'); + const slug = 'commit-tutorial'; + const sha = 'a'.repeat(40); + // begin → append(files + sourceCommits) → commit, per the existing append helper in this file + await publishViaSession({ files: { [slug]: '

x

' }, sourceCommits: { [slug]: sha } }); + const row = await SELECT.one.from(ContentCurrent).where({ slug }); + expect(row.sourceCommit).toBe(sha); +}); +``` + +(If no `publishViaSession` helper exists, inline the begin/append/commit `project.axios.post` calls the sibling append test already uses, adding `sourceCommits` to the append body.) + +- [ ] **Step 4: Run test to verify it fails** + +Run: `npx vitest run --project unit test/lib/content-store.test.js -t sourceCommit` +Expected: FAIL (`sourceCommit` undefined / column absent in payload path). + +- [ ] **Step 5: Thread `sourceCommits` through the handler and session** + +In `srv/lib/content-store.js` `appendHandler` (~line 1935), add `sourceCommits` to the destructure and forward it: + +```js +const { sessionId, files, metadata, bodyTexts, branchSpecs, sources, sourceCommits } = req.body || {}; +... +const result = await sessionHelpers.appendToSession({ sessionId, files, metadata, bodyTexts, branchSpecs, sources, sourceCommits }); +``` + +In `srv/lib/content-publish-session.js` `appendToSession` (~line 150), accept `sourceCommits = {}` and stamp it on the per-slug ContentFiles entry (~lines 186–196): + +```js +entries.push({ + slug, version: session.version, content: compressed, contentHash, + sizeBytes: decompressed.length, compressedBytes: compressed.length, + mimeType: 'text/html', sourceContent, sourceHash, + sourceCommit: sourceCommits[slug] || null, +}); +``` + +Then ensure the ContentFiles→ContentCurrent promotion copies `sourceCommit`. Locate the ContentCurrent upsert in `content-publish-session.js` (the mutable-current write, Option B) and add `sourceCommit` to its column set the same way `sourceHash` is carried. + +- [ ] **Step 6: Run test to verify it passes** + +Run: `npx vitest run --project unit test/lib/content-store.test.js -t sourceCommit` +Expected: PASS. + +- [ ] **Step 7: Deploy-check the migration compiles for production** + +Run: `npx cds build --production > /dev/null && echo BUILD_OK` +Expected: `BUILD_OK` (confirms the new column generates a clean migration table; do NOT hand-author `.hdbmigrationtable`). + +- [ ] **Step 8: Commit** + +```bash +git add db/_content-shape.cds srv/lib/content-publish-session.js srv/lib/content-store.js test/lib/content-store.test.js +git commit -m "feat(2245): persist per-slug sourceCommit through publish append" +``` + +--- + +### Task 5: Provenance envelope builder + signer + cache + +**Files:** +- Create: `srv/lib/provenance-envelope.js` +- Test: `test/unit/provenance-envelope.test.js` + +**Interfaces:** +- Consumes: `getSigningKey()` (Task 2), `deriveConfidence()` (Task 3). +- Produces: `async buildEnvelope({ slug, contentHash, sourceCommit, builtAt, report, now })` → `{ jws, claims } | null` (null = fail-open / no key). Signs a compact JWS (JWT) with claims per spec. Caches the JWS keyed by `${slug}:${contentHash}:${report?.runAt || 'none'}`. Exposes `__clearEnvelopeCacheForTest()`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/provenance-envelope.test.js +import { describe, it, expect, beforeAll, afterEach } from 'vitest'; +import { generateKeyPair, exportPKCS8, importJWK, jwtVerify, decodeJwt } from 'jose'; +import { buildEnvelope, __clearEnvelopeCacheForTest } from '../../srv/lib/provenance-envelope.js'; +import { getJwks, __setKeyForTest, __resetKeysForTest } from '../../srv/lib/provenance-keys.js'; + +let pem; +beforeAll(async () => { + const { privateKey } = await generateKeyPair('EdDSA', { crv: 'Ed25519', extractable: true }); + pem = await exportPKCS8(privateKey); +}); +afterEach(() => { __resetKeysForTest(); __clearEnvelopeCacheForTest(); }); + +const base = { + slug: 'my-tutorial', contentHash: 'c'.repeat(64), sourceCommit: 'a'.repeat(40), + builtAt: '2026-09-10T00:00:00.000Z', + report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: '2026-09-05T00:00:00.000Z', model: 'gpt-x' }, + now: Date.UTC(2026, 8, 11), +}; + +describe('buildEnvelope', () => { + it('returns null when no signing key (fail-open)', async () => { + __setKeyForTest(null); + expect(await buildEnvelope(base)).toBeNull(); + }); + + it('signs a verifiable JWS with two distinct claim groups', async () => { + __setKeyForTest(pem); + const { jws, claims } = await buildEnvelope(base); + const pub = await importJWK((await getJwks()).keys[0], 'EdDSA'); + const { payload } = await jwtVerify(jws, pub, { issuer: 'https://developers.sap.com' }); + expect(payload.sub).toBe('my-tutorial'); + expect(payload.contentHash).toBe(base.contentHash); + expect(payload.provenance).toMatchObject({ sourceRepo: 'sap-tutorials/Tutorials', sourceCommit: base.sourceCommit, builtAt: base.builtAt }); + expect(payload.freshness).toMatchObject({ confidence: 'high', lastScanned: base.report.runAt, openHighCount: 0, detectorModel: 'gpt-x' }); + expect(payload.exp - payload.iat).toBe(86400); + expect(claims.freshness.confidence).toBe('high'); + }); + + it('tampered payload fails verification', async () => { + __setKeyForTest(pem); + const { jws } = await buildEnvelope(base); + const pub = await importJWK((await getJwks()).keys[0], 'EdDSA'); + const [h, , s] = jws.split('.'); + const forged = Buffer.from(JSON.stringify({ ...decodeJwt(jws), contentHash: 'f'.repeat(64) })).toString('base64url'); + await expect(jwtVerify(`${h}.${forged}.${s}`, pub)).rejects.toThrow(); + }); + + it('emits unknown confidence + null sourceCommit honestly', async () => { + __setKeyForTest(pem); + const { claims } = await buildEnvelope({ ...base, sourceCommit: null, report: null }); + expect(claims.freshness.confidence).toBe('unknown'); + expect(claims.provenance.sourceCommit).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/provenance-envelope.test.js` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement `srv/lib/provenance-envelope.js`** + +```js +import { SignJWT } from 'jose'; +import { getSigningKey } from './provenance-keys.js'; +import { deriveConfidence } from './provenance-freshness.js'; + +const ISS = 'https://developers.sap.com'; +const SOURCE_REPO = 'sap-tutorials/Tutorials'; +const TTL_SECONDS = 86400; +const _cache = new Map(); // key -> { jws, claims } + +function cacheKey({ slug, contentHash, report }) { + return `${slug}:${contentHash}:${report?.runAt || 'none'}`; +} + +export async function buildEnvelope({ slug, contentHash, sourceCommit, builtAt, report, now = Date.now() }) { + const signer = await getSigningKey(); + if (!signer) return null; // fail-open: no key configured + + const key = cacheKey({ slug, contentHash, report }); + const hit = _cache.get(key); + if (hit && hit.claims.exp * 1000 > now) return hit; + + const iat = Math.floor(now / 1000); + const claims = { + iss: ISS, sub: slug, iat, exp: iat + TTL_SECONDS, + contentHash, + provenance: { sourceRepo: SOURCE_REPO, sourceCommit: sourceCommit ?? null, builtAt: builtAt ?? null }, + freshness: { + confidence: deriveConfidence({ report, now }), + lastScanned: report?.runAt ?? null, + openHighCount: report?.openHighCount ?? 0, + detectorModel: report?.model ?? null, + }, + }; + + try { + const { iss, sub, iat: _i, exp, ...rest } = claims; + const jws = await new SignJWT(rest) + .setProtectedHeader({ alg: 'EdDSA', kid: signer.kid, typ: 'application/tutorial-provenance+jws' }) + .setIssuer(ISS).setSubject(slug).setIssuedAt(iat).setExpirationTime(claims.exp) + .sign(signer.key); + const envelope = { jws, claims }; + _cache.set(key, envelope); + return envelope; + } catch (e) { + console.warn('[provenance-envelope] signing failed, fail-open:', e.message); + return null; + } +} + +export function __clearEnvelopeCacheForTest() { _cache.clear(); } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/provenance-envelope.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/provenance-envelope.js test/unit/provenance-envelope.test.js +git commit -m "feat(2245): signed provenance envelope builder + cache" +``` + +--- + +### Task 6: Serve-time data loader (slug → contentHash + sourceCommit + report) + +**Files:** +- Create: `srv/lib/provenance-data.js` +- Test: `test/unit/provenance-data.test.js` (uses `cds.test` in-memory) + +**Interfaces:** +- Produces: `async loadProvenanceInputs(slug)` → `{ contentHash, sourceCommit, builtAt, report } | null` (null when the tutorial content row is absent). `report` shape: `{ status, openHighCount, openMediumCount, runAt, model } | null`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/provenance-data.test.js +import { describe, it, expect, beforeAll, beforeEach } from 'vitest'; +import cds from '@sap/cds'; +import { loadProvenanceInputs } from '../../srv/lib/provenance-data.js'; + +const project = cds.test('serve', '--project', '.', '--in-memory'); + +describe('loadProvenanceInputs', () => { + let ContentCurrent, Tutorials, FreshnessReport, FreshnessFinding; + beforeAll(() => { ({ ContentCurrent, Tutorials, FreshnessReport, FreshnessFinding } = cds.entities('com.sap.developers.ims')); }); + beforeEach(async () => { + await DELETE.from(ContentCurrent); await DELETE.from(FreshnessFinding); + await DELETE.from(FreshnessReport); await DELETE.from(Tutorials); + }); + + it('returns null for unknown slug', async () => { + expect(await loadProvenanceInputs('nope')).toBeNull(); + }); + + it('joins content row, source commit, and current freshness report', async () => { + const tid = cds.utils.uuid(); + await INSERT.into(Tutorials).entries({ ID: tid, slug: 'demo' }); + await INSERT.into(ContentCurrent).entries({ slug: 'demo', contentHash: 'h'.repeat(64), sourceCommit: 'a'.repeat(40), modifiedAt: '2026-09-10T00:00:00.000Z' }); + await INSERT.into(FreshnessReport).entries({ ID: cds.utils.uuid(), tutorial_ID: tid, status: 'DONE', openHighCount: 0, runAt: '2026-09-05T00:00:00.000Z', model: 'm1' }); + const out = await loadProvenanceInputs('demo'); + expect(out.contentHash).toBe('h'.repeat(64)); + expect(out.sourceCommit).toBe('a'.repeat(40)); + expect(out.report).toMatchObject({ status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: '2026-09-05T00:00:00.000Z', model: 'm1' }); + }); + + it('counts open medium findings', async () => { + const tid = cds.utils.uuid(); + await INSERT.into(Tutorials).entries({ ID: tid, slug: 'demo2' }); + await INSERT.into(ContentCurrent).entries({ slug: 'demo2', contentHash: 'h'.repeat(64) }); + const rid = cds.utils.uuid(); + await INSERT.into(FreshnessReport).entries({ ID: rid, tutorial_ID: tid, status: 'DONE', openHighCount: 0, runAt: '2026-09-05T00:00:00.000Z' }); + await INSERT.into(FreshnessFinding).entries([ + { ID: cds.utils.uuid(), report_ID: rid, tutorial_ID: tid, severity: 'Medium', disposition: 'OPEN' }, + { ID: cds.utils.uuid(), report_ID: rid, tutorial_ID: tid, severity: 'Medium', disposition: 'DISMISSED' }, + ]); + const out = await loadProvenanceInputs('demo2'); + expect(out.report.openMediumCount).toBe(1); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/provenance-data.test.js` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement `srv/lib/provenance-data.js`** + +```js +import cds from '@sap/cds'; + +export async function loadProvenanceInputs(rawSlug) { + const slug = String(rawSlug || '').toLowerCase(); + const { ContentCurrent, Tutorials, FreshnessReport, FreshnessFinding } = cds.entities('com.sap.developers.ims'); + try { + const content = await SELECT.one.from(ContentCurrent).columns('contentHash', 'sourceCommit', 'modifiedAt').where({ slug }); + if (!content) return null; + + const tut = await SELECT.one.from(Tutorials).columns('ID').where({ slug }); + let report = null; + if (tut) { + const rep = await SELECT.one.from(FreshnessReport) + .columns('status', 'openHighCount', 'runAt', 'model').where({ tutorial_ID: tut.ID }); + if (rep) { + const med = await SELECT.one.from(FreshnessFinding) + .columns('count(*) as n').where({ tutorial_ID: tut.ID, severity: 'Medium', disposition: 'OPEN' }); + report = { status: rep.status, openHighCount: rep.openHighCount || 0, openMediumCount: med?.n || 0, runAt: rep.runAt, model: rep.model }; + } + } + return { contentHash: content.contentHash, sourceCommit: content.sourceCommit || null, builtAt: content.modifiedAt || null, report }; + } catch (e) { + console.warn('[provenance-data] load failed, fail-open:', e.message); + return null; + } +} +``` + +> **Executor note:** confirm the `Tutorials` slug column name and that `ContentCurrent` exposes `modifiedAt` (from `managed`). If `count(*) as n` misbehaves under the CI Node/CDS combo, use `cds.entities(NS)` (already done) and fall back to fetching finding rows and counting in JS — see memory `ci-node-version-mismatch`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/provenance-data.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add srv/lib/provenance-data.js test/unit/provenance-data.test.js +git commit -m "feat(2245): serve-time provenance input loader" +``` + +--- + +### Task 7: Provenance endpoint + JWKS route + +**Files:** +- Modify: `srv/server.js` (register two routes in the `cds.on('bootstrap')` block) +- Create: `srv/lib/provenance-handlers.js` (route handlers) +- Test: `test/lib/provenance-endpoint.test.js` (`cds.test` HTTP) + +**Interfaces:** +- Consumes: `isFlagEnabled` (Task 1), `buildEnvelope` (Task 5), `loadProvenanceInputs` (Task 6), `getJwks` (Task 2). +- Produces: `provenanceHandler(req,res)` and `jwksHandler(req,res)` exported from `provenance-handlers.js`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/lib/provenance-endpoint.test.js +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; +import cds from '@sap/cds'; +import { generateKeyPair, exportPKCS8, importJWK, jwtVerify } from 'jose'; +import { __setKeyForTest, __resetKeysForTest } from '../../srv/lib/provenance-keys.js'; +import { __setFlagForTest, __resetFlagsForTest } from '../../srv/lib/feature-flags/db-flags.js'; + +const project = cds.test('serve', '--project', '.', '--in-memory'); + +describe('provenance endpoint', () => { + let ContentCurrent, Tutorials; + beforeAll(async () => { + ({ ContentCurrent, Tutorials } = cds.entities('com.sap.developers.ims')); + const { privateKey } = await generateKeyPair('EdDSA', { crv: 'Ed25519', extractable: true }); + __setKeyForTest(await exportPKCS8(privateKey)); + }); + afterAll(() => { __resetKeysForTest(); __resetFlagsForTest(); }); + beforeEach(async () => { + await DELETE.from(ContentCurrent); await DELETE.from(Tutorials); + await INSERT.into(Tutorials).entries({ ID: cds.utils.uuid(), slug: 'demo' }); + await INSERT.into(ContentCurrent).entries({ slug: 'demo', contentHash: 'h'.repeat(64), sourceCommit: 'a'.repeat(40) }); + }); + + it('404s when flag OFF', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', false); + await expect(project.axios.get('/content/tutorials/demo/provenance')).rejects.toMatchObject({ response: { status: 404 } }); + }); + + it('serves a verifiable JWS when flag ON', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', true); + const res = await project.axios.get('/content/tutorials/demo/provenance'); + expect(res.status).toBe(200); + const jwks = await project.axios.get('/.well-known/tutorial-provenance/jwks.json'); + const pub = await importJWK(jwks.data.keys[0], 'EdDSA'); + const { payload } = await jwtVerify(res.data.jws, pub, { issuer: 'https://developers.sap.com' }); + expect(payload.sub).toBe('demo'); + expect(payload.provenance.sourceCommit).toBe('a'.repeat(40)); + }); + + it('404s for unknown slug when flag ON', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', true); + await expect(project.axios.get('/content/tutorials/nope/provenance')).rejects.toMatchObject({ response: { status: 404 } }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/lib/provenance-endpoint.test.js` +Expected: FAIL (routes not registered → likely wildcard swallow / 200 HTML or 404 with wrong body). + +- [ ] **Step 3: Implement `srv/lib/provenance-handlers.js`** + +```js +import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { buildEnvelope } from './provenance-envelope.js'; +import { loadProvenanceInputs } from './provenance-data.js'; +import { getJwks } from './provenance-keys.js'; + +export async function provenanceHandler(req, res) { + if (!isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED')) return res.status(404).end(); + const slug = String(req.params.slug || '').toLowerCase(); + const inputs = await loadProvenanceInputs(slug); + if (!inputs) return res.status(404).json({ error: 'not_found' }); + const envelope = await buildEnvelope({ slug, ...inputs }); + if (!envelope) return res.status(503).json({ error: 'attestation_unavailable' }); + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'public, max-age=60, s-maxage=600'); + res.json({ jws: envelope.jws, jwks_url: '/.well-known/tutorial-provenance/jwks.json' }); +} + +export async function jwksHandler(req, res) { + if (!isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED')) return res.status(404).end(); + const jwks = await getJwks(); + res.setHeader('Content-Type', 'application/jwk-set+json; charset=utf-8'); + res.setHeader('Cache-Control', 'public, max-age=300, s-maxage=3600'); + res.json(jwks); +} +``` + +- [ ] **Step 4: Register the routes in `srv/server.js`** + +In the `cds.on('bootstrap')` block, import the handlers at the top (near line 32) and register the provenance route **before** the `app.get('/content/tutorials/*slug', serveHandler)` wildcard (~line 760), and the JWKS route alongside the other `/.well-known/*` handlers (~line 1034): + +```js +// before the /content/tutorials/*slug wildcard: +app.get('/content/tutorials/:slug/provenance', provenanceHandler); +// alongside other .well-known routes: +app.get('/.well-known/tutorial-provenance/jwks.json', jwksHandler); +``` + +> **Executor note:** Express 5 route ordering — the `:slug/provenance` path has an extra segment so it will not collide with the single-segment `.md` regex, but it MUST precede the `*slug` wildcard or the wildcard captures `demo/provenance` as the slug. Verify with the test. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `npx vitest run --project unit test/lib/provenance-endpoint.test.js` +Expected: PASS. + +- [ ] **Step 6: srv-qa cp-list audit** + +`provenance-handlers.js` is now reachable from `server.js` but NOT from `content-store.js`, so it is not in the content-store transitive set. Still, confirm `.deploy/mta.yaml` `srv-qa` `cp` list includes the four new `srv/lib/provenance-*.js` files if QA boots `server.js`. Add any missing ones. + +- [ ] **Step 7: Commit** + +```bash +git add srv/lib/provenance-handlers.js srv/server.js test/lib/provenance-endpoint.test.js .deploy/mta.yaml +git commit -m "feat(2245): provenance JWS endpoint + JWKS route" +``` + +--- + +### Task 8: Advisory headers on the HTML serve path + +**Files:** +- Modify: `srv/lib/content-store.js` (`ContentCache.set/get` to carry advisory meta; both 200 branches of `serveStoredSlug`) +- Test: `test/lib/provenance-headers.test.js` (`cds.test` HTTP) + +**Interfaces:** +- Consumes: `isFlagEnabled` (Task 1), `loadProvenanceInputs` + `deriveConfidence` (Tasks 3/6). +- Produces: `X-Freshness-Confidence` and `X-Content-Provenance` headers on `/content/tutorials/:slug` (both cache-miss and cache-hit paths) when the flag is ON. + +- [ ] **Step 1: Write the failing test** + +```js +// test/lib/provenance-headers.test.js +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; +import cds from '@sap/cds'; +import { gzipSync } from 'node:zlib'; +import { __setFlagForTest, __resetFlagsForTest } from '../../srv/lib/feature-flags/db-flags.js'; + +const project = cds.test('serve', '--project', '.', '--in-memory'); +const b64gz = html => gzipSync(Buffer.from(html)).toString('base64'); + +describe('advisory provenance headers', () => { + beforeAll(() => { process.env.CONTENT_API_KEY = 'k'; }); + afterAll(() => __resetFlagsForTest()); + beforeEach(async () => { + const { ContentCurrent, ContentFiles, ContentManifest } = cds.entities('com.sap.developers.ims'); + await DELETE.from(ContentCurrent); await DELETE.from(ContentFiles); await DELETE.from(ContentManifest); + // publish 'demo' so it is servable (reuse the publish helper/flow from content-store.test.js) + }); + + it('omits headers when flag OFF', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', false); + const res = await project.axios.get('/content/tutorials/demo'); + expect(res.headers['x-freshness-confidence']).toBeUndefined(); + }); + + it('emits headers on cache-miss AND cache-hit when flag ON', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', true); + const miss = await project.axios.get('/content/tutorials/demo'); // fills LRU + expect(miss.headers['x-freshness-confidence']).toBe('unknown'); + expect(miss.headers['x-content-provenance']).toBe('/content/tutorials/demo/provenance'); + const hit = await project.axios.get('/content/tutorials/demo'); // LRU hit + expect(hit.headers['x-content-source']).toBe('cache'); + expect(hit.headers['x-freshness-confidence']).toBe('unknown'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/lib/provenance-headers.test.js` +Expected: FAIL (headers absent). + +- [ ] **Step 3: Implement — carry advisory meta in the LRU and set headers in both branches** + +In `srv/lib/content-store.js`: +- Extend `ContentCache.set(key, buffer, hash, advisory)` to store `advisory` on the entry (default `null`); `get` returns it. +- Add a small helper near the top: + +```js +import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { loadProvenanceInputs } from './provenance-data.js'; +import { deriveConfidence } from './provenance-freshness.js'; + +async function computeAdvisory(slug) { + if (!isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED')) return null; + try { + const inputs = await loadProvenanceInputs(slug); + if (!inputs) return null; + return { confidence: deriveConfidence({ report: inputs.report }), url: `/content/tutorials/${slug}/provenance` }; + } catch { return null; } +} + +function setAdvisoryHeaders(res, advisory) { + if (!advisory) return; + res.setHeader('X-Freshness-Confidence', advisory.confidence); + res.setHeader('X-Content-Provenance', advisory.url); +} +``` + +- In the fresh-DB-read branch (~lines 1089–1097): compute `const advisory = await computeAdvisory(slug);`, pass it into `cache.set(slug, decompressed, meta.contentHash, advisory)`, and call `setAdvisoryHeaders(res, advisory)` before `res.send`. +- In the cache-hit branch (~lines 1012–1026): call `setAdvisoryHeaders(res, cached.advisory)` before `res.send` (no DB hit — advisory is whatever was cached; refreshes on next TTL miss, acceptable for a hint). + +> **Executor note:** `content-store.js` is the load-bearing content module — keep the new imports lazy-safe. `provenance-data.js` imports `cds` only; no AI SDK. Re-run the `srv-qa` cp-list audit: `provenance-data.js`, `provenance-freshness.js`, and `feature-flags/db-flags.js` are now transitively reachable from `content-store.js` and MUST be in the `srv-qa` `cp` list (`db-flags` already is; add the two `provenance-*` if absent). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/lib/provenance-headers.test.js` +Expected: PASS. + +- [ ] **Step 5: Run the full content-store suite to check no regression** + +Run: `npx vitest run --project unit test/lib/content-store.test.js` +Expected: PASS (existing serve/publish tests unaffected). + +- [ ] **Step 6: Commit** + +```bash +git add srv/lib/content-store.js test/lib/provenance-headers.test.js .deploy/mta.yaml +git commit -m "feat(2245): advisory freshness/provenance headers on tutorial serve" +``` + +--- + +### Task 9: Thread commit SHA from fetch → publish client + +**Files:** +- Modify: `scripts/parsers/github.ts` (surface `lastCommitSha` in the returned metadata already — confirm it reaches `fetch-tutorials.ts`) +- Modify: `scripts/fetch-tutorials.ts` (carry `ghMeta.lastCommitSha` per slug into a commit map written for publish) +- Modify: `scripts/publish-content.ts` (build `sourceCommitsAll`, pass `sourceCommits` in `appendBatch`) +- Modify: `scripts/lib/publish-client.ts` (`appendBatch` forwards `sourceCommits` in the POST body) +- Test: `test/unit/publish-content-source-commit.test.js` (or extend an existing publish-client test) + +**Interfaces:** +- Consumes: `lastCommitSha` (git commit SHA) from `fetchGitHubMeta` (Task-independent; already computed). +- Produces: the append POST body carries `sourceCommits: Record`, consumed by Task 4's `appendHandler`. + +- [ ] **Step 1: Write the failing test** + +```js +// test/unit/publish-content-source-commit.test.js +import { describe, it, expect } from 'vitest'; +import { buildAppendBody } from '../../scripts/lib/publish-client.ts'; + +describe('appendBatch body', () => { + it('includes sourceCommits when provided', () => { + const body = buildAppendBody({ sessionId: 's', files: { a: 'x' }, sourceCommits: { a: 'sha1' } }); + expect(body.sourceCommits).toEqual({ a: 'sha1' }); + }); + it('omits sourceCommits key cleanly when absent', () => { + const body = buildAppendBody({ sessionId: 's', files: { a: 'x' } }); + expect(body.sourceCommits).toBeUndefined(); + }); +}); +``` + +> If `appendBatch` builds its body inline, extract a pure `buildAppendBody(opts)` helper first (small refactor) so it is unit-testable, then have `appendBatch` call it. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run --project unit test/unit/publish-content-source-commit.test.js` +Expected: FAIL (no `buildAppendBody` / no `sourceCommits`). + +- [ ] **Step 3: Implement the threading** + +- `scripts/lib/publish-client.ts`: add/extract `buildAppendBody(opts)` that spreads `sessionId, files, metadata, bodyTexts, branchSpecs, sources` and conditionally `...(opts.sourceCommits ? { sourceCommits: opts.sourceCommits } : {})`. `appendBatch` POSTs `buildAppendBody(...)`. +- `scripts/fetch-tutorials.ts`: where `ghMeta` is obtained per slug (~line 962–974), record `sourceCommits[slug] = ghMeta.lastCommitSha` into a map available to the publish step (persist alongside the existing publish inputs — mirror how `sourceHashes` is surfaced). +- `scripts/publish-content.ts`: build `sourceCommitsAll` (like `sourcesAll`, ~lines 1166–1168) and pass `sourceCommits: pickEntries(sourceCommitsAll, batch)` in the `appendBatch(...)` call (~lines 1188–1199). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run --project unit test/unit/publish-content-source-commit.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/lib/publish-client.ts scripts/fetch-tutorials.ts scripts/publish-content.ts test/unit/publish-content-source-commit.test.js +git commit -m "feat(2245): thread source commit SHA through publish pipeline" +``` + +--- + +### Task 10: Docs + operational wiring + full-suite verification + +**Files:** +- Modify: `docs/developers/reference/tutorials-ims-gotchas.md` (add a "Signed provenance envelope" entry) +- Modify: `CLAUDE.md` "Top Gotchas" (one-line pointer, per repo convention) +- Modify: `docs/developers/operations/testing-endpoints.md` (document the two new public endpoints + the flag) + +**Interfaces:** none (documentation + verification only). + +- [ ] **Step 1: Document the feature** + +Add to `tutorials-ims-gotchas.md`: the `PROVENANCE_ENVELOPE_ENABLED` flag (DB config, DEV-first, default OFF), the two endpoints, the `PROVENANCE_SIGNING_KEY` credstore secret (Ed25519 PKCS8 PEM), key-rotation-via-JWKS note, and the fail-open contract. Add the CLAUDE.md one-liner pointing to it. Document endpoints in `testing-endpoints.md` (anonymous, `@requires` not applicable — Express routes, not a CAP service). + +- [ ] **Step 2: Generate a DEV signing key + record the credstore step** + +Document (do not commit any key) how to generate the key for DEV: + +```bash +node -e "import('jose').then(async j=>{const {privateKey}=await j.generateKeyPair('EdDSA',{crv:'Ed25519',extractable:true});console.log(await j.exportPKCS8(privateKey))})" +``` + +Store the PEM in the target env's BTP Credential Store as `PROVENANCE_SIGNING_KEY` via `/admin-ui/#secrets` (per repo secret-rotation flow). Never in source or `.mtaext`. + +- [ ] **Step 3: Run the full unit suite** + +Run: `npm test` +Expected: PASS (all `--project unit` tests, including the new provenance suites and the registry drift test). + +- [ ] **Step 4: Production build sanity** + +Run: `npx cds build --production > /dev/null && echo BUILD_OK` +Expected: `BUILD_OK` (migration table for `sourceCommit` generated cleanly). + +- [ ] **Step 5: Commit** + +```bash +git add docs/ CLAUDE.md +git commit -m "docs(2245): signed provenance envelope endpoints, flag, key handling" +``` + +--- + +## Self-Review + +**Spec coverage:** +- A (JWS/EdDSA) → Tasks 2, 5. *Note:* implemented as compact JWS (JWT via `SignJWT`) rather than flattened JWS — still standard, still `jose`-verifiable; the spec's "flattened" wording is satisfied by an equivalent compact serialization. +- B (confidence derivation) → Task 3 (+ `openMediumCount` sourced in Task 6). +- C (keys + JWKS) → Tasks 2, 7. +- D (endpoint + advisory headers) → Tasks 7, 8; cache-hit correctness → Task 8 Step 3. +- E (sign-on-first-serve + cache keyed by contentHash+runAt) → Task 5. +- F (sourceCommit plumbing) → Tasks 4 (schema/server) + 9 (fetch/client). +- G (flag + fail-open) → Task 1; fail-open asserted in Tasks 2/5/6/7/8. +- H (tests) → each task is TDD; full suite in Task 10. + +**Placeholder scan:** No TBD/TODO. Two explicit executor-verification notes (Task 6 slug column / `count(*)`, Task 7 route ordering) are confirm-against-reality checks with concrete fallbacks, not placeholders. + +**Type consistency:** `buildEnvelope({slug, contentHash, sourceCommit, builtAt, report, now})` — Task 6's `loadProvenanceInputs` returns exactly `{contentHash, sourceCommit, builtAt, report}`, spread into `buildEnvelope` in Task 7. `report` shape `{status, openHighCount, openMediumCount, runAt, model}` is consistent across Tasks 3/5/6. `getSigningKey()→{key,kid}` and `getJwks()→{keys}` consistent across Tasks 2/5/7. Flag key `'PROVENANCE_ENVELOPE_ENABLED'` consistent across Tasks 1/7/8. diff --git a/docs/superpowers/specs/2026-09-11-2245-signed-provenance-envelope-design.md b/docs/superpowers/specs/2026-09-11-2245-signed-provenance-envelope-design.md new file mode 100644 index 000000000..b7819e299 --- /dev/null +++ b/docs/superpowers/specs/2026-09-11-2245-signed-provenance-envelope-design.md @@ -0,0 +1,145 @@ +# Signed Tutorial Provenance & Freshness Attestation + +- **Issue:** [#2245](https://github.com/sap-tutorials/tutorials-ims/issues/2245) — "Executable, self-verifying, self-healing tutorials" (idea #1: signed freshness/provenance signal) +- **Date:** 2026-09-11 +- **Status:** Design approved (brainstorm); pending spec review → implementation plan +- **Scope:** Idea #1 only. Ideas #2–4 (assert blocks, tutorials-as-Skills, self-healing) are out of scope and depend on this as their trust spine. + +## Problem + +Staleness is the #1 cause of wrong AI SAP advice. When an AI agent fetches a tutorial (HTML or `.md`) to scaffold or answer, the content carries **no machine-readable signal** for the agent to judge how much to trust it or whether it is current. There is also no way for a third party to prove a given tutorial payload genuinely came from SAP unmodified. + +We already own every hard piece: a shipped freshness detector (`srv/lib/freshness-detector.js`), a HANA-backed content serve path with content hashing, a source-commit SHA captured at fetch time, and a `.well-known` middleware seam. This design exposes a **cryptographically signed provenance + freshness attestation** an agent can fetch and verify. + +## Goals + +- A per-tutorial signed envelope any third-party agent can verify **offline** against a published public key, with no shared secret. +- Two **distinct** claims — factual provenance and derived staleness — never collapsed into one opaque score. +- Cheap advisory headers on the existing serve path so an agent doing a `HEAD`/`GET` gets a hint and a pointer without parsing crypto. +- Fail-open: the attestation never blocks or degrades content delivery. + +## Non-goals (YAGNI) + +- No automated **key-rotation job** — the format is rotation-ready (JWKS lists multiple keys) but rotation stays a manual op for now. +- No **per-step** provenance — attestation is per-tutorial. +- No envelopes for homepage / content-pages / concepts — **tutorials only**. +- No signing of the HTML bytes inline (no JSON-LD embed) — the envelope is a separate resource. +- Not idea #2/#3/#4. No executable assertions, no `SKILL.md` generation, no self-healing PRs. + +## Approved design + +### A. Envelope format — JWS / EdDSA (Ed25519) + +Use a standard **flattened JWS** (RFC 7515) with `alg: EdDSA`, `crv: Ed25519`, rather than a bespoke signature. Any consuming agent verifies with an off-the-shelf JOSE library plus the published JWKS — no custom crypto for us to document or for them to reimplement. + +JWS protected header: + +```json +{ "alg": "EdDSA", "kid": "", "typ": "application/tutorial-provenance+jws" } +``` + +Payload (claims): + +```jsonc +{ + "iss": "https://developers.sap.com", + "sub": "", // lowercase-canonical slug + "iat": 1757600000, + "exp": 1757686400, // TTL bounds staleness of the ATTESTATION itself (see E) + "contentHash": "", // binds the signature to exact served bytes (= ETag) + "provenance": { // FACTUAL claim — no judgment + "sourceRepo": "sap-tutorials/Tutorials", + "sourceCommit": "", + "builtAt": "" + }, + "freshness": { // JUDGMENT claim — derived (see B) + "confidence": "high|medium|low|unknown", + "lastScanned": "", + "openHighCount": 0, + "detectorModel": "" // transparency: which LLM judged + } +} +``` + +The two claim groups are kept as separate objects on purpose: `provenance` is verifiable fact; `freshness` is an LLM-derived judgment. An agent may weight them independently (e.g. trust an old-but-clean tutorial while down-weighting a recently-scanned-but-flagged one). + +### B. Freshness confidence derivation + +Deterministic mapping from the latest `FreshnessReport` for the tutorial — **no LLM call at serve time**. Thresholds are constants (candidate values below; confirm in review): + +| confidence | condition | +|-----------|-----------| +| `high` | latest scan ≤ 30d old **and** `openHighCount == 0` **and** no open Medium finding | +| `medium` | clean but aging (scan 30–90d old), **or** only open Low/Medium findings | +| `low` | any open **High** finding, **or** latest scan > 90d old | +| `unknown` | never scanned, no report, or freshness feature disabled | + +`unknown` is honest — we never fake `high` in the absence of a scan. "Open" respects `FreshnessFinding` disposition (ACCEPTED/DISMISSED/FIXED findings do not count against confidence; only OPEN does). + +### C. Keys — Ed25519 + JWKS + +- **Private key** stored in the BTP Credential Store (same seam as `CONTENT_API_KEY`), loaded once at boot. Never in source or committed env. Absent key ⇒ feature fails open (disabled), logged once. +- **`kid`** identifies the signing key in the JWS header. +- **Public keys** published as a JWKS document at `GET /.well-known/tutorial-provenance/jwks.json`, reusing the existing `.well-known` approuter-middleware seam (`docs/.../2026-08-28-well-known-oauth-discovery-design.md`). Anonymous, cacheable. +- **Rotation-ready:** JWKS may list multiple public keys; we sign with the newest `kid`. Old public keys remain published until all envelopes signed under them have expired. No rotation automation built now. + +### D. Endpoint + serve integration + +- **`GET /content/tutorials/:slug/provenance`** → returns the flattened JWS as `application/jose+json` (or a thin wrapper `{ jws, jwks_url }`). Anonymous, CDN-frontable, tagged with the **same `Edge-Cache-Tag`** as the tutorial so it purges together on republish. +- **Advisory (unsigned) headers** added to the existing HTML serve path (`serveStoredSlug`, `srv/lib/content-store.js`): + - `X-Freshness-Confidence: high|medium|low|unknown` + - `X-Content-Provenance: /content/tutorials/:slug/provenance` + These are hints, not trust anchors — an agent that wants assurance fetches and verifies the JWS. +- **Cache-hit correctness:** the advisory values must be stored **inside the LRU-cached object** alongside `{ buffer, hash }`, because the cache-hit branch bypasses the DB read. Otherwise cache hits would silently drop the headers (this is the gotcha flagged during grounding). + +### E. Signing lifecycle + +Sign **on first serve**, cache the JWS keyed by `(contentHash, freshnessReportRunAt)`. Re-sign only when the content changes (new `contentHash`) or the freshness report changes (`runAt` advances). No coupling to the publish transaction for signing. `exp` = `iat + ATTESTATION_TTL` (candidate 24h) so an attestation cannot be replayed indefinitely; expiry forces a re-sign that re-reads current freshness. + +### F. Data plumbing for `sourceCommit` + +The source SHA (`currentSha`) is captured in `scripts/fetch-tutorials.ts` (written to `.tutorial-cache/.sha`, used as the cache-invalidation key) but discarded afterward. Plumb it through: + +1. `fetch-tutorials.ts` — retain `currentSha` per slug into the publish input. +2. `scripts/publish-content.ts` → `POST /content/publish/append` payload — add `sourceCommit`. +3. New `sourceCommit : String(64)` column on the content aspect in `db/_content-shape.cds` (carried by `ContentCurrent` / `ContentManifest`). Migration via `cds build --production`. +4. Pre-existing rows: `sourceCommit` is null ⇒ the `provenance.sourceCommit` claim is emitted as `null` (honest; not fabricated). + +> Review question F: store `sourceCommit` on `ContentCurrent` (per-serve read, no extra query) or on `ContentManifest` only (one row per publish, needs a join at serve)? Default: `ContentCurrent` for cheap serve-time read. + +### G. Feature flag + fail-open + +- DB-config flag **`PROVENANCE_ENVELOPE_ENABLED`** registered in `srv/.../feature-flags/registry.js` (`kind:'db'`, `dev-only` first, default OFF). Per project rule, feature flags are DB config, never env (blue-green drops `cf set-env`). +- Flag OFF ⇒ `/provenance` returns 404, no advisory headers emitted. +- **Fail-open everywhere:** any signing error, missing key, or freshness lookup failure ⇒ content serves normally; the envelope endpoint returns 503; nothing throws into the content path. + +### H. Testing + +Deterministic unit coverage (no live LLM, no live HANA — SQLite/in-memory per project test conventions): + +- Confidence derivation table (each row of B, incl. disposition handling). +- Envelope claim shape and required fields. +- JWS round-trips: verifies against the matching public key; **tamper** (mutated payload) ⇒ verification fails; wrong `kid` ⇒ fails. +- JWKS document shape and that the served public key matches the signing key. +- Flag OFF path (404, no headers); missing report ⇒ `unknown`; null `sourceCommit` ⇒ null claim. +- Cache-hit path still emits advisory headers. + +## Components & seams (existing code to attach to) + +| Concern | Seam | +|---|---| +| Freshness data | `srv/lib/freshness-detector.js`, `db/tutorial-freshness.cds` (`FreshnessReport`/`FreshnessFinding`) | +| Serve path + headers + LRU | `srv/lib/content-store.js` (`serveStoredSlug`), header sites; `srv/server.js:~760` route | +| Content identity | existing `contentHash`/ETag on the served row | +| Source SHA | `scripts/fetch-tutorials.ts` (`currentSha`), `scripts/publish-content.ts`, `/content/publish/append` | +| Schema | `db/_content-shape.cds` (content aspect) | +| Key distribution | `.well-known` middleware seam (2026-08-28 design) | +| Flag | `feature-flags/registry.js` (`kind:'db'`) | +| New module | `srv/lib/provenance-envelope.js` (build + sign + cache), `srv/lib/provenance-keys.js` (key load + JWKS) | + +## Review questions — RESOLVED (defaults approved 2026-09-11) + +1. **B** — thresholds 30d (`high`) / 90d (`low`) and attestation TTL 24h ship as **constants** in the new module (not DB-config for v1). +2. **C** — JWKS served at **`/.well-known/tutorial-provenance/jwks.json`** via the existing `.well-known` middleware seam. +3. **F** — `sourceCommit` stored on **`ContentCurrent`** for cheap serve-time read. +4. **Crypto** — **Node native `crypto`** Ed25519 (`sign`/`verify` with `'Ed25519'`) + minimal flattened-JWS assembly; **no new runtime dependency** (confirm `jose` is not already resolvable at plan time — if it is, reuse it). diff --git a/scripts/check-srv-qa-route-drift.ts b/scripts/check-srv-qa-route-drift.ts index edc0118bd..6a7b06d14 100644 --- a/scripts/check-srv-qa-route-drift.ts +++ b/scripts/check-srv-qa-route-drift.ts @@ -71,6 +71,14 @@ const SRV_QA_SERVER = join(REPO_ROOT, 'srv-qa', 'server.js'); * Format: 'METHOD /path' */ const ALLOWLIST_ONLY_ON_SRV: Record = { + 'GET /content/tutorials/:slug/provenance': + 'Signed provenance envelope (#2245) — an anonymous, public, read-only prod content ' + + 'surface that emits a JWS over PUBLISHED-tutorial freshness/provenance. Feature-flagged ' + + '(PROVENANCE_ENVELOPE_ENABLED, DB config, default OFF, DEV-first) and fail-open. It does ' + + 'not fit the QA channel: srv-qa serves /content/tutorials/*slug entirely behind ' + + 'requireAuthorScope (author-draft preview), the PROVENANCE_SIGNING_KEY credstore secret is ' + + 'not provisioned for srv-qa, and provenance is meaningful only for published content, not ' + + 'in-flight -Contribution drafts. Re-evaluate if QA ever gains a published-content trust surface.', 'POST /content/code-check-specs': 'AI code-check (#171) — gated behind ChatSettings.codeCheckEnabled feature flag; ' + 'not yet wired for QA author-preview. Re-evaluate when credstore-backed ChatSettings ' + diff --git a/scripts/fetch-tutorials.ts b/scripts/fetch-tutorials.ts index 8668d4349..d8f53bb6a 100644 --- a/scripts/fetch-tutorials.ts +++ b/scripts/fetch-tutorials.ts @@ -963,6 +963,12 @@ async function main() { lastUpdated = ghMeta.lastUpdated createdAt = ghMeta.createdAt contributors = ghMeta.contributors + // #2245: persist commit SHA sidecar so publish-content can thread it into + // the append body's sourceCommits map. Uses lowercase-canonical slug to + // match the sidecar convention (see validate-answer.json, codecheck.json). + if (ghMeta.lastCommitSha) { + writeFileSync(join(CACHE_DIR, `${t.slug.toLowerCase()}.commit-sha`), ghMeta.lastCommitSha, 'utf-8') + } cacheHits++ console.log(`${label} [cached]`) } else { @@ -972,6 +978,11 @@ async function main() { lastUpdated = ghMeta.lastUpdated createdAt = ghMeta.createdAt contributors = ghMeta.contributors + // #2245: persist commit SHA sidecar so publish-content can thread it into + // the append body's sourceCommits map. + if (ghMeta.lastCommitSha) { + writeFileSync(join(CACHE_DIR, `${t.slug.toLowerCase()}.commit-sha`), ghMeta.lastCommitSha, 'utf-8') + } if (cacheStatus === 'cached') cacheHits++ else if (cacheStatus === 'refreshed') cacheRefreshes++ diff --git a/scripts/lib/publish-client.ts b/scripts/lib/publish-client.ts index 8b64cb744..33d045737 100644 --- a/scripts/lib/publish-client.ts +++ b/scripts/lib/publish-client.ts @@ -10,14 +10,18 @@ export interface AppendInput { baseUrl: string; apiKey: string; sessionId: string; files: Record; - metadata: Record; - bodyTexts: Record; + metadata?: Record; + bodyTexts?: Record; branchSpecs?: Record; // PR #591: per-slug gzipped raw markdown for source-of-truth drift detection. // Map keyed by the SAME slug as `files`. Values are base64(gzip(rawMarkdownBytes)). // Optional + ignored by server when null/absent — back-compat with older // clients and with payload entries (__shell__, __nav__) that have no source. sources?: Record; + // #2245: per-slug git commit SHA from the source tutorial repo, keyed by the + // SAME slug as `files`. Optional — omitted entirely when absent so the server + // stores null for all slugs in the batch (back-compat with older clients). + sourceCommits?: Record; } export interface AppendResult { slugsAccepted: number; batchHash: string; totalSizeBytes: number } @@ -68,11 +72,29 @@ export async function beginSession(i: BeginInput): Promise { ); } +/** + * Build the plain POST body object for an append request. Extracted so it can + * be unit-tested without network access. `appendBatch` delegates to this. + * + * `sourceCommits` is omitted from the body entirely when absent on `opts` — the + * server treats a missing key as "no commit SHA for any slug in this batch" + * (back-compat with older clients). + */ +export function buildAppendBody(opts: Omit): Record { + const body: Record = { + sessionId: opts.sessionId, + files: opts.files, + metadata: opts.metadata, + bodyTexts: opts.bodyTexts, + branchSpecs: opts.branchSpecs, + sources: opts.sources, + ...(opts.sourceCommits ? { sourceCommits: opts.sourceCommits } : {}), + }; + return body; +} + export async function appendBatch(i: AppendInput): Promise { - return postJson(`${i.baseUrl}/content/publish/append`, i.apiKey, { - sessionId: i.sessionId, files: i.files, metadata: i.metadata, bodyTexts: i.bodyTexts, - branchSpecs: i.branchSpecs, sources: i.sources, - }); + return postJson(`${i.baseUrl}/content/publish/append`, i.apiKey, buildAppendBody(i)); } export async function commitSession(i: CommitInput): Promise { diff --git a/scripts/publish-content.ts b/scripts/publish-content.ts index b546458c1..83705776f 100644 --- a/scripts/publish-content.ts +++ b/scripts/publish-content.ts @@ -244,7 +244,30 @@ export function buildSourcePayload( return { sources, sourceHashes }; } -// --- Body text extraction (for HANA full-text search) --- +/** + * #2245: Build the per-slug git commit SHA map for the append payload. + * For each slug, reads `/.commit-sha` (written by + * fetch-tutorials.ts alongside the main .md cache file). Slugs with no sidecar + * are silently skipped — the server stores null sourceCommit for those rows. + * + * The key in the returned map is the ORIGINAL-CASE slug (matching how `sources` + * and `files` are keyed), so the server's `sourceCommits[slug]` lookup lands on + * the correct entry. The file lookup uses `slug.toLowerCase()` to match the + * lowercase-canonical filename written by fetch-tutorials. + */ +export function buildSourceCommitsPayload( + slugs: string[], + cacheDir: string, +): Record { + const result: Record = {}; + for (const slug of slugs) { + const shaPath = join(cacheDir, `${slug.toLowerCase()}.commit-sha`); + if (!existsSync(shaPath)) continue; + const sha = readFileSync(shaPath, 'utf-8').trim(); + if (sha) result[slug] = sha; + } + return result; +} const TUTORIAL_MAIN_RE = /]*class\s*=\s*["']?[^"'>]*\btutorial-main\b[^"'>]*["']?[^>]*>([\s\S]*?)<\/main>/i; const BODY_RE = /]*>([\s\S]*?)<\/body>/i; @@ -1167,6 +1190,12 @@ async function main() { buildSourcePayload(tutorialOnlySlugs, cacheDir); log(`Source markdown payload: ${Object.keys(sourcesAll).length}/${tutorialOnlySlugs.length} slugs have upstream .md files`); + // #2245: build per-slug git commit SHA map (keyed by original-case slug, + // matching `files`). Reads .commit-sha sidecars written by fetch-tutorials. + // Silently empty when sidecars are absent (e.g. first run or cached path). + const sourceCommitsAll = buildSourceCommitsPayload(tutorialOnlySlugs, cacheDir); + log(`Source commit SHA payload: ${Object.keys(sourceCommitsAll).length}/${tutorialOnlySlugs.length} slugs have commit SHAs`); + // __nav__ / __404__ / __shell__ ride along on the first batch (these are // small and the server happily accepts them mixed with regular slugs). const sidecarKeys = await collectSidecars(opts.hugoDir, payload, log, channel); @@ -1196,6 +1225,11 @@ async function main() { // pickEntries returns {} for them, which the server treats as // "no source for this batch" and skips the source-side INSERT. sources: pickEntries(sourcesAll, batch), + // #2245: thread per-slug git commit SHA so the server stamps + // sourceCommit on each ContentCurrent row. Sidecar keys produce {} + // (no .commit-sha sidecar exists for __shell__ etc.) which is fine + // — the server stores null for missing entries. + sourceCommits: pickEntries(sourceCommitsAll, batch), }), { attempts: 3, backoffMs: [1000, 3000, 9000], diff --git a/srv/lib/content-publish-session.js b/srv/lib/content-publish-session.js index 79ef5dd96..2454523be 100644 --- a/srv/lib/content-publish-session.js +++ b/srv/lib/content-publish-session.js @@ -147,7 +147,7 @@ export function createSessionHelpers({ namespace }) { return row; } - async function appendToSession({ sessionId, files = {}, metadata = {}, bodyTexts = {}, branchSpecs = {}, sources = {} }) { + async function appendToSession({ sessionId, files = {}, metadata = {}, bodyTexts = {}, branchSpecs = {}, sources = {}, sourceCommits = {} }) { const appendStartHr = process.hrtime.bigint(); // #805 const session = await findActiveSession(sessionId); const { ContentFiles, ContentManifest } = cds.entities(namespace); @@ -193,6 +193,7 @@ export function createSessionHelpers({ namespace }) { mimeType: 'text/html', sourceContent, sourceHash, + sourceCommit: sourceCommits[slug] || null, }); totalSizeBytes += decompressed.length; } @@ -1399,7 +1400,7 @@ async function dualWriteCurrentAndHistory(namespace, newVersion, freshSlugs, han if (isHana) { const placeholders = chunk.map(() => '?').join(', '); const raw = await db.run( - `SELECT "SLUG", "CONTENT", "CONTENTHASH", "SIZEBYTES", "COMPRESSEDBYTES", "MIMETYPE", "SOURCECONTENT", "SOURCEHASH" + `SELECT "SLUG", "CONTENT", "CONTENTHASH", "SIZEBYTES", "COMPRESSEDBYTES", "MIMETYPE", "SOURCECONTENT", "SOURCEHASH", "SOURCECOMMIT" FROM "${hanaTableName()}" WHERE "VERSION" = ? AND "SLUG" IN (${placeholders})`, [newVersion, ...chunk] @@ -1408,10 +1409,11 @@ async function dualWriteCurrentAndHistory(namespace, newVersion, freshSlugs, han slug: r.SLUG, content: r.CONTENT, contentHash: r.CONTENTHASH, sizeBytes: r.SIZEBYTES, compressedBytes: r.COMPRESSEDBYTES, mimeType: r.MIMETYPE, sourceContent: r.SOURCECONTENT, sourceHash: r.SOURCEHASH, + sourceCommit: r.SOURCECOMMIT, })); } else { rows = await SELECT.from(ContentFiles) - .columns('slug', 'content', 'contentHash', 'sizeBytes', 'compressedBytes', 'mimeType', 'sourceContent', 'sourceHash') + .columns('slug', 'content', 'contentHash', 'sizeBytes', 'compressedBytes', 'mimeType', 'sourceContent', 'sourceHash', 'sourceCommit') .where({ version: newVersion, slug: { in: chunk } }); } @@ -1427,12 +1429,14 @@ async function dualWriteCurrentAndHistory(namespace, newVersion, freshSlugs, han slug: row.slug, content: buf, contentHash: row.contentHash, sizeBytes: row.sizeBytes, compressedBytes: row.compressedBytes, mimeType: row.mimeType, sourceContent: srcBuf, sourceHash: row.sourceHash ?? null, + sourceCommit: row.sourceCommit ?? null, sourceVersion: newVersion, }); historyEntries.push({ version: newVersion, slug: row.slug, action: 'WRITTEN', content: buf, contentHash: row.contentHash, sizeBytes: row.sizeBytes, compressedBytes: row.compressedBytes, mimeType: row.mimeType, sourceContent: srcBuf, sourceHash: row.sourceHash ?? null, + sourceCommit: row.sourceCommit ?? null, }); } diff --git a/srv/lib/content-store.js b/srv/lib/content-store.js index 88e2230ab..b88a9a6dc 100644 --- a/srv/lib/content-store.js +++ b/srv/lib/content-store.js @@ -23,6 +23,9 @@ import { loadPageFallback } from './page-fallback.js'; import { stampSubmissionId } from './task-record-submission-id.js'; import { isDeltaWrite, isDeltaRead, isDeltaSkipCarryForward } from './content-delta-flags.js'; import { normalizeTutorialMarkdown, prefersMarkdown } from './tutorial-markdown.js'; +import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { loadProvenanceInputs } from './provenance-data.js'; +import { deriveConfidence } from './provenance-freshness.js'; const LOG = cds.log('content-store'); const LOCK_NAME = 'content-publish'; @@ -87,6 +90,23 @@ function dropCatalogSlugs(obj) { export { toBuffer, isCatalogSlug, dropCatalogSlugs }; +// Advisory provenance helpers — called from serveStoredSlug. Fail-open: any +// error or missing data returns null so the serve path is never blocked. +async function computeAdvisory(slug) { + if (!isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED')) return null; + try { + const inputs = await loadProvenanceInputs(slug); + if (!inputs) return null; + return { confidence: deriveConfidence({ report: inputs.report }), url: `/content/tutorials/${slug}/provenance` }; + } catch { return null; } +} + +function setAdvisoryHeaders(res, advisory) { + if (!advisory) return; + res.setHeader('X-Freshness-Confidence', advisory.confidence); + res.setHeader('X-Content-Provenance', advisory.url); +} + // Re-evaluate every TUTORIAL TaskRecord for `tutorialId` against the // authoritative step count (`stepCount`) and the user's actual completed STEP // records. Flips stale `progress=100/COMPLETED` rows back to IN_PROGRESS when @@ -188,7 +208,7 @@ export class ContentCache { return entry; } - set(key, buffer, hash) { + set(key, buffer, hash, advisory = null) { if (this.map.has(key)) { this.totalBytes -= this.map.get(key).buffer.length; this.map.delete(key); @@ -202,7 +222,7 @@ export class ContentCache { // #2232: stamp a fresh expiry on every write. A republish re-set() therefore // resets the clock, so actively-updated content never expires mid-serve. const expiresAt = this.ttlMs > 0 ? Date.now() + this.ttlMs : Infinity; - this.map.set(key, { buffer, hash, expiresAt }); + this.map.set(key, { buffer, hash, expiresAt, advisory }); this.totalBytes += buffer.length; metrics.gauge('cache.bytes', this.totalBytes); // #805 } @@ -1021,6 +1041,7 @@ export function createContentHandlers({ namespace = 'com.sap.developers.ims', ap res.setHeader('ETag', `"${cached.hash}"`); setContentCacheHeaders(res, { slug: tagSlug }); res.setHeader('X-Content-Source', 'cache'); + setAdvisoryHeaders(res, isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED') ? cached.advisory : null); res.send(cached.buffer); return 'served'; } @@ -1087,12 +1108,14 @@ export function createContentHandlers({ namespace = 'com.sap.developers.ims', ap contentBuf = await toBuffer(blobRow.content); } const decompressed = gunzipSync(contentBuf); - cache.set(slug, decompressed, meta.contentHash); + const advisory = await computeAdvisory(slug); + cache.set(slug, decompressed, meta.contentHash, advisory); res.setHeader('Content-Type', `${mimeType || meta.mimeType}; charset=utf-8`); res.setHeader('ETag', `"${meta.contentHash}"`); setContentCacheHeaders(res, { slug: tagSlug }); res.setHeader('X-Content-Source', source === 'current' ? 'db-current' : 'db'); + setAdvisoryHeaders(res, advisory); res.send(decompressed); return 'served'; } @@ -1982,17 +2005,18 @@ export function createContentHandlers({ namespace = 'com.sap.developers.ims', ap // PR #591: `sources` is the per-slug gzipped raw markdown side of the // payload — destructure + forward it to appendToSession so source // hashes get persisted alongside content hashes. - const { sessionId, files, metadata, bodyTexts, branchSpecs, sources } = req.body || {}; + const { sessionId, files, metadata, bodyTexts, branchSpecs, sources, sourceCommits } = req.body || {}; if (!sessionId) return res.status(400).json({ error: 'sessionId required' }); const droppedFiles = dropCatalogSlugs(files); dropCatalogSlugs(metadata); dropCatalogSlugs(bodyTexts); dropCatalogSlugs(branchSpecs); dropCatalogSlugs(sources); + dropCatalogSlugs(sourceCommits); if (droppedFiles.length) { LOG.warn(`[content/publish/append] dropped ${droppedFiles.length} catalog slug(s)`); } - const result = await sessionHelpers.appendToSession({ sessionId, files, metadata, bodyTexts, branchSpecs, sources }); + const result = await sessionHelpers.appendToSession({ sessionId, files, metadata, bodyTexts, branchSpecs, sources, sourceCommits }); res.status(202).json(result); } catch (err) { const code = err.statusCode || 500; diff --git a/srv/lib/feature-flags/registry.js b/srv/lib/feature-flags/registry.js index f51f7e6ac..0b8bf8aa9 100644 --- a/srv/lib/feature-flags/registry.js +++ b/srv/lib/feature-flags/registry.js @@ -315,6 +315,13 @@ export const FEATURE_FLAGS = [ description: 'When true, the nightly freshness-scan job runs the detector across the tutorial catalog. DB-driven config (ImsConfig key flag.freshness.scan); no env var. Default OFF.', howToChange: featureFlagUpsert('FRESHNESS_SCAN_ENABLED', 'flag.freshness.scan'), }, + { + key: 'PROVENANCE_ENVELOPE_ENABLED', label: 'Signed provenance & freshness envelope', category: 'Content', + kind: 'db', imsConfigKey: 'flag.provenance.envelope', + valueType: 'boolean', default: false, status: 'dev-only', + description: 'When true, serves the signed provenance JWS at /content/tutorials/:slug/provenance, publishes the JWKS at /.well-known/tutorial-provenance/jwks.json, and emits advisory X-Freshness-Confidence / X-Content-Provenance headers. DB-driven config (ImsConfig key flag.provenance.envelope); no env var. Default OFF.', + howToChange: featureFlagUpsert('PROVENANCE_ENVELOPE_ENABLED', 'flag.provenance.envelope'), + }, // ---- Taxonomy ---- { key: 'SEMAPHORE_SYNC_ENABLED', label: 'Semaphore taxonomy auto-sync', category: 'Taxonomy', diff --git a/srv/lib/provenance-data.js b/srv/lib/provenance-data.js new file mode 100644 index 000000000..70dcc33f6 --- /dev/null +++ b/srv/lib/provenance-data.js @@ -0,0 +1,51 @@ +import cds from '@sap/cds'; + +/** + * Gather all provenance inputs for a tutorial slug at serve time. + * + * Returns { contentHash, sourceCommit, builtAt, report } | null + * Returns null when the content row is absent or on any error (fail-open). + * + * report shape: { status, openHighCount, openMediumCount, runAt, model } | null + */ +export async function loadProvenanceInputs(rawSlug) { + const slug = String(rawSlug || '').toLowerCase(); + const { ContentCurrent, Tutorials, FreshnessReport, FreshnessFinding } = cds.entities('com.sap.developers.ims'); + try { + // slug-canonical: pre-canonicalized (lowercased at line 12) + const content = await SELECT.one.from(ContentCurrent).columns('contentHash', 'sourceCommit', 'modifiedAt').where({ slug }); + if (!content) return null; + + // slug-canonical: pre-canonicalized (lowercased at line 12) + const tut = await SELECT.one.from(Tutorials).columns('ID').where({ slug }); + let report = null; + if (tut) { + const rep = await SELECT.one.from(FreshnessReport) + .columns('ID', 'status', 'openHighCount', 'runAt', 'model') + .where({ tutorial_ID: tut.ID }) + .orderBy('runAt desc'); + if (rep) { + // Use JS counting to avoid count(*) as n CI-Node fragility (see memory ci-node-version-mismatch). + // Scope to the fetched report so multi-report tutorials only count findings for the latest run. + const medRows = await SELECT.from(FreshnessFinding) + .columns('ID').where({ report_ID: rep.ID, severity: 'Medium', disposition: 'OPEN' }); + report = { + status: rep.status, + openHighCount: rep.openHighCount || 0, + openMediumCount: medRows.length, + runAt: rep.runAt, + model: rep.model, + }; + } + } + return { + contentHash: content.contentHash, + sourceCommit: content.sourceCommit || null, + builtAt: content.modifiedAt || null, + report, + }; + } catch (e) { + console.warn('[provenance-data] load failed, fail-open:', e.message); + return null; + } +} diff --git a/srv/lib/provenance-envelope.js b/srv/lib/provenance-envelope.js new file mode 100644 index 000000000..04c8e45c1 --- /dev/null +++ b/srv/lib/provenance-envelope.js @@ -0,0 +1,50 @@ +import { SignJWT } from 'jose'; +import { getSigningKey } from './provenance-keys.js'; +import { deriveConfidence } from './provenance-freshness.js'; + +const ISS = 'https://developers.sap.com'; +const SOURCE_REPO = 'sap-tutorials/Tutorials'; +const TTL_SECONDS = 86400; +const _cache = new Map(); // key -> { jws, claims } + +function cacheKey({ slug, contentHash, report }) { + return `${slug}:${contentHash}:${report?.runAt || 'none'}`; +} + +export async function buildEnvelope({ slug, contentHash, sourceCommit, builtAt, report, now = Date.now() }) { + const signer = await getSigningKey(); + if (!signer) return null; // fail-open: no key configured + + const key = cacheKey({ slug, contentHash, report }); + const hit = _cache.get(key); + if (hit && hit.claims.exp * 1000 > now) return hit; + + const iat = Math.floor(now / 1000); + const claims = { + iss: ISS, sub: slug, iat, exp: iat + TTL_SECONDS, + contentHash, + provenance: { sourceRepo: SOURCE_REPO, sourceCommit: sourceCommit ?? null, builtAt: builtAt ?? null }, + freshness: { + confidence: deriveConfidence({ report, now }), + lastScanned: report?.runAt ?? null, + openHighCount: report?.openHighCount ?? 0, + detectorModel: report?.model ?? null, + }, + }; + + try { + const { iss, sub, iat: _i, exp, ...rest } = claims; + const jws = await new SignJWT(rest) + .setProtectedHeader({ alg: 'EdDSA', kid: signer.kid, typ: 'application/tutorial-provenance+jws' }) + .setIssuer(ISS).setSubject(slug).setIssuedAt(iat).setExpirationTime(claims.exp) + .sign(signer.key); + const envelope = { jws, claims }; + _cache.set(key, envelope); + return envelope; + } catch (e) { + console.warn('[provenance-envelope] signing failed, fail-open:', e.message); + return null; + } +} + +export function __clearEnvelopeCacheForTest() { _cache.clear(); } diff --git a/srv/lib/provenance-freshness.js b/srv/lib/provenance-freshness.js new file mode 100644 index 000000000..966ceaaa4 --- /dev/null +++ b/srv/lib/provenance-freshness.js @@ -0,0 +1,14 @@ +export const FRESH_MAX_AGE_DAYS = 30; +export const STALE_AGE_DAYS = 90; +const DAY = 86400000; + +export function deriveConfidence({ report, now = Date.now() }) { + if (!report || report.status !== 'DONE' || !report.runAt) return 'unknown'; + const ageDays = (now - new Date(report.runAt).getTime()) / DAY; + const high = report.openHighCount || 0; + const medium = report.openMediumCount || 0; + if (high > 0) return 'low'; + if (ageDays > STALE_AGE_DAYS) return 'low'; + if (ageDays > FRESH_MAX_AGE_DAYS || medium > 0) return 'medium'; + return 'high'; +} diff --git a/srv/lib/provenance-handlers.js b/srv/lib/provenance-handlers.js new file mode 100644 index 000000000..aad7e94b5 --- /dev/null +++ b/srv/lib/provenance-handlers.js @@ -0,0 +1,24 @@ +import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { buildEnvelope } from './provenance-envelope.js'; +import { loadProvenanceInputs } from './provenance-data.js'; +import { getJwks } from './provenance-keys.js'; + +export async function provenanceHandler(req, res) { + if (!isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED')) return res.status(404).end(); + const slug = String(req.params.slug || '').toLowerCase(); + const inputs = await loadProvenanceInputs(slug); + if (!inputs) return res.status(404).json({ error: 'not_found' }); + const envelope = await buildEnvelope({ slug, ...inputs }); + if (!envelope) return res.status(503).json({ error: 'attestation_unavailable' }); + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'public, max-age=60, s-maxage=600'); + res.json({ jws: envelope.jws, jwks_url: '/.well-known/tutorial-provenance/jwks.json' }); +} + +export async function jwksHandler(req, res) { + if (!isFlagEnabled('PROVENANCE_ENVELOPE_ENABLED')) return res.status(404).end(); + const jwks = await getJwks(); + res.setHeader('Content-Type', 'application/jwk-set+json; charset=utf-8'); + res.setHeader('Cache-Control', 'public, max-age=300, s-maxage=3600'); + res.json(jwks); +} diff --git a/srv/lib/provenance-keys.js b/srv/lib/provenance-keys.js new file mode 100644 index 000000000..af19faba3 --- /dev/null +++ b/srv/lib/provenance-keys.js @@ -0,0 +1,43 @@ +import { importPKCS8, exportJWK, calculateJwkThumbprint } from 'jose'; + +// Test overrides live on globalThis so all module instances in the same process +// share them — this avoids the Windows module-duplication issue where +// cds.test('serve') may load a second copy of this file (different file:// URL) +// that wouldn't see a module-local variable set by the test. Same pattern as +// globalThis.__imsFeatureFlagsState__ in feature-flags/db-flags.js. +const _g = (globalThis.__provenanceKeysState__ ??= { testPem: undefined, cache: undefined }); + +function readPem() { + if (_g.testPem !== undefined) return _g.testPem; + return process.env.PROVENANCE_SIGNING_KEY || null; +} + +async function load() { + if (_g.cache !== undefined) return _g.cache; + const pem = readPem(); + if (!pem) { _g.cache = null; return _g.cache; } + try { + const key = await importPKCS8(pem, 'EdDSA', { extractable: true }); + const priv = await exportJWK(key); + const jwk = { kty: priv.kty, crv: priv.crv, x: priv.x }; // public-only + const kid = await calculateJwkThumbprint(jwk); + _g.cache = { key, kid, jwk: { ...jwk, use: 'sig', alg: 'EdDSA', kid } }; + } catch (e) { + console.warn('[provenance-keys] failed to load signing key, disabling:', e.message); + _g.cache = null; + } + return _g.cache; +} + +export async function getSigningKey() { + const c = await load(); + return c ? { key: c.key, kid: c.kid } : null; +} + +export async function getJwks() { + const c = await load(); + return { keys: c ? [c.jwk] : [] }; +} + +export function __setKeyForTest(pem) { _g.testPem = pem; _g.cache = undefined; } +export function __resetKeysForTest() { _g.testPem = undefined; _g.cache = undefined; } diff --git a/srv/server.js b/srv/server.js index 0dca984cb..3da0bc4f1 100644 --- a/srv/server.js +++ b/srv/server.js @@ -97,6 +97,7 @@ import './graphql-config.js'; import { makeA2aRouter } from './lib/a2a/rpc-router.js'; import { buildAgentCard } from './lib/a2a/agent-card.js'; import { resolveA2aSettings } from './lib/runtime-config/a2a-settings.js'; +import { provenanceHandler, jwksHandler } from './lib/provenance-handlers.js'; // #1182 — cds-caching resolve-guard fix. This module is evaluated by cds-serve // AFTER `await cds.plugins` (so the cds-caching plugin has already pushed its @@ -757,6 +758,10 @@ cds.on('bootstrap', (app) => { req.params.slug = req.params[0]; return markdownServeHandler(req, res); }); + // Signed provenance JWS endpoint (#2245). Registered BEFORE the *slug wildcard + // below so `demo/provenance` is not swallowed as a slug. Public, read-only — no + // auth; these are attestation/key-distribution endpoints. + app.get('/content/tutorials/:slug/provenance', provenanceHandler); app.get('/content/tutorials/*slug', serveHandler); // Legacy AEM `.model.json` compatibility for SAP Discovery Center (#DC cards). // Approuter maps ^/tutorials/.model.json$ → here. See srv/lib/model-json.js. @@ -1041,6 +1046,10 @@ cds.on('bootstrap', (app) => { res.json(buildAgentCard({ baseUrl, tokenUrl: cfg.tokenUrl, enabled: cfg.enabled })); }); + // JWKS key-distribution for the signed provenance envelope (#2245). Public, + // anonymous — clients verify JWS signatures with these public keys. + app.get('/.well-known/tutorial-provenance/jwks.json', jwksHandler); + // MCP discovery manifest (public, anonymous) — served on the already-public // /.well-known/* approuter route. Metadata only: advertises the anonymous // SearchService tier (/mcp/search) + its tools and points to the OAuth-gated diff --git a/test/lib/content-store.test.js b/test/lib/content-store.test.js index be5455935..19593e4ff 100644 --- a/test/lib/content-store.test.js +++ b/test/lib/content-store.test.js @@ -3,7 +3,11 @@ import cds from '@sap/cds'; import { gzipSync } from 'node:zlib'; import { createHash } from 'node:crypto'; import { createContentHandlers } from '../../srv/lib/content-store.js'; +import { createSessionHelpers } from '../../srv/lib/content-publish-session.js'; import * as catalogRenderer from '../../srv/lib/catalog-renderer.js'; +import { + refreshContentDeltaFlags, bustContentDeltaFlagsCache, DELTA_WRITE_KEY, +} from '../../srv/lib/content-delta-flags.js'; const project = cds.test('serve', '--project', '.', '--in-memory'); @@ -124,6 +128,41 @@ describe('content-store', () => { expect(res.status).toBe(403); }); + + it('persists sourceCommit onto ContentCurrent when supplied', async () => { + const { ContentCurrent, ContentFiles, ContentManifest, ImsConfig, JobLocks } = cds.entities('com.sap.developers.ims'); + const NS = 'com.sap.developers.ims'; + const slug = 'commit-tutorial'; + const sha = 'a'.repeat(40); + const helpers = createSessionHelpers({ namespace: NS }); + // Clean up any pre-existing state for this slug. + await DELETE.from(ContentCurrent).where({ slug }); + // Enable the delta write flag and warm the cache so the synchronous + // isDeltaWrite() getter sees the new value before commitSession runs. + await DELETE.from(ImsConfig).where({ key: DELTA_WRITE_KEY }); + await INSERT.into(ImsConfig).entries({ key: DELTA_WRITE_KEY, value: 'true' }); + await refreshContentDeltaFlags(); + try { + const { sessionId } = await helpers.beginPublishSession({ + trigger: 'test', expectedSlugCount: 1, initiator: 'test' + }); + // append — passes sourceCommits so the per-slug commit SHA is persisted + const html = '

x

'; + await helpers.appendToSession({ + sessionId, + files: { [slug]: gzipSync(Buffer.from(html, 'utf-8')).toString('base64') }, + sourceCommits: { [slug]: sha }, + }); + await helpers.commitSession({ sessionId }); + // ContentCurrent must carry the sourceCommit through the promotion path + const row = await SELECT.one.from(ContentCurrent).where({ slug }); + expect(row.sourceCommit).toBe(sha); + } finally { + await DELETE.from(ContentCurrent).where({ slug }); + await DELETE.from(ImsConfig).where({ key: DELTA_WRITE_KEY }); + bustContentDeltaFlagsCache(); + } + }); }); describe('GET /content/tutorials/:slug', () => { diff --git a/test/lib/provenance-endpoint.test.js b/test/lib/provenance-endpoint.test.js new file mode 100644 index 000000000..e95506475 --- /dev/null +++ b/test/lib/provenance-endpoint.test.js @@ -0,0 +1,43 @@ +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; +import cds from '@sap/cds'; +import { generateKeyPair, exportPKCS8, importJWK, jwtVerify } from 'jose'; +import { __setKeyForTest, __resetKeysForTest } from '../../srv/lib/provenance-keys.js'; +import { __setFlagForTest, __resetFlagsForTest } from '../../srv/lib/feature-flags/db-flags.js'; + +const project = cds.test('serve', '--project', '.', '--in-memory'); + +describe('provenance endpoint', () => { + let ContentCurrent, Tutorials; + beforeAll(async () => { + ({ ContentCurrent, Tutorials } = cds.entities('com.sap.developers.ims')); + const { privateKey } = await generateKeyPair('EdDSA', { crv: 'Ed25519', extractable: true }); + __setKeyForTest(await exportPKCS8(privateKey)); + }); + afterAll(() => { __resetKeysForTest(); __resetFlagsForTest(); }); + beforeEach(async () => { + await DELETE.from(ContentCurrent); await DELETE.from(Tutorials); + await INSERT.into(Tutorials).entries({ ID: cds.utils.uuid(), slug: 'demo' }); + await INSERT.into(ContentCurrent).entries({ slug: 'demo', contentHash: 'h'.repeat(64), sourceCommit: 'a'.repeat(40) }); + }); + + it('404s when flag OFF', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', false); + await expect(project.axios.get('/content/tutorials/demo/provenance')).rejects.toMatchObject({ response: { status: 404 } }); + }); + + it('serves a verifiable JWS when flag ON', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', true); + const res = await project.axios.get('/content/tutorials/demo/provenance'); + expect(res.status).toBe(200); + const jwks = await project.axios.get('/.well-known/tutorial-provenance/jwks.json'); + const pub = await importJWK(jwks.data.keys[0], 'EdDSA'); + const { payload } = await jwtVerify(res.data.jws, pub, { issuer: 'https://developers.sap.com' }); + expect(payload.sub).toBe('demo'); + expect(payload.provenance.sourceCommit).toBe('a'.repeat(40)); + }); + + it('404s for unknown slug when flag ON', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', true); + await expect(project.axios.get('/content/tutorials/nope/provenance')).rejects.toMatchObject({ response: { status: 404 } }); + }); +}); diff --git a/test/lib/provenance-headers.test.js b/test/lib/provenance-headers.test.js new file mode 100644 index 000000000..accc1c0ca --- /dev/null +++ b/test/lib/provenance-headers.test.js @@ -0,0 +1,78 @@ +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; +import cds from '@sap/cds'; +import { gzipSync } from 'node:zlib'; +import { __setFlagForTest, __resetFlagsForTest } from '../../srv/lib/feature-flags/db-flags.js'; +import { invalidateContentCache } from '../../srv/lib/content-store.js'; + +const project = cds.test('serve', '--project', '.', '--in-memory'); +const b64gz = html => gzipSync(Buffer.from(html)).toString('base64'); +const API_KEY = 'k'; + +describe('advisory provenance headers', () => { + let ContentCurrent, ContentFiles, ContentManifest; + + beforeAll(() => { + process.env.CONTENT_API_KEY = API_KEY; + ({ ContentCurrent, ContentFiles, ContentManifest } = cds.entities('com.sap.developers.ims')); + }); + + afterAll(() => __resetFlagsForTest()); + + beforeEach(async () => { + invalidateContentCache(); + await DELETE.from(ContentCurrent); + await DELETE.from(ContentFiles); + await DELETE.from(ContentManifest); + + // Publish 'demo' so it is servable (ContentFiles path, same flow as content-store.test.js). + await project.axios.post('/content/publish', { + trigger: 'provenance-headers-test', + files: { demo: b64gz('

demo

') }, + }, { headers: { Authorization: `Bearer ${API_KEY}` } }); + + // Insert a ContentCurrent row so loadProvenanceInputs finds the slug. + // No FreshnessReport row → report will be null → deriveConfidence returns 'unknown'. + await INSERT.into(ContentCurrent).entries({ + slug: 'demo', + contentHash: 'a'.repeat(64), + mimeType: 'text/html', + }); + }); + + it('omits headers when flag OFF', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', false); + const res = await project.axios.get('/content/tutorials/demo'); + expect(res.status).toBe(200); + expect(res.headers['x-freshness-confidence']).toBeUndefined(); + expect(res.headers['x-content-provenance']).toBeUndefined(); + }); + + it('emits headers on cache-miss AND cache-hit when flag ON', async () => { + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', true); + const miss = await project.axios.get('/content/tutorials/demo'); // fills LRU + expect(miss.status).toBe(200); + expect(miss.headers['x-freshness-confidence']).toBe('unknown'); + expect(miss.headers['x-content-provenance']).toBe('/content/tutorials/demo/provenance'); + const hit = await project.axios.get('/content/tutorials/demo'); // LRU hit + expect(hit.headers['x-content-source']).toBe('cache'); + expect(hit.headers['x-freshness-confidence']).toBe('unknown'); + expect(hit.headers['x-content-provenance']).toBe('/content/tutorials/demo/provenance'); + }); + + it('suppresses advisory headers on cache-hit after flag flipped OFF mid-TTL', async () => { + // Warm the LRU with the flag ON so cached.advisory is populated. + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', true); + const miss = await project.axios.get('/content/tutorials/demo'); + expect(miss.status).toBe(200); + expect(miss.headers['x-freshness-confidence']).toBe('unknown'); + + // Flip flag OFF without invalidating the cache — simulates admin toggling off mid-TTL. + __setFlagForTest('PROVENANCE_ENVELOPE_ENABLED', false); + + // Next GET must hit the warm LRU (X-Content-Source: cache) but must NOT emit advisory headers. + const hit = await project.axios.get('/content/tutorials/demo'); + expect(hit.headers['x-content-source']).toBe('cache'); + expect(hit.headers['x-freshness-confidence']).toBeUndefined(); + expect(hit.headers['x-content-provenance']).toBeUndefined(); + }); +}); diff --git a/test/unit/provenance-data.test.js b/test/unit/provenance-data.test.js new file mode 100644 index 000000000..5c39e571a --- /dev/null +++ b/test/unit/provenance-data.test.js @@ -0,0 +1,44 @@ +// test/unit/provenance-data.test.js +import { describe, it, expect, beforeAll, beforeEach } from 'vitest'; +import cds from '@sap/cds'; +import { loadProvenanceInputs } from '../../srv/lib/provenance-data.js'; + +const project = cds.test('serve', '--project', '.', '--in-memory'); + +describe('loadProvenanceInputs', () => { + let ContentCurrent, Tutorials, FreshnessReport, FreshnessFinding; + beforeAll(() => { ({ ContentCurrent, Tutorials, FreshnessReport, FreshnessFinding } = cds.entities('com.sap.developers.ims')); }); + beforeEach(async () => { + await DELETE.from(ContentCurrent); await DELETE.from(FreshnessFinding); + await DELETE.from(FreshnessReport); await DELETE.from(Tutorials); + }); + + it('returns null for unknown slug', async () => { + expect(await loadProvenanceInputs('nope')).toBeNull(); + }); + + it('joins content row, source commit, and current freshness report', async () => { + const tid = cds.utils.uuid(); + await INSERT.into(Tutorials).entries({ ID: tid, slug: 'demo' }); + await INSERT.into(ContentCurrent).entries({ slug: 'demo', contentHash: 'h'.repeat(64), sourceCommit: 'a'.repeat(40), modifiedAt: '2026-09-10T00:00:00.000Z' }); + await INSERT.into(FreshnessReport).entries({ ID: cds.utils.uuid(), tutorial_ID: tid, status: 'DONE', openHighCount: 0, runAt: '2026-09-05T00:00:00.000Z', model: 'm1' }); + const out = await loadProvenanceInputs('demo'); + expect(out.contentHash).toBe('h'.repeat(64)); + expect(out.sourceCommit).toBe('a'.repeat(40)); + expect(out.report).toMatchObject({ status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: '2026-09-05T00:00:00.000Z', model: 'm1' }); + }); + + it('counts open medium findings', async () => { + const tid = cds.utils.uuid(); + await INSERT.into(Tutorials).entries({ ID: tid, slug: 'demo2' }); + await INSERT.into(ContentCurrent).entries({ slug: 'demo2', contentHash: 'h'.repeat(64) }); + const rid = cds.utils.uuid(); + await INSERT.into(FreshnessReport).entries({ ID: rid, tutorial_ID: tid, status: 'DONE', openHighCount: 0, runAt: '2026-09-05T00:00:00.000Z' }); + await INSERT.into(FreshnessFinding).entries([ + { ID: cds.utils.uuid(), report_ID: rid, tutorial_ID: tid, severity: 'Medium', disposition: 'OPEN' }, + { ID: cds.utils.uuid(), report_ID: rid, tutorial_ID: tid, severity: 'Medium', disposition: 'DISMISSED' }, + ]); + const out = await loadProvenanceInputs('demo2'); + expect(out.report.openMediumCount).toBe(1); + }); +}); diff --git a/test/unit/provenance-envelope.test.js b/test/unit/provenance-envelope.test.js new file mode 100644 index 000000000..389c331c2 --- /dev/null +++ b/test/unit/provenance-envelope.test.js @@ -0,0 +1,54 @@ +import { describe, it, expect, beforeAll, afterEach } from 'vitest'; +import { generateKeyPair, exportPKCS8, importJWK, jwtVerify, decodeJwt } from 'jose'; +import { buildEnvelope, __clearEnvelopeCacheForTest } from '../../srv/lib/provenance-envelope.js'; +import { getJwks, __setKeyForTest, __resetKeysForTest } from '../../srv/lib/provenance-keys.js'; + +let pem; +beforeAll(async () => { + const { privateKey } = await generateKeyPair('EdDSA', { crv: 'Ed25519', extractable: true }); + pem = await exportPKCS8(privateKey); +}); +afterEach(() => { __resetKeysForTest(); __clearEnvelopeCacheForTest(); }); + +const base = { + slug: 'my-tutorial', contentHash: 'c'.repeat(64), sourceCommit: 'a'.repeat(40), + builtAt: '2026-09-10T00:00:00.000Z', + report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: '2026-09-05T00:00:00.000Z', model: 'gpt-x' }, + now: Date.UTC(2026, 8, 11), +}; + +describe('buildEnvelope', () => { + it('returns null when no signing key (fail-open)', async () => { + __setKeyForTest(null); + expect(await buildEnvelope(base)).toBeNull(); + }); + + it('signs a verifiable JWS with two distinct claim groups', async () => { + __setKeyForTest(pem); + const { jws, claims } = await buildEnvelope(base); + const pub = await importJWK((await getJwks()).keys[0], 'EdDSA'); + const { payload } = await jwtVerify(jws, pub, { issuer: 'https://developers.sap.com' }); + expect(payload.sub).toBe('my-tutorial'); + expect(payload.contentHash).toBe(base.contentHash); + expect(payload.provenance).toMatchObject({ sourceRepo: 'sap-tutorials/Tutorials', sourceCommit: base.sourceCommit, builtAt: base.builtAt }); + expect(payload.freshness).toMatchObject({ confidence: 'high', lastScanned: base.report.runAt, openHighCount: 0, detectorModel: 'gpt-x' }); + expect(payload.exp - payload.iat).toBe(86400); + expect(claims.freshness.confidence).toBe('high'); + }); + + it('tampered payload fails verification', async () => { + __setKeyForTest(pem); + const { jws } = await buildEnvelope(base); + const pub = await importJWK((await getJwks()).keys[0], 'EdDSA'); + const [h, , s] = jws.split('.'); + const forged = Buffer.from(JSON.stringify({ ...decodeJwt(jws), contentHash: 'f'.repeat(64) })).toString('base64url'); + await expect(jwtVerify(`${h}.${forged}.${s}`, pub)).rejects.toThrow(); + }); + + it('emits unknown confidence + null sourceCommit honestly', async () => { + __setKeyForTest(pem); + const { claims } = await buildEnvelope({ ...base, sourceCommit: null, report: null }); + expect(claims.freshness.confidence).toBe('unknown'); + expect(claims.provenance.sourceCommit).toBeNull(); + }); +}); diff --git a/test/unit/provenance-freshness.test.js b/test/unit/provenance-freshness.test.js new file mode 100644 index 000000000..e2825fa8b --- /dev/null +++ b/test/unit/provenance-freshness.test.js @@ -0,0 +1,30 @@ +import { describe, it, expect } from 'vitest'; +import { deriveConfidence } from '../../srv/lib/provenance-freshness.js'; + +const DAY = 86400000; +const now = Date.UTC(2026, 8, 11); +const ago = d => new Date(now - d * DAY).toISOString(); + +describe('deriveConfidence', () => { + it('unknown when no report', () => { + expect(deriveConfidence({ report: null, now })).toBe('unknown'); + }); + it('unknown when report not DONE', () => { + expect(deriveConfidence({ report: { status: 'FAILED', openHighCount: 0, openMediumCount: 0, runAt: ago(1) }, now })).toBe('unknown'); + }); + it('high: fresh scan, no high, no medium', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: ago(10) }, now })).toBe('high'); + }); + it('medium: clean but aging (30-90d)', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: ago(45) }, now })).toBe('medium'); + }); + it('medium: fresh scan but only medium findings', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 2, runAt: ago(5) }, now })).toBe('medium'); + }); + it('low: any open high finding', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 1, openMediumCount: 0, runAt: ago(1) }, now })).toBe('low'); + }); + it('low: scan older than 90d even if clean', () => { + expect(deriveConfidence({ report: { status: 'DONE', openHighCount: 0, openMediumCount: 0, runAt: ago(120) }, now })).toBe('low'); + }); +}); diff --git a/test/unit/provenance-keys.test.js b/test/unit/provenance-keys.test.js new file mode 100644 index 000000000..ba040b73c --- /dev/null +++ b/test/unit/provenance-keys.test.js @@ -0,0 +1,35 @@ +import { describe, it, expect, beforeAll, afterEach } from 'vitest'; +import { generateKeyPair, exportPKCS8, jwtVerify, importJWK } from 'jose'; +import { getSigningKey, getJwks, __setKeyForTest, __resetKeysForTest } from '../../srv/lib/provenance-keys.js'; + +let pem; +beforeAll(async () => { + const { privateKey } = await generateKeyPair('EdDSA', { crv: 'Ed25519', extractable: true }); + pem = await exportPKCS8(privateKey); +}); +afterEach(() => __resetKeysForTest()); + +describe('provenance-keys', () => { + it('returns null signer when no key configured', async () => { + __setKeyForTest(null); + expect(await getSigningKey()).toBeNull(); + expect((await getJwks()).keys).toEqual([]); + }); + + it('loads an Ed25519 signer and publishes a matching public JWK', async () => { + __setKeyForTest(pem); + const signer = await getSigningKey(); + expect(signer).not.toBeNull(); + expect(signer.kid).toMatch(/.+/); + const jwks = await getJwks(); + expect(jwks.keys).toHaveLength(1); + expect(jwks.keys[0]).toMatchObject({ kty: 'OKP', crv: 'Ed25519', use: 'sig', alg: 'EdDSA', kid: signer.kid }); + expect(jwks.keys[0].d).toBeUndefined(); // never leak the private scalar + // round-trip: sign with signer, verify with the published public JWK + const { SignJWT } = await import('jose'); + const jws = await new SignJWT({ t: 1 }).setProtectedHeader({ alg: 'EdDSA', kid: signer.kid }).sign(signer.key); + const pub = await importJWK(jwks.keys[0], 'EdDSA'); + const { payload } = await jwtVerify(jws, pub); + expect(payload.t).toBe(1); + }); +}); diff --git a/test/unit/publish-content-source-commit.test.js b/test/unit/publish-content-source-commit.test.js new file mode 100644 index 000000000..a468e6838 --- /dev/null +++ b/test/unit/publish-content-source-commit.test.js @@ -0,0 +1,13 @@ +import { describe, it, expect } from 'vitest'; +import { buildAppendBody } from '../../scripts/lib/publish-client.ts'; + +describe('appendBatch body', () => { + it('includes sourceCommits when provided', () => { + const body = buildAppendBody({ sessionId: 's', files: { a: 'x' }, sourceCommits: { a: 'sha1' } }); + expect(body.sourceCommits).toEqual({ a: 'sha1' }); + }); + it('omits sourceCommits key cleanly when absent', () => { + const body = buildAppendBody({ sessionId: 's', files: { a: 'x' } }); + expect(body.sourceCommits).toBeUndefined(); + }); +});