diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md new file mode 100644 index 0000000..6f7f97a --- /dev/null +++ b/DOCUMENTATION.md @@ -0,0 +1,37 @@ +# Keyverse Documentation Map + +Keyverse already has strong feature-specific specifications, doctoring, federation/onboarding, topology, and operations records. This index makes the cross-cutting product and architecture graph explicit without replacing those slice documents. + +| Area | Canonical document | +|---|---| +| Product requirements | [`docs/PRD.md`](docs/PRD.md) | +| Technical requirements | [`docs/TRD.md`](docs/TRD.md) | +| Architecture | [`ARCHITECTURE.md`](ARCHITECTURE.md) | +| Topology | [`docs/topology.md`](docs/topology.md) | +| UML/runtime/authority flows | [`docs/UML.md`](docs/UML.md) | +| Logical/physical ERD | [`docs/ERD.md`](docs/ERD.md) | +| Threat model | [`docs/THREAT_MODEL.md`](docs/THREAT_MODEL.md) | +| Test strategy | [`docs/TEST_STRATEGY.md`](docs/TEST_STRATEGY.md) | +| Operability/recovery/release | [`docs/OPERABILITY.md`](docs/OPERABILITY.md) | +| Requirements/evidence traceability | [`docs/TRACEABILITY.md`](docs/TRACEABILITY.md) | +| Architecture decisions | [`docs/adr/README.md`](docs/adr/README.md) | +| Federation onboarding | [`docs/federation-onboarding.md`](docs/federation-onboarding.md) | +| RP onboarding | [`docs/rp-onboarding.md`](docs/rp-onboarding.md) | +| Account merge/unification | [`docs/merge-unification-flow.md`](docs/merge-unification-flow.md) | +| Standards/APA 7 evidence | [`docs/doctoring/`](docs/doctoring/) and [`docs/papers/`](docs/papers/) | +| Operations | [`docs/operations/`](docs/operations/) | +| Security reporting | [`SECURITY.md`](SECURITY.md) | +| Agent instructions | [`AGENTS.md`](AGENTS.md) | +| Agent context | [`CLAUDE.md`](CLAUDE.md) | +| Product overview | [`README.md`](README.md) | +| Change history | [`CHANGELOG.md`](CHANGELOG.md) | + +## Maturity vocabulary + +- **implemented-main** — present on protected main with source/tests. +- **active-PR** — implemented only on an open PR and not yet a protected-main claim. +- **deployment-owned** — private tenant/customer secret/configuration behavior owned by deployment controller/secret store. +- **external-system** — Keycloak/ADFS/LDAP/external OIDC/HR/IGA behavior not implemented by Keyverse itself. +- **planned** — accepted target without executable implementation. + +Open PR #72 OIDC RP claim mapper profile and PR #74 hourly GitHub API remediation remain active-PR until merged. Keyverse's current protected-main desired-state/reconciliation capabilities are documented independently from those changes. \ No newline at end of file diff --git a/docs/ERD.md b/docs/ERD.md new file mode 100644 index 0000000..9044318 --- /dev/null +++ b/docs/ERD.md @@ -0,0 +1,189 @@ +# Keyverse Logical and Persistence ERD + +**Status:** Accepted cross-cutting data model. Exact Keycloak internal schema remains Keycloak-owned. +**Last reviewed:** 2026-08-09 + +Keyverse persists its own configuration, desired-state, receipts, merge audit, and user-operation locks while Keycloak/PostgreSQL owns canonical IdP users/sessions/clients/federation runtime state. This ERD models Keyverse-owned durable records and their relation to external Keycloak identities without pretending to own Keycloak's internal tables. + +```mermaid +erDiagram + IDP_CONFIG_ENTRY }o--|| TENANT_DEPLOYMENT : scoped_to + FEDERATION_SOURCE }o--|| TENANT_DEPLOYMENT : scoped_to + DIRECTORY_FEDERATION_SOURCE }o--|| TENANT_DEPLOYMENT : scoped_to + RELYING_PARTY_SOURCE }o--|| TENANT_DEPLOYMENT : scoped_to + + FEDERATION_SOURCE ||--o{ FEDERATION_APPLY_RECEIPT : produces + DIRECTORY_FEDERATION_SOURCE ||--o{ DIRECTORY_FEDERATION_APPLY_RECEIPT : produces + RELYING_PARTY_SOURCE ||--o{ RELYING_PARTY_APPLY_RECEIPT : produces + + KEYCLOAK_USER_REFERENCE ||--o{ ACCOUNT_MERGE_AUDIT : survivor_or_duplicate + KEYCLOAK_USER_REFERENCE ||--o| USER_OPERATION_LOCK_STATE : guarded_by + KEYCLOAK_USER_REFERENCE ||--o{ EXTERNAL_IDENTITY_LINK : owns + EXTERNAL_IDENTITY_LINK }o--|| FEDERATION_SOURCE : originates_from + + TENANT_DEPLOYMENT { + uuid tenant_deployment_id PK + text deployment_name + text deployment_status_code + timestamptz created_at + } + + IDP_CONFIG_ENTRY { + uuid idp_config_entry_id PK + uuid tenant_deployment_id FK + text config_key + text protected_value_ref + text config_version + timestamptz updated_at + } + + FEDERATION_SOURCE { + uuid federation_source_id PK + uuid tenant_deployment_id FK + text federation_alias + text protocol_code + jsonb secret_free_desired_state + text desired_state_hash + text lifecycle_status_code + timestamptz updated_at + } + + FEDERATION_APPLY_RECEIPT { + uuid federation_apply_receipt_id PK + uuid federation_source_id FK + uuid apply_attempt_id UK + text desired_state_hash + text keycloak_resource_id + text observed_state_hash + text apply_outcome_code + timestamptz observed_at + } + + DIRECTORY_FEDERATION_SOURCE { + uuid directory_federation_source_id PK + uuid tenant_deployment_id FK + text directory_alias + jsonb private_desired_state + text desired_state_hash + text lifecycle_status_code + timestamptz updated_at + } + + DIRECTORY_FEDERATION_APPLY_RECEIPT { + uuid directory_federation_apply_receipt_id PK + uuid directory_federation_source_id FK + uuid apply_attempt_id UK + text desired_state_hash + text keycloak_component_id + text observed_state_hash + text apply_outcome_code + timestamptz observed_at + } + + RELYING_PARTY_SOURCE { + uuid relying_party_source_id PK + uuid tenant_deployment_id FK + text client_id + jsonb secret_free_desired_state + text desired_state_hash + text lifecycle_status_code + timestamptz updated_at + } + + RELYING_PARTY_APPLY_RECEIPT { + uuid relying_party_apply_receipt_id PK + uuid relying_party_source_id FK + uuid apply_attempt_id UK + text desired_state_hash + text keycloak_client_uuid + text observed_state_hash + text apply_outcome_code + timestamptz observed_at + } + + KEYCLOAK_USER_REFERENCE { + uuid keycloak_user_reference_id PK + uuid tenant_deployment_id FK + text keycloak_user_uuid + text lifecycle_status_code + } + + EXTERNAL_IDENTITY_LINK { + uuid external_identity_link_id PK + uuid keycloak_user_reference_id FK + uuid federation_source_id FK + text external_subject_hash + boolean email_verified + } + + ACCOUNT_MERGE_AUDIT { + uuid account_merge_audit_id PK + uuid survivor_user_reference_id FK + uuid duplicate_user_reference_id FK + text match_evidence_code + text operation_outcome_code + uuid actor_identity_id + timestamptz occurred_at + } + + USER_OPERATION_LOCK_STATE { + uuid user_operation_lock_state_id PK + uuid keycloak_user_reference_id FK + text operation_type_code + text lock_owner_token + timestamptz acquired_at + timestamptz lease_expires_at + } +``` + +## Logical uniqueness constraints + +UUID primary identifiers are globally unique. Human/provider identifiers are scoped to the owning tenant or federation source and MUST NOT be interpreted as global keys. + +| Entity | Required logical uniqueness | +|---|---| +| `IDP_CONFIG_ENTRY` | `(tenant_deployment_id, config_key)` | +| `FEDERATION_SOURCE` | `(tenant_deployment_id, federation_alias)` | +| `DIRECTORY_FEDERATION_SOURCE` | `(tenant_deployment_id, directory_alias)` | +| `RELYING_PARTY_SOURCE` | `(tenant_deployment_id, client_id)` | +| `KEYCLOAK_USER_REFERENCE` | `(tenant_deployment_id, keycloak_user_uuid)` | +| `EXTERNAL_IDENTITY_LINK` | `(federation_source_id, external_subject_hash)` | + +`federation_source_id` defines the identity-provider scope for the external-subject uniqueness rule. Within one federation source, one normalized/hashed external subject may link to at most one Keycloak user reference. This prevents one issuer/provider subject from being attached to multiple users while still allowing unrelated providers to use the same subject string. + +Physical migrations must enforce these constraints in the owning Keyverse store. Documentation labels such as `client_id`, `federation_alias`, or Keycloak UUID never authorize cross-tenant lookup by themselves. + +## Identity and authorization rules + +- Keycloak UUIDs, federation aliases, RP client IDs, email values, and external subjects are data identifiers, not authorization by themselves. +- Exact external identity key is `(identity_provider, subject)`; verified email may support matching under policy but unverified email never authorizes linking. +- `tenant_deployment_id` is explicit in Keyverse-owned records; deployment/customer separation must not be inferred from realm/resource names. +- Secrets are referenced through protected values/handles where possible; secret-free desired-state tables must never gain client/bind credentials accidentally. + +## Desired-state and receipt invariant + +```mermaid +flowchart LR + PRIVATE[Private rendered input] + VALID[Preflight validation] + INTENT[Versioned desired-state source] + REMOTE[Keycloak live state] + RECEIPT[Version-bound apply receipt] + + PRIVATE --> VALID + VALID --> INTENT + INTENT --> REMOTE + REMOTE --> RECEIPT +``` + +Every apply receipt records the exact `desired_state_hash` that was acted on as well as the canonical `observed_state_hash`, outcome, unique `apply_attempt_id`, and observation time. A receipt is current for a source only when its `desired_state_hash` equals that source's current desired-state hash. The latest current receipt is the greatest `observed_at` among receipts for that exact desired-state hash; a receipt for an older hash is historical evidence and cannot establish convergence for a newer desired state. + +Retry handling is idempotency-aware. Reusing the same `apply_attempt_id` must return/reuse the same receipt rather than create a second logical attempt. A retry under a new attempt ID may create another receipt, but it remains a distinct attempt and must still bind to the exact desired-state hash. Delete flows that require remote-first semantics cannot remove local desired state before remote deletion succeeds. + +## Keycloak ownership + +Users, sessions, roles, groups, credentials, WebAuthn material, IdP runtime representation, LDAP storage components, and RP clients ultimately live in Keycloak's schema/API. Keyverse stores controlled references/intent/receipts but does not duplicate or directly edit unsupported Keycloak internal tables. + +## Migration acceptance + +Changes to Keyverse-owned persistence require migrations/rollback, transaction/concurrency tests, indexes/constraints, tenant isolation, secret/logging tests, backup/restore impact, and ERD/operability/ADR synchronization. Keycloak upgrades require supported schema migration through Keycloak, not custom manipulation of its private database tables. \ No newline at end of file diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md new file mode 100644 index 0000000..c8a5e33 --- /dev/null +++ b/docs/OPERABILITY.md @@ -0,0 +1,97 @@ +# Keyverse Operability, Recovery, and Release Guide + +**Status:** Accepted cross-cutting operating baseline +**Last reviewed:** 2026-08-09 + +Feature-specific procedures under `docs/operations/`, federation/RP onboarding, and deployment READMEs remain authoritative for their slices. This guide defines the shared operating model and evidence needed before declaring the identity platform healthy or release-ready. + +## Health model + +Distinguish these conditions: + +1. **process liveness:** Keycloak/admin process responds; +2. **component readiness:** database/config/bootstrap and core dependencies are usable; +3. **desired-state convergence:** configured federation/directory/RP state matches Keycloak; +4. **protocol acceptance:** controlled login/logout/token/SCIM/bind/search behavior succeeds; +5. **downstream authorization acceptance:** RP accepts expected issuer/audience/claims and applies its own authorization policy. + +A lower-level green state never implies a higher-level state. + +## Key SLIs + +- Keycloak/admin readiness and latency; +- login/passkey success/error rates; +- SCIM mutation success/conflict/retry/lock contention; +- account merge/link outcomes and rollback/tombstone anomalies; +- desired-state drift and reconciliation age; +- federation/LDAP/RP apply/re-observation failures; +- duplicate remote resource detections; +- user-operation lock wait/expiry/recovery; +- token issuer/audience/claim acceptance failures; +- database/storage availability and transaction errors; +- secret/config bootstrap failures; +- hourly governance run outcomes without false-green classification. + +Do not put raw tokens, secrets, passwords/bind credentials, protected private payloads, or unnecessary PII into metrics/logs. + +## Federation onboarding runbook + +1. render private tenant configuration from approved KV/secret source; +2. run authenticated side-effect-free Keyverse preflight; +3. review exact policy result; +4. persist/apply desired state through the owning reconciliation path; +5. verify exact post-mutation Keycloak state/receipt; +6. for LDAP/AD perform controlled bind/search/login acceptance after explicit apply; +7. for SAML/OIDC perform controlled login/issuer/subject/email/trust checks; +8. monitor convergence/errors; +9. retain rollback data until acceptance criteria expire. + +## RP onboarding runbook + +1. submit secret-free client representation; +2. preflight redirects/origins/logout/PKCE/scopes/type; +3. reconcile exact Keycloak client and receipt; +4. provision confidential secret through the separate secret-management path if needed; +5. configure RP securely; +6. run authorization-code/PKCE login/logout/token audience/claim acceptance; +7. validate downstream authorization separately from authentication. + +PR #72's mapper profile requires the same acceptance after merge: operators must test the **Naruon** product login/token/authorization journey using the `naruon-web` RP client ID and verify the expected audience and bounded claims. Mapper unit tests alone do not prove Naruon product authorization readiness. + +## Account merge recovery + +Merge and SCIM full replacement (`PUT`) must hold the shared operation lock. Protected-main `PATCH active=false` is not currently inside that shared-lock guarantee and must not be treated as transactionally serialized with merge. On failure, classify whether state changed in Keycloak, Keyverse audit, linked identities, or tombstone status. Re-observe before retry. Never infer a retry is safe solely from the previous HTTP response. Preserve survivor and duplicate lineage in audit. + +## Desired-state recovery + +On controller/API crash after intent but before receipt: + +- read persisted desired state; +- query exact remote Keycloak state; +- classify converged, absent, duplicate, or drifted; +- reconcile idempotently; +- write receipt only after exact re-observation and bind it to the desired-state version/hash that was applied. + +On delete, keep local intent until remote-first deletion has succeeded where required. + +## Database/backup + +Back up Keycloak PostgreSQL and Keyverse-owned configuration/audit/intent/receipt state according to deployment RPO/RTO. Restore through supported Keycloak/database procedures, then run reconciliation and controlled authentication/provisioning acceptance. Do not edit unsupported Keycloak internal tables as a normal recovery technique. + +## Upgrade/rollback + +- review Keycloak release/migration notes and Keyverse CHANGELOG/ADRs; +- rehearse database migration and Helm/Compose upgrade; +- validate realm/config/template compatibility; +- run merge/SCIM/federation/RP suites; +- canary controlled login/provisioning; +- roll back application/config where safe and use supported DB backup/restore for incompatible schema migrations; +- re-run convergence and protocol acceptance after rollback. + +## Automation incident RCA + +PR #74 demonstrates that a workflow can appear successful while doing no useful work if a GitHub API gate fails open. Scheduled governance must classify transport failure separately from a valid empty/unhealthy result, fit its time budget, keep provider secrets in the broker phase only, and require exact `success` for protected evidence. After PR #74 merges, operational closure requires a real protected-main scheduled/manual run. + +## Release gate + +Release only after protected-head CI/security/review, 100% coverage/docstrings, realm/package/deployment validation, migrations/rollback/backup, passkey/federation/SCIM/RP controlled acceptance, secret scan, SBOM/provenance/image digest, runbooks/support, and CHANGELOG/version artifacts are coherent. A merged PR is not a release by itself. \ No newline at end of file diff --git a/docs/PRD.md b/docs/PRD.md new file mode 100644 index 0000000..958f0ce --- /dev/null +++ b/docs/PRD.md @@ -0,0 +1,118 @@ +# Keyverse Product Requirements Document + +**Status:** Accepted cross-cutting product baseline for protected `main` at `c8968ec1e68fab16d0ad8216fb5c8fd0b385e95f` +**Last reviewed:** 2026-08-09 + +## 1. Product purpose + +Keyverse is the ContextualWisdomLab ecosystem identity control plane: a standalone and embeddable Keycloak-based IdP plus Keyverse-owned control services for passwordless authentication policy, external federation desired state, SCIM provisioning, account unification, relying-party lifecycle, safe deployment preflight/reconciliation, and auditable user operations. + +Its job is to let CWL products consume stable standards-based identity without each product reimplementing Keycloak Admin REST, federation trust policy, credential handling, merge semantics, or provisioning safety. + +## 2. Current protected-main capabilities + +- portable Keycloak `cwl` realm with passwordless-first WebAuthn and no password authenticator for ecosystem-local accounts; +- OIDC/OAuth relying-party service for CWL applications and SAML brokering for external identity providers; +- inbound SCIM v2 shim for lifecycle provisioning; +- account linking/unification and survivor-wins merge with verified-email policy and tombstone behavior; +- user-operation locking across merge and SCIM full-replacement (`PUT`) paths; +- password-free registration enrollment action flow; +- deterministic, side-effect-free SAML/OIDC federation preflight and durable desired-state reconciliation; +- deterministic LDAPS-only directory preflight and durable Keycloak component desired-state reconciliation; +- secret-free OIDC relying-party desired-state preflight/reconciliation with exact Keycloak client identity and receipts; +- standalone Compose and Helm deployment modes with readiness probes; +- configuration/secret bootstrap via KV/DB boundary rather than application environment as runtime source of truth; +- 100% production statement/branch/docstring quality gates and protected review/security workflows. + +The current SCIM `PATCH active=false` deprovisioning path is not protected by the shared cross-process user-operation lock used by merge and full replacement. It must not be represented as transactionally serialized with merge until a source change and concurrency regression prove that boundary. + +## 3. Active-PR boundaries + +- PR #72 adds a closed OIDC RP mapper profile for exactly one audience mapper plus bounded `role`, `org`, and `workspace` hardcoded claims; it remains **active-PR** and is not protected-main behavior until merged. +- PR #74 repairs the hourly product-development GitHub API/egress/time-budget/evidence boundary; it remains **active-PR** operational-governance work until merged and then proven by a protected-main run. + +## 4. Primary users + +- **CWL application team:** onboard an OIDC relying party without owning IdP internals. +- **Enterprise deployment/identity engineer:** connect SAML/OIDC/LDAP/AD sources through explicit safe desired-state/apply workflows. +- **IGA/HR integration engineer:** provision/deprovision users through SCIM. +- **Identity administrator:** link or merge accounts with deterministic verified-evidence rules and audit. +- **Security/SRE:** operate Keycloak/PostgreSQL/admin service with controlled secrets, readiness, rollback, and immutable evidence. + +## 5. Product invariants + +1. Keyverse is the identity hub; employer/customer directories are external federation sources, not the hub. +2. Unverified email never authorizes automatic account linking or merge. +3. Exact `(identity_provider, subject)` is stronger identity evidence than email. +4. Portable realm configuration contains no customer-specific federation secret/configuration and no confidential RP secret. +5. Preflight is side-effect-free and does not perform DNS/network/bind/search/store/Keycloak mutation unless its endpoint explicitly owns apply/reconciliation. +6. Private rendered apply payloads and credentials never appear in public responses/logs/source/templates. +7. Desired state is recorded before external mutation where the lifecycle requires recoverable intent. +8. Mutation receipts are written only after exact live re-observation confirms the result. +9. Duplicate remote identity/client/component matches fail closed; Keyverse does not pick an arbitrary duplicate. +10. Delete is remote-first where a stale desired-state receipt would otherwise falsely claim deletion. +11. Relying-party credential provisioning is a separate secret-management responsibility from secret-free client desired state. +12. Passwordless-local identity must not silently fall back to a password authenticator. +13. Runtime application code consumes configuration from the approved KV/DB boundary; environment is bootstrap transport only. +14. Tenant/application authorization must not be inferred from client ID, UUID, email, or federation source name alone. + +## 6. Functional requirements + +### PRD-FR-001 Passwordless authentication + +The portable local account flow SHALL require passwordless WebAuthn/passkey enrollment/authentication policy and SHALL not include an ordinary password authenticator for ecosystem-local accounts. + +### PRD-FR-002 Federation + +Keyverse SHALL support deployment-owned external SAML/OIDC and LDAP/AD onboarding through closed schemas, side-effect-free preflight, explicit trust/email-link policy, durable desired state where owned, reconciliation, redacted observability, and controlled acceptance evidence. + +### PRD-FR-003 Account unification + +Account matching and merge SHALL follow exact subject → verified email → explicit operator link precedence. Merge SHALL preserve a canonical survivor, disable/tombstone duplicates, retain auditable lineage, and coordinate full-replacement SCIM writes through the shared user-operation lock. Any additional SCIM read-modify-write operation may claim the same serialization guarantee only after it uses that lock and has a concurrency regression covering merge/tombstone interaction. + +### PRD-FR-004 SCIM + +Inbound SCIM SHALL map authoritative enterprise lifecycle operations into Keycloak while preserving Keyverse merge/tombstone invariants and failing closed on unsafe identity ambiguity. Protected-main currently serializes merge with full SCIM `PUT` replacement; the `PATCH active=false` path is a narrower deprovisioning path and is not yet part of that shared-lock guarantee. + +### PRD-FR-005 Relying-party lifecycle + +RP registration SHALL validate exact HTTPS redirect/origin/logout and authorization-code + PKCE profile, store secret-free desired state, reconcile an exact Keycloak client, re-observe before receipt, and keep confidential secret placement separate. Native loopback redirect profiles are not part of the current protected-main RP trust contract unless separately introduced with an Accepted ADR and synchronized security/test/traceability rules. + +### PRD-FR-006 Claims/profile lifecycle + +Optional claim expansion SHALL be closed and least-privilege. New audience/claim mapper profiles require explicit typed policy, no script/user-attribute/group/regex arbitrary mapper classes unless separately accepted, and downstream authorization acceptance tests before claiming application readiness. + +### PRD-FR-007 Configuration/secrets + +Secrets SHALL be sourced through private deployment-controlled stores/handles. Logs, responses, CLI args, checked-in templates, and desired-state records must not contain raw secrets unless the exact durable store is designed to own them encrypted. + +### PRD-FR-008 Deployment and readiness + +Compose/Helm deployments SHALL expose component readiness that distinguishes Keycloak/admin/database/configuration reachability from full end-to-end login/federation acceptance. A green preflight is not a successful login claim. + +### PRD-FR-009 Audit and recovery + +Privileged identity and desired-state operations SHALL produce auditable intent/outcome evidence sufficient for reconciliation/rollback without exposing protected secret values. + +## 7. Security/privacy requirements + +- passkey/federation/SCIM/OIDC/SAML/JWT behaviors follow current standards and Keycloak-supported contracts; +- verified-email and issuer/subject/audience validation are explicit; +- dynamic privileged path segments and remote Location identifiers are validated before transport/use; +- directory binds use LDAPS under the current accepted profile; +- secret/PII handling uses least privilege, encryption, bounded retention, audit, controlled export, and private apply payloads rather than destructive blanket masking; +- automation credentials/reviewer/merge/release authority remain separated. + +## 8. Non-goals + +- replacing Keycloak with a proprietary identity engine; +- embedding employer/customer ADFS/LDAP secrets in portable realm config; +- allowing every RP to administer Keycloak directly; +- automatically linking on unverified email; +- treating preflight as proof of external network/login success; +- storing confidential RP secrets in secret-free desired-state templates; +- giving an autonomous model authority to approve/merge/release identity policy. + +## 9. Quality and release + +Release requires current protected-head exact CI/security/review, 100% production statement/branch/docstring gates, package/realm/Compose/Helm/template validation, migration/rollback and backup/recovery evidence, SBOM/provenance/image digest, controlled login/federation/SCIM/RP acceptance, and CHANGELOG/version/artifact consistency. No current repository version should be promoted merely because a feature PR is green. \ No newline at end of file diff --git a/docs/TEST_STRATEGY.md b/docs/TEST_STRATEGY.md new file mode 100644 index 0000000..8619386 --- /dev/null +++ b/docs/TEST_STRATEGY.md @@ -0,0 +1,92 @@ +# Keyverse Test Strategy + +**Status:** Accepted quality baseline +**Last reviewed:** 2026-08-09 + +## Mandatory gates + +- Ruff and Python compilation; +- public production docstrings 100%; +- production statement coverage 100%; +- production branch coverage 100%; +- complete pytest suite; +- realm/template/package/Compose/Helm validation as applicable; +- exact-current-head CodeQL, Semgrep, Security Scan, review, and branch protection. + +Skipped, cancelled, absent, stale, predecessor-head, synthetic-only, rate-limited, or failed evidence is never passing. + +## Authentication and account tests + +- passwordless browser flow contains WebAuthn passwordless and no password authenticator; +- registration creates no password and rolls back if enrollment initialization fails; +- exact `(IdP, subject)` matching; +- verified-email matching only; +- unverified email never auto-links/merges; +- explicit link behavior; +- survivor/duplicate tombstone semantics; +- merge idempotency and rollback; +- merge/SCIM shared lock concurrency. + +## SCIM tests + +Use realistic create/read/update/replace/delete lifecycle, authoritative/deprovisioning behavior, invalid identifiers, tombstones, retry/idempotency, hostile strings, and concurrent user mutation. Verify safe mapping to Keycloak and no accidental resurrection of merged duplicates. + +## Federation preflight/reconciliation + +### SAML/OIDC + +- closed schema and protocol-specific required fields; +- untrusted/invalid issuer/redirect/trust-email policy; +- no discovery/metadata/network/store/Keycloak side effects during preflight; +- exact remote lookup/duplicate handling; +- desired-state-before-mutation and exact post-mutation receipt; +- delete/recovery behavior; +- public response/log redaction. + +### LDAP/AD + +- LDAPS-only current profile; +- read-only mode and Kerberos disabled; +- `trustEmail=false`; +- RFC-valid DN and bounded config; +- no DNS/socket/bind/search/store/Keycloak side effects during preflight; +- exact component reconciliation, duplicates, remote-first delete; +- controlled real integration lane for bind/search/login after explicit apply, using synthetic test identities/approved environment. + +## Relying-party tests + +- authorization code + PKCE S256; +- redirect/origin/logout exact-policy matrices; +- public/confidential consistency; +- exact portable scopes; +- clientId key and Keycloak UUID integrity; +- exact Keycloak search behavior; +- duplicate/zero/one remote client classification; +- create/update/delete/re-observation/receipt; +- secret-free desired state and separate secret provision; +- controlled authorization-code/login/logout/token audience acceptance. + +PR #72 mapper tests remain active-PR evidence until merged. They should cover exact audience mapper, bounded `role`/`org`/`workspace`, Keycloak-generated mapper IDs/order, and rejection of scripts/arbitrary claims/classes. + +## Deployment and persistence tests + +- PostgreSQL/KV migrations and rollback for Keyverse-owned records; +- configuration bootstrap and runtime store behavior; +- user-operation locks across processes; +- desired-state idempotency/concurrency; +- secret scanning and redacted logs; +- Compose health/readiness; +- Helm template/install/upgrade/rollback where supported; +- backup/restore of Keyverse-owned audit/config/intent/receipts and supported Keycloak database recovery procedure. + +## Security/adversarial tests + +Mirror `docs/THREAT_MODEL.md`: malicious IdP/LDAP URLs, path/resource IDs, duplicate Keycloak objects, protocol confusion, account linking attacks, secret reflection, oversized payloads, stale receipts, SCIM races, redirect manipulation, mapper overreach, and automation credential/check-classification failures. + +## Documentation contract + +CI should require PRD, TRD, Architecture, UML, ERD, Threat Model, Test Strategy, Operability, Traceability, ADR index, README, AGENTS, CLAUDE, CHANGELOG, and discoverable `docs/doctoring/`, `docs/papers/`, and `docs/operations/` research/standards/runbook records. It must assert PR #72/#74 remain active-PR claims until integrated. + +## Release acceptance + +Release evidence includes protected-head identity protocol tests, real deployment acceptance in an approved environment, SBOM/image provenance/digests, migrations/rollback, readiness plus login/federation/SCIM/RP tests, and independent review. Preflight unit success is never treated as full external federation readiness. \ No newline at end of file diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md new file mode 100644 index 0000000..509054c --- /dev/null +++ b/docs/THREAT_MODEL.md @@ -0,0 +1,83 @@ +# Keyverse Threat Model + +**Status:** Accepted baseline for protected-main identity control plane +**Last reviewed:** 2026-08-09 + +## Trust boundaries + +```mermaid +flowchart LR + USER[User / workforce identity] + EXT[External IdP / LDAP / HR-IGA] + EDGE[WAF/public edge] + KC[Keycloak] + KV[Keyverse admin/SCIM] + DEPLOY[Private deployment controller] + STORE[(PostgreSQL/KV)] + RP[Relying parties] + + USER --> EDGE + EXT --> EDGE + EDGE --> KC + EDGE --> KV + KC --> STORE + KV --> STORE + DEPLOY --> KV + DEPLOY --> KC + KC --> RP +``` + +## Threat inventory + +| Threat | Impact | Required controls | +|---|---|---| +| unverified-email linking | account takeover | never auto-link/merge on unverified email; exact issuer/subject precedence | +| issuer/subject confusion | cross-IdP identity collision | provider-scoped subject identity and closed federation config | +| SAML/OIDC/LDAP secret disclosure | tenant compromise | private deployment payloads, redacted responses/logs, protected secret store | +| malicious federation endpoint | SSRF/credential exfiltration | side-effect-free preflight, explicit apply egress/TLS policy, approved hosts | +| insecure LDAP | credential disclosure/tampering | LDAPS-only current profile, bounded timeout, read-only, Kerberos disabled | +| duplicate Keycloak resources | wrong object mutated | exact search and fail-closed duplicate classification | +| forged Location/resource ID | privileged path misuse | validate resource UUID/path before follow-up transport | +| desired-state/remote divergence | false operational status | persist intent, exact re-observation, canonical receipt, reconciliation | +| local-first delete | false deletion / drift | remote-first deletion where required | +| SCIM/merge race | lost updates/account resurrection | shared cross-process user-operation lock and transaction tests | +| tombstone reprovisioning | duplicate account resurrection | survivor pointer + disabled duplicate policy | +| password fallback | weakens passwordless policy | portable flow contains no password authenticator | +| RP redirect/origin mistake | auth-code/token theft | exact HTTPS/PKCE/client policy; separate native loopback profile | +| arbitrary protocol mapper | excessive claims/code execution | closed mapper classes/claims; PR #72 active-PR until merged | +| raw secret in desired state | leakage and poor rotation | secret-free RP source + separate credential provisioning | +| automation credential exposure | repository/provider compromise | isolated OpenCode/broker/verification/publication and reviewer separation | +| stale/false-green CI | unverified identity policy lands | exact-head checks, success-only evidence, fail-closed API gate | + +## STRIDE interpretation + +- **Spoofing:** external identities require exact issuer/provider + subject and protocol validation; unverified email is insufficient. +- **Tampering:** desired state and receipts are versioned/auditable; duplicates fail closed; mutations re-observe live state. +- **Repudiation:** merge, provisioning, federation, RP, and deployment mutations require durable intent/outcome evidence and actor/correlation context. +- **Information disclosure:** credentials, private payloads, bind DNs, tokens, provider error bodies, and protected identity data are minimized/redacted at public boundaries. +- **Denial of service:** API bodies, directory/provider configs, retries/timeouts, SCIM mutation rate, external lookups, queues, and automation loops are bounded. +- **Elevation of privilege:** public client IDs, email, UUIDs, or model output never create admin/reviewer/release authority. + +## Current protected-main versus active PR + +Protected main already has passwordless realm policy, account unification/SCIM, federation/directory/RP desired state, and deployment boundaries. PR #72 expands RP mappers; PR #74 repairs hourly automation. Those threat-surface changes remain active-PR until integrated and then require protected-main operational acceptance. + +## Required security tests + +- verified versus unverified email linking; +- issuer/subject collisions and explicit link; +- merge/SCIM concurrency and tombstone behavior; +- passwordless registration rollback; +- SAML/OIDC/LDAP/RP closed-schema hostile inputs; +- no preflight DNS/socket/HTTP/Keycloak/store side effects; +- LDAPS/read-only/trustEmail=false current directory policy; +- duplicate remote resource classification; +- Keycloak Location/resource-ID path validation; +- remote-first delete/re-observation/receipt integrity; +- secret redaction and template scanning; +- RP redirect/origin/logout/PKCE/audience/claim acceptance; +- automation secret isolation, egress, exact-head check classification, and independent review authority. + +## Review triggers + +Revisit for a new authenticator, linking evidence source, external protocol, mapper class, directory write mode, Kerberos, secret ownership change, admin API exposure, data-store boundary, new tenant model, or altered development/release credentials. \ No newline at end of file diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md new file mode 100644 index 0000000..87483a8 --- /dev/null +++ b/docs/TRACEABILITY.md @@ -0,0 +1,40 @@ +# Keyverse Requirements and Evidence Traceability + +**Status:** Accepted cross-cutting baseline +**Last reviewed:** 2026-08-09 + +| Requirement / decision | Standards / authoritative basis | Source/evidence boundary | Maturity | +|---|---|---|---| +| passwordless local accounts | WebAuthn/FIDO2 + Keycloak supported flow; research/standards records | realm validator + deployment tests | implemented-main | +| exact subject then verified-email match | OIDC federation / NIST federation guidance; merge documentation | account-unification matching/merge tests | implemented-main | +| unverified email never auto-links | security/product invariant | merge/federation tests | implemented-main | +| SCIM inbound lifecycle | RFC 7643/7644; protocol documentation | SCIM service/lifecycle tests | implemented-main | +| SAML/OIDC federation desired state | SAML/OIDC/Keycloak docs | preflight/reconciliation/receipt tests | implemented-main | +| LDAPS directory profile | LDAP RFC 4511–4515 + Keycloak component docs | directory preflight/reconciliation tests | implemented-main | +| secret-free RP desired state | OAuth/OIDC/PKCE/Keycloak client docs | RP preflight/reconciliation/integrity tests | implemented-main | +| RP audience/role/org/workspace mapper profile | OIDC/JWT audience + Keycloak mapper docs | PR #72 research/tests | active-PR | +| merge/SCIM PUT shared operation lock | concurrency/data-integrity decision; ADR-0006 | merge + full-replacement lock/concurrency tests | implemented-main | +| SCIM PATCH active=false shared-lock parity | ADR-0006 boundary | current PATCH source has no shared-lock proof | gap-not-claimed | +| intent before mutation, receipt after re-observation | desired-state/recovery decision | federation/directory/RP reconciliation tests | implemented-main | +| receipt bound to exact desired-state version/hash | threat/recovery contract; ERD | persistence/migration/idempotency evidence required | accepted-contract | +| remote-first deletion | consistency/recovery decision | delete/reconciliation tests | implemented-main | +| secrets from KV/DB, env bootstrap only | architecture/security decision | config/bootstrap/template validation | implemented-main | +| work-conserving fail-closed hourly API gate | automation safety decision | PR #74 workflow tests/exact-head evidence | active-PR | +| 100% production statement/branch/docstring | CWL quality contract | CI/pytest/interrogate | implemented-main | + +## Research, standards, and operations records + +`docs/doctoring/`, `docs/papers/`, and `docs/operations/` are the authoritative research/standards/runbook record for OIDC/OAuth/JWT, SCIM, SAML, LDAP, WebAuthn/passkeys, Keycloak behavior, relying-party lifecycle, and automation changes. This matrix does not duplicate full bibliographic entries. + +## Maturity rules + +- `implemented-main`: source and representative tests exist on protected main. +- `active-PR`: source/evidence exists only on an open PR; do not advertise as released/current behavior. +- `accepted-contract`: architecture/data contract is accepted, but physical migration/runtime evidence is still required before claiming enforcement. +- `gap-not-claimed`: a known boundary is intentionally documented as not guaranteed by protected-main behavior. +- Architecture diagrams/plans/PR bodies alone cannot promote maturity. +- Queued, cancelled, stale, skipped-required, predecessor-head, or rate-limited checks/reviews are historical/non-passing evidence. + +## Change rule + +Every material identity/federation/SCIM/RP/security/automation PR should update affected rows and link its research/standards/operations evidence. If a decision is superseded, preserve historical ADR/research records and point to the replacement. \ No newline at end of file diff --git a/docs/TRD.md b/docs/TRD.md new file mode 100644 index 0000000..eea7e75 --- /dev/null +++ b/docs/TRD.md @@ -0,0 +1,83 @@ +# Keyverse Technical Requirements Document + +**Status:** Accepted cross-cutting technical baseline for protected main +**Last reviewed:** 2026-08-09 + +## 1. Architecture objective + +Keyverse separates portable Keycloak realm policy, Keyverse-owned identity control logic, customer/deployment secrets, remote apply/reconciliation, and downstream relying-party authorization so that each trust boundary can be tested and recovered independently. + +## 2. Runtime components + +- **Keycloak engine:** OIDC/OAuth, SAML brokering, WebAuthn, users/sessions/roles/groups, external IdP and LDAP component execution, RP clients. +- **Account-unification FastAPI service:** merge/link, SCIM, federation/directory/RP validation and desired-state/reconciliation, audit/locking boundaries. +- **PostgreSQL/KV:** Keycloak state plus Keyverse configuration, intent, receipts, merge audit, and user-operation locks. +- **Deployment controller:** private configuration rendering, egress/TLS policy, explicit apply, controlled acceptance, rollback. +- **Compose/Helm:** standalone deployment topology and probes. + +## 3. Trust/authority rules + +- Portable realm policy may contain public client/scope/authentication definitions but no tenant/customer federation private values. +- Deterministic preflight never performs network, DNS, Keycloak, bind/search, file, or store mutation. +- Apply/reconciliation endpoints use exact remote identity lookup and fail on zero/multiple states according to operation semantics. +- Desired-state intent is persisted before external mutation where recovery requires it; receipt is persisted only after exact re-observation and binds the desired-state hash/version acted on. +- RP desired state remains separate from confidential client material. +- Deployment controller, not public API, owns private bind/client and certificate material. + +## 4. Identity evidence + +Identity matching precedence is exact `(identity_provider, subject)` → verified email → explicit operator link. Unverified email is never sufficient. Merged users become disabled tombstones pointing to the survivor, preventing accidental resurrection/reprovisioning as independent people. + +## 5. Concurrency and transactions + +User merge and SCIM full replacement (`PUT`) share one cross-process operation-lock boundary. Protected-main `PATCH active=false` is currently outside that shared-lock guarantee and must not be represented as serialized with merge. Any PATCH or future SCIM read-modify-write operation that can affect tombstone/survivor invariants must join the same lock boundary and add a concurrency regression before the stronger guarantee is promoted. Desired-state records and apply receipts require deterministic keys, transaction-safe update semantics, exact desired-version binding, and reconciliation after crash/retry. Remote deletion precedes local desired-state removal when local-first deletion could falsely report success. + +## 6. Federation requirements + +### SAML / external OIDC + +Closed validated payload; no metadata/discovery fetch during preflight; strict issuer/entity/redirect/trust-email policy; explicit apply through Keyverse; redacted status. + +### LDAP / AD + +Current accepted profile is LDAPS-only, read-only, Kerberos-disabled, `trustEmail=false`, bounded timeouts, closed Keycloak component shape, RFC-valid DN values, no network side effect during preflight. Controlled bind/search/login acceptance occurs only after deployment apply. + +### RP clients + +Authorization code + PKCE S256, exact HTTPS redirect/origin/logout rules, exact scope policy, secret-free desired state, exact client lookup and UUID integrity, post-mutation re-observation, separate confidential-material provisioning. Native loopback redirects are not part of the protected-main RP profile; introducing them requires a separately accepted trust-policy change plus synchronized product, threat, test, and traceability evidence. + +PR #72 claim mapper behavior remains active-PR until merged. + +## 7. API/error boundary + +Authenticated operator APIs accept closed versioned schemas. Errors must not echo private values, raw provider responses, or arbitrary Keycloak Location/header content. Remote resource IDs parsed from Keycloak are validated before use in privileged paths. + +## 8. Persistence/data model + +Current architecture owns PostgreSQL/KV state for configuration, desired-state sources, apply receipts, merge audit, and operation locks. Database objects use descriptive two-word-or-longer `snake_case` names. `docs/ERD.md` defines tenant-scoped uniqueness, receipt identity/version binding, relationships, and lifecycle; migrations must preserve tenant/identity/audit integrity. + +## 9. Security and privacy + +Use standards-backed OIDC/OAuth/SAML/SCIM/LDAP/WebAuthn/JWT validation, least privilege, encrypted stores/transport, bounded retention/export, controlled admin egress, and auditable privileged outcomes. Do not mask identity fields in ways that break identity matching; protect them through access and lifecycle controls. + +## 10. Quality gates + +- Ruff / compile; +- Interrogate/public docstrings 100%; +- production statement and branch coverage 100%; +- realistic merge/SCIM/federation/RP concurrency and hostile-input tests; +- package, realm, deployment template, Compose/Helm validation; +- exact-current-head CodeQL/Semgrep/security/review evidence; +- no queued/cancelled/skipped/stale evidence counted as passing. + +## 11. Operability + +Readiness is component/lifecycle specific. Preflight success does not imply Keycloak apply, bind/search, login, logout, token audience, SCIM provisioning, or downstream authorization success. `docs/OPERABILITY.md` defines acceptance and recovery. + +## 12. Automation boundary + +Autonomous development uses NVIDIA NIM/OpenCode through an isolated model phase. Model execution has no publication/reviewer/release authority. PR #74 is active remediation of this boundary and must be proven again after protected-main merge. + +## 13. Change control + +Changes to identity matching, passwordless policy, federation trust, private-value ownership, desired-state/reconciliation order, persistence, external admin authority, or automation authority require an ADR plus PRD/TRD/Architecture/UML/ERD/Threat/Test/Operability/Traceability reconciliation. \ No newline at end of file diff --git a/docs/UML.md b/docs/UML.md new file mode 100644 index 0000000..56c9ac5 --- /dev/null +++ b/docs/UML.md @@ -0,0 +1,150 @@ +# Keyverse UML and Runtime Views + +**Status:** Accepted protected-main diagrams with active-PR items labelled. +**Last reviewed:** 2026-08-09 + +## Component and authority view + +```mermaid +flowchart LR + USER[User / workforce identity] + EXT[External IdPs / LDAP / HR-IGA] + EDGE[WAF / public edge] + KC[Keycloak engine] + KCAPI[Keycloak Admin REST API] + ADMIN[Account-unification + SCIM API] + DEPLOY[Private deployment controller] + KV[(KV / secret manager)] + KCDB[(Keycloak-owned PostgreSQL)] + KVS[(Keyverse-owned config / intent / receipt / audit store)] + RP[CWL relying parties] + + USER --> EDGE + EXT --> EDGE + EDGE --> KC + EDGE --> ADMIN + KC --> KCDB + KCAPI --> KC + ADMIN --> KCAPI + ADMIN --> KVS + DEPLOY --> KV + DEPLOY --> ADMIN + DEPLOY --> KCAPI + KC --> RP +``` + +The two storage nodes are authority boundaries, even when a deployment places them on the same physical database service. Keycloak owns and migrates its internal schema. Keyverse reads/writes only its own supported store and reaches Keycloak user/client/federation state through the supported Admin API rather than direct private-table access. + +## Federation desired-state sequence + +```mermaid +sequenceDiagram + actor Operator + participant Deploy as Deployment controller + participant Keyverse + participant Store as Keyverse state store + participant Keycloak as Keycloak Admin API/engine + + Operator->>Deploy: private rendered federation payload + Deploy->>Keyverse: authenticated preflight + Keyverse->>Keyverse: local closed-schema validation + Keyverse-->>Deploy: redacted readiness result + Deploy->>Keyverse: desired-state apply/reconcile + Keyverse->>Store: persist versioned intent + Keyverse->>Keycloak: exact lookup / create or update + Keycloak-->>Keyverse: live remote state + Keyverse->>Keyverse: canonical re-observation + Keyverse->>Store: write desired-version-bound receipt + Keyverse-->>Deploy: redacted outcome + Deploy->>Operator: controlled acceptance evidence +``` + +## RP registration sequence + +```mermaid +sequenceDiagram + actor AppOwner + participant Deploy as Deployment controller + participant Keyverse + participant Store as Desired-state store + participant Keycloak as Keycloak Admin API/engine + participant Secret as Secret-management port + participant App as Relying party + + AppOwner->>Deploy: secret-free client representation + Deploy->>Keyverse: preflight + Keyverse-->>Deploy: policy result + Deploy->>Keyverse: reconcile desired state + Keyverse->>Store: versioned intent + Keyverse->>Keycloak: exact client search/create/update + Keycloak-->>Keyverse: exact live representation + Keyverse->>Store: version-bound apply receipt + opt confidential client + Deploy->>Secret: provision secret separately + Secret-->>App: controlled credential placement + end + Deploy->>App: run login/logout/token acceptance +``` + +PR #72 extends this sequence with a closed mapper profile; it remains active-PR. + +## Account merge state view + +```mermaid +stateDiagram-v2 + [*] --> distinct_accounts + distinct_accounts --> candidate_link: exact subject / verified email / operator evidence + candidate_link --> rejected: unsafe or ambiguous evidence + candidate_link --> locked: acquire shared user-operation lock + locked --> merging + merging --> survivor_active + merging --> rollback_required: downstream/transaction failure + survivor_active --> duplicate_tombstoned + duplicate_tombstoned --> [*] + rollback_required --> distinct_accounts + rejected --> [*] +``` + +Unverified email cannot enter `candidate_link` by itself. + +## SCIM / merge concurrency authority + +```mermaid +flowchart LR + PUT[SCIM full replacement PUT] + PATCH[SCIM PATCH active=false — current narrower path] + MERGE[Merge/link mutation] + LOCK[user_operation_lock_state] + USER[Keycloak user state] + AUDIT[account_merge_audit / operation evidence] + + PUT --> LOCK + MERGE --> LOCK + LOCK --> USER + PATCH -. not currently in shared-lock guarantee .-> USER + USER --> AUDIT +``` + +Protected `main` guarantees the shared cross-process lock for merge/link and full SCIM replacement. The current `PATCH active=false` path is explicitly not represented as serialized with merge. Extending that guarantee is a source-and-concurrency-test change, not a documentation relabel. + +## Automation authority + +```mermaid +flowchart LR + MODEL[OpenCode model process] + VERIFY[credential-free verifier] + PUB[bounded PR publisher] + REVIEW[independent review/security] + MAIN[protected main] + + MODEL --> VERIFY + VERIFY --> PUB + PUB --> REVIEW + REVIEW --> MAIN +``` + +PR #74 changes exact hourly gate implementation but not this authority separation. + +## Maintenance rule + +Update these views whenever Keycloak/Keyverse/deployment-controller ownership, identity matching, desired-state lifecycle, secret boundary, persistence, or protected automation authority changes. Active-PR items must not be relabelled as protected-main until integrated. \ No newline at end of file diff --git a/docs/adr/0001-keycloak-hub.md b/docs/adr/0001-keycloak-hub.md new file mode 100644 index 0000000..1dd0af9 --- /dev/null +++ b/docs/adr/0001-keycloak-hub.md @@ -0,0 +1,6 @@ +# ADR-0001: Keep Keycloak and Keyverse as the ecosystem identity hub + +**Status:** Accepted +**Date:** 2026-08-09 + +Keyverse uses Keycloak as the standards-based identity engine and adds CWL-owned control services around it. Employer/customer ADFS, LDAP/AD, external OIDC, and HR/IGA are federation/provisioning sources rather than peer hubs. CWL relying parties trust the Keyverse/Keycloak boundary instead of administering those external systems directly. Customer-specific federation remains deployment data, not portable realm code. \ No newline at end of file diff --git a/docs/adr/0002-passwordless-local-accounts.md b/docs/adr/0002-passwordless-local-accounts.md new file mode 100644 index 0000000..0ff62ca --- /dev/null +++ b/docs/adr/0002-passwordless-local-accounts.md @@ -0,0 +1,6 @@ +# ADR-0002: Keep ecosystem-local accounts passwordless-first + +**Status:** Accepted +**Date:** 2026-08-09 + +The portable local browser flow uses WebAuthn/passkeys and does not include an ordinary password authenticator. Registration creates no password and uses a controlled enrollment action. External federation may rely on its upstream authentication policy, but Keyverse does not silently add a local password fallback for ecosystem-local accounts. Changing this boundary requires explicit security/product review and migration evidence. \ No newline at end of file diff --git a/docs/adr/0003-identity-matching.md b/docs/adr/0003-identity-matching.md new file mode 100644 index 0000000..4045434 --- /dev/null +++ b/docs/adr/0003-identity-matching.md @@ -0,0 +1,6 @@ +# ADR-0003: Use exact external subject, then verified email, then explicit link + +**Status:** Accepted +**Date:** 2026-08-09 + +Account matching precedence is exact `(identity_provider, subject)`, then verified email under policy, then explicit operator link. Unverified email never authorizes automatic linking or merge. Merged duplicate accounts remain disabled tombstones with survivor lineage. This decision is shared by account unification, federation, and SCIM so one path cannot weaken another's identity evidence. \ No newline at end of file diff --git a/docs/adr/0004-desired-state-reconciliation.md b/docs/adr/0004-desired-state-reconciliation.md new file mode 100644 index 0000000..35ba95c --- /dev/null +++ b/docs/adr/0004-desired-state-reconciliation.md @@ -0,0 +1,6 @@ +# ADR-0004: Use side-effect-free preflight and re-observed desired-state reconciliation + +**Status:** Accepted +**Date:** 2026-08-09 + +Federation, directory, and relying-party onboarding separate deterministic local preflight from external apply. Where Keyverse owns desired state, intent is persisted before remote mutation, duplicate remote matches fail closed, and a canonical apply receipt is written only after exact live re-observation. Delete uses remote-first ordering where local-first deletion could create false success. Preflight success never means external login/bind/provisioning success. \ No newline at end of file diff --git a/docs/adr/0005-secret-ownership.md b/docs/adr/0005-secret-ownership.md new file mode 100644 index 0000000..5e79284 --- /dev/null +++ b/docs/adr/0005-secret-ownership.md @@ -0,0 +1,6 @@ +# ADR-0005: Separate portable configuration from deployment-private values + +**Status:** Accepted +**Date:** 2026-08-09 + +Portable realm configuration and ordinary desired-state records contain only the fields needed for reproducible identity policy. Deployment-specific confidential values remain owned by the deployment controller and its approved configuration store. Public repository artifacts, ordinary responses, and routine logs do not copy those private values. This keeps the portable realm reusable across tenants and supports controlled rotation and rollback. \ No newline at end of file diff --git a/docs/adr/0006-user-operation-lock.md b/docs/adr/0006-user-operation-lock.md new file mode 100644 index 0000000..1a912ad --- /dev/null +++ b/docs/adr/0006-user-operation-lock.md @@ -0,0 +1,23 @@ +# ADR-0006: Share one user-operation lock across merge and SCIM full replacement + +**Status:** Accepted +**Date:** 2026-08-09 + +## Context + +Account merge/link operations and a SCIM full user replacement can target the same Keycloak user. The full replacement path reads tombstone state and then writes the user representation, so it must not race a merge that creates the tombstone between those operations. + +Protected `main` also supports the narrower `PATCH active=false` deprovisioning path. That PATCH path currently performs its read/deactivate/read sequence outside the shared cross-process lock. This ADR therefore must not imply that every SCIM mutation is serialized with merge. + +## Decision + +Keyverse uses one cross-process user-operation lock boundary for account merge/link and SCIM `PUT /Users/{id}` full replacement. Those operations serialize consistently, preserve tombstone/survivor invariants, and can be retried or recovered from observed durable state. + +The current SCIM `PATCH active=false` path is explicitly outside this Accepted shared-lock guarantee. If PATCH or any future SCIM read-modify-write operation can affect tombstone, survivor, or reactivation invariants, it must join the same lock boundary and add a concurrency regression before documentation may claim equivalent serialization. + +## Consequences + +- Merge and SCIM full replacement share one documented concurrency authority. +- The protected-main PATCH behavior remains usable but must not be described as transactionally serialized with merge. +- Expanding the lock guarantee requires a source/test change, not a documentation-only promotion. +- Clustered deployments must provide the same shared-lock semantics for every operation included in this boundary. \ No newline at end of file diff --git a/docs/adr/0007-automation-authority.md b/docs/adr/0007-automation-authority.md new file mode 100644 index 0000000..8e1b264 --- /dev/null +++ b/docs/adr/0007-automation-authority.md @@ -0,0 +1,6 @@ +# ADR-0007: Separate autonomous development from review, merge, and release authority + +**Status:** Accepted +**Date:** 2026-08-09 + +Autonomous development may inspect exact repository state, produce a bounded patch, and submit ordinary reviewable work after independent verification. It cannot create its own qualifying approval, bypass branch protection, merge protected main, tag, or publish a release. Model-provider credentials remain separate from reviewer, publication, and release credentials. PR #74 refines the hourly implementation while preserving this authority boundary. \ No newline at end of file diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..31b4b18 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,19 @@ +# Keyverse Architecture Decision Record Index + +`Accepted` means the decision governs architecture; it does not imply an active PR has merged or a customer deployment has completed acceptance. + +| ADR | Decision | Status | +|---|---|---| +| [0001](0001-keycloak-hub.md) | Keep Keycloak/Keyverse as the ecosystem identity hub | Accepted | +| [0002](0002-passwordless-local-accounts.md) | Passwordless-first local accounts without password authenticator | Accepted | +| [0003](0003-identity-matching.md) | Exact external subject → verified email → explicit link matching precedence | Accepted | +| [0004](0004-desired-state-reconciliation.md) | Side-effect-free preflight plus intent/reconcile/re-observe/receipt lifecycle | Accepted | +| [0005](0005-secret-ownership.md) | Deployment/KV owns secrets; portable desired state remains secret-minimized | Accepted | +| [0006](0006-user-operation-lock.md) | Merge and SCIM full replacement share one user-operation lock boundary | Accepted | +| [0007](0007-automation-authority.md) | Autonomous development remains separate from review/merge/release authority | Accepted | + +## ADR triggers + +Create or update an ADR for changes to authenticator policy, federation hub ownership, identity matching evidence, merge/tombstone semantics, SCIM authority, directory write/trust policy, RP credential/claim ownership, desired-state mutation order, persistent state, secret handling, or autonomous/release authority. + +Each implementation PR should reconcile PRD/TRD/Architecture/UML/ERD/Threat/Test/Operability/Traceability and the relevant `docs/doctoring/`, `docs/papers/`, or `docs/operations/` research/standards/runbook record when those contracts move. \ No newline at end of file diff --git a/docs/merge-unification-flow.md b/docs/merge-unification-flow.md index 35a88f9..55d96f0 100644 --- a/docs/merge-unification-flow.md +++ b/docs/merge-unification-flow.md @@ -80,18 +80,27 @@ resolve to the survivor. can set `active: true`. Its tombstone check and replacement PUT therefore execute inside the **same user-operation lock** used by the complete merge transaction. A merge cannot create `merged_into_user_id` between those two Admin API calls, -and SCIM cannot wipe a newly-created tombstone or reactivate the duplicate. +and SCIM PUT cannot wipe a newly-created tombstone or reactivate the duplicate. + +Protected `main` does **not** currently extend that shared-lock guarantee to +`PATCH /scim/v2/Users/{id}`. The implemented PATCH surface only supports the +`active=false` deprovisioning shape and executes its read/deactivate/read path +without `user_operation_locks.hold(user_id)`. Operators and documentation must +therefore not treat PATCH and merge as transactionally serialized. If PATCH is +expanded or needs the same tombstone/survivor concurrency guarantee, the source +path must join the shared lock and add a merge/PATCH race regression before the +stronger contract is promoted. Standalone deployments use a dedicated SQLite sidecar lock database and hold a `BEGIN IMMEDIATE` transaction for the complete critical section. This provides a crash-safe mutex shared by every worker/process using the same database path; process death closes the connection and releases the lock. The current backend -serializes all user mutations conservatively rather than risking a multi-user -deadlock. Lock acquisition waits up to 10 seconds, then returns retryable HTTP -`503` without performing a partial mutation. A clustered Postgres deployment -must provide the same `UserOperationLocks` contract (for example, ordered -advisory locks) and wire one shared instance into both the merge service and SCIM -router. +serializes operations that participate in the shared lock conservatively rather +than risking a multi-user deadlock. Lock acquisition waits up to 10 seconds, +then returns retryable HTTP `503` without performing a partial mutation. A +clustered Postgres deployment must provide the same `UserOperationLocks` +contract and wire one shared instance into the merge service and every SCIM +operation that claims this serialization guarantee. ## Audit @@ -122,4 +131,4 @@ production swaps in a Postgres-backed sink. All config/secrets (Keycloak server URL, realm, service-account client id + secret, conflict policy) come from the KV/DB store via the bootstrap pointer — -see `services/account_unification/app/config.py`. No runtime `os.getenv`. +see `services/account_unification/app/config.py`. No runtime `os.getenv`. \ No newline at end of file diff --git a/tests/test_documentation_contract.py b/tests/test_documentation_contract.py new file mode 100644 index 0000000..5fbc211 --- /dev/null +++ b/tests/test_documentation_contract.py @@ -0,0 +1,105 @@ +"""Contract tests for Keyverse's canonical product and architecture documents.""" + +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +REQUIRED_DOCUMENTS = ( + "DOCUMENTATION.md", + "docs/PRD.md", + "docs/TRD.md", + "ARCHITECTURE.md", + "docs/UML.md", + "docs/ERD.md", + "docs/THREAT_MODEL.md", + "docs/TEST_STRATEGY.md", + "docs/OPERABILITY.md", + "docs/TRACEABILITY.md", + "docs/adr/README.md", + "README.md", + "AGENTS.md", + "CLAUDE.md", + "CHANGELOG.md", +) +GOVERNING_ADRS = ( + "0001-keycloak-hub.md", + "0002-passwordless-local-accounts.md", + "0003-identity-matching.md", + "0004-desired-state-reconciliation.md", + "0005-secret-ownership.md", + "0006-user-operation-lock.md", + "0007-automation-authority.md", +) + + +def _read(relative_path: str) -> str: + """Read one repository document using the canonical UTF-8 encoding.""" + + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def _row_with(text: str, marker: str) -> str: + """Return the Markdown table row containing ``marker``.""" + + for line in text.splitlines(): + if line.startswith("|") and marker in line: + return line + raise AssertionError(f"missing Markdown table row containing {marker!r}") + + +def test_canonical_identity_documents_exist() -> None: + """Keep product, architecture, safety, and operating memory discoverable.""" + + missing = [path for path in REQUIRED_DOCUMENTS if not (ROOT / path).is_file()] + assert not missing, f"missing canonical documentation: {missing}" + + +def test_documentation_map_links_cross_cutting_contracts() -> None: + """Require the documentation map to link every canonical record.""" + + documentation = _read("DOCUMENTATION.md") + for path in REQUIRED_DOCUMENTS[1:]: + assert f"]({path})" in documentation, ( + f"documentation map does not link {path}" + ) + + +def test_active_pr_features_are_not_promoted_to_main() -> None: + """Keep OIDC mapper and hourly-remediation PRs labelled as active work.""" + + prd = _read("docs/PRD.md") + traceability = _read("docs/TRACEABILITY.md") + assert any( + "PR #72" in line and "active-PR" in line + for line in prd.splitlines() + ) + assert any( + "PR #74" in line and "active-PR" in line + for line in prd.splitlines() + ) + mapper_row = _row_with(traceability, "RP audience/role/org/workspace mapper profile") + hourly_row = _row_with(traceability, "work-conserving fail-closed hourly API gate") + assert mapper_row.rstrip().endswith("| active-PR |") + assert "PR #72" in mapper_row + assert hourly_row.rstrip().endswith("| active-PR |") + assert "PR #74" in hourly_row + + +def test_erd_keeps_keycloak_internal_schema_external() -> None: + """Prevent the logical ERD from claiming ownership of Keycloak internals.""" + + erd = _read("docs/ERD.md") + assert "Keycloak internal schema remains Keycloak-owned" in erd + assert "does not duplicate or directly edit unsupported Keycloak internal tables" in erd + + +def test_adr_index_contains_governing_identity_decisions() -> None: + """Keep every indexed architecture decision present and reviewable.""" + + index = _read("docs/adr/README.md") + for adr in GOVERNING_ADRS: + adr_path = ROOT / "docs" / "adr" / adr + assert adr_path.is_file(), f"ADR file is missing: {adr}" + assert f"]({adr})" in index, f"ADR index does not link {adr}"