diff --git a/.github/workflows/validate-defaults.yaml b/.github/workflows/validate-defaults.yaml index e4fa7477..0f6935cf 100644 --- a/.github/workflows/validate-defaults.yaml +++ b/.github/workflows/validate-defaults.yaml @@ -26,9 +26,9 @@ jobs: with: yq-version: v4.30.7 - - name: Validate clusterGroupName is simple + - name: Validate clusterGroupName is azure run: | - if [ "$(yq '.main.clusterGroupName' values-global.yaml)" != "simple" ]; then - echo "main.clusterGroupName must be 'simple'" + if [ "$(yq '.main.clusterGroupName' values-global.yaml)" != "azure" ]; then + echo "main.clusterGroupName must be 'azure'" exit 1 fi diff --git a/AGENTS.md b/AGENTS.md index 6bad2fdc..34f5083f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,10 +45,10 @@ Use the **first** approach that fits your requirement: ├── rhdp/ # Red Hat Demo Platform tooling ├── scripts/ # Utility scripts ├── values-global.yaml # Global configuration -├── values-simple.yaml # Cluster group: simple +├── values-azure.yaml # Cluster group: azure ├── values-baremetal.yaml # Cluster group: baremetal ├── values-trusted-hub.yaml # Cluster group: trusted-hub -├── values-spoke.yaml # Cluster group: spoke +├── values-azure-spoke.yaml # Cluster group: azure-spoke └── values-secret.yaml.template # Secrets template (never commit filled-in copy) ``` @@ -59,8 +59,8 @@ These charts are published independently and consumed from the `charts.validated | Chart Name | Repository | Purpose | |---|---|---| | `trustee` | `validatedpatterns/trustee-chart` | Trustee / KBS configuration | -| `sandboxed-policies` | `validatedpatterns/sandboxed-policies-chart` | ACM policies hub → spoke | -| `sandboxed-containers` | `validatedpatterns/sandboxed-containers-chart` | Sandboxed runtime on spoke | +| `sandboxed-policies` | `validatedpatterns/sandboxed-policies-chart` | ACM policies hub → azure-spoke | +| `sandboxed-containers` | `validatedpatterns/sandboxed-containers-chart` | Sandboxed runtime on azure-spoke | Changes to companion charts require a release (Git tag) before the pattern can consume them. Update the `chartVersion:` field in the values files to pick up new releases. @@ -70,11 +70,10 @@ Set via `main.clusterGroupName` in `values-global.yaml`. | Cluster Group | Values File | Role | Description | |---|---|---|---| -| `simple` | `values-simple.yaml` | Hub (single cluster) | All components on one Azure cluster | -| `baremetal` | `values-baremetal.yaml` | Hub (single cluster) | TDX/SNP + LVM storage on bare metal | -| `baremetal-gpu` | `values-baremetal-gpu.yaml` | Hub (single cluster) | Bare metal + NVIDIA H100 GPU support | +| `azure` | `values-azure.yaml` | Hub (single cluster) | All components on one Azure cluster | +| `baremetal` | `values-baremetal.yaml` | Hub (single cluster) | Bare metal (hardware profile gated: intel-tdx, amd-snp, intel-tdx-gpu, amd-snp-gpu) | | `trusted-hub` | `values-trusted-hub.yaml` | Multi-cluster hub | Trustee + ACM policies | -| `spoke` | `values-spoke.yaml` | Multi-cluster spoke | Sandbox runtime + workloads | +| `azure-spoke` | `values-azure-spoke.yaml` | Multi-cluster spoke | Sandbox runtime + workloads (Azure) | ## Values File Hierarchy diff --git a/Makefile b/Makefile index 26a89e0e..fadd93d3 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,19 @@ include Makefile-common +##@ Key Management +.PHONY: cache-keys +cache-keys: ## Download Red Hat signing keys from official sources to ~/.coco-pattern/ + @mkdir -p ~/.coco-pattern + @echo "Fetching Red Hat sigstore public key (release key 3)..." + @curl -fsSL https://www.redhat.com/security/data/63405576.txt -o ~/.coco-pattern/SIGSTORE-redhat-release3 + @echo " Cached at ~/.coco-pattern/SIGSTORE-redhat-release3" + @echo "Fetching Red Hat GPG public key (release key 2)..." + @curl -fsSL https://access.redhat.com/security/data/fd431d51.txt -o ~/.coco-pattern/RPM-GPG-KEY-redhat-release + @echo " Cached at ~/.coco-pattern/RPM-GPG-KEY-redhat-release" + @echo "Done. Verify fingerprints at https://access.redhat.com/security/team/key/" + + ##@ Reference Value Collection .PHONY: collect-firmware-refvals collect-firmware-refvals: ## Collect firmware reference values (bare metal, default) @@ -12,3 +25,37 @@ collect-firmware-refvals: ## Collect firmware reference values (bare metal, defa .PHONY: collect-azure-refvals collect-azure-refvals: ## Collect PCR reference values (Azure) @scripts/collect-firmware-refvals.sh --platform azure + +##@ Hardware Detection +.PHONY: detect-hardware +detect-hardware: ## Detect hardware profile from cluster nodes (requires KUBECONFIG or oc login) + @echo "Detecting hardware profile from cluster nodes..." + @echo "---" + @CPU_VENDOR=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-model\.vendor_id}' 2>/dev/null) && \ + TDX_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.tdx\.enabled}' 2>/dev/null) && \ + SNP_ENABLED=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.feature\.node\.kubernetes\.io/cpu-security\.sev\.snp}' 2>/dev/null) && \ + GPU_PRESENT=$$(oc get nodes -o jsonpath='{.items[0].metadata.labels.nvidia\.com/gpu\.present}' 2>/dev/null) && \ + echo "CPU Vendor: $${CPU_VENDOR:-unknown}" && \ + echo "TDX Enabled: $${TDX_ENABLED:-false}" && \ + echo "SNP Enabled: $${SNP_ENABLED:-false}" && \ + echo "GPU Present: $${GPU_PRESENT:-false}" && \ + echo "---" && \ + if [ "$${CPU_VENDOR}" = "Intel" ] && [ "$${TDX_ENABLED}" = "true" ]; then \ + if [ "$${GPU_PRESENT}" = "true" ]; then \ + echo "Recommended profile: intel-tdx-gpu"; \ + else \ + echo "Recommended profile: intel-tdx"; \ + fi; \ + elif [ "$${CPU_VENDOR}" = "AuthenticAMD" ] || [ "$${SNP_ENABLED}" = "true" ]; then \ + if [ "$${GPU_PRESENT}" = "true" ]; then \ + echo "Recommended profile: amd-snp-gpu"; \ + else \ + echo "Recommended profile: amd-snp"; \ + fi; \ + else \ + echo "Could not determine hardware profile."; \ + echo "Ensure NFD operator is running and node labels are populated."; \ + echo "Set global.hardware.profile manually in values-global.yaml"; \ + fi && \ + echo "" && \ + echo "To apply: edit values-global.yaml and set global.hardware.profile to the recommended value." diff --git a/README.md b/README.md index de7c94b6..88fd1599 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,21 @@ Confidential containers use hardware-backed Trusted Execution Environments (TEEs The pattern provides four deployment topologies: -1. **Single cluster** (`simple` clusterGroup) — deploys all components (Trustee, Vault, ACM, sandboxed containers, workloads) in one cluster on Azure. This breaks the RACI separation expected in a remote attestation architecture but simplifies testing and demonstrations. +1. **Single cluster** (`azure` clusterGroup) — deploys all components (Trustee, Vault, ACM, sandboxed containers, workloads) in one cluster on Azure. This breaks the RACI separation expected in a remote attestation architecture but simplifies testing and demonstrations. -2. **Multi-cluster** (`trusted-hub` + `spoke` clusterGroups) — separates the trusted zone from the untrusted workload zone: +2. **Multi-cluster** (`trusted-hub` + `azure-spoke` clusterGroups) — separates the trusted zone from the untrusted workload zone: - **Hub** (`trusted-hub`): Runs Trustee (KBS + attestation service), HashiCorp Vault, ACM, and cert-manager. This cluster is the trust anchor. - - **Spoke** (`spoke`): Runs the sandboxed containers operator and confidential workloads. The spoke is imported into ACM and managed from the hub. + - **Spoke** (`azure-spoke`): Runs the sandboxed containers operator and confidential workloads. The spoke is imported into ACM and managed from the hub. 3. **Bare metal** (`baremetal` clusterGroup) — deploys all components on bare metal hardware with Intel TDX or AMD SEV-SNP support. NFD (Node Feature Discovery) auto-detects the CPU architecture and configures the appropriate runtime. Supports SNO (Single Node OpenShift) and multi-node clusters. -4. **Bare metal with GPU** (`baremetal-gpu` clusterGroup) — extends the bare metal topology with NVIDIA H100 confidential GPU support. Adds the NVIDIA GPU Operator, IOMMU kernel configuration, and a sample CUDA workload for CC GPU verification. Requires NVIDIA H100 GPUs with confidential computing firmware. + Hardware-specific operators (GPU, Intel device plugins, DCAP) are controlled by `global.hardware.profile`: + - `intel-tdx` — Intel TDX without GPU + - `amd-snp` — AMD SEV-SNP without GPU + - `intel-tdx-gpu` — Intel TDX with NVIDIA H100 GPU + - `amd-snp-gpu` — AMD SEV-SNP with NVIDIA H100 GPU -The topology is controlled by the `main.clusterGroupName` field in `values-global.yaml`. +The topology is controlled by the `main.clusterGroupName` field in `values-global.yaml`. For bare metal deployments, also set `global.hardware.profile` to match your hardware configuration. Azure deployments use peer-pods, which provision confidential VMs (`Standard_DCas_v5` family) directly on the Azure hypervisor. Bare metal deployments use layered images and hardware TEE features directly. @@ -81,7 +85,7 @@ These scripts generate the cryptographic material and attestation reference valu ### Single cluster deployment (Azure) -1. Set `main.clusterGroupName: simple` in `values-global.yaml` +1. Set `main.clusterGroupName: azure` in `values-global.yaml` 2. Ensure your Azure configuration is populated in `values-global.yaml` (see `global.azure.*` fields) 3. `./pattern.sh make install` 4. Wait for the cluster to reboot all nodes (the sandboxed containers operator triggers a MachineConfig update). Monitor progress in the ArgoCD UI. @@ -92,17 +96,20 @@ These scripts generate the cryptographic material and attestation reference valu 2. Deploy the hub cluster: `./pattern.sh make install` 3. Wait for ACM (`MultiClusterHub`) to reach `Running` state on the hub 4. Provision a second OpenShift 4.19.28+ cluster on Azure for the spoke -5. Import the spoke into ACM with label `clusterGroup=spoke` +5. Import the spoke into ACM with label `clusterGroup=azure-spoke` (see [importing a cluster](https://validatedpatterns.io/learn/importing-a-cluster/)) -6. ACM will automatically deploy the `spoke` clusterGroup applications (sandboxed containers, workloads) to the imported cluster +6. ACM will automatically deploy the `azure-spoke` clusterGroup applications (sandboxed containers, workloads) to the imported cluster ### Bare metal deployment 1. Set `main.clusterGroupName: baremetal` in `values-global.yaml` -2. Run `bash scripts/gen-secrets.sh` to generate KBS keys and PCCS secrets -3. For Intel TDX: uncomment the PCCS secrets in `~/values-secret-coco-pattern.yaml` and provide your Intel PCS API key -4. `./pattern.sh make install` -5. Wait for the cluster to reboot nodes (MachineConfig updates for TDX kernel parameters and vsock) +2. Set `global.hardware.profile` to match your hardware (default: `intel-tdx`) + - Run `make detect-hardware` after NFD is deployed to detect your hardware profile automatically + - Options: `intel-tdx`, `amd-snp`, `intel-tdx-gpu`, `amd-snp-gpu` +3. Run `bash scripts/gen-secrets.sh` to generate KBS keys and PCCS secrets +4. For Intel TDX: uncomment the PCCS secrets in `~/values-secret-coco-pattern.yaml` and provide your Intel PCS API key +5. `./pattern.sh make install` +6. Wait for the cluster to reboot nodes (MachineConfig updates for TDX/SEV-SNP kernel parameters and vsock) > **Note:** Bare metal support is currently tested on SNO (Single Node OpenShift) configurations. Multi-node bare metal clusters are expected to work but have not been validated yet. @@ -117,20 +124,15 @@ The system auto-detects your hardware: Optional: pin PCCS to a specific node with `bash scripts/get-pccs-node.sh` and set `baremetal.pccs.nodeSelector` in the baremetal chart values. -### Bare metal GPU deployment +For GPU-enabled deployments (`intel-tdx-gpu` or `amd-snp-gpu` profiles): -1. Set `main.clusterGroupName: baremetal-gpu` in `values-global.yaml` -2. Run `bash scripts/gen-secrets.sh` to generate KBS keys and PCCS secrets -3. For Intel TDX: uncomment the PCCS secrets in `~/values-secret-coco-pattern.yaml` and provide your Intel PCS API key -4. `./pattern.sh make install` -5. Wait for the cluster to reboot nodes (MachineConfig updates for TDX/SEV-SNP kernel parameters, vsock, and IOMMU) -6. Approve the GPU Operator install plan when it appears (uses `installPlanApproval: Manual`) - -> **Note:** The `baremetal-gpu` topology deploys IOMMU MachineConfig on all nodes and will trigger reboots. For clusters without GPUs, use the `baremetal` topology instead. The GPU workload deployment will remain Pending on non-GPU systems but is otherwise harmless. +- IOMMU MachineConfig is deployed on all nodes and will trigger reboots +- Approve the GPU Operator install plan when it appears (uses `installPlanApproval: Manual`) +- A sample CUDA workload (`gpu-workload`) is deployed for CC GPU verification ## Sample applications -Two sample applications are deployed on the cluster running confidential workloads (the single cluster in `simple` mode, or the spoke in multi-cluster mode): +Two sample applications are deployed on the cluster running confidential workloads (the single cluster in `azure` mode, or the spoke in multi-cluster mode): - **hello-openshift**: Three pods demonstrating CoCo security boundaries: - `standard` — a regular Kubernetes pod (no confidential computing) diff --git a/ansible/install-deps.yaml b/ansible/get-azure-deps.yaml similarity index 92% rename from ansible/install-deps.yaml rename to ansible/get-azure-deps.yaml index 8a927b90..7359cfe4 100644 --- a/ansible/install-deps.yaml +++ b/ansible/get-azure-deps.yaml @@ -1,4 +1,4 @@ -- name: Retrieve Credentials for AAP on OpenShift +- name: Install Azure collection dependencies become: false connection: local hosts: localhost diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index c9de0a5f..3bd18999 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -132,7 +132,9 @@ ansible.builtin.shell: | set -o pipefail initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 - PCR8_HASH=$(echo -n "${initial_pcr}{{ raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + hash_input="${initial_pcr}{{ raw_hash.stdout }}" + hash_cmd='import sys,hashlib; print(hashlib.sha256(bytes.fromhex(sys.stdin.read())).hexdigest())' + PCR8_HASH=$(echo -n "${hash_input}" | python3 -c "${hash_cmd}") && echo $PCR8_HASH register: pcr8_hash changed_when: false @@ -140,7 +142,9 @@ ansible.builtin.shell: | set -o pipefail initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 - PCR8_HASH=$(echo -n "${initial_pcr}{{ debug_raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + hash_input="${initial_pcr}{{ debug_raw_hash.stdout }}" + hash_cmd='import sys,hashlib; print(hashlib.sha256(bytes.fromhex(sys.stdin.read())).hexdigest())' + PCR8_HASH=$(echo -n "${hash_input}" | python3 -c "${hash_cmd}") && echo $PCR8_HASH register: debug_pcr8_hash changed_when: false diff --git a/ansible/initdata-debug.toml.tpl b/ansible/initdata-debug.toml.tpl index b49fae8f..a463c3b2 100644 --- a/ansible/initdata-debug.toml.tpl +++ b/ansible/initdata-debug.toml.tpl @@ -24,6 +24,7 @@ kbs_cert = """{{ trustee_cert }}""" [image] image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_flavour }}' +authenticated_registry_credentials_uri = 'kbs:///default/credential/regcred' ''' "policy.rego" = ''' diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index 3fd1ecc3..daf579bf 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -24,6 +24,7 @@ kbs_cert = """{{ trustee_cert }}""" [image] image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_flavour }}' +authenticated_registry_credentials_uri = 'kbs:///default/credential/regcred' ''' "policy.rego" = ''' diff --git a/charts/all/baremetal/templates/nfd-instance.yaml b/charts/all/baremetal/templates/nfd-instance.yaml index 97ce9ee1..ec967a1b 100644 --- a/charts/all/baremetal/templates/nfd-instance.yaml +++ b/charts/all/baremetal/templates/nfd-instance.yaml @@ -5,7 +5,7 @@ metadata: namespace: openshift-nfd spec: operand: - image: registry.redhat.io/openshift4/ose-node-feature-discovery-rhel9:v4.20 + image: registry.redhat.io/openshift4/ose-node-feature-discovery-rhel9:v{{ .Values.global.clusterVersion }} imagePullPolicy: Always servicePort: 12000 workerConfig: diff --git a/charts/all/intel-dcap/templates/pccs-secrets-eso.yaml b/charts/all/intel-dcap/templates/pccs-secrets-eso.yaml index 5ee91ab9..b8bb2f55 100644 --- a/charts/all/intel-dcap/templates/pccs-secrets-eso.yaml +++ b/charts/all/intel-dcap/templates/pccs-secrets-eso.yaml @@ -1,5 +1,5 @@ --- -apiVersion: "external-secrets.io/v1beta1" +apiVersion: "external-secrets.io/v1" kind: ExternalSecret metadata: name: pccs-secrets-eso diff --git a/charts/all/intel-dcap/templates/pccs-tls-eso.yaml b/charts/all/intel-dcap/templates/pccs-tls-eso.yaml index a7212ae1..0d82feb6 100644 --- a/charts/all/intel-dcap/templates/pccs-tls-eso.yaml +++ b/charts/all/intel-dcap/templates/pccs-tls-eso.yaml @@ -1,5 +1,5 @@ --- -apiVersion: "external-secrets.io/v1beta1" +apiVersion: "external-secrets.io/v1" kind: ExternalSecret metadata: name: pccs-tls-eso diff --git a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml index d67faecd..99d0a405 100644 --- a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml +++ b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml @@ -16,7 +16,7 @@ spec: labels: app: gpu-vectoradd annotations: - coco.io/initdata-configmap: debug-initdata + coco.io/initdata-configmap: initdata {{- if .Values.defaultMemory }} io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} {{- end }} @@ -32,6 +32,15 @@ spec: - | /opt/cuda-samples/Samples/0_Introduction/vectorAdd/build/vectorAdd sleep 36000 + volumeMounts: + - name: initdata + mountPath: /opt/confidential-containers/initdata + readOnly: true resources: limits: nvidia.com/pgpu: 1 + volumes: + - name: initdata + configMap: + name: initdata + optional: false diff --git a/charts/coco-supported/hello-openshift/README.md b/charts/coco-supported/hello-openshift/README.md new file mode 100644 index 00000000..416848cc --- /dev/null +++ b/charts/coco-supported/hello-openshift/README.md @@ -0,0 +1,40 @@ +# hello-openshift + +Three deployments demonstrating confidential containers on Red Hat httpd-24, each with a different security posture. + +All three use `registry.redhat.io/ubi9/httpd-24` on port 8080 with custom web content served from ConfigMaps. + +## Deployments + +### standard + +Non-confidential baseline. Runs without `kata-cc` runtime — standard OCI container for comparison. + +| Field | Value | +|-------|-------| +| Route | `http://standard-hello-openshift.apps./` | +| Runtime | default (runc) | +| Initdata | none | +| Web content | `standard-web-content` ConfigMap | + +### insecure-policy + +Confidential container using `kata-cc` runtime with the `debug-initdata` ConfigMap (insecure/debug attestation policy). Useful for development and troubleshooting — allows `oc exec` and relaxed attestation. + +| Field | Value | +|-------|-------| +| Route | `http://insecure-policy-hello-openshift.apps./` | +| Runtime | `kata-cc` | +| Initdata | `debug-initdata` (insecure policy) | +| Web content | `insecure-policy-web-content` ConfigMap | + +### secure + +Confidential container using `kata-cc` runtime with the production `initdata` ConfigMap (signed/verified attestation policy). Blocks `oc exec` and enforces attestation. + +| Field | Value | +|-------|-------| +| Route | `http://secure-hello-openshift.apps./` | +| Runtime | `kata-cc` | +| Initdata | `initdata` (production policy) | +| Web content | `secure-web-content` ConfigMap | diff --git a/charts/coco-supported/hello-openshift/templates/_helpers.tpl b/charts/coco-supported/hello-openshift/templates/_helpers.tpl index ab49aab2..162bbf9d 100644 --- a/charts/coco-supported/hello-openshift/templates/_helpers.tpl +++ b/charts/coco-supported/hello-openshift/templates/_helpers.tpl @@ -51,16 +51,8 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Determine runtime class name. -If runtimeClassName is explicitly set, use it. -Otherwise, detect from cluster platform: "kata-remote" for Azure/AWS, "kata-cc" for other platforms. +Determine runtime class name from global config. */}} {{- define "hello-openshift.runtimeClassName" -}} -{{- if .Values.runtimeClassName -}} -{{- .Values.runtimeClassName -}} -{{- else if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") -}} -kata-remote -{{- else -}} -kata-cc -{{- end -}} +{{- .Values.global.coco.runtimeClassName -}} {{- end }} diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-configmap.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-configmap.yaml new file mode 100644 index 00000000..caeee6d9 --- /dev/null +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: insecure-policy-web-content + labels: + app: insecure-policy +data: + index.html: | + + + + Hello OpenShift + + +

Hello, OpenShift!

+

This confidential container is running with an insecure (debug) policy on Red Hat httpd.

+ + diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml index 6770443e..936804dc 100644 --- a/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml @@ -4,6 +4,8 @@ metadata: name: insecure-policy labels: app: insecure-policy + annotations: + argocd.argoproj.io/sync-wave: "10" spec: replicas: 1 selector: @@ -19,9 +21,15 @@ spec: runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} containers: - name: hello-openshift - image: quay.io/openshift/origin-hello-openshift + image: registry.redhat.io/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2 ports: - - containerPort: 8888 + - containerPort: 8080 + volumeMounts: + - name: web-content + mountPath: /var/www/html + - name: initdata + mountPath: /opt/confidential-containers/initdata + readOnly: true securityContext: privileged: false allowPrivilegeEscalation: false @@ -31,3 +39,11 @@ spec: - ALL seccompProfile: type: RuntimeDefault + volumes: + - name: web-content + configMap: + name: insecure-policy-web-content + - name: initdata + configMap: + name: debug-initdata + optional: false diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-route.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-route.yaml index 9bc9c8a2..54cac6b9 100644 --- a/charts/coco-supported/hello-openshift/templates/insecure-policy-route.yaml +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-route.yaml @@ -4,7 +4,7 @@ metadata: name: insecure-policy spec: port: - targetPort: 8888 + targetPort: 8080 to: kind: Service name: insecure-policy diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-svc.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-svc.yaml index d96410f3..34102ee0 100644 --- a/charts/coco-supported/hello-openshift/templates/insecure-policy-svc.yaml +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-svc.yaml @@ -4,10 +4,10 @@ metadata: name: insecure-policy spec: ports: - - name: 8888-tcp - port: 8888 + - name: 8080-tcp + port: 8080 protocol: TCP - targetPort: 8888 + targetPort: 8080 selector: app: insecure-policy sessionAffinity: None diff --git a/charts/coco-supported/hello-openshift/templates/secure-configmap.yaml b/charts/coco-supported/hello-openshift/templates/secure-configmap.yaml new file mode 100644 index 00000000..98cb6942 --- /dev/null +++ b/charts/coco-supported/hello-openshift/templates/secure-configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: secure-web-content + labels: + app: secure +data: + index.html: | + + + + Hello OpenShift + + +

Hello, OpenShift!

+

This confidential container is running on Red Hat httpd.

+ + diff --git a/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml index 4fa15564..35edb6b7 100644 --- a/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml @@ -4,6 +4,8 @@ metadata: name: secure labels: app: secure + annotations: + argocd.argoproj.io/sync-wave: "10" spec: replicas: 1 selector: @@ -16,13 +18,20 @@ spec: annotations: peerpods: "true" coco.io/initdata-configmap: initdata + io.katacontainers.config.runtime.create_container_timeout: "900" spec: runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} containers: - name: hello-openshift - image: quay.io/openshift/origin-hello-openshift + image: registry.redhat.io/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2 ports: - - containerPort: 8888 + - containerPort: 8080 + volumeMounts: + - name: web-content + mountPath: /var/www/html + - name: initdata + mountPath: /opt/confidential-containers/initdata + readOnly: true securityContext: privileged: false allowPrivilegeEscalation: false @@ -32,3 +41,11 @@ spec: - ALL seccompProfile: type: RuntimeDefault + volumes: + - name: web-content + configMap: + name: secure-web-content + - name: initdata + configMap: + name: initdata + optional: false diff --git a/charts/coco-supported/hello-openshift/templates/secure-route.yaml b/charts/coco-supported/hello-openshift/templates/secure-route.yaml index 7e1364fc..3164176b 100644 --- a/charts/coco-supported/hello-openshift/templates/secure-route.yaml +++ b/charts/coco-supported/hello-openshift/templates/secure-route.yaml @@ -4,7 +4,7 @@ metadata: name: secure spec: port: - targetPort: 8888 + targetPort: 8080 to: kind: Service name: secure diff --git a/charts/coco-supported/hello-openshift/templates/secure-svc.yaml b/charts/coco-supported/hello-openshift/templates/secure-svc.yaml index cff85a42..3658d9b2 100644 --- a/charts/coco-supported/hello-openshift/templates/secure-svc.yaml +++ b/charts/coco-supported/hello-openshift/templates/secure-svc.yaml @@ -4,10 +4,10 @@ metadata: name: secure spec: ports: - - name: 8888-tcp - port: 8888 + - name: 8080-tcp + port: 8080 protocol: TCP - targetPort: 8888 + targetPort: 8080 selector: app: secure sessionAffinity: None diff --git a/charts/coco-supported/hello-openshift/templates/standard-configmap.yaml b/charts/coco-supported/hello-openshift/templates/standard-configmap.yaml new file mode 100644 index 00000000..68d04112 --- /dev/null +++ b/charts/coco-supported/hello-openshift/templates/standard-configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: standard-web-content + labels: + app: standard +data: + index.html: | + + + + Hello OpenShift + + +

Hello, OpenShift!

+

This is a standard (non-confidential) container running on Red Hat httpd.

+ + diff --git a/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml index b8b5689b..57cf264e 100644 --- a/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml @@ -16,9 +16,12 @@ spec: spec: containers: - name: hello-openshift - image: quay.io/openshift/origin-hello-openshift + image: registry.redhat.io/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2 ports: - - containerPort: 8888 + - containerPort: 8080 + volumeMounts: + - name: web-content + mountPath: /var/www/html securityContext: privileged: false allowPrivilegeEscalation: false @@ -28,3 +31,7 @@ spec: - ALL seccompProfile: type: RuntimeDefault + volumes: + - name: web-content + configMap: + name: standard-web-content diff --git a/charts/coco-supported/hello-openshift/templates/standard-route.yaml b/charts/coco-supported/hello-openshift/templates/standard-route.yaml index 01218aa7..1c2e4581 100644 --- a/charts/coco-supported/hello-openshift/templates/standard-route.yaml +++ b/charts/coco-supported/hello-openshift/templates/standard-route.yaml @@ -4,7 +4,7 @@ metadata: name: standard spec: port: - targetPort: 8888 + targetPort: 8080 to: kind: Service name: standard diff --git a/charts/coco-supported/hello-openshift/templates/standard-svc.yaml b/charts/coco-supported/hello-openshift/templates/standard-svc.yaml index d7e49607..4c1be365 100644 --- a/charts/coco-supported/hello-openshift/templates/standard-svc.yaml +++ b/charts/coco-supported/hello-openshift/templates/standard-svc.yaml @@ -4,10 +4,10 @@ metadata: name: standard spec: ports: - - name: 8888-tcp - port: 8888 + - name: 8080-tcp + port: 8080 protocol: TCP - targetPort: 8888 + targetPort: 8080 selector: app: standard sessionAffinity: None diff --git a/charts/coco-supported/kbs-access-curl/Chart.yaml b/charts/coco-supported/kbs-access-curl/Chart.yaml new file mode 100644 index 00000000..a6bdfb57 --- /dev/null +++ b/charts/coco-supported/kbs-access-curl/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +description: Demonstrates accessing keys within the KBS using httpd to serve the secret. +keywords: +- pattern +- confidential-containers +name: kbs-access-curl +version: 0.1.0 diff --git a/charts/coco-supported/kbs-access-curl/README.md b/charts/coco-supported/kbs-access-curl/README.md new file mode 100644 index 00000000..26ebf7dc --- /dev/null +++ b/charts/coco-supported/kbs-access-curl/README.md @@ -0,0 +1,31 @@ +# kbs-access-curl + +Demonstrates KBS secret retrieval using the Confidential Data Hub (CDH) init container pattern. An init container fetches a secret from the KBS via the CDH API and writes it to a shared volume. The main httpd container then serves it. + +## How it works + +1. Init container `curl` runs inside the confidential VM +2. Fetches secret from CDH: `curl http://127.0.0.1:8006/cdh/resource/default/kbsres1/key3` +3. Writes result to `/var/www/html/secret.txt` (shared emptyDir volume) +4. Main `httpd-24` container serves the file + +## Endpoints + +| Path | Content | +|------|---------| +| `/` | Default httpd test page (no custom index.html) | +| `/secret.txt` | KBS secret value retrieved by the init container | + +## Route + +```text +http://kbs-access-curl-kbs-access.apps./secret.txt +``` + +## Configuration + +| Field | Value | +|-------|-------| +| Runtime | `kata-cc` | +| Initdata | `debug-initdata` | +| KBS resource path | `default/kbsres1/key3` | diff --git a/charts/coco-supported/kbs-access-curl/templates/deployment.yaml b/charts/coco-supported/kbs-access-curl/templates/deployment.yaml new file mode 100644 index 00000000..821b4397 --- /dev/null +++ b/charts/coco-supported/kbs-access-curl/templates/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbs-access-curl + labels: + app: kbs-access-curl + annotations: + argocd.argoproj.io/sync-wave: "10" +spec: + replicas: 1 + selector: + matchLabels: + app: kbs-access-curl + template: + metadata: + labels: + app: kbs-access-curl + annotations: + peerpods: "true" + coco.io/initdata-configmap: debug-initdata + {{- if .Values.defaultMemory }} + io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} + {{- end }} + io.katacontainers.config.runtime.create_container_timeout: "900" + spec: + runtimeClassName: {{ .Values.global.coco.runtimeClassName }} + containers: + - name: httpd + image: registry.redhat.io/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2 + ports: + - containerPort: 8080 + volumeMounts: + - name: shared-volume + mountPath: /var/www/html + - name: initdata + mountPath: /opt/confidential-containers/initdata + readOnly: true + initContainers: + - name: curl + image: registry.access.redhat.com/ubi9/ubi:latest + command: ['sh', '-c', 'curl -s http://127.0.0.1:8006/cdh/resource/default/kbsres1/key3 > /var/www/html/secret.txt'] + volumeMounts: + - name: shared-volume + mountPath: /var/www/html + volumes: + - name: shared-volume + emptyDir: {} + - name: initdata + configMap: + name: debug-initdata + optional: false diff --git a/charts/coco-supported/kbs-access/templates/secure-route.yaml b/charts/coco-supported/kbs-access-curl/templates/route.yaml similarity index 65% rename from charts/coco-supported/kbs-access/templates/secure-route.yaml rename to charts/coco-supported/kbs-access-curl/templates/route.yaml index dba755f2..37e7f37a 100644 --- a/charts/coco-supported/kbs-access/templates/secure-route.yaml +++ b/charts/coco-supported/kbs-access-curl/templates/route.yaml @@ -1,12 +1,12 @@ apiVersion: route.openshift.io/v1 kind: Route metadata: - name: secure + name: kbs-access-curl spec: port: - targetPort: 5000 + targetPort: 8080 to: kind: Service - name: secure + name: kbs-access-curl weight: 100 wildcardPolicy: None diff --git a/charts/coco-supported/kbs-access/templates/secure-svc.yaml b/charts/coco-supported/kbs-access-curl/templates/service.yaml similarity index 54% rename from charts/coco-supported/kbs-access/templates/secure-svc.yaml rename to charts/coco-supported/kbs-access-curl/templates/service.yaml index 618f2f41..1def982f 100644 --- a/charts/coco-supported/kbs-access/templates/secure-svc.yaml +++ b/charts/coco-supported/kbs-access-curl/templates/service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: secure + name: kbs-access-curl spec: ports: - - name: 5000-tcp - port: 5000 + - name: 8080-tcp + port: 8080 protocol: TCP - targetPort: 5000 + targetPort: 8080 selector: - app: secure + app: kbs-access-curl sessionAffinity: None type: ClusterIP diff --git a/charts/coco-supported/kbs-access/values.yaml b/charts/coco-supported/kbs-access-curl/values.yaml similarity index 100% rename from charts/coco-supported/kbs-access/values.yaml rename to charts/coco-supported/kbs-access-curl/values.yaml diff --git a/charts/coco-supported/kbs-access-sealed/Chart.yaml b/charts/coco-supported/kbs-access-sealed/Chart.yaml new file mode 100644 index 00000000..af4957df --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: kbs-access-sealed +description: Test workload that retrieves secrets from KBS and serves via httpd +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/charts/coco-supported/kbs-access-sealed/README.md b/charts/coco-supported/kbs-access-sealed/README.md new file mode 100644 index 00000000..ecd87b75 --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/README.md @@ -0,0 +1,32 @@ +# kbs-access-sealed + +Demonstrates KBS secret access using a Kubernetes Secret mounted as a volume. The secret is sealed (encrypted) and only decryptable inside the confidential VM via attestation. The httpd container serves the decrypted secret content directly. + +## How it works + +1. `kbs-sealed-secret` Secret is mounted at `/var/www/html` +2. Each key in the Secret becomes a file in the webroot +3. The `httpd-24` container serves these files + +## Endpoints + +| Path | Content | +|------|---------| +| `/` | Default httpd test page (Secret files don't include `index.html`) | +| `/secret-key` | Decrypted secret value from `kbs-sealed-secret` | + +## Route + +```text +https://kbs-access-sealed-kbs-access.apps./secret-key +``` + +Note: This route uses TLS edge termination with redirect. + +## Configuration + +| Field | Value | +|-------|-------| +| Runtime | `kata-cc` | +| Initdata | `initdata` (production policy) | +| Secret | `kbs-sealed-secret` (key: `secret-key`) | diff --git a/charts/coco-supported/kbs-access-sealed/templates/deployment.yaml b/charts/coco-supported/kbs-access-sealed/templates/deployment.yaml new file mode 100644 index 00000000..71464bdb --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/templates/deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbs-access-sealed + namespace: {{ .Values.global.namespace }} + labels: + app: kbs-access-sealed + annotations: + argocd.argoproj.io/sync-wave: "10" +spec: + replicas: 1 + selector: + matchLabels: + app: kbs-access-sealed + template: + metadata: + labels: + app: kbs-access-sealed + annotations: + coco.io/initdata-configmap: initdata + spec: + runtimeClassName: {{ .Values.global.coco.runtimeClassName }} + containers: + - name: httpd + image: registry.redhat.io/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2 + ports: + - containerPort: 8080 + protocol: TCP + volumeMounts: + - name: secret-data + mountPath: /var/www/html + readOnly: true + - name: initdata + mountPath: /opt/confidential-containers/initdata + readOnly: true + resources: + limits: + memory: "2304Mi" + cpu: "200m" + requests: + memory: "128Mi" + cpu: "100m" + volumes: + - name: secret-data + secret: + secretName: kbs-sealed-secret + - name: initdata + configMap: + name: initdata + optional: false diff --git a/charts/coco-supported/kbs-access-sealed/templates/route.yaml b/charts/coco-supported/kbs-access-sealed/templates/route.yaml new file mode 100644 index 00000000..a6c23ec9 --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/templates/route.yaml @@ -0,0 +1,20 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: kbs-access-sealed + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "10" + labels: + app: kbs-access-sealed +spec: + to: + kind: Service + name: kbs-access-sealed + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/charts/coco-supported/kbs-access-sealed/templates/sealed-secret.yaml b/charts/coco-supported/kbs-access-sealed/templates/sealed-secret.yaml new file mode 100644 index 00000000..949df91d --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/templates/sealed-secret.yaml @@ -0,0 +1,30 @@ +# NOTE: This is a placeholder Kubernetes Secret that contains a KBS resource pointer. +# +# In production, this secret should be created using coco-tools to generate the +# sealed secret resource identifier that points to the actual secret stored in KBS. +# +# The KBS resource pointer format is typically: +# kbs:/// +# +# Example workflow: +# 1. Store secret in KBS: coco-tools kbs add-secret +# 2. Generate resource pointer: coco-tools kbs get-pointer +# 3. Update this secret with the pointer (base64 encoded) +# +# For testing purposes, this contains a literal secret value. +# Replace with KBS pointer in production deployments. + +apiVersion: v1 +kind: Secret +metadata: + name: kbs-sealed-secret + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "5" + labels: + app: kbs-access-sealed +type: Opaque +data: + # TODO: Replace with KBS resource pointer generated via coco-tools + # Example: secret-key: a2JzOi8va2JzLXNlcnZpY2Uua2JzLXN5c3RlbS9zZWNyZXRzL215LXNlY3JldA== + secret-key: VGhpcyBpcyBhIHRlc3Qgc2VjcmV0IGZyb20gS0JT diff --git a/charts/coco-supported/kbs-access-sealed/templates/service.yaml b/charts/coco-supported/kbs-access-sealed/templates/service.yaml new file mode 100644 index 00000000..d02ad3ee --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: kbs-access-sealed + namespace: {{ .Values.global.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "10" + labels: + app: kbs-access-sealed +spec: + selector: + app: kbs-access-sealed + ports: + - name: http + port: 8080 + targetPort: 8080 + protocol: TCP + type: ClusterIP diff --git a/charts/coco-supported/kbs-access-sealed/values.yaml b/charts/coco-supported/kbs-access-sealed/values.yaml new file mode 100644 index 00000000..a592f64e --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/values.yaml @@ -0,0 +1,4 @@ +# Default values for kbs-access-sealed chart +# Override via values-global.yaml or values-baremetal.yaml + +# Empty defaults - all configuration handled by global values diff --git a/charts/coco-supported/kbs-access/Chart.yaml b/charts/coco-supported/kbs-access/Chart.yaml deleted file mode 100644 index 0c784c2a..00000000 --- a/charts/coco-supported/kbs-access/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -description: Demonstrates accessing keys within the KBS. -keywords: -- pattern -name: kbs-access -version: 0.0.1 diff --git a/charts/coco-supported/kbs-access/README.md b/charts/coco-supported/kbs-access/README.md deleted file mode 100644 index d8e52b7e..00000000 --- a/charts/coco-supported/kbs-access/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Notes use of external image - -This chart currently uses an [image hosted on ghcr.io](ghcr.io/butler54/kbs-access-app:latest) built from the [following repository](https://github.com/butler54/coco-kbs-access). - -Using separate repository for build rather than integrated content is discouraged by validated patterns. - -The separate repository is because Coco (via the Kata guest components) must be served by an image registry using a TLS connection with a well known CA (as of today). - -This chart will be updated as that position changes. diff --git a/charts/coco-supported/kbs-access/templates/environment.yaml b/charts/coco-supported/kbs-access/templates/environment.yaml deleted file mode 100644 index 3f5bc49e..00000000 --- a/charts/coco-supported/kbs-access/templates/environment.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kbsref - namespace: kbs-access -data: - FILEPATH: "/output/kbsres1.txt" \ No newline at end of file diff --git a/charts/coco-supported/kbs-access/templates/secure-deployment.yaml b/charts/coco-supported/kbs-access/templates/secure-deployment.yaml deleted file mode 100644 index e53e874f..00000000 --- a/charts/coco-supported/kbs-access/templates/secure-deployment.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: secure - labels: - app: secure -spec: - replicas: 1 - selector: - matchLabels: - app: secure - template: - metadata: - labels: - app: secure - annotations: - peerpods: "true" - coco.io/initdata-configmap: initdata - {{- if .Values.defaultMemory }} - io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} - {{- end }} - spec: - runtimeClassName: {{ if .Values.runtimeClassName }}{{ .Values.runtimeClassName }}{{ else if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") }}kata-remote{{ else }}kata-cc{{ end }} - containers: - - name: python-access - image: ghcr.io/butler54/kbs-access-app:latest - ports: - - containerPort: 5000 - volumeMounts: - - name: output-volume - mountPath: /output - envFrom: - - configMapRef: - name: kbsref - initContainers: - - name: curl - image: registry.access.redhat.com/ubi9/ubi:latest - command: ['sh', '-c', 'curl -s http://127.0.0.1:8006/cdh/resource/default/kbsres1/key3 > /output/kbsres1.txt'] - volumeMounts: - - name: output-volume - mountPath: /output - volumes: - - name: output-volume - emptyDir: {} diff --git a/docs/CONTAINER-SIGNING-BLOCKER.md b/docs/CONTAINER-SIGNING-BLOCKER.md new file mode 100644 index 00000000..5cea3234 --- /dev/null +++ b/docs/CONTAINER-SIGNING-BLOCKER.md @@ -0,0 +1,277 @@ +# Container Signing Policy Enforcement - Upstream Blocker + +**Status**: ⛔ BLOCKED - Waiting for upstream fix +**Issue**: Red Hat container image signature verification fails with sigstore policies +**Upstream PR**: [guest-components#1398](https://github.com/confidential-containers/guest-components/pull/1398) + +## Summary + +Container image signature verification for Red Hat images (registry.redhat.io, registry.access.redhat.com) is currently **not working** in Confidential Containers due to a bug in the `image-rs` sigstore implementation. + +**Current Configuration**: `securityPolicyFlavour: "insecure"` (no signature verification) +**Target Configuration**: `securityPolicyFlavour: "redhat-secure-sigstore"` (blocked) + +## Root Cause + +**Bug**: image-rs does not base64-decode the `keyData` field for cosign/sigstore signatures. + +**Location**: `image-rs/src/signature/policy/cosign/mod.rs` line 69 + +**Current code**: + +```rust +(Some(key_data), None) => key_data.as_bytes().to_vec(), // ❌ Wrong: treats base64 string as raw bytes +``` + +**Expected code**: + +```rust +(Some(key_data), None) => { + use base64::Engine; + base64::engine::general_purpose::STANDARD.decode(key_data)? // ✅ Correct: decode base64 first +} +``` + +**Impact**: The cryptographic verifier receives base64-encoded text instead of decoded PEM key bytes, causing all signature verification attempts to fail with "rejected by sigstoreSigned rule". + +## Evidence + +### Podman Works (Golang containers/image) + +```bash +# Test on RHEL 10.2 with identical policy +podman pull --signature-policy=policy.json \ + registry.redhat.io/ubi9/httpd-24:latest + +# Result: ✅ SUCCESS +# - Signature found in registry +# - Verification passed +# - Image pulled successfully +``` + +### image-rs Fails (Rust rewrite) + +```bash +# CoCo pod with identical policy +Image: registry.redhat.io/ubi9/httpd-24@sha256:68a91ff... +Policy: kbs:///default/security-policy/redhat-secure-sigstore +Key: Embedded base64-encoded PEM public key + +# Result: ❌ FAIL +# Error: Image policy rejected: Denied by policy: rejected by `sigstoreSigned` rule +# Pod status: CreateContainerError +``` + +**The error message proves repository matching works** - if matching failed, the error would be "no matching policy" not "rejected by sigstoreSigned rule". + +## Upstream Fix Status + +**PR**: [guest-components#1398](https://github.com/confidential-containers/guest-components/pull/1398) +**Repository**: confidential-containers/guest-components +**Component**: image-rs (used by attestation-agent in kata guest VMs) + +**Required for**: + +- Red Hat build of trustee-operator +- OpenShift Sandboxed Containers +- Confidential Containers on OpenShift + +**Waiting on**: + +1. PR merge to guest-components +2. Release of updated guest-components version +3. Integration into Red Hat build of trustee +4. Update of kata guest image with fixed image-rs + +## Infrastructure Ready for Future Enablement + +All required infrastructure is **already deployed** and tested: + +### ✅ Deployed Components + +1. **Sigstore public key** + - Source: `/etc/pki/sigstore/SIGSTORE-redhat-release3` from RHEL 10.2 + - Key ID: `4096R/E60D446E63405576` (issued 2024-09-20) + - Location: `coco-pattern/keys/SIGSTORE-redhat-release3` + +2. **KBS secret** + - Secret: `sigstore-keys` in `trustee-operator-system` namespace + - Field: `redhat-release3` + - KBS URI: `kbs:///default/sigstore-keys/redhat-release3` + +3. **Policy template** + - Policy: `redhat-secure-sigstore` in `values-secret.yaml.template` + - Type: `sigstoreSigned` with embedded `keyData` + - Registries: `registry.redhat.io`, `registry.access.redhat.com` + +4. **Makefile targets** + - `make cache-sigstore-keys` - Cache key to `~/.coco-pattern/` + +### ✅ Verified Working + +**Podman verification successful** (2026-07-03): + +- Platform: RHEL 10.2 jump host +- Image: registry.access.redhat.com/ubi9/ubi-minimal:latest +- Policy: sigstoreSigned with Red Hat sigstore key +- Result: Signature verification passed ✅ + +## Policy Configuration + +### Current (Insecure) + +```yaml +# values-global.yaml +global: + coco: + securityPolicyFlavour: "insecure" # ⚠️ No signature verification +``` + +### Target (When Fix Lands) + +```yaml +# values-global.yaml +global: + coco: + securityPolicyFlavour: "redhat-secure-sigstore" # ✅ Sigstore verification +``` + +### Policy Details + +```json +{ + "default": [{"type": "insecureAcceptAnything"}], + "transports": { + "docker": { + "registry.redhat.io": [ + { + "type": "sigstoreSigned", + "keyData": "YXJ0aWZhY3RzIHRoYXQgYXJlIHNpZ3N0b3JlLWVuYWJsZWQuCi0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBMEFTeXVIMlRMV3ZCVXFQSFo0SXAKNzVnN0VuY0JrZ1FIZEpuanp4QVc1S1FUTWgvc2lCb0IvQm9TcnRpUE13bkNoYlRDblFPSVFlWnVEaUZuaHVKNwpNL0QzYjdKb1gwbTEyM05jQ1NuNjdtQWRqQmE2Qmc2a3VrWmdDUDRaVVplRVNhaldYL0VqeWxGY1JGT1hXNTdwClJEQ0VONDJKL2pZbFZxdCtnOStHcmtlcjhTejg2SDNsMHRicU9kamJ6L1Z4SFlod0YwY3RVTUhzeVZSRHEyUVAKdHF6TlhsbWxNaFMvUG9GcjZSNHUvN0hDbi9LK0xlZ2NPMmZBRk9iNDBLdktTS0tWRDZsZXdVWkVyaG9wMUNnSgpYakR0R21tTzlkR01GNzFtZjZIRWZhS1NkeStFRTZpU0YyQTJWdjlRaEJhd01pcTJrT3pFaUxnNG5BZEpUOHdnClpyTUFtUENxR0lzWE5HWjQvUStZVHd3bGNlM2dscWI1TDl0Zk5vekVkU1I5Tjg1REVTZlFMUUVkWTNDYWx3S00KQlQxT0VoRVgxd0hSQ1U0ZHJNT2VqNkJOVzBWdHNjR3RIbUNyczc0alBlemh3TlQ4eXBreVMrVDB6VDRUc3k2ZgpWWGtKOFlTSHllblN6TUIyT3AyYnZzRTNnclkrczc0V2hHOVVJQTZEQnhjVGllMTVOU3pLd2Z6YW9OV09EY0xGCnA3Qlk4YWFIRTJNcUZ4WUZYK0lianBrUVJmYWVRUXNvdURGZENrWEVGVmZQcGJEMmRrNkZsZWFNVFB1eXh0SVQKZ2pWRXRHUUsycUdDRkdpUUhGZDRoZlYrZUNBNjNKcm8xejB6b0JNNUJiSUlRMytlVkZ3dDNBbFpwNVVWd3I2ZApzZWNxa2kveXJtdjNZMGRxWjlWT24zVUNBd0VBQVE9PQotLS0tLUVORCBQVUJMSUMgS0VZLS0tLS0K", + "signedIdentity": {"type": "matchRepository"} + } + ], + "registry.access.redhat.com": [ + { + "type": "sigstoreSigned", + "keyData": "YXJ0aWZhY3RzIHRoYXQgYXJlIHNpZ3N0b3JlLWVuYWJsZWQuCi0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBMEFTeXVIMlRMV3ZCVXFQSFo0SXAKNzVnN0VuY0JrZ1FIZEpuanp4QVc1S1FUTWgvc2lCb0IvQm9TcnRpUE13bkNoYlRDblFPSVFlWnVEaUZuaHVKNwpNL0QzYjdKb1gwbTEyM05jQ1NuNjdtQWRqQmE2Qmc2a3VrWmdDUDRaVVplRVNhaldYL0VqeWxGY1JGT1hXNTdwClJEQ0VONDJKL2pZbFZxdCtnOStHcmtlcjhTejg2SDNsMHRicU9kamJ6L1Z4SFlod0YwY3RVTUhzeVZSRHEyUVAKdHF6TlhsbWxNaFMvUG9GcjZSNHUvN0hDbi9LK0xlZ2NPMmZBRk9iNDBLdktTS0tWRDZsZXdVWkVyaG9wMUNnSgpYakR0R21tTzlkR01GNzFtZjZIRWZhS1NkeStFRTZpU0YyQTJWdjlRaEJhd01pcTJrT3pFaUxnNG5BZEpUOHdnClpyTUFtUENxR0lzWE5HWjQvUStZVHd3bGNlM2dscWI1TDl0Zk5vekVkU1I5Tjg1REVTZlFMUUVkWTNDYWx3S00KQlQxT0VoRVgxd0hSQ1U0ZHJNT2VqNkJOVzBWdHNjR3RIbUNyczc0alBlemh3TlQ4eXBreVMrVDB6VDRUc3k2ZgpWWGtKOFlTSHllblN6TUIyT3AyYnZzRTNnclkrczc0V2hHOVVJQTZEQnhjVGllMTVOU3pLd2Z6YW9OV09EY0xGCnA3Qlk4YWFIRTJNcUZ4WUZYK0lianBrUVJmYWVRUXNvdURGZENrWEVGVmZQcGJEMmRrNkZsZWFNVFB1eXh0SVQKZ2pWRXRHUUsycUdDRkdpUUhGZDRoZlYrZUNBNjNKcm8xejB6b0JNNUJiSUlRMytlVkZ3dDNBbFpwNVVWd3I2ZApzZWNxa2kveXJtdjNZMGRxWjlWT24zVUNBd0VBQVE9PQotLS0tLUVORCBQVUJMSUMgS0VZLS0tLS0K", + "signedIdentity": {"type": "matchRepository"} + } + ] + } + } +} +``` + +The `keyData` field contains the Red Hat sigstore public key (release key 3) base64-encoded. + +## Why GPG Signatures Don't Work Either + +Red Hat dual-signs all container images: + +1. **GPG signatures** - Stored on separate HTTPS lookaside servers +2. **Sigstore signatures** - Stored as OCI artifacts in the registry + +**GPG approach blocked**: image-rs does not support HTTP/HTTPS for fetching signatures from lookaside servers (tracked in [image-rs#9](https://github.com/confidential-containers/image-rs/issues/9)). + +**Sigstore approach blocked**: This base64-decode bug. + +## Re-Enabling Signature Verification + +When the upstream fix is available: + +### 1. Verify Fix is Available + +```bash +# Check guest-components release notes for the fix +# Confirm Red Hat trustee-operator includes updated image-rs +``` + +### 2. Update Configuration + +```bash +cd ~/coco-pattern +git pull origin dev/phase1-modernization + +# Edit values-global.yaml +# Change: securityPolicyFlavour: "insecure" +# To: securityPolicyFlavour: "redhat-secure-sigstore" +``` + +### 3. Deploy Updated Pattern + +```bash +export KUBECONFIG=~/node-02-output/421_build/auth/kubeconfig +./pattern.sh make install +``` + +### 4. Verify Signature Enforcement + +```bash +# Delete confidential pod to force recreation +oc delete pod -n hello-openshift -l app=secure + +# Check pod starts successfully +oc get pods -n hello-openshift -l app=secure + +# Verify policy is active +oc get pod -n hello-openshift -l app=secure -o yaml | grep -A 5 init_data + +# Should show: image_security_policy_uri = 'kbs:///default/security-policy/redhat-secure-sigstore' +``` + +### 5. Test with Unsigned Image + +```bash +# Create test deployment with unsigned custom image +oc apply -n hello-openshift -f - <