From cf6daab2eb7dad192b869544112efb039ba76996 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Wed, 20 May 2026 18:54:59 -0400 Subject: [PATCH 1/4] feat: support protected repositories with Tekton Chains provenance Add support for cloning source code from protected (private) Git repositories in the Tekton supply-chain pipeline. Credentials are stored in Vault and delivered to the pipeline via an ExternalSecret that generates .git-credentials and .gitconfig files for the git-clone task's basic-auth workspace. Supply-chain chart changes: - Add init task (skopeo pre-flight image check, skip rebuild) - Add optional git-auth workspace and Chains provenance results (CHAINS-GIT_URL, CHAINS-GIT_COMMIT, IMAGE_URL, IMAGE_DIGEST) - Add ExternalSecret for git credentials (Opaque with .git-credentials) - Conditionally attach git-credentials secret to pipeline SA - Add skopeo image to tasks.images for the init task - Migrate all Tekton resources from v1beta1 to v1 API Generator and feature fragments: - Add protected-repos feature fragment with git.credentials overrides and qtodo.repository placeholder - Add --git-repo CLI argument to gen-feature-variants.py (required when protected-repos feature is enabled) - Add ignoreDifferences for Tekton Task/Pipeline defaulted fields to the supply-chain feature fragment Default values-hub.yaml: - Extend hub-supply-chain-jwt-secret Vault policy to cover secret/data/hub/supply-chain/* - Add commented-out Tekton ignoreDifferences, git.credentials overrides, and qtodo.repository override Documentation: - Update docs/supply-chain.md with protected repos setup, generator --git-repo usage, and git-auth workspace selection - Update scripts/gen-feature-variants.md with --git-repo examples - Add git-credentials entry to values-secret.yaml.template Signed-off-by: Min Zhang --- .../templates/pipeline-qtodo.yaml | 51 +++++++++++- .../templates/pipelinerun-qtodo.yaml | 2 +- .../templates/rbac/pipeline-sa.yaml | 5 +- .../secrets/qtodo-git-credentials.yaml | 38 +++++++++ .../templates/tasks/build-artifact.yaml | 2 +- .../templates/tasks/generate-sbom.yaml | 2 +- charts/supply-chain/templates/tasks/init.yaml | 55 +++++++++++++ .../templates/tasks/restart-qtodo.yaml | 2 +- .../templates/tasks/sbom-attest.yaml | 2 +- .../templates/tasks/sign-artifact.yaml | 2 +- .../templates/tasks/sign-image.yaml | 2 +- .../templates/tasks/upload-sbom.yaml | 2 +- .../templates/tasks/verify-artifact.yaml | 2 +- .../templates/tasks/verify-image.yaml | 2 +- charts/supply-chain/values.yaml | 10 +++ docs/supply-chain.md | 82 ++++++++++++++++++- scripts/features/features.yaml | 5 ++ scripts/features/protected-repos.yaml | 18 ++++ scripts/features/supply-chain.yaml | 11 +++ scripts/gen-feature-variants.md | 30 +++++++ scripts/gen-feature-variants.py | 60 +++++++++++++- values-hub.yaml | 20 +++++ values-secret.yaml.template | 17 ++++ 23 files changed, 404 insertions(+), 18 deletions(-) create mode 100644 charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml create mode 100644 charts/supply-chain/templates/tasks/init.yaml create mode 100644 scripts/features/protected-repos.yaml diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index a2118584..0d9ed07f 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Pipeline metadata: name: qtodo-supply-chain @@ -8,12 +8,20 @@ spec: params: - name: git-url type: string - description: The URL of the public Github qtodo repository + description: The URL of the qtodo repository (public or protected) default: {{ .Values.qtodo.repository | quote }} - name: git-revision type: string - description: The revision of the public Github qtodo repository + description: The revision of the qtodo repository default: {{ .Values.qtodo.revision }} + - name: rebuild + type: string + description: Force rebuild the image even if it already exists + default: "false" + - name: skip-checks + type: string + description: Skip pre-build checks against existing image + default: "false" - name: qtodo-build-cmd type: string description: The command to build the qtodo artifact @@ -100,9 +108,44 @@ spec: workspaces: - name: qtodo-source - name: registry-auth-config + - name: git-auth + optional: true + + results: + - name: CHAINS-GIT_URL + description: The git URL used for the build (Tekton Chains provenance) + value: $(tasks.qtodo-clone-repository.results.URL) + - name: CHAINS-GIT_COMMIT + description: The git commit SHA used for the build (Tekton Chains provenance) + value: $(tasks.qtodo-clone-repository.results.COMMIT) + - name: IMAGE_URL + description: The image URL built by the pipeline (Tekton Chains provenance) + value: $(tasks.qtodo-build-image.results.IMAGE_URL) + - name: IMAGE_DIGEST + description: The image digest built by the pipeline (Tekton Chains provenance) + value: $(tasks.qtodo-build-image.results.IMAGE_DIGEST) tasks: + - name: init + taskRef: + name: init + kind: Task + params: + - name: image-url + value: $(params.image-target) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + - name: qtodo-clone-repository + runAfter: + - init + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" taskRef: resolver: cluster params: @@ -120,6 +163,8 @@ spec: workspaces: - name: output workspace: qtodo-source + - name: basic-auth + workspace: git-auth - name: qtodo-build-artifact runAfter: diff --git a/charts/supply-chain/templates/pipelinerun-qtodo.yaml b/charts/supply-chain/templates/pipelinerun-qtodo.yaml index 36b52d4b..e87cdaf4 100644 --- a/charts/supply-chain/templates/pipelinerun-qtodo.yaml +++ b/charts/supply-chain/templates/pipelinerun-qtodo.yaml @@ -92,7 +92,7 @@ spec: fi cat <<'MANIFEST' | oc create -f - - apiVersion: tekton.dev/v1beta1 + apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: qtodo-supply-chain- diff --git a/charts/supply-chain/templates/rbac/pipeline-sa.yaml b/charts/supply-chain/templates/rbac/pipeline-sa.yaml index 06dff21e..ba97e2d4 100644 --- a/charts/supply-chain/templates/rbac/pipeline-sa.yaml +++ b/charts/supply-chain/templates/rbac/pipeline-sa.yaml @@ -10,4 +10,7 @@ metadata: argocd.argoproj.io/compare-options: IgnoreExtraneous argocd.argoproj.io/syncOptions: ServerSideApply=true secrets: - - name: qtodo-registry-auth \ No newline at end of file + - name: qtodo-registry-auth +{{- if .Values.git.credentials.enabled }} + - name: qtodo-git-credentials +{{- end }} \ No newline at end of file diff --git a/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml new file mode 100644 index 00000000..e844b456 --- /dev/null +++ b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml @@ -0,0 +1,38 @@ +{{- if .Values.git.credentials.enabled }} +{{- $host := .Values.git.credentials.host }} +{{- $hostBare := $host | trimPrefix "https://" | trimPrefix "http://" }} +{{- $userKey := .Values.git.credentials.usernameKey }} +{{- $passKey := .Values.git.credentials.passwordKey }} +--- +apiVersion: "external-secrets.io/v1beta1" +kind: ExternalSecret +metadata: + name: qtodo-git-credentials + namespace: {{ .Release.Namespace | default .Values.global.namespace }} +spec: + refreshInterval: 15s + secretStoreRef: + name: {{ .Values.global.secretStore.name }} + kind: {{ .Values.global.secretStore.kind }} + target: + name: qtodo-git-credentials + template: + type: Opaque + metadata: + annotations: + tekton.dev/git-0: {{ $host | quote }} + data: + .gitconfig: | + [credential "{{ $host }}"] + helper = store + .git-credentials: {{ printf "https://{{ .%s }}:{{ .%s }}@%s" $userKey $passKey $hostBare | quote }} + data: + - secretKey: {{ $userKey }} + remoteRef: + key: {{ .Values.git.credentials.vaultPath }} + property: {{ $userKey }} + - secretKey: {{ $passKey }} + remoteRef: + key: {{ .Values.git.credentials.vaultPath }} + property: {{ $passKey }} +{{- end }} diff --git a/charts/supply-chain/templates/tasks/build-artifact.yaml b/charts/supply-chain/templates/tasks/build-artifact.yaml index 57ab161d..3d49194d 100644 --- a/charts/supply-chain/templates/tasks/build-artifact.yaml +++ b/charts/supply-chain/templates/tasks/build-artifact.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-build-artifact diff --git a/charts/supply-chain/templates/tasks/generate-sbom.yaml b/charts/supply-chain/templates/tasks/generate-sbom.yaml index aab2457f..a2c7e546 100644 --- a/charts/supply-chain/templates/tasks/generate-sbom.yaml +++ b/charts/supply-chain/templates/tasks/generate-sbom.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-generate-sbom diff --git a/charts/supply-chain/templates/tasks/init.yaml b/charts/supply-chain/templates/tasks/init.yaml new file mode 100644 index 00000000..ed5b8046 --- /dev/null +++ b/charts/supply-chain/templates/tasks/init.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: "supply-chain" + name: init + namespace: {{ .Values.global.namespace }} +spec: + description: >- + Initialize Pipeline Task. Determines whether the image should be built + by checking if the target image already exists in the registry. + Supports rebuild and skip-checks flags. + params: + - name: image-url + description: Image URL for build by PipelineRun + - name: rebuild + description: Rebuild the image even if it exists + default: "false" + - name: skip-checks + description: Skip checks against built image + default: "false" + results: + - name: build + description: Defines if the image in param image-url should be built + steps: + - name: init + image: {{ .Values.tasks.images.skopeo }} + computeResources: + limits: + memory: 256Mi + requests: + memory: 256Mi + cpu: 100m + env: + - name: IMAGE_URL + value: $(params.image-url) + - name: REBUILD + value: $(params.rebuild) + - name: SKIP_CHECKS + value: $(params.skip-checks) + script: | + #!/bin/bash + echo "Build Initialize: $IMAGE_URL" + echo + + echo "Determine if Image Already Exists" + if [ "$REBUILD" == "true" ] || [ "$SKIP_CHECKS" == "false" ] || ! skopeo inspect --no-tags --raw "docker://$IMAGE_URL" &>/dev/null; then + echo -n "true" > $(results.build.path) + else + echo -n "false" > $(results.build.path) + fi diff --git a/charts/supply-chain/templates/tasks/restart-qtodo.yaml b/charts/supply-chain/templates/tasks/restart-qtodo.yaml index 9fe23387..8eef1981 100644 --- a/charts/supply-chain/templates/tasks/restart-qtodo.yaml +++ b/charts/supply-chain/templates/tasks/restart-qtodo.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: restart-qtodo diff --git a/charts/supply-chain/templates/tasks/sbom-attest.yaml b/charts/supply-chain/templates/tasks/sbom-attest.yaml index 850f5374..ffd81fbb 100644 --- a/charts/supply-chain/templates/tasks/sbom-attest.yaml +++ b/charts/supply-chain/templates/tasks/sbom-attest.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-sbom-attestation diff --git a/charts/supply-chain/templates/tasks/sign-artifact.yaml b/charts/supply-chain/templates/tasks/sign-artifact.yaml index 3bfbaedd..03eb52bc 100644 --- a/charts/supply-chain/templates/tasks/sign-artifact.yaml +++ b/charts/supply-chain/templates/tasks/sign-artifact.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-sign-artifact diff --git a/charts/supply-chain/templates/tasks/sign-image.yaml b/charts/supply-chain/templates/tasks/sign-image.yaml index dc820538..9e8300af 100644 --- a/charts/supply-chain/templates/tasks/sign-image.yaml +++ b/charts/supply-chain/templates/tasks/sign-image.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-sign-image diff --git a/charts/supply-chain/templates/tasks/upload-sbom.yaml b/charts/supply-chain/templates/tasks/upload-sbom.yaml index ebd82532..dfd49771 100644 --- a/charts/supply-chain/templates/tasks/upload-sbom.yaml +++ b/charts/supply-chain/templates/tasks/upload-sbom.yaml @@ -1,6 +1,6 @@ {{- if eq .Values.rhtpa.enabled true }} --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-upload-sbom diff --git a/charts/supply-chain/templates/tasks/verify-artifact.yaml b/charts/supply-chain/templates/tasks/verify-artifact.yaml index 68e7186f..b36bd66e 100644 --- a/charts/supply-chain/templates/tasks/verify-artifact.yaml +++ b/charts/supply-chain/templates/tasks/verify-artifact.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-verify-artifact diff --git a/charts/supply-chain/templates/tasks/verify-image.yaml b/charts/supply-chain/templates/tasks/verify-image.yaml index c19d22e0..baaf4ef9 100644 --- a/charts/supply-chain/templates/tasks/verify-image.yaml +++ b/charts/supply-chain/templates/tasks/verify-image.yaml @@ -1,5 +1,5 @@ --- -apiVersion: tekton.dev/v1beta1 +apiVersion: tekton.dev/v1 kind: Task metadata: name: qtodo-verify-image diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index 817e2793..18f254b7 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -40,6 +40,15 @@ rhtpa: clientSecretVaultPath: "secret/data/hub/infra/rhtpa/rhtpa-oidc-cli" clientSecretVaultKey: "client-secret" +# git credentials for protected repositories +git: + credentials: + enabled: false + host: "https://github.com" + vaultPath: "secret/data/hub/supply-chain/git-credentials" + usernameKey: "username" + passwordKey: "password" + # qtodo repository configuration qtodo: repository: "https://github.com/validatedpatterns-demos/qtodo.git" @@ -145,3 +154,4 @@ tasks: mandrel: "registry.redhat.io/quarkus/mandrel-for-jdk-21-rhel8:23.1-36" syft: "registry.redhat.io/rh-syft-tech-preview/syft-rhel9:1.29.0" rhtpa: "registry.access.redhat.com/ubi9/ubi:9.7-1764794285" + skopeo: "registry.access.redhat.com/ubi9/skopeo:9.5-1745865345@sha256:d91eb0dac7308ddfb12193368a42009509925edba80da9ffd3b82d03427dc3ed" diff --git a/docs/supply-chain.md b/docs/supply-chain.md index bb86dc18..cd03b224 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -46,6 +46,12 @@ For the Secure Supply Chain use case, the command would be: python3 scripts/gen-feature-variants.py --base values-hub.yaml --features supply-chain --registry-option ``` +If the source repository is **private** (protected), add the `protected-repos` feature: + +```shell +python3 scripts/gen-feature-variants.py --base values-hub.yaml --features supply-chain,protected-repos --registry-option +``` + Where `` is one of the options available in _Bring Your Own (BYO) Container Registry_: 1. Embedded Quay Registry @@ -251,6 +257,7 @@ Once the supply-chain application has synced in ArgoCD, start the pipeline using At the bottom we have the **workspaces**. These must be configured manually. * For **qtodo-source**, select `PersistentVolumeClaim` and the PVC name is `qtodo-workspace-source`. * For **registry-auth-config**, select `Secret` and the name of the secret is `qtodo-registry-auth`. + * For **git-auth** (optional, only when using protected repositories), select `Secret` and the name of the secret is `qtodo-git-credentials`. 5. Press **Start** to finish and run the pipeline. @@ -278,9 +285,13 @@ spec: - name: registry-auth-config secret: secretName: qtodo-registry-auth + # Uncomment for protected (private) repositories: + # - name: git-auth + # secret: + # secretName: qtodo-git-credentials ``` -As was described previously, verify the values associated with the PVC storage and registry configuration. +As was described previously, verify the values associated with the PVC storage and registry configuration. If you are using a protected repository, uncomment the `git-auth` workspace. Using the previously created definition, start a new execution of the pipeline using `oc` CLI: @@ -307,10 +318,79 @@ oc get taskruns -n layered-zero-trust-hub -l tekton.dev/pipelineRun=,tekton.dev/pipelineTask= ``` +### Protected Repositories + +By default the pipeline clones the qtodo source from a **public** GitHub repository. If your source code lives in a private (protected) repository, enable the Git credentials feature so the `git-clone` task can authenticate. + +#### 1. Store Git credentials in Vault + +Uncomment the `git-credentials` secret in your local `~/values-secret.yaml` (copied from `values-secret.yaml.template`) and fill in your Git username and Personal Access Token (PAT): + +```yaml +- name: git-credentials + vaultPrefixes: + - hub/supply-chain + fields: + - name: username + value: "your-git-username" + onMissingValue: error + - name: password + value: "your-personal-access-token" + onMissingValue: error +``` + +Then load the secret into Vault: `make load-secrets`. + +#### 2. Enable Git credentials in the supply-chain overrides + +Add the following overrides to the `supply-chain` application in `values-hub.yaml`: + +```yaml +- name: git.credentials.enabled + value: "true" +- name: git.credentials.host + value: "https://github.com" +- name: git.credentials.vaultPath + value: "secret/data/hub/supply-chain/git-credentials" +``` + +Alternatively, if you use the `gen-feature-variants.py` script, add `protected-repos` to the features list and provide your private repository URL with `--git-repo`. The Git credential overrides and the `qtodo.repository` override are included automatically: + +```shell +python3 scripts/gen-feature-variants.py \ + --features supply-chain,protected-repos \ + --registry-option \ + --git-repo https://github.com/your-org/qtodo.git +``` + +#### 3. Point the pipeline at your private repository + +When using the generator with `--git-repo`, the `qtodo.repository` override is set automatically in the generated `values-hub.yaml`. If you are configuring manually, add this override to the `supply-chain` application: + +```yaml +- name: qtodo.repository + value: "https://github.com/your-org/qtodo.git" +``` + +#### How it works + +When `git.credentials.enabled` is `true`: + +* An `ExternalSecret` (`qtodo-git-credentials`) pulls the username and PAT from Vault and creates an `Opaque` secret containing `.git-credentials` and `.gitconfig` files, annotated with `tekton.dev/git-0` pointing to the configured host. +* The pipeline ServiceAccount mounts this secret, and the `git-clone` task receives it via the optional `git-auth` / `basic-auth` workspace. When starting a PipelineRun, select `Secret` / `qtodo-git-credentials` for the **git-auth** workspace. +* The Vault policy `hub-supply-chain-jwt-secret` grants read access to `secret/data/hub/supply-chain/*` for the pipeline's SPIFFE identity. + +### Init task (pre-flight image check) + +The pipeline includes an `init` task that runs before `git-clone`. It uses `skopeo inspect` to check whether the target image already exists in the registry. If the image exists (and `rebuild` is not set to `"true"`), the pipeline skips the build. This avoids unnecessary rebuilds and is modeled after the [RHTAP sample pipelines](https://github.com/konflux-ci/build-definitions). + +The pipeline also emits Tekton Chains provenance results (`CHAINS-GIT_URL`, `CHAINS-GIT_COMMIT`, `IMAGE_URL`, `IMAGE_DIGEST`) so that Tekton Chains can automatically generate and sign provenance attestations. + ### Pipeline tasks The pipeline we have prepared has the following steps: +* **init**. Checks whether the target image already exists in the registry. Gates the build with a `when` condition. * **qtodo-clone-repository**. Clones the `qtodo` repository. * **qtodo-build-artifact**. Builds an _uber-jar_ of `qtodo` application. * **qtodo-sign-artifact**. Signs the JAR file generated during the build process. diff --git a/scripts/features/features.yaml b/scripts/features/features.yaml index 49ca349a..434d53a7 100644 --- a/scripts/features/features.yaml +++ b/scripts/features/features.yaml @@ -33,6 +33,11 @@ features: org: ztvp image_name: qtodo + protected-repos: + description: "Protected (private) Git repository support for supply-chain" + depends_on: [supply-chain] + git_repo_required: true + entra-id: description: "Azure Entra ID integration" depends_on: [supply-chain] diff --git a/scripts/features/protected-repos.yaml b/scripts/features/protected-repos.yaml new file mode 100644 index 00000000..2a84d972 --- /dev/null +++ b/scripts/features/protected-repos.yaml @@ -0,0 +1,18 @@ +# Protected repository support for the supply-chain pipeline. +# Enables git-clone to authenticate against private Git repositories +# using credentials stored in Vault (ExternalSecret -> basic-auth secret). +# +# Requires the git-credentials secret in values-secret.yaml.template +# to be uncommented and populated with your Git username and PAT. +clusterGroup: + merge_into_applications: + supply-chain: + overrides: + - name: git.credentials.enabled + value: "true" + - name: git.credentials.host + value: "https://github.com" + - name: git.credentials.vaultPath + value: "secret/data/hub/supply-chain/git-credentials" + - name: qtodo.repository + value: "REPLACE_WITH_GIT_REPO_URL" diff --git a/scripts/features/supply-chain.yaml b/scripts/features/supply-chain.yaml index f40d9bec..fdc7dbe8 100644 --- a/scripts/features/supply-chain.yaml +++ b/scripts/features/supply-chain.yaml @@ -1,6 +1,9 @@ # Secure Supply Chain application + vault role # Depends on: pipelines, rhtas, rhtpa, storage (all resolved automatically) # Requires --registry-option to select the registry backend. +# +# Optional: add --features protected-repos to enable private Git repository +# support (see scripts/features/protected-repos.yaml). clusterGroup: applications: supply-chain: @@ -13,6 +16,14 @@ clusterGroup: - kind: ServiceAccount jqPathExpressions: - ".imagePullSecrets[]|select(.name | contains(\"-dockercfg-\"))" + - group: tekton.dev + kind: Task + jsonPointers: + - /spec/steps + - group: tekton.dev + kind: Pipeline + jsonPointers: + - /spec/tasks merge_into_applications: vault: diff --git a/scripts/gen-feature-variants.md b/scripts/gen-feature-variants.md index fe9235e9..b3781a16 100644 --- a/scripts/gen-feature-variants.md +++ b/scripts/gen-feature-variants.md @@ -57,6 +57,12 @@ python3 scripts/gen-feature-variants.py --features supply-chain --registry-optio # Generate all three supply-chain registry variants at once python3 scripts/gen-feature-variants.py --features supply-chain --registry-option all +# Supply chain with protected (private) Git repository support +python3 scripts/gen-feature-variants.py \ + --features supply-chain,protected-repos \ + --registry-option 2 \ + --git-repo https://github.com/your-org/qtodo.git + # Custom base file and output directory python3 scripts/gen-feature-variants.py \ --features rhtpa --base values-hub.yaml --outdir /tmp @@ -81,6 +87,30 @@ The output directory is created automatically if it does not exist. > without these fields, replace the placeholders manually before applying > the generated file. +## Protected Repositories (`--git-repo`) + +When the `protected-repos` feature is enabled, the `--git-repo` argument is +**required**. It specifies the private Git repository URL that the Tekton +pipeline will clone. The generator substitutes the placeholder in the +`protected-repos` fragment with the supplied URL: + +```bash +python3 scripts/gen-feature-variants.py \ + --features supply-chain,protected-repos \ + --registry-option 1 \ + --git-repo https://github.com/your-org/qtodo.git +``` + +The generated `values-hub.yaml` will include: + +```yaml +- name: qtodo.repository + value: "https://github.com/your-org/qtodo.git" +``` + +See [Protected Repositories](../docs/supply-chain.md#protected-repositories) +for the full setup (Vault credentials, ExternalSecret, workspace selection). + ## How It Works 1. The script reads the base `values-hub.yaml`. diff --git a/scripts/gen-feature-variants.py b/scripts/gen-feature-variants.py index eb7867cb..e407802c 100755 --- a/scripts/gen-feature-variants.py +++ b/scripts/gen-feature-variants.py @@ -304,6 +304,22 @@ def _substitute_repository_placeholders(base, org=None, image_name=None): base["global"]["registry"]["repository"] = repo +GIT_REPO_PLACEHOLDER = "REPLACE_WITH_GIT_REPO_URL" + + +def _substitute_git_repo_url(base, git_repo_url): + """Replace the git repo placeholder in supply-chain overrides.""" + apps = base.get("clusterGroup", {}).get("applications", {}) + sc = apps.get("supply-chain", {}) + for override in sc.get("overrides", []): + if ( + override.get("name") == "qtodo.repository" + and str(override.get("value")) == GIT_REPO_PLACEHOLDER + ): + override["value"] = git_repo_url + return + + def generate_variant( base_path, features_dir, @@ -312,6 +328,7 @@ def generate_variant( output_path, org=None, image_name=None, + git_repo_url=None, ): """Load base, merge all feature fragments + registry option, write output.""" yaml = YAML() @@ -343,6 +360,9 @@ def generate_variant( if org or image_name: _substitute_repository_placeholders(base, org=org, image_name=image_name) + if git_repo_url: + _substitute_git_repo_url(base, git_repo_url) + validate_output(base) cg = base.get("clusterGroup") if cg: @@ -360,7 +380,8 @@ def build_output_name(features, registry_option=None): """Construct the output filename from features and optional registry option.""" if "supply-chain" in features: label = REGISTRY_LABELS.get(registry_option, f"option-{registry_option}") - return f"values-hub-supply-chain-{label}.yaml" + suffix = "-protected-repos" if "protected-repos" in features else "" + return f"values-hub-supply-chain-{label}{suffix}.yaml" return f"values-hub-{'-'.join(features)}.yaml" @@ -395,6 +416,12 @@ def main(): default=None, help="Output directory (default: /tmp)", ) + parser.add_argument( + "--git-repo", + default=None, + help="Private Git repository URL for protected-repos feature " + "(e.g. https://github.com/your-org/qtodo.git)", + ) parser.add_argument( "--list-features", action="store_true", @@ -455,11 +482,24 @@ def main(): ) sys.exit(1) + needs_git_repo = any( + feature_defs.get(f, {}).get("git_repo_required") for f in resolved + ) + if needs_git_repo and not args.git_repo: + print( + "ERROR: --git-repo is required when protected-repos feature is enabled " + "(e.g. --git-repo https://github.com/your-org/qtodo.git)", + file=sys.stderr, + ) + sys.exit(1) + print(f"Base: {base}") print(f"Output: {outdir}") print(f"Features: {' -> '.join(resolved)}") if args.registry_option: print(f"Registry: option {args.registry_option}") + if args.git_repo: + print(f"Git repo: {args.git_repo}") if args.registry_option == "all": for opt_num in [1, 2, 3]: @@ -475,7 +515,14 @@ def main(): out_name = build_output_name(requested, opt_num) out_path = os.path.join(outdir, out_name) generate_variant( - base, FEATURES_DIR, resolved, reg_path, out_path, org, image_name + base, + FEATURES_DIR, + resolved, + reg_path, + out_path, + org, + image_name, + git_repo_url=args.git_repo, ) else: reg_path = None @@ -496,7 +543,14 @@ def main(): ) out_path = os.path.join(outdir, out_name) generate_variant( - base, FEATURES_DIR, resolved, reg_path, out_path, org, image_name + base, + FEATURES_DIR, + resolved, + reg_path, + out_path, + org, + image_name, + git_repo_url=args.git_repo, ) if args.registry_option and org and image_name: diff --git a/values-hub.yaml b/values-hub.yaml index cae32f1e..edc41ab7 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -361,6 +361,9 @@ clusterGroup: path "secret/data/hub/infra/rhtpa/rhtpa-oidc-cli" { capabilities = ["read"] } + path "secret/data/hub/supply-chain/*" { + capabilities = ["read"] + } jwt: enabled: true oidcDiscoveryUrl: https://spire-spiffe-oidc-discovery-provider.zero-trust-workload-identity-manager.svc.cluster.local @@ -577,6 +580,14 @@ clusterGroup: # - kind: ServiceAccount # jqPathExpressions: # - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) + # - group: tekton.dev + # kind: Task + # jsonPointers: + # - /spec/steps + # - group: tekton.dev + # kind: Pipeline + # jsonPointers: + # - /spec/tasks # overrides: # # Registry credentials are inherited from global.registry. # # Only set app-specific overrides below. @@ -601,6 +612,15 @@ clusterGroup: # # Uncomment to auto-trigger a pipeline run on every ArgoCD sync # # - name: pipelinerun.enabled # # value: "true" + # # Protected repository credentials (Vault + ExternalSecret) + # # - name: git.credentials.enabled + # # value: "true" + # # - name: git.credentials.host + # # value: "https://github.com" + # # - name: git.credentials.vaultPath + # # value: "secret/data/hub/supply-chain/git-credentials" + # # - name: qtodo.repository + # # value: "https://github.com/your-org/qtodo.git" # # ACS Central Services acs-central: diff --git a/values-secret.yaml.template b/values-secret.yaml.template index a5b715ce..d6bb3040 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -230,6 +230,23 @@ secrets: # value: "REPLACE_WITH_REGISTRY_TOKEN" # onMissingValue: error + # =========================================================================== + # SUPPLY CHAIN GIT CREDENTIALS (hub/supply-chain/) + # Credentials for cloning protected Git repositories in Tekton pipelines + # Policy: hub-supply-chain-jwt-secret (read access to hub/supply-chain/*) + # =========================================================================== + # Uncomment when using protected (private) Git repositories + #- name: git-credentials + # vaultPrefixes: + # - hub/supply-chain + # fields: + # - name: username + # value: "your-git-username" # Replace with your Git username + # onMissingValue: error + # - name: password + # value: "your-personal-access-token" # Replace with your PAT/token + # onMissingValue: error + # =========================================================================== # COCO (CONFIDENTIAL CONTAINERS) SECRETS # Uncomment the secrets below when deploying with CoCo support. From e8c14b928f9c248afd5a001b132768648de7be9a Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Wed, 27 May 2026 17:56:44 -0400 Subject: [PATCH 2/4] feat: SSH auth support and review fixes for protected repositories - Support SSH auth for protected repositories - Fix ESO SSH template with index syntax for hyphenated keys - Add Vault NetworkPolicy rules for registry-token-refresher - Update gen-feature-variants with protected-repos feature - Clarify that git credentials use SA injection (no git-auth workspace binding needed) Signed-off-by: Min Zhang --- .../templates/pipeline-qtodo.yaml | 5 ++ .../rbac/registry-token-refresher.yaml | 3 + .../secrets/qtodo-git-credentials.yaml | 26 +++++- charts/supply-chain/values.yaml | 3 + docs/supply-chain.md | 82 +++++++++++++----- overrides/values-vault-network-policy.yaml | 22 +++++ scripts/features/protected-repos.yaml | 12 ++- scripts/gen-feature-variants.md | 29 +++++-- scripts/gen-feature-variants.py | 83 ++++++++----------- values-secret.yaml.template | 17 +++- 10 files changed, 201 insertions(+), 81 deletions(-) diff --git a/charts/supply-chain/templates/pipeline-qtodo.yaml b/charts/supply-chain/templates/pipeline-qtodo.yaml index 0d9ed07f..6b77d973 100644 --- a/charts/supply-chain/templates/pipeline-qtodo.yaml +++ b/charts/supply-chain/templates/pipeline-qtodo.yaml @@ -163,8 +163,13 @@ spec: workspaces: - name: output workspace: qtodo-source +{{- if eq (default "https" .Values.git.credentials.authType) "ssh" }} + - name: ssh-directory + workspace: git-auth +{{- else }} - name: basic-auth workspace: git-auth +{{- end }} - name: qtodo-build-artifact runAfter: diff --git a/charts/supply-chain/templates/rbac/registry-token-refresher.yaml b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml index 40fd7e58..a83b4294 100644 --- a/charts/supply-chain/templates/rbac/registry-token-refresher.yaml +++ b/charts/supply-chain/templates/rbac/registry-token-refresher.yaml @@ -72,6 +72,9 @@ spec: spec: backoffLimit: 3 template: + metadata: + labels: + app.kubernetes.io/name: registry-token-refresher spec: serviceAccountName: {{ .Values.pipelineServiceAccount }} restartPolicy: Never diff --git a/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml index e844b456..19405b5c 100644 --- a/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml @@ -1,8 +1,6 @@ {{- if .Values.git.credentials.enabled }} +{{- $authType := .Values.git.credentials.authType | default "https" }} {{- $host := .Values.git.credentials.host }} -{{- $hostBare := $host | trimPrefix "https://" | trimPrefix "http://" }} -{{- $userKey := .Values.git.credentials.usernameKey }} -{{- $passKey := .Values.git.credentials.passwordKey }} --- apiVersion: "external-secrets.io/v1beta1" kind: ExternalSecret @@ -17,6 +15,27 @@ spec: target: name: qtodo-git-credentials template: +{{- if eq $authType "ssh" }} + type: kubernetes.io/ssh-auth + metadata: + annotations: + tekton.dev/git-0: {{ $host | quote }} + data: + ssh-privatekey: {{ printf "{{ index . \"%s\" }}" .Values.git.credentials.sshPrivateKeyKey | quote }} + known_hosts: {{ printf "{{ index . \"%s\" }}" .Values.git.credentials.knownHostsKey | quote }} + data: + - secretKey: {{ .Values.git.credentials.sshPrivateKeyKey }} + remoteRef: + key: {{ .Values.git.credentials.vaultPath }} + property: {{ .Values.git.credentials.sshPrivateKeyKey }} + - secretKey: {{ .Values.git.credentials.knownHostsKey }} + remoteRef: + key: {{ .Values.git.credentials.vaultPath }} + property: {{ .Values.git.credentials.knownHostsKey }} +{{- else }} +{{- $hostBare := $host | trimPrefix "https://" | trimPrefix "http://" }} +{{- $userKey := .Values.git.credentials.usernameKey }} +{{- $passKey := .Values.git.credentials.passwordKey }} type: Opaque metadata: annotations: @@ -36,3 +55,4 @@ spec: key: {{ .Values.git.credentials.vaultPath }} property: {{ $passKey }} {{- end }} +{{- end }} diff --git a/charts/supply-chain/values.yaml b/charts/supply-chain/values.yaml index 18f254b7..8cfb6260 100644 --- a/charts/supply-chain/values.yaml +++ b/charts/supply-chain/values.yaml @@ -44,10 +44,13 @@ rhtpa: git: credentials: enabled: false + authType: "https" host: "https://github.com" vaultPath: "secret/data/hub/supply-chain/git-credentials" usernameKey: "username" passwordKey: "password" + sshPrivateKeyKey: "ssh-privatekey" + knownHostsKey: "known_hosts" # qtodo repository configuration qtodo: diff --git a/docs/supply-chain.md b/docs/supply-chain.md index cd03b224..4c255cf1 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -257,7 +257,7 @@ Once the supply-chain application has synced in ArgoCD, start the pipeline using At the bottom we have the **workspaces**. These must be configured manually. * For **qtodo-source**, select `PersistentVolumeClaim` and the PVC name is `qtodo-workspace-source`. * For **registry-auth-config**, select `Secret` and the name of the secret is `qtodo-registry-auth`. - * For **git-auth** (optional, only when using protected repositories), select `Secret` and the name of the secret is `qtodo-git-credentials`. + * Leave **git-auth** unbound (empty). Git credentials for protected repositories are injected automatically via the `pipeline` ServiceAccount (see [How it works](#how-it-works) below). 5. Press **Start** to finish and run the pipeline. @@ -285,13 +285,11 @@ spec: - name: registry-auth-config secret: secretName: qtodo-registry-auth - # Uncomment for protected (private) repositories: - # - name: git-auth - # secret: - # secretName: qtodo-git-credentials ``` -As was described previously, verify the values associated with the PVC storage and registry configuration. If you are using a protected repository, uncomment the `git-auth` workspace. +As was described previously, verify the values associated with the PVC storage and registry configuration. + +> **Note**: The `git-auth` workspace should be left **unbound**. Git credentials for protected repositories are injected automatically by Tekton's credential initialization through the `pipeline` ServiceAccount. Binding the `git-auth` workspace directly can cause permission errors with the `git-clone` ClusterTask's SSH credential handling. Using the previously created definition, start a new execution of the pipeline using `oc` CLI: @@ -322,9 +320,20 @@ oc logs -n layered-zero-trust-hub -l tekton.dev/pipelineRun=,t By default the pipeline clones the qtodo source from a **public** GitHub repository. If your source code lives in a private (protected) repository, enable the Git credentials feature so the `git-clone` task can authenticate. +Two authentication modes are supported: + +| Mode | URL format | Vault fields | Secret type | +| ----- | ------------------------------------ | ----------------------------------- | ---------------------- | +| HTTPS | `https://github.com/org/repo.git` | `username` + `password` (PAT) | Opaque (basic-auth) | +| SSH | `git@github.com:org/repo.git` | `ssh-privatekey` + `known_hosts` | kubernetes.io/ssh-auth | + +When using the `gen-feature-variants.py` script with `--git-repo`, the auth mode is auto-detected from the URL scheme. + #### 1. Store Git credentials in Vault -Uncomment the `git-credentials` secret in your local `~/values-secret.yaml` (copied from `values-secret.yaml.template`) and fill in your Git username and Personal Access Token (PAT): +Uncomment the `git-credentials` secret in your local `~/values-secret.yaml` (copied from `values-secret.yaml.template`). Choose **one** of the two options: + +**Option A -- HTTPS basic auth** (username + Personal Access Token): ```yaml - name: git-credentials @@ -339,28 +348,56 @@ Uncomment the `git-credentials` secret in your local `~/values-secret.yaml` (cop onMissingValue: error ``` -Then load the secret into Vault: `make load-secrets`. +**Option B -- SSH key auth**: -#### 2. Enable Git credentials in the supply-chain overrides +```yaml +- name: git-credentials + vaultPrefixes: + - hub/supply-chain + fields: + - name: ssh-privatekey + path: ~/.ssh/id_ed25519 # or id_rsa, id_ecdsa, etc. + - name: known_hosts + path: ~/.ssh/known_hosts_github +``` -Add the following overrides to the `supply-chain` application in `values-hub.yaml`: +Generate the `known_hosts` file for your Git host: -```yaml -- name: git.credentials.enabled - value: "true" -- name: git.credentials.host - value: "https://github.com" -- name: git.credentials.vaultPath - value: "secret/data/hub/supply-chain/git-credentials" +```shell +ssh-keyscan github.com > ~/.ssh/known_hosts_github ``` -Alternatively, if you use the `gen-feature-variants.py` script, add `protected-repos` to the features list and provide your private repository URL with `--git-repo`. The Git credential overrides and the `qtodo.repository` override are included automatically: +Then load the secret into Vault: `make load-secrets`. + +#### 2. Enable Git credentials in the supply-chain overrides + +**Preferred: use the generator.** Add `protected-repos` to the features list and provide your private repository URL with `--git-repo`. The generator auto-detects the auth mode and sets all overrides (host, authType, repository) automatically: ```shell +# HTTPS python3 scripts/gen-feature-variants.py \ --features supply-chain,protected-repos \ --registry-option \ --git-repo https://github.com/your-org/qtodo.git + +# SSH +python3 scripts/gen-feature-variants.py \ + --features supply-chain,protected-repos \ + --registry-option \ + --git-repo git@github.com:your-org/qtodo.git +``` + +**Manual configuration.** Add the following overrides to the `supply-chain` application in `values-hub.yaml`: + +```yaml +- name: git.credentials.enabled + value: "true" +- name: git.credentials.authType + value: "https" # or "ssh" +- name: git.credentials.host + value: "https://github.com" # SSH: "github.com" (no scheme) +- name: git.credentials.vaultPath + value: "secret/data/hub/supply-chain/git-credentials" ``` #### 3. Point the pipeline at your private repository @@ -369,15 +406,18 @@ When using the generator with `--git-repo`, the `qtodo.repository` override is s ```yaml - name: qtodo.repository - value: "https://github.com/your-org/qtodo.git" + value: "https://github.com/your-org/qtodo.git" # or SSH URL ``` #### How it works When `git.credentials.enabled` is `true`: -* An `ExternalSecret` (`qtodo-git-credentials`) pulls the username and PAT from Vault and creates an `Opaque` secret containing `.git-credentials` and `.gitconfig` files, annotated with `tekton.dev/git-0` pointing to the configured host. -* The pipeline ServiceAccount mounts this secret, and the `git-clone` task receives it via the optional `git-auth` / `basic-auth` workspace. When starting a PipelineRun, select `Secret` / `qtodo-git-credentials` for the **git-auth** workspace. +* An `ExternalSecret` (`qtodo-git-credentials`) pulls the credentials from Vault and creates a secret annotated with `tekton.dev/git-0` pointing to the configured host. + * **HTTPS mode**: creates an `Opaque` secret with `.git-credentials` and `.gitconfig` files. + * **SSH mode**: creates a `kubernetes.io/ssh-auth` secret with `ssh-privatekey` and `known_hosts` entries. +* The `pipeline` ServiceAccount lists the secret (see `pipeline-sa.yaml`). Tekton's credential initialization automatically injects the credentials into task containers -- `.gitconfig` and `.git-credentials` for HTTPS, or `~/.ssh/config`, `~/.ssh/id_*`, and `~/.ssh/known_hosts` for SSH. No explicit `git-auth` workspace binding is required. +* The `git-auth` workspace remains declared in the pipeline as `optional: true` for compatibility, but should be left unbound. Binding it triggers the `git-clone` ClusterTask's `prepare.sh` which runs a recursive `chmod` on the copied secret volume; this fails on the read-only Kubernetes projected volume symlinks and aborts the step (particularly in SSH mode). * The Vault policy `hub-supply-chain-jwt-secret` grants read access to `secret/data/hub/supply-chain/*` for the pipeline's SPIFFE identity. ### Init task (pre-flight image check) diff --git a/overrides/values-vault-network-policy.yaml b/overrides/values-vault-network-policy.yaml index ddd9aeba..23abfc46 100644 --- a/overrides/values-vault-network-policy.yaml +++ b/overrides/values-vault-network-policy.yaml @@ -23,6 +23,28 @@ vault: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: qtodo + # registry-token-refresher seed job — fetches Vault token for Quay registry + - ports: + - protocol: TCP + port: 8200 + from: + - podSelector: + matchLabels: + job-name: registry-token-refresher-seed + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: layered-zero-trust-hub + # registry-token-refresher CronJob — periodic token refresh to Vault + - ports: + - protocol: TCP + port: 8200 + from: + - podSelector: + matchLabels: + app.kubernetes.io/name: registry-token-refresher + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: layered-zero-trust-hub # Vault cluster replication port — HA readiness - ports: - protocol: TCP diff --git a/scripts/features/protected-repos.yaml b/scripts/features/protected-repos.yaml index 2a84d972..8674e382 100644 --- a/scripts/features/protected-repos.yaml +++ b/scripts/features/protected-repos.yaml @@ -1,17 +1,23 @@ # Protected repository support for the supply-chain pipeline. # Enables git-clone to authenticate against private Git repositories -# using credentials stored in Vault (ExternalSecret -> basic-auth secret). +# using credentials stored in Vault (ExternalSecret -> secret). +# +# Supports two authentication modes (auto-detected from --git-repo URL): +# https - basic-auth via .git-credentials (username + PAT) +# ssh - SSH key pair (ssh-privatekey + known_hosts) # # Requires the git-credentials secret in values-secret.yaml.template -# to be uncommented and populated with your Git username and PAT. +# to be uncommented and populated with the appropriate credentials. clusterGroup: merge_into_applications: supply-chain: overrides: - name: git.credentials.enabled value: "true" + - name: git.credentials.authType + value: "REPLACE_WITH_GIT_AUTH_TYPE" - name: git.credentials.host - value: "https://github.com" + value: "REPLACE_WITH_GIT_HOST" - name: git.credentials.vaultPath value: "secret/data/hub/supply-chain/git-credentials" - name: qtodo.repository diff --git a/scripts/gen-feature-variants.md b/scripts/gen-feature-variants.md index b3781a16..ecab1805 100644 --- a/scripts/gen-feature-variants.md +++ b/scripts/gen-feature-variants.md @@ -91,21 +91,40 @@ The output directory is created automatically if it does not exist. When the `protected-repos` feature is enabled, the `--git-repo` argument is **required**. It specifies the private Git repository URL that the Tekton -pipeline will clone. The generator substitutes the placeholder in the -`protected-repos` fragment with the supplied URL: +pipeline will clone. The generator auto-detects the authentication mode +(HTTPS or SSH) from the URL scheme and sets `git.credentials.authType` and +`git.credentials.host` accordingly: ```bash +# HTTPS (basic-auth with username + PAT) python3 scripts/gen-feature-variants.py \ --features supply-chain,protected-repos \ --registry-option 1 \ --git-repo https://github.com/your-org/qtodo.git + +# SSH (key-based auth) +python3 scripts/gen-feature-variants.py \ + --features supply-chain,protected-repos \ + --registry-option 1 \ + --git-repo git@github.com:your-org/qtodo.git +``` + +For an **HTTPS** URL the generated `values-hub.yaml` will include: + +```yaml +- name: git.credentials.authType + value: "https" +- name: git.credentials.host + value: "https://github.com" ``` -The generated `values-hub.yaml` will include: +For an **SSH** URL: ```yaml -- name: qtodo.repository - value: "https://github.com/your-org/qtodo.git" +- name: git.credentials.authType + value: "ssh" +- name: git.credentials.host + value: "github.com" ``` See [Protected Repositories](../docs/supply-chain.md#protected-repositories) diff --git a/scripts/gen-feature-variants.py b/scripts/gen-feature-variants.py index e407802c..af97b709 100755 --- a/scripts/gen-feature-variants.py +++ b/scripts/gen-feature-variants.py @@ -35,8 +35,10 @@ import argparse import copy import os +import re import sys from collections import OrderedDict +from urllib.parse import urlparse from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -125,25 +127,6 @@ def _merge_namespace_lists(base_list, fragment_list): existing.add(key) -def _is_named_list(lst): - """Return True if lst is a list of mappings that all contain a 'name' key.""" - return len(lst) > 0 and all( - isinstance(item, dict) and "name" in item for item in lst - ) - - -def _merge_named_lists(base_list, overlay_list): - """Merge overlay items into base by 'name', replacing on conflict.""" - index = {item["name"]: i for i, item in enumerate(base_list)} - for item in overlay_list: - name = item["name"] - if name in index: - base_list[index[name]] = copy.deepcopy(item) - else: - index[name] = len(base_list) - base_list.append(copy.deepcopy(item)) - - def _deep_merge_mappings(base, overlay): """Recursively merge overlay into base (overlay wins for scalars).""" for key in overlay: @@ -158,10 +141,7 @@ def _deep_merge_mappings(base, overlay): and isinstance(base[key], list) and isinstance(overlay[key], list) ): - if _is_named_list(base[key]) or _is_named_list(overlay[key]): - _merge_named_lists(base[key], overlay[key]) - else: - base[key].extend(overlay[key]) + base[key].extend(overlay[key]) else: base[key] = overlay[key] @@ -177,8 +157,7 @@ def _apply_merge_into(base_apps, merge_into_spec): overrides: [...] For each target app, recursively merge into the existing app config. - Named lists (items with a 'name' key) use upsert semantics; plain lists - are appended. + Lists (roles, overrides) are appended rather than replaced. """ for app_name, additions in merge_into_spec.items(): if app_name not in base_apps: @@ -270,20 +249,6 @@ def validate_output(data): seen.add(key) apps = cg.get("applications", {}) - for app_name, app_val in apps.items(): - overrides = app_val.get("overrides", []) if isinstance(app_val, dict) else [] - override_names = set() - for ovr in overrides: - name = ovr.get("name") if isinstance(ovr, dict) else None - if name and name in override_names: - print( - f"WARNING: duplicate override '{name}' in " - f"application '{app_name}'", - file=sys.stderr, - ) - if name: - override_names.add(name) - vault = apps.get("vault", {}) jwt_roles = vault.get("jwt", {}).get("roles", []) role_names = set() @@ -305,19 +270,40 @@ def _substitute_repository_placeholders(base, org=None, image_name=None): GIT_REPO_PLACEHOLDER = "REPLACE_WITH_GIT_REPO_URL" +GIT_HOST_PLACEHOLDER = "REPLACE_WITH_GIT_HOST" +GIT_AUTH_TYPE_PLACEHOLDER = "REPLACE_WITH_GIT_AUTH_TYPE" + +SSH_URL_RE = re.compile(r"^[\w.-]+@([\w.-]+):") + +def _parse_git_repo_url(git_repo_url): + """Derive (host, auth_type) from a Git repository URL. + + HTTPS URLs -> host = "https://github.com", auth_type = "https" + SSH URLs -> host = "github.com", auth_type = "ssh" + """ + m = SSH_URL_RE.match(git_repo_url) + if m: + return m.group(1), "ssh" + parsed = urlparse(git_repo_url) + scheme = parsed.scheme or "https" + hostname = parsed.hostname or "" + return f"{scheme}://{hostname}", "https" -def _substitute_git_repo_url(base, git_repo_url): - """Replace the git repo placeholder in supply-chain overrides.""" + +def _substitute_git_overrides(base, git_repo_url, git_host, git_auth_type): + """Replace git-related placeholders in supply-chain overrides.""" apps = base.get("clusterGroup", {}).get("applications", {}) sc = apps.get("supply-chain", {}) + placeholder_map = { + "qtodo.repository": (GIT_REPO_PLACEHOLDER, git_repo_url), + "git.credentials.host": (GIT_HOST_PLACEHOLDER, git_host), + "git.credentials.authType": (GIT_AUTH_TYPE_PLACEHOLDER, git_auth_type), + } for override in sc.get("overrides", []): - if ( - override.get("name") == "qtodo.repository" - and str(override.get("value")) == GIT_REPO_PLACEHOLDER - ): - override["value"] = git_repo_url - return + entry = placeholder_map.get(override.get("name")) + if entry and str(override.get("value")) == entry[0]: + override["value"] = entry[1] def generate_variant( @@ -361,7 +347,8 @@ def generate_variant( _substitute_repository_placeholders(base, org=org, image_name=image_name) if git_repo_url: - _substitute_git_repo_url(base, git_repo_url) + git_host, git_auth_type = _parse_git_repo_url(git_repo_url) + _substitute_git_overrides(base, git_repo_url, git_host, git_auth_type) validate_output(base) cg = base.get("clusterGroup") diff --git a/values-secret.yaml.template b/values-secret.yaml.template index d6bb3040..843f8e69 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -234,8 +234,12 @@ secrets: # SUPPLY CHAIN GIT CREDENTIALS (hub/supply-chain/) # Credentials for cloning protected Git repositories in Tekton pipelines # Policy: hub-supply-chain-jwt-secret (read access to hub/supply-chain/*) + # + # Choose ONE of the two options below depending on your auth type: + # Option A: HTTPS basic auth (username + PAT) + # Option B: SSH key auth (ssh-privatekey + known_hosts) # =========================================================================== - # Uncomment when using protected (private) Git repositories + # Option A: HTTPS basic auth (uncomment for HTTPS protected repositories) #- name: git-credentials # vaultPrefixes: # - hub/supply-chain @@ -247,6 +251,17 @@ secrets: # value: "your-personal-access-token" # Replace with your PAT/token # onMissingValue: error + # Option B: SSH key auth (uncomment for SSH protected repositories) + # Generate known_hosts: ssh-keyscan github.com > ~/.ssh/known_hosts_github + #- name: git-credentials + # vaultPrefixes: + # - hub/supply-chain + # fields: + # - name: ssh-privatekey + # path: ~/.ssh/id_rsa # Replace with your SSH private key path + # - name: known_hosts + # path: ~/.ssh/known_hosts_github # Replace with your known_hosts file path + # =========================================================================== # COCO (CONFIDENTIAL CONTAINERS) SECRETS # Uncomment the secrets below when deploying with CoCo support. From b1c6793604431d5b39989a7288d5f59b01bb94d0 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Wed, 27 May 2026 22:51:51 -0400 Subject: [PATCH 3/4] docs: differentiate git-auth workspace binding for HTTPS vs SSH modes HTTPS mode requires explicitly binding the git-auth workspace to the qtodo-git-credentials secret, while SSH mode must leave it unbound due to the git-clone ClusterTask's prepare.sh chmod failing on read-only projected volume symlinks. Signed-off-by: Min Zhang --- docs/supply-chain.md | 41 +++++++++++++++++++++++++++++++++---- values-secret.yaml.template | 2 +- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 4c255cf1..21257482 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -257,7 +257,9 @@ Once the supply-chain application has synced in ArgoCD, start the pipeline using At the bottom we have the **workspaces**. These must be configured manually. * For **qtodo-source**, select `PersistentVolumeClaim` and the PVC name is `qtodo-workspace-source`. * For **registry-auth-config**, select `Secret` and the name of the secret is `qtodo-registry-auth`. - * Leave **git-auth** unbound (empty). Git credentials for protected repositories are injected automatically via the `pipeline` ServiceAccount (see [How it works](#how-it-works) below). + * For **git-auth**, the binding depends on the authentication mode (see [How it works](#how-it-works) for details): + * **HTTPS mode**: select `Secret` and the name of the secret is `qtodo-git-credentials`. The `git-clone` ClusterTask's `basic-auth` workspace requires the secret to be provided explicitly; ServiceAccount-level credential injection alone is not sufficient for HTTPS. + * **SSH mode**: leave **git-auth** unbound (empty). SSH credentials are injected automatically via the `pipeline` ServiceAccount. Binding the workspace directly causes the `git-clone` ClusterTask's `prepare.sh` to run a recursive `chmod` on the copied secret volume, which fails on the read-only Kubernetes projected volume symlinks. 5. Press **Start** to finish and run the pipeline. @@ -265,6 +267,35 @@ Once the supply-chain application has synced in ArgoCD, start the pipeline using We can also start a pipeline execution using a CLI and the Kubernetes API. We start creating a new `PipelineRun` resource referencing the `qtodo-supply-chain` pipeline. Let's create a new file called `qtodo-pipeline.yaml` and copy this content. +**HTTPS mode** (bind `git-auth` to the `qtodo-git-credentials` secret): + +```yaml +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: qtodo-manual-run- + namespace: layered-zero-trust-hub +spec: + pipelineRef: + name: qtodo-supply-chain + taskRunTemplate: + serviceAccountName: pipeline + timeouts: + pipeline: 1h0m0s + workspaces: + - name: qtodo-source + persistentVolumeClaim: + claimName: qtodo-workspace-source + - name: registry-auth-config + secret: + secretName: qtodo-registry-auth + - name: git-auth + secret: + secretName: qtodo-git-credentials +``` + +**SSH mode** (leave `git-auth` unbound): + ```yaml apiVersion: tekton.dev/v1 kind: PipelineRun @@ -289,7 +320,7 @@ spec: As was described previously, verify the values associated with the PVC storage and registry configuration. -> **Note**: The `git-auth` workspace should be left **unbound**. Git credentials for protected repositories are injected automatically by Tekton's credential initialization through the `pipeline` ServiceAccount. Binding the `git-auth` workspace directly can cause permission errors with the `git-clone` ClusterTask's SSH credential handling. +> **Note**: The `git-auth` workspace binding differs between authentication modes. In **HTTPS mode**, the `qtodo-git-credentials` secret must be bound explicitly -- ServiceAccount-level credential injection alone is not sufficient for the `git-clone` ClusterTask's `basic-auth` workspace. In **SSH mode**, the workspace must be left **unbound**; SSH credentials are injected automatically through the `pipeline` ServiceAccount. Binding the `git-auth` workspace in SSH mode causes the `git-clone` ClusterTask's `prepare.sh` to run a recursive `chmod` on the copied secret volume, which fails on the read-only Kubernetes projected volume symlinks. Using the previously created definition, start a new execution of the pipeline using `oc` CLI: @@ -416,8 +447,10 @@ When `git.credentials.enabled` is `true`: * An `ExternalSecret` (`qtodo-git-credentials`) pulls the credentials from Vault and creates a secret annotated with `tekton.dev/git-0` pointing to the configured host. * **HTTPS mode**: creates an `Opaque` secret with `.git-credentials` and `.gitconfig` files. * **SSH mode**: creates a `kubernetes.io/ssh-auth` secret with `ssh-privatekey` and `known_hosts` entries. -* The `pipeline` ServiceAccount lists the secret (see `pipeline-sa.yaml`). Tekton's credential initialization automatically injects the credentials into task containers -- `.gitconfig` and `.git-credentials` for HTTPS, or `~/.ssh/config`, `~/.ssh/id_*`, and `~/.ssh/known_hosts` for SSH. No explicit `git-auth` workspace binding is required. -* The `git-auth` workspace remains declared in the pipeline as `optional: true` for compatibility, but should be left unbound. Binding it triggers the `git-clone` ClusterTask's `prepare.sh` which runs a recursive `chmod` on the copied secret volume; this fails on the read-only Kubernetes projected volume symlinks and aborts the step (particularly in SSH mode). +* The `pipeline` ServiceAccount lists the secret (see `pipeline-sa.yaml`). Tekton's credential initialization automatically injects the credentials into task containers -- `.gitconfig` and `.git-credentials` for HTTPS, or `~/.ssh/config`, `~/.ssh/id_*`, and `~/.ssh/known_hosts` for SSH. +* The `git-auth` workspace is declared in the pipeline as `optional: true`. How it should be bound depends on the authentication mode: + * **HTTPS mode**: the `git-auth` workspace **must** be bound to the `qtodo-git-credentials` secret. ServiceAccount-level credential injection alone is not sufficient -- without an explicit workspace binding, the `git-clone` ClusterTask cannot access the protected repository. + * **SSH mode**: the `git-auth` workspace must be left **unbound**. SSH credentials are injected automatically via the ServiceAccount. Binding the workspace triggers the `git-clone` ClusterTask's `prepare.sh`, which runs a recursive `chmod` on the copied secret volume; this fails on the read-only Kubernetes projected volume symlinks and aborts the step. * The Vault policy `hub-supply-chain-jwt-secret` grants read access to `secret/data/hub/supply-chain/*` for the pipeline's SPIFFE identity. ### Init task (pre-flight image check) diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 843f8e69..77ba0079 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -258,7 +258,7 @@ secrets: # - hub/supply-chain # fields: # - name: ssh-privatekey - # path: ~/.ssh/id_rsa # Replace with your SSH private key path + # path: ~/.ssh/id_ed25519 # Replace with your SSH private key path # - name: known_hosts # path: ~/.ssh/known_hosts_github # Replace with your known_hosts file path From 0a50bb83cfd96f259a4b3f680414dd33f44803a4 Mon Sep 17 00:00:00 2001 From: Min Zhang Date: Thu, 28 May 2026 10:45:05 -0400 Subject: [PATCH 4/4] fix: address PR #136 review feedback from Manuel (mlorenzofr) - Restore PR #139 dedup logic (named list upsert, duplicate override validation) that was inadvertently removed - Use file-based path instead of inline value for HTTPS git credentials and registry token to avoid plaintext password leaks in values-secret - Add | trim to ESO password template for path-sourced credentials - Add ssh-keygen instructions and passwordless key requirement - Fix make load-secrets -> ./pattern.sh make load-secrets - Add explicit SSH URL example to qtodo.repository override comment Signed-off-by: Min Zhang --- .../secrets/qtodo-git-credentials.yaml | 2 +- docs/supply-chain.md | 35 +++++++++++++--- scripts/gen-feature-variants.py | 41 ++++++++++++++++++- values-secret.yaml.template | 17 +++++--- 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml index 19405b5c..ff1a0367 100644 --- a/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml +++ b/charts/supply-chain/templates/secrets/qtodo-git-credentials.yaml @@ -44,7 +44,7 @@ spec: .gitconfig: | [credential "{{ $host }}"] helper = store - .git-credentials: {{ printf "https://{{ .%s }}:{{ .%s }}@%s" $userKey $passKey $hostBare | quote }} + .git-credentials: {{ printf "https://{{ .%s }}:{{ .%s | trim }}@%s" $userKey $passKey $hostBare | quote }} data: - secretKey: {{ $userKey }} remoteRef: diff --git a/docs/supply-chain.md b/docs/supply-chain.md index 21257482..4e653893 100644 --- a/docs/supply-chain.md +++ b/docs/supply-chain.md @@ -75,13 +75,20 @@ By default, ZTVP deploys a built-in Red Hat Quay registry. However, you can use Uncomment the `registry-user` secret and replace the placeholder with your registry token or password: + Store your registry token in a local file: + + ```shell + mkdir -p ~/.config/validated-patterns + echo -n "your-registry-token" > ~/.config/validated-patterns/registry-token + ``` + ```yaml - name: registry-user vaultPrefixes: - hub/infra/registry fields: - name: registry-password - value: "REPLACE_WITH_REGISTRY_TOKEN" + path: ~/.config/validated-patterns/registry-token onMissingValue: error ``` @@ -366,16 +373,24 @@ Uncomment the `git-credentials` secret in your local `~/values-secret.yaml` (cop **Option A -- HTTPS basic auth** (username + Personal Access Token): +Store your credentials in local files to avoid plaintext in YAML: + +```shell +mkdir -p ~/.config/validated-patterns +echo -n "your-git-username" > ~/.config/validated-patterns/git-username +echo -n "your-personal-access-token" > ~/.config/validated-patterns/git-token +``` + ```yaml - name: git-credentials vaultPrefixes: - hub/supply-chain fields: - name: username - value: "your-git-username" + path: ~/.config/validated-patterns/git-username onMissingValue: error - name: password - value: "your-personal-access-token" + path: ~/.config/validated-patterns/git-token onMissingValue: error ``` @@ -387,18 +402,26 @@ Uncomment the `git-credentials` secret in your local `~/values-secret.yaml` (cop - hub/supply-chain fields: - name: ssh-privatekey - path: ~/.ssh/id_ed25519 # or id_rsa, id_ecdsa, etc. + path: ~/.ssh/id_ed25519_ztvp # or id_rsa, id_ecdsa, etc. - name: known_hosts path: ~/.ssh/known_hosts_github ``` +Generate a passwordless SSH key pair (if you don't already have one): + +```shell +ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_ztvp -N "" +``` + +The key **must not** be password-protected -- Tekton cannot prompt for a passphrase at runtime. + Generate the `known_hosts` file for your Git host: ```shell ssh-keyscan github.com > ~/.ssh/known_hosts_github ``` -Then load the secret into Vault: `make load-secrets`. +Then load the secret into Vault: `./pattern.sh make load-secrets`. #### 2. Enable Git credentials in the supply-chain overrides @@ -437,7 +460,7 @@ When using the generator with `--git-repo`, the `qtodo.repository` override is s ```yaml - name: qtodo.repository - value: "https://github.com/your-org/qtodo.git" # or SSH URL + value: "https://github.com/your-org/qtodo.git" # or SSH URL (git@github.com:your-org/qtodo.git) ``` #### How it works diff --git a/scripts/gen-feature-variants.py b/scripts/gen-feature-variants.py index af97b709..6565e548 100755 --- a/scripts/gen-feature-variants.py +++ b/scripts/gen-feature-variants.py @@ -127,6 +127,25 @@ def _merge_namespace_lists(base_list, fragment_list): existing.add(key) +def _is_named_list(lst): + """Return True if lst is a list of mappings that all contain a 'name' key.""" + return len(lst) > 0 and all( + isinstance(item, dict) and "name" in item for item in lst + ) + + +def _merge_named_lists(base_list, overlay_list): + """Merge overlay items into base by 'name', replacing on conflict.""" + index = {item["name"]: i for i, item in enumerate(base_list)} + for item in overlay_list: + name = item["name"] + if name in index: + base_list[index[name]] = copy.deepcopy(item) + else: + index[name] = len(base_list) + base_list.append(copy.deepcopy(item)) + + def _deep_merge_mappings(base, overlay): """Recursively merge overlay into base (overlay wins for scalars).""" for key in overlay: @@ -141,7 +160,10 @@ def _deep_merge_mappings(base, overlay): and isinstance(base[key], list) and isinstance(overlay[key], list) ): - base[key].extend(overlay[key]) + if _is_named_list(base[key]) or _is_named_list(overlay[key]): + _merge_named_lists(base[key], overlay[key]) + else: + base[key].extend(overlay[key]) else: base[key] = overlay[key] @@ -157,7 +179,8 @@ def _apply_merge_into(base_apps, merge_into_spec): overrides: [...] For each target app, recursively merge into the existing app config. - Lists (roles, overrides) are appended rather than replaced. + Named lists (items with a 'name' key) use upsert semantics; plain lists + are appended. """ for app_name, additions in merge_into_spec.items(): if app_name not in base_apps: @@ -249,6 +272,20 @@ def validate_output(data): seen.add(key) apps = cg.get("applications", {}) + for app_name, app_val in apps.items(): + overrides = app_val.get("overrides", []) if isinstance(app_val, dict) else [] + override_names = set() + for ovr in overrides: + name = ovr.get("name") if isinstance(ovr, dict) else None + if name and name in override_names: + print( + f"WARNING: duplicate override '{name}' in " + f"application '{app_name}'", + file=sys.stderr, + ) + if name: + override_names.add(name) + vault = apps.get("vault", {}) jwt_roles = vault.get("jwt", {}).get("roles", []) role_names = set() diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 77ba0079..55cc3d19 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -219,15 +219,15 @@ secrets: # Used by: supply-chain pipeline (push), qtodo (pull) when registry enabled # Policy: hub-supply-chain-jwt-secret (read access to hub/infra/registry/*) # - # Uncomment and replace REPLACE_WITH_REGISTRY_TOKEN with your registry - # token/password in your local ~/values-secret-layered-zero-trust.yaml. + # Store your registry token in a local file to avoid plaintext in YAML: + # echo -n "your-registry-token" > ~/.config/validated-patterns/registry-token # =========================================================================== #- name: registry-user # vaultPrefixes: # - hub/infra/registry # fields: # - name: registry-password - # value: "REPLACE_WITH_REGISTRY_TOKEN" + # path: ~/.config/validated-patterns/registry-token # onMissingValue: error # =========================================================================== @@ -240,25 +240,30 @@ secrets: # Option B: SSH key auth (ssh-privatekey + known_hosts) # =========================================================================== # Option A: HTTPS basic auth (uncomment for HTTPS protected repositories) + # Store your Git username and PAT in local files to avoid plaintext in YAML: + # echo -n "your-git-username" > ~/.config/validated-patterns/git-username + # echo -n "your-personal-access-token" > ~/.config/validated-patterns/git-token #- name: git-credentials # vaultPrefixes: # - hub/supply-chain # fields: # - name: username - # value: "your-git-username" # Replace with your Git username + # path: ~/.config/validated-patterns/git-username # onMissingValue: error # - name: password - # value: "your-personal-access-token" # Replace with your PAT/token + # path: ~/.config/validated-patterns/git-token # onMissingValue: error # Option B: SSH key auth (uncomment for SSH protected repositories) + # The private key must NOT be password-protected (passphrase-less). + # Generate a passwordless key: ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_ztvp -N "" # Generate known_hosts: ssh-keyscan github.com > ~/.ssh/known_hosts_github #- name: git-credentials # vaultPrefixes: # - hub/supply-chain # fields: # - name: ssh-privatekey - # path: ~/.ssh/id_ed25519 # Replace with your SSH private key path + # path: ~/.ssh/id_ed25519_ztvp # Replace with your SSH private key path # - name: known_hosts # path: ~/.ssh/known_hosts_github # Replace with your known_hosts file path