diff --git a/charts/rh-keycloak/templates/wait-for-eso-webhook-netpol.yaml b/charts/rh-keycloak/templates/wait-for-eso-webhook-netpol.yaml new file mode 100644 index 00000000..8b9cf7b8 --- /dev/null +++ b/charts/rh-keycloak/templates/wait-for-eso-webhook-netpol.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.esoWebhookCheck.enabled (eq (.Values.rhbk.defaultDenyNetworkPolicy.enabled | toString) "true") }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: wait-for-eso-webhook-netpol + namespace: {{ .Release.Namespace }} + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + podSelector: + matchLabels: + app: wait-for-eso-webhook + policyTypes: + - Egress + egress: + - ports: + - protocol: UDP + port: 5353 + - protocol: TCP + port: 5353 + to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + - ports: + - protocol: TCP + port: 6443 +{{- end }} diff --git a/charts/rh-keycloak/templates/wait-for-eso-webhook.yaml b/charts/rh-keycloak/templates/wait-for-eso-webhook.yaml new file mode 100644 index 00000000..15e15e45 --- /dev/null +++ b/charts/rh-keycloak/templates/wait-for-eso-webhook.yaml @@ -0,0 +1,74 @@ +{{- if .Values.esoWebhookCheck.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: wait-for-eso-webhook + namespace: {{ .Release.Namespace }} + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: wait-for-eso-webhook-{{ .Release.Namespace }} + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +rules: +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: wait-for-eso-webhook-{{ .Release.Namespace }} + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: wait-for-eso-webhook-{{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: wait-for-eso-webhook + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: wait-for-eso-webhook + namespace: {{ .Release.Namespace }} + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + backoffLimit: 3 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + app: wait-for-eso-webhook + spec: + serviceAccountName: wait-for-eso-webhook + restartPolicy: Never + containers: + - name: wait + image: {{ .Values.esoWebhookCheck.image }} + command: + - /bin/bash + - -ce + - | + echo "Waiting for ESO webhook endpoints in external-secrets namespace..." + until oc get endpointslice -n external-secrets \ + -l=kubernetes.io/service-name=external-secrets-webhook \ + -o jsonpath='{.items[0].endpoints[0].addresses[0]}' 2>/dev/null + do + echo "ESO webhook not ready, retrying in 5s..." + sleep 5 + done + echo "ESO webhook is ready." +{{- end }} diff --git a/charts/rh-keycloak/values.yaml b/charts/rh-keycloak/values.yaml index 94bcface..6df89cfd 100644 --- a/charts/rh-keycloak/values.yaml +++ b/charts/rh-keycloak/values.yaml @@ -1,3 +1,8 @@ +# PreSync Job waits for the ESO webhook to have ready endpoints before syncing ExternalSecrets. +esoWebhookCheck: + enabled: true + image: registry.redhat.io/openshift4/ose-cli-rhel9:latest + # PostSync Job deletes secrets labeled for cleanup after realm import consumes them. cleanup: enabled: true diff --git a/docs/SYNC-WAVE-INVENTORY.md b/docs/SYNC-WAVE-INVENTORY.md index e2a47083..5181a7a8 100644 --- a/docs/SYNC-WAVE-INVENTORY.md +++ b/docs/SYNC-WAVE-INVENTORY.md @@ -23,11 +23,12 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | 26 | └ ztvp-certificates | chart | distribution-policy (3 resources) | | 26 | └ openshift-storage | ns | Namespace + OperatorGroup | | 26 | └ rhtpa-operator | ns | Namespace + OperatorGroup | +| 26 | └ external-secrets-operator | ns | Namespace + OperatorGroup | | 27 | └ odf | sub | ODF operator install | | 27 | └ rhtpa-operator | sub | RHTPA operator install | +| 27 | └ external-secrets | sub | External Secrets Operator install | | 28 | └ quay-operator | sub | Quay operator install | | 29 | └ rhtas-operator | sub | RHTAS operator install | -| 30 | golang-external-secrets | **App** | | | 30 | zero-trust-workload-identity-manager | **App** | | | 31 | └ rhtpa-operator | chart | ingress-ca-job (SA, Role, RoleBinding, ConfigMap, Job) | | 32 | └ rhtpa-operator | chart | operator-rolebinding (2 bindings) | @@ -101,7 +102,8 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | rhtpa-operator (subscription) | -4 | 27 | After OperatorGroup (26) | | quay-operator (subscription) | -3 | 28 | After ODF operator | | rhtas-operator (subscription) | -2 | 29 | After Quay operator | -| golang-external-secrets | — | 30 | After Vault (newly added) | +| external-secrets-operator (OperatorGroup) | — | 26 | Before operator subscription (newly added) | +| external-secrets (subscription) | — | 27 | After OperatorGroup (26) (newly added) | | zero-trust-workload-identity-manager | — | 30 | After Vault/certs (newly added) | | quay-enterprise (namespace) | 1 | 32 | Before NooBaa and Quay components | | trusted-artifact-signer (namespace) | 1 | 32 | Auto-created by RHTAS operator | @@ -278,3 +280,4 @@ Resources without an explicit sync-wave default to wave 0. These include: pipeli - Template waves are resolved **locally within each app sync**, not globally. A template wave of 32 inside acs-central (app wave 41) does not conflict with a template wave of 32 inside noobaa-mcg (app wave 36); they run independently. - Sync waves control **Application creation order**, not readiness. A later wave means the Application resource is submitted to the hub later, but the earlier app's pods may not be fully running yet. For hard readiness gates, use Argo CD health checks or resource hooks. - **Externalized charts**: Five charts (certmanager, compliance-scanning, keycloak/RHBK, quay-registry, ZTWIM) are maintained in standalone repositories. Their resource-level sync-wave annotations are managed there and pinned via `chartVersion` in `values-hub.yaml`. Application-level sync-waves remain in this repository. +- **External Secrets Operator**: The ESO webhook must be ready before applications with ExternalSecret resources sync. The `rh-keycloak` chart includes a PreSync hook Job (`wait-for-eso-webhook`) that polls the webhook endpoints until they are available, preventing ExternalSecret creation failures during initial deployment. Argo CD health checks for `ClusterSecretStore` and `Application` resources provide additional sync-wave ordering guarantees. diff --git a/values-hub.yaml b/values-hub.yaml index 13a3490d..940edcde 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -57,6 +57,8 @@ clusterGroup: external-secrets-operator: operatorGroup: true targetNamespaces: [] + annotations: + argocd.argoproj.io/sync-wave: "26" # Create before operator subscription external-secrets: keycloak-system: operatorGroup: true @@ -121,6 +123,8 @@ clusterGroup: name: openshift-external-secrets-operator namespace: external-secrets-operator channel: stable-v1 + annotations: + argocd.argoproj.io/sync-wave: "27" # Install after OperatorGroup (26) cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -688,6 +692,19 @@ clusterGroup: return hs group: k8s.keycloak.org kind: KeycloakRealmImport + - check: | + local hs = {} + hs.status = "Progressing" + hs.message = "Waiting for application status" + if obj.status ~= nil and obj.status.health ~= nil then + hs.status = obj.status.health.status + if obj.status.health.message ~= nil then + hs.message = obj.status.health.message + end + end + return hs + group: argoproj.io + kind: Application resourceExclusions: | - apiGroups: - internal.open-cluster-management.io