From 525677655f333295e1ff3dad6137500624db2cbc Mon Sep 17 00:00:00 2001 From: xnoto Date: Fri, 31 Jul 2026 18:07:27 -0600 Subject: [PATCH] chore(headlamp): remove routing and legacy auth --- AGENTS.md | 23 +++++- README.md | 105 +++++++++++++++++++++++-- bootstrap/argocd-config.yaml | 6 -- bootstrap/kustomization.yaml | 2 +- bootstrap/oidc-rbac.yaml | 4 +- docs/kubeconfig.example.yaml | 31 ++++++++ operators/headlamp/kustomization.yaml | 1 - operators/headlamp/tunnel-binding.yaml | 16 ---- workloads/apps/kustomization.yaml | 3 - 9 files changed, 153 insertions(+), 38 deletions(-) create mode 100644 docs/kubeconfig.example.yaml delete mode 100644 operators/headlamp/tunnel-binding.yaml diff --git a/AGENTS.md b/AGENTS.md index 22ee526..91e6629 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,11 @@ PostSync: ci-token-sync, wait-for-* jobs Sync waves order resources within a single ArgoCD Application — they are **not** global across Applications. The App-of-Apps structure plus `wait-for-*` post-sync jobs enforces cross-Application ordering. +The active Applications reconcile `bootstrap/secrets`, `operators`, and +`workloads/apps`; they do **not** reconcile the rest of `bootstrap/`. Changes +to the ArgoCD CR, OIDC RBAC, or CI ServiceAccount require a separately reviewed +bootstrap apply or node provisioning. + ## Domain Architecture | Domain | Path | TLS | @@ -20,7 +25,7 @@ Sync waves order resources within a single ArgoCD Application — they are **not | `.makeitwork.cloud` | HTTP via cloudflare-operator `TunnelBinding` | Cloudflare edge | | `k3s.makeitwork.cloud` | TCP via `ClusterTunnel` to kube-apiserver, gated by Cloudflare Access | Cloudflare edge | -There is no in-cluster ingress controller and no public IP. Every external entry point — public web, kubectl, everything — is a Cloudflare Tunnel managed by cloudflare-operator. Legacy hostnames `api.makeitwork.cloud` and `*.apps.makeitwork.cloud` are not in use. +There is no in-cluster ingress controller and no public IP. Every external entry point — public web, kubectl, everything — uses a Cloudflare Tunnel managed by cloudflare-operator. App CNAMEs are declared in `tfroot-cloudflare` and must stay aligned with the routes here. Legacy hostnames `api.makeitwork.cloud` and `*.apps.makeitwork.cloud` are not in use. ## Key Namespaces @@ -43,11 +48,21 @@ The Cloudflare API token lives in `cert-manager/cloudflare-api-token` and is ref ## Cloudflare Tunnel DNS -Public `*.makeitwork.cloud` DNS records are operator-managed from `TunnelBinding` resources. +App DNS and routes have coordinated owners: -- Keep `tunnelRef.disableDNSUpdates: false` so the operator owns CNAMEs +- `tfroot-cloudflare/cf-tunnels.tf` declares CNAMEs +- `TunnelBinding` resources declare routes and the operator-managed ownership TXT records - `subjects[].name` must match the real Kubernetes `Service` name in the same namespace; if it doesn't exist, status reports `http_status:404` -- Ownership is tracked in `_managed.` TXT records — deleting a CNAME without removing its matching TXT record causes update-by-stale-id failures (`Record does not exist. (81044)`) +- Remove and reconcile the `TunnelBinding` before removing its Terraform hostname; deleting only the CNAME leaves a stale `_managed.` TXT record and causes update-by-stale-id failures (`Record does not exist. (81044)`) + +## kubectl Access + +Use the dedicated `makeitworkcloud-k3s` kubeconfig and the Cloudflare/Dex OIDC +procedure in `README.md#kubectl-access`. Never use or modify an unrelated +production or staging context. If no Make IT Work Cloud context is configured, +ask for the approved public server CA and create the non-secret exec kubeconfig +from `docs/kubeconfig.example.yaml`; do not copy the k3s admin kubeconfig off +the node. ## SOPS / KSOPS diff --git a/README.md b/README.md index d37a8ed..0d5b0d6 100644 --- a/README.md +++ b/README.md @@ -34,15 +34,110 @@ There is no in-cluster ingress controller and no public IP. Every external entry ### TunnelBinding DNS -Public DNS under `*.makeitwork.cloud` is owned by cloudflare-operator from `TunnelBinding` resources in this repo. +App hostnames have two coordinated declarative control points: -- Keep `tunnelRef.disableDNSUpdates: false` so the operator manages CNAMEs -- `subjects[].name` must match the real `Service` name in the same namespace -- The operator stores ownership in `_managed.` TXT records; deleting a CNAME without removing its matching TXT record yields Cloudflare error `81044` +- `tfroot-cloudflare/cf-tunnels.tf` declares the CNAME at Cloudflare. +- Each `TunnelBinding` declares the in-cluster route and cloudflare-operator + tracks it with a `_managed.` TXT record. + +The hostname lists must stay aligned. To retire a route, remove and reconcile +the `TunnelBinding` first so the operator can clear its ownership record, then +remove the Terraform hostname and review a narrow OpenTofu plan. Do not delete +only the CNAME: a stale managed TXT record causes Cloudflare error `81044`. +`subjects[].name` must match the real Kubernetes `Service` name in the same +namespace. ## Authentication -GitHub OAuth provides SSO for ArgoCD, Grafana, and kubectl/Headlamp (via OIDC). Cluster-admin RBAC for the maintainer GitHub team is defined in `bootstrap/oidc-rbac.yaml`. CI uses a dedicated `ci-deployer` ServiceAccount whose token is synced to GitHub Actions secrets by a PostSync job. +GitHub OAuth provides SSO for ArgoCD, Grafana, Forgejo, and kubectl. Dex issues +kubectl tokens for the public `kubectl` client defined in +`bootstrap/argocd-config.yaml`; the API server validates that audience and maps +the `makeitworkcloud:admins` GitHub team to cluster-admin through +`bootstrap/oidc-rbac.yaml`. CI uses a separate `ci-deployer` ServiceAccount. + +### kubectl access + +Access has two independent gates: + +1. Cloudflare Access authorizes the TCP connection to + `k3s.makeitwork.cloud` using the policy in `tfroot-cloudflare`. +2. `kubelogin` obtains a Dex token. Dex includes the requested `email` and + `groups` claims, which the API server and Kubernetes RBAC validate. + +Install `cloudflared`, `kubectl`, and the `kubectl oidc-login` plugin. Confirm +all three commands are available before continuing. Start the local TCP proxy +in a dedicated terminal: + +```bash +cloudflared access tcp --hostname k3s.makeitwork.cloud --url localhost:6443 +``` + +If port 6443 is occupied, choose another local port and use the same port in +the kubeconfig server URL. + +Create a dedicated kubeconfig such as `~/.kube/makeitworkcloud-k3s.yaml`, mode +`0600`. Its cluster entry must point to `https://127.0.0.1:6443` and trust the +public k3s server CA supplied by an administrator. That CA may be copied from +`/var/lib/rancher/k3s/server/tls/server-ca.crt` on the k3s VM through an +approved channel. The kubeconfig must not contain a token, client certificate, +or client key. Do not copy `/etc/rancher/k3s/k3s.yaml` off the node: it contains +cluster-admin client credentials. + +Copy `docs/kubeconfig.example.yaml` to that dedicated path, replace its +`certificate-authority` placeholder with the absolute path to the approved CA +file, and set mode `0600`. Its user exec credential is: + +```yaml +user: + exec: + apiVersion: client.authentication.k8s.io/v1 + command: kubectl + interactiveMode: IfAvailable + args: + - oidc-login + - get-token + - --oidc-issuer-url=https://argocd.makeitwork.cloud/api/dex + - --oidc-client-id=kubectl + - --oidc-extra-scope=email + - --oidc-extra-scope=groups + - --oidc-pkce-method=S256 + - --token-cache-storage=keyring +``` + +Give this cluster a distinct context name such as `makeitworkcloud-k3s`; never +reuse an unrelated production or staging context. With the proxy running, +first confirm the dedicated file's current context, server, and user name +without displaying credentials: + +```bash +export KUBECONFIG="$HOME/.kube/makeitworkcloud-k3s.yaml" +kubectl config current-context +kubectl config view --minify \ + -o jsonpath='{.clusters[0].cluster.server}{"\n"}{.users[0].name}{"\n"}' +``` + +The expected context is `makeitworkcloud-k3s`, the server is +`https://127.0.0.1:6443`, and the user is the dedicated OIDC exec user. Then +verify the authenticated identity before performing any change: + +```bash +kubectl --context makeitworkcloud-k3s auth whoami +kubectl --context makeitworkcloud-k3s auth can-i '*' '*' --all-namespaces +``` + +`auth whoami` should show your email and the `makeitworkcloud:admins` group; +`auth can-i` should return `yes`. When finished, stop the local `cloudflared` +process. OIDC and Cloudflare Access sessions expire independently; do not run a +global credential-cache cleanup unless you have reviewed its scope. + +For break-glass access, SSH to the k3s VM through `hero.makeitwork.cloud` and +run kubectl there with `/etc/rancher/k3s/k3s.yaml`; see `tfroot-libvirt`. + +The App-of-Apps Applications reconcile `bootstrap/secrets`, `operators`, and +`workloads/apps`. They do not reconcile the rest of `bootstrap/`. Changes to +the ArgoCD CR, OIDC RBAC, or CI ServiceAccount therefore require a separately +reviewed bootstrap apply or node provisioning; an ordinary ArgoCD sync is not +enough. ## SOPS / KSOPS diff --git a/bootstrap/argocd-config.yaml b/bootstrap/argocd-config.yaml index 6c736d6..480624b 100644 --- a/bootstrap/argocd-config.yaml +++ b/bootstrap/argocd-config.yaml @@ -118,12 +118,6 @@ spec: redirectURIs: - http://localhost:8000 - http://localhost:18000 - # Headlamp uses Dex as its OIDC provider (Dex bridges to GitHub). - - id: headlamp - name: Headlamp - secret: $dex.headlamp.clientSecret - redirectURIs: - - https://headlamp.makeitwork.cloud/oidc-callback # Grafana uses Dex via GF_AUTH_GENERIC_OAUTH_*, replacing its # built-in GitHub OAuth so all cluster apps share one IdP. - id: grafana diff --git a/bootstrap/kustomization.yaml b/bootstrap/kustomization.yaml index 39ef060..bc02a8f 100644 --- a/bootstrap/kustomization.yaml +++ b/bootstrap/kustomization.yaml @@ -6,7 +6,7 @@ resources: # NOTE: cluster-admin ClusterRoleBinding is managed by ansible-role-crc # (ArgoCD cannot grant itself permissions it doesn't have) - argocd-config.yaml - # Wave 0: GitHub-team-to-cluster-admin RBAC for OIDC users (Headlamp, kubectl) + # Wave 0: GitHub-team-to-cluster-admin RBAC for kubectl OIDC users - oidc-rbac.yaml # Wave 0: CI/CD service account with cluster-admin for GitHub Actions - ci-service-account.yaml diff --git a/bootstrap/oidc-rbac.yaml b/bootstrap/oidc-rbac.yaml index 6a51340..46aa771 100644 --- a/bootstrap/oidc-rbac.yaml +++ b/bootstrap/oidc-rbac.yaml @@ -1,10 +1,10 @@ --- # Maps the GitHub team `makeitworkcloud:admins` (surfaced as a `groups` claim # by Dex's GitHub connector) to cluster-admin so OIDC-authenticated users -# can use kubectl/Headlamp without a separate per-user binding. +# can use kubectl without a separate per-user binding. # # Pairs with the k3s apiserver flags in tfroot-libvirt cloud-init that point -# at the same Dex issuer: --oidc-issuer-url, --oidc-client-id=headlamp, +# at the same Dex issuer: --oidc-issuer-url, --oidc-client-id=kubectl, # --oidc-username-claim=email, --oidc-groups-claim=groups. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/docs/kubeconfig.example.yaml b/docs/kubeconfig.example.yaml new file mode 100644 index 0000000..b7371d3 --- /dev/null +++ b/docs/kubeconfig.example.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: Config +preferences: {} +clusters: + - name: makeitworkcloud-k3s + cluster: + server: https://127.0.0.1:6443 + certificate-authority: /absolute/path/to/makeitworkcloud-k3s-server-ca.crt +users: + - name: makeitworkcloud-oidc + user: + exec: + apiVersion: client.authentication.k8s.io/v1 + command: kubectl + interactiveMode: IfAvailable + args: + - oidc-login + - get-token + - --oidc-issuer-url=https://argocd.makeitwork.cloud/api/dex + - --oidc-client-id=kubectl + - --oidc-extra-scope=email + - --oidc-extra-scope=groups + - --oidc-pkce-method=S256 + - --token-cache-storage=keyring +contexts: + - name: makeitworkcloud-k3s + context: + cluster: makeitworkcloud-k3s + user: makeitworkcloud-oidc +current-context: makeitworkcloud-k3s diff --git a/operators/headlamp/kustomization.yaml b/operators/headlamp/kustomization.yaml index 42f9ed2..699eece 100644 --- a/operators/headlamp/kustomization.yaml +++ b/operators/headlamp/kustomization.yaml @@ -4,6 +4,5 @@ kind: Kustomization resources: - namespace.yaml - application.yaml - - tunnel-binding.yaml generators: - ksops-headlamp-secrets.yaml diff --git a/operators/headlamp/tunnel-binding.yaml b/operators/headlamp/tunnel-binding.yaml deleted file mode 100644 index 5ad3a6d..0000000 --- a/operators/headlamp/tunnel-binding.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: networking.cfargotunnel.com/v1alpha1 -kind: TunnelBinding -metadata: - name: headlamp - namespace: headlamp - annotations: - argocd.argoproj.io/sync-wave: "1" -subjects: - - name: headlamp - spec: - fqdn: headlamp.makeitwork.cloud - target: http://headlamp.headlamp.svc:80 -tunnelRef: - kind: ClusterTunnel - name: cluster-apps diff --git a/workloads/apps/kustomization.yaml b/workloads/apps/kustomization.yaml index d0dc1a7..242e985 100644 --- a/workloads/apps/kustomization.yaml +++ b/workloads/apps/kustomization.yaml @@ -12,6 +12,3 @@ resources: - grafana-app.yaml - kubectl-tunnel-app.yaml - ../argocd # Direct include - TunnelBinding for ArgoCD - # Note: Headlamp is installed by gitops-operators (operators/headlamp/) since - # the chart's Application CR + namespace/Secret/TunnelBinding all live together - # there. No separate workloads-app entry needed.