diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml new file mode 100644 index 0000000..52d4d16 --- /dev/null +++ b/.github/workflows/on-pr-colima-smoke.yaml @@ -0,0 +1,170 @@ +name: colima backend smoke + +# End-to-end check of `hops local` on the colima backend on macOS. +# Mirrors on-pr-kind-smoke.yaml with --backend colima and kubectl --context colima. +# +# Runner constraints: +# - Colima needs nested virtualization (Lima VM). Pin macos-15-intel — that +# image is known to support nested virt for Colima/Lima on GHA. +# - Do NOT use bare macos-latest alone (tracks arm images; arm64 GHA macOS +# historically fails with HV_UNSUPPORTED for nested virt). +# - Prefer localhost:30500 for registry traffic, not the Colima VM IP +# (macOS 15 Local Network Privacy can block non-root VM-IP access). +# +# Install: brew provides colima/docker/kubectl/helm only. Do not pre-start +# colima here — hops local start --backend colima owns cluster bring-up. +# +# Sizing: hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard +# macos-15-intel runners (~4 CPU / ~14 GiB). Pass explicit smaller sizes +# so the VZ VM can allocate; stop/start resume uses the persisted profile. + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + workflow_dispatch: + +jobs: + colima-smoke: + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test-colima') + # Nested-virt-capable Intel pin (not bare macos-latest). + runs-on: macos-15-intel + # Nested virt + cold Crossplane pulls can take well over 30m. + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Install colima, docker, kubectl, helm + run: | + set -euxo pipefail + # Tools only — do not `colima start` here so the smoke exercises hops. + brew install colima docker kubectl helm + colima version + docker version --format '{{.Client.Version}}' || docker --version + kubectl version --client + helm version --short + + - name: Build hops + run: cargo build + + - name: Host resources (for sizing) + run: | + set -euxo pipefail + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + + - name: hops local start --backend colima + run: | + set -euxo pipefail + # Fit macos-15-intel (~4 CPU / ~14 GiB RAM). Defaults (8/16/60) OOM VZ. + # Memory 10 (not 8): at 8Gi CoreDNS/metrics-server thrash and smoke pods + # sit in ContainerCreating without IPs even after images pull. Leave ~4Gi + # for host macOS + VZ. Disk 40: registry PVC requests 20Gi. + ./target/debug/hops-cli local start --backend colima \ + --cpus 3 --memory 10 --disk 40 + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Wait for node + CoreDNS (nested virt settles) + run: | + set -euxo pipefail + # After cold start the control plane is still soft: probes time out and + # new pods stick in ContainerCreating. Wait for basics before registry. + kubectl --context colima wait --for=condition=Ready nodes --all --timeout=120s + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=kube-dns --timeout=180s || \ + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=coredns --timeout=180s || true + kubectl --context colima -n kube-system get pods -o wide || true + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + docker pull public.ecr.aws/docker/library/busybox:stable + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + # Service-name pull: containerd resolves via its certs.d alias. + kubectl --context colima run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # localhost:30500 pull: what provider runtime pods reference. + kubectl --context colima run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # Nested virt: image pull can succeed while CNI/IP assignment lags. + if ! kubectl --context colima wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then + echo "==== smoke pods not Ready ====" + kubectl --context colima get pods smoke-svc-name smoke-localhost -o wide || true + kubectl --context colima describe pod smoke-svc-name smoke-localhost || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-svc-name || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-localhost || true + exit 1 + fi + kubectl --context colima delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + ./target/debug/hops-cli local stop + ./target/debug/hops-cli local start + # After colima VM stop/start, docker container IDs are gone and pods + # often sit in Error until kubelet recreates them. Wait generously and + # force a rollout restart if Available still stalls. + if ! kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s; then + echo "deployments not Available after resume; restarting..." + kubectl --context colima -n crossplane-system get pods -o wide || true + kubectl --context colima -n crossplane-system rollout restart \ + deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true + kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s + fi + ./target/debug/hops-cli local doctor + + # Capture live cluster state BEFORE destroy so failures are diagnosable. + - name: Debug dump on failure + if: failure() + run: | + set +e + echo "==== host ====" + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + echo "==== colima ====" + colima status || true + colima list || true + echo "==== cluster ====" + kubectl --context colima get nodes,pods -A -o wide || true + echo "==== smoke pods (default) ====" + kubectl --context colima describe pod smoke-svc-name smoke-localhost 2>/dev/null || true + echo "==== crossplane-system ====" + kubectl --context colima describe pods -n crossplane-system || true + kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -100 || true + echo "==== lima ha.stderr / serial ====" + for f in \ + "$HOME/.colima/_lima/colima/ha.stderr.log" \ + "$HOME/.colima/_lima/colima/ha.stdout.log" \ + "$HOME/.colima/_lima/colima/serial.log" \ + "$HOME/.colima/_lima/colima/serialv.log" + do + if [ -f "$f" ]; then + echo "----- $f -----" + tail -n 200 "$f" || true + fi + done + ./target/debug/hops-cli local doctor || true + + - name: hops local destroy + if: always() + run: ./target/debug/hops-cli local destroy || true diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml new file mode 100644 index 0000000..3696675 --- /dev/null +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -0,0 +1,341 @@ +name: dory backend smoke + +# Real stock-Dory + hops integration on a self-hosted Apple Silicon Mac. +# +# Why self-hosted (not macos-15 / Colima): +# - Stock Dory needs dory-hv (Hypervisor.framework). Public GHA VMs cannot nest it. +# - An earlier "green" dory-smoke spent ~10m building a fork, then used Colima as +# ~/.dory/*.sock — that was not product Dory. This workflow refuses that lie. +# +# Trigger: label `test-dory` on the PR, or workflow_dispatch. +# Offline Mac: the job stays queued until the runner is online. +# +# Do-not-interrupt contract (personal Mac while you work): +# - Reuses your already-running Dory engine + dory-k8s +# - Never mutates desktop defaults (no global context switching, no merge into +# ~/.kube/config). Session is entirely env-driven: +# DOCKER_HOST=unix://$HOME/.dory/dory.sock +# KUBECONFIG= +# HOPS_DORY_DESKTOP=0 +# - Never destroy / stop dory-k8s, never kill Dory +# - Caps cargo parallelism and runs the build under nice(1) +# +# Runner setup (one-time on the Mac): +# mkdir -p ~/actions-runners/hops-cli && cd ~/actions-runners/hops-cli +# # download latest macOS arm64 runner from actions/runner releases +# ./config.sh --url https://github.com/hops-ops/hops-cli \ +# --token \ +# --name "$(hostname -s)-dory" \ +# --labels macOS,ARM64,hops-dory \ +# --work _work +# ./run.sh # on-demand; omit svc install so jobs only run when you start it +# +# Prerequisites: Dory engine healthy, Kubernetes enabled (dory-k8s), helm + cargo +# + `up` (Upbound CLI) on the runner user's PATH (path-based config install). + +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + workflow_dispatch: + +concurrency: + group: dory-smoke-self-hosted + cancel-in-progress: false + +jobs: + dory-smoke: + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test-dory') + runs-on: [self-hosted, macOS, ARM64, hops-dory] + timeout-minutes: 90 + env: + # Do not rewrite ~/.kube/config or change the machine docker default. + HOPS_DORY_DESKTOP: "0" + CARGO_BUILD_JOBS: "2" + CARGO_TERM_COLOR: always + steps: + - name: Checkout hops-cli + uses: actions/checkout@v4 + + - name: Require Apple Silicon + run: | + set -euxo pipefail + arch="$(uname -m)" + echo "arch=$arch host=$(hostname -s)" + if [ "$arch" != "arm64" ] && [ "$arch" != "arm64e" ]; then + echo "Stock Dory requires Apple Silicon (got $arch)." >&2 + exit 1 + fi + + - name: Env-only Dory session (private KUBECONFIG + DOCKER_HOST) + run: | + # No set -x: avoid dumping full PATH / home internals into Actions logs. + set -euo pipefail + sock="$HOME/.dory/dory.sock" + export PATH="$HOME/.dory/bin:/Applications/Dory.app/Contents/Helpers:$PATH" + echo "$HOME/.dory/bin" >> "$GITHUB_PATH" + echo "/Applications/Dory.app/Contents/Helpers" >> "$GITHUB_PATH" + + if [ ! -d /Applications/Dory.app ]; then + echo "Dory.app missing. Once: brew install --cask Augani/dory/dory" >&2 + exit 1 + fi + command -v helm >/dev/null || { echo "helm missing (brew install helm)" >&2; exit 1; } + command -v up >/dev/null || { echo "up CLI missing (needed for hops config install --path)" >&2; exit 1; } + + # Prefer a toolchain cargo that actually runs (shim may be broken). + resolve_cargo_bin() { + if command -v cargo >/dev/null 2>&1 && cargo --version >/dev/null 2>&1; then + command -v cargo + return 0 + fi + local c + for c in \ + "$HOME/.rustup/toolchains/stable-aarch64-apple-darwin/bin/cargo" \ + "$HOME/.rustup/toolchains/stable-x86_64-apple-darwin/bin/cargo" \ + "$HOME/.rustup/toolchains/"*/bin/cargo + do + if [ -x "$c" ] && "$c" --version >/dev/null 2>&1; then + printf '%s\n' "$c" + return 0 + fi + done + return 1 + } + CARGO_BIN="$(resolve_cargo_bin)" || { + echo "cargo missing or broken (install rustup stable toolchain)" >&2 + exit 1 + } + echo "cargo ok ($("$CARGO_BIN" --version))" + echo "$(dirname "$CARGO_BIN")" >> "$GITHUB_PATH" + export PATH="$(dirname "$CARGO_BIN"):$PATH" + + if [ ! -S "$sock" ] || ! docker -H "unix://$sock" info >/dev/null 2>&1; then + echo "waking Dory engine..." + command -v dory >/dev/null 2>&1 && { dory engine wake || dory engine start || true; } >/dev/null 2>&1 || true + if [ ! -S "$sock" ]; then + open -g -j -a Dory 2>/dev/null || true + fi + fi + + ready=0 + for i in $(seq 1 36); do + if [ -S "$sock" ] && docker -H "unix://$sock" info >/dev/null 2>&1; then + echo "dory engine ready" + ready=1 + break + fi + sleep 5 + done + if [ "$ready" -ne 1 ]; then + echo "Dory engine not ready (is Dory.app running?)" >&2 + exit 1 + fi + + if [ -L "$sock" ]; then + target="$(readlink "$sock" || true)" + case "$target" in + *colima*) + echo "refusing Colima-backed dory.sock" >&2 + exit 1 + ;; + esac + fi + + export DOCKER_HOST="unix://$sock" + echo "DOCKER_HOST=unix://$sock" >> "$GITHUB_ENV" + + if ! docker inspect -f '{{.State.Running}}' dory-k8s 2>/dev/null | grep -q true; then + if docker inspect dory-k8s >/dev/null 2>&1; then + echo "starting dory-k8s..." + docker start dory-k8s >/dev/null + else + echo "dory-k8s missing — enable Kubernetes in the Dory app" >&2 + exit 1 + fi + fi + + for i in $(seq 1 60); do + if docker exec dory-k8s kubectl get nodes --no-headers 2>/dev/null | grep -q ' Ready'; then + echo "dory-k8s Ready" + break + fi + if [ "$i" -eq 60 ]; then + echo "dory-k8s did not become Ready" >&2 + exit 1 + fi + sleep 2 + done + + # Job-private kubeconfig only — never touch ~/.kube/config. + kube="$RUNNER_TEMP/hops-dory-ci.kubeconfig" + if [ -f "$HOME/.kube/dory-config" ]; then + cp "$HOME/.kube/dory-config" "$kube" + else + docker exec dory-k8s cat /etc/rancher/k3s/k3s.yaml >"$kube" + if curl -sk --connect-timeout 2 https://127.0.0.1:6443/ >/dev/null 2>&1; then + sed -i.bak 's#server:.*#server: https://127.0.0.1:6443#' "$kube" || true + rm -f "${kube}.bak" + fi + fi + chmod 600 "$kube" + echo "KUBECONFIG=$kube" >> "$GITHUB_ENV" + export KUBECONFIG="$kube" + + # Minimal proof (no docker info / full node dump — personal Mac). + kubectl get nodes --no-headers | awk '{print "node", $1, $2}' + echo "env session ready (DOCKER_HOST + private KUBECONFIG)" + + # No Swatinem/rust-cache: on a personal Mac runner the post-job save + # tars ~100MB+ of target/ and can hang (slow upload, or a password prompt + # with no TTY). Local cargo builds here in ~20s without it. + + - name: Build hops (nice, limited jobs) + run: | + set -euxo pipefail + nice -n 10 cargo build -j "${CARGO_BUILD_JOBS}" + + - name: Host resources + run: | + set -euxo pipefail + sysctl -n hw.ncpu || true + sysctl hw.memsize || true + uname -m + uptime || true + # DOCKER_HOST + KUBECONFIG already in env from prior step + dory readiness || true + + - name: hops local start --backend dory + run: | + set -euxo pipefail + # HOPS_DORY_DESKTOP=0 → env session only (no global context changes) + ./target/debug/hops-cli local start --backend dory + + - name: hops local doctor + run: | + set -euxo pipefail + ./target/debug/hops-cli local doctor + + - name: Wait for node + CoreDNS + run: | + set -euxo pipefail + # Uses job KUBECONFIG only (current-context inside that file). + kubectl wait --for=condition=Ready nodes --all --timeout=120s + kubectl -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=kube-dns --timeout=180s || \ + kubectl -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=coredns --timeout=180s || true + kubectl -n kube-system get pods -o wide || true + kubectl -n crossplane-system get pods -o wide || true + + - name: Registry round-trip (engine push + in-cluster pull) + run: | + set -euxo pipefail + + NODE_IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' dory-k8s)" + test -n "$NODE_IP" + PUSH="${NODE_IP}:30500" + echo "engine push endpoint: $PUSH" + + # Pull + re-materialize as a *single-platform* image for this host. + # A plain `docker pull` of busybox:stable keeps multi-arch index + # metadata; `docker push` then warns: + # Not all multiplatform-content is present and only the available + # single-platform image was pushed + case "$(uname -m)" in + arm64|arm64e|aarch64) PLATFORM=linux/arm64 ;; + x86_64|amd64) PLATFORM=linux/amd64 ;; + *) PLATFORM="linux/$(uname -m)" ;; + esac + SRC=public.ecr.aws/docker/library/busybox:stable + for attempt in 1 2 3 4 5; do + if docker pull --platform "$PLATFORM" "$SRC"; then + break + fi + echo "docker pull failed (attempt $attempt); sleeping..." + sleep $((attempt * 15)) + if [ "$attempt" -eq 5 ]; then + exit 1 + fi + done + # Fresh single-platform tag (no incomplete multi-arch index). + docker build --platform "$PLATFORM" --provenance=false --sbom=false \ + -t "${PUSH}/smoke/busybox:ci" - </dev/null || true)" + installed="$(kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Installed")].status}' 2>/dev/null || true)" + if [ "$healthy" = "True" ] && [ "$installed" = "True" ]; then + echo "Configuration hops-ops-config-smoke Healthy after ${i} attempts" + break + fi + if [ "$i" -eq 90 ]; then + kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke -o yaml || true + kubectl get configurationrevisions.pkg.crossplane.io -o wide || true + kubectl get functions.pkg.crossplane.io -o wide || true + exit 1 + fi + sleep 5 + done + + # Wait for XRD to be established. + kubectl wait --for=condition=Established \ + crd/configsmokes.ci.hops.ops.com.ai --timeout=180s + + # Apply CI XR into hops-ci only (namespaced ConfigMap). + kubectl apply -f "$FIXTURE/local/ci-xr.yaml" + + # XR Ready (namespaced composite). + kubectl -n hops-ci wait --for=condition=Ready \ + configsmoke.ci.hops.ops.com.ai/hops-ci-smoke --timeout=300s + + # Prove the ConfigMap exists with expected data. + kubectl -n hops-ci get configmap hops-ci-smoke -o yaml + smoke_val="$(kubectl -n hops-ci get configmap hops-ci-smoke \ + -o jsonpath='{.data.smoke}')" + test "$smoke_val" = "ok" + + - name: Soft cleanup (CI fixture only) + if: always() + run: | + set +e + kubectl delete pod smoke-svc-name smoke-nodeport --wait=false 2>/dev/null || true + # Remove only hops-ci smoke resources — leave Dory / dory-k8s / Crossplane. + kubectl -n hops-ci delete configsmoke hops-ci-smoke --wait=false 2>/dev/null || true + kubectl -n hops-ci delete configmap hops-ci-smoke --wait=false 2>/dev/null || true + kubectl -n hops-ci delete object hops-ci-smoke --wait=false 2>/dev/null || true + # Leave Configuration package installed (idempotent next run) or drop it: + # kubectl delete configuration.pkg.crossplane.io hops-ops-config-smoke --wait=false diff --git a/.github/workflows/on-pr-kind-smoke.yaml b/.github/workflows/on-pr-kind-smoke.yaml new file mode 100644 index 0000000..9f63322 --- /dev/null +++ b/.github/workflows/on-pr-kind-smoke.yaml @@ -0,0 +1,126 @@ +name: kind backend smoke + +# End-to-end check of `hops local` on the kind backend (ubuntu runners). +# Registry round-trip + path-based config install (same fixture as dory smoke). + +on: + pull_request: + +jobs: + kind-smoke: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Install up CLI + run: | + set -euxo pipefail + # Required for `hops config install --path` (up project build). + curl -sL "https://cli.upbound.io" | sh + echo "$HOME/bin" >> "$GITHUB_PATH" + echo "$HOME/.up/bin" >> "$GITHUB_PATH" + # Installer may place the binary in different locations. + if [ -x "$HOME/bin/up" ]; then + echo "$HOME/bin" >> "$GITHUB_PATH" + elif [ -x /usr/local/bin/up ]; then + true + else + # Fall back: download latest release asset if shell installer differs. + mkdir -p "$HOME/bin" + curl -sL "https://cli.upbound.io/stable/current/linux_amd64/up" -o "$HOME/bin/up" + chmod +x "$HOME/bin/up" + echo "$HOME/bin" >> "$GITHUB_PATH" + fi + up version + + - name: Build hops + run: cargo build + + - name: hops local start --backend kind + run: ./target/debug/hops-cli local start --backend kind + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + # Single-platform materialization — avoid multiplatform push warnings. + PLATFORM=linux/amd64 + SRC=public.ecr.aws/docker/library/busybox:stable + docker pull --platform "$PLATFORM" "$SRC" + docker build --platform "$PLATFORM" --provenance=false --sbom=false \ + -t localhost:30500/smoke/busybox:ci - </dev/null || true)" + installed="$(kubectl --context kind-hops get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Installed")].status}' 2>/dev/null || true)" + if [ "$healthy" = "True" ] && [ "$installed" = "True" ]; then + echo "Configuration hops-ops-config-smoke Healthy after ${i} attempts" + break + fi + if [ "$i" -eq 90 ]; then + kubectl --context kind-hops get configuration.pkg.crossplane.io hops-ops-config-smoke -o yaml || true + kubectl --context kind-hops get configurationrevisions.pkg.crossplane.io -o wide || true + kubectl --context kind-hops get functions.pkg.crossplane.io -o wide || true + exit 1 + fi + sleep 5 + done + + kubectl --context kind-hops wait --for=condition=Established \ + crd/configsmokes.ci.hops.ops.com.ai --timeout=180s + + kubectl --context kind-hops apply -f "$FIXTURE/local/ci-xr.yaml" + + kubectl --context kind-hops -n hops-ci wait --for=condition=Ready \ + configsmoke.ci.hops.ops.com.ai/hops-ci-smoke --timeout=300s + + kubectl --context kind-hops -n hops-ci get configmap hops-ci-smoke -o yaml + smoke_val="$(kubectl --context kind-hops -n hops-ci get configmap hops-ci-smoke \ + -o jsonpath='{.data.smoke}')" + test "$smoke_val" = "ok" + + - name: Soft cleanup (CI fixture only) + if: always() + run: | + set +e + kubectl --context kind-hops delete pod smoke-svc-name smoke-localhost --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete configsmoke hops-ci-smoke --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete configmap hops-ci-smoke --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete object hops-ci-smoke --wait=false 2>/dev/null || true + + - name: hops local destroy + if: always() + run: ./target/debug/hops-cli local destroy || true diff --git a/README.md b/README.md index c1547e8..22703c9 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ This tool supports three related workflows: -- Local cluster setup on Colima +- Local cluster setup on colima or kind - Configuration package install/uninstall against the connected cluster - XR observe/manage/adopt/orphan workflows for existing infrastructure For local development, it can also: -- Install and manage Colima +- Install and manage a local cluster backend (colima or kind) - Start a local k8s cluster with Crossplane installed via Helm - Install the Kubernetes and Helm Crossplane providers - Deploy an in-cluster OCI registry (`crossplane-system/registry`) @@ -56,7 +56,7 @@ See "Releases" for available versions and changenotes. - `up` (Upbound CLI, used by `up project build`) - `aws` CLI v2 (used by `local aws` to export profile credentials) -Note: `hops-cli local install` installs `colima` through Homebrew. +Note: `hops-cli local install` installs the selected backend (`colima` or `kind`) through Homebrew. ## Build @@ -87,7 +87,7 @@ hops service --help `hops-cli` is organized into a few command groups: - `local` - - Manage a local Colima-based control plane, install providers, and bootstrap AWS or GitHub provider auth. + - Manage a local control plane (colima or kind backend), install providers, and bootstrap AWS or GitHub provider auth. - `config` - Build, install, reload, and uninstall Crossplane configuration packages against the connected cluster. - `secrets` @@ -192,7 +192,8 @@ Examples: ## Create a Local Control Plane ```bash -# 1) Install Colima (via Homebrew) +# 1) Install the backend (via Homebrew). Defaults to colima on macOS; +# pass --backend kind to use kind on any docker daemon. hops local install # 2) Start local k8s + Crossplane + providers + local registry @@ -211,6 +212,239 @@ hops local zitadel --source-context pat-local --domain auth.ops.com.ai hops config install --repo hops-ops/aws-auto-eks-cluster --version v0.11.0 ``` +### Cluster backends + +`hops local` supports three backends behind the same commands: + +- **colima** — a VM running dockerd + k3s. macOS/Linux; supports `--cpus`, + `--memory`, `--disk`, and `hops local resize`. +- **kind** — cluster nodes as docker containers on any reachable docker + daemon: Docker Desktop, colima's dockerd, or CI runners. No VM of its own, + so sizing flags don't apply (size the docker daemon instead); requires + kind >= v0.27. +- **dory** — [dory](https://augani.github.io/dory) stock app: shared Apple + Silicon engine + product k3s. Enable Kubernetes **in the Dory app** (hops + does not fork Dory or call `dory k8s enable`). Package installs use the same + **in-cluster** `registry:2` as other backends (Crossplane pulls + `registry.crossplane-system.svc.cluster.local:5000`). Docker push uses the + k3s **NodePort** on the engine docker bridge (`{dory-k8s-ip}:30500`) because + dockerd runs *inside* the engine — Mac `localhost` is the wrong plane. The + VM is sized in the Dory app, so hops sizing flags don't apply. + +Select with the global `--backend` flag: + +```bash +hops local start --backend kind +``` + +The chosen backend is persisted to `~/.hops/local/backend` on a successful +start, so later commands (`stop`, `destroy`, `doctor`, package installs) +target the same cluster without the flag. Resolution order: `--backend` flag > +persisted choice > existing cluster detection (colima wins) > platform +default (macOS: colima, otherwise kind). + +Unless `--context` is given, kubectl commands automatically use the backend's +kubeconfig context (`colima`, `kind-hops`, or `hops-dory`), regardless of your +current-context. + +#### Using dory + +Stock Dory only (brew cask / [Dory.app](https://augani.github.io/dory)). No hops +fork of Dory required. + +```bash +# 1. Open Dory.app — engine healthy (not "needs attention") +# 2. Enable Kubernetes in the app; wait until the cluster is running +# (product container is usually named dory-k8s) +# 3. Bootstrap Crossplane + local package registry +hops local start --backend dory +``` + +On start/activate, hops: + +- merges stock `~/.kube/dory-config` into `~/.kube/config` as context **`hops-dory`** + (override with `--name ` or `HOPS_DORY_NAME`; persisted in `~/.hops/local/dory-name`) +- runs `kubectl config use-context hops-dory` +- creates/uses a docker context of the same name → `unix://$HOME/.dory/dory.sock` + +So you should **not** need: + +```bash +export KUBECONFIG=$HOME/.kube/dory-config +export DOCKER_HOST=unix://$HOME/.dory/dory.sock +``` + +```bash +hops local start --backend dory # name defaults to hops-dory +hops local start --backend dory --name mine # custom kube+docker context name + +kubectl get nodes # context hops-dory +docker info # context hops-dory +hops local doctor +hops local github -o hops-ops +hops config install --path … --backend dory +``` + +Alternatively, use kind on Dory's docker socket (no product k3s): + +```bash +docker context use dory # product context from Dory.app +hops local start --backend kind +``` + +**CI:** `.github/workflows/on-pr-dory-smoke.yaml` runs on a **self-hosted** +Apple Silicon Mac labeled `hops-dory` (opt-in with PR label `test-dory` or +`workflow_dispatch`). Stock Dory only — no fork build, no Colima sock. +Offline runner → job waits. The session is **env-only** so your desktop +defaults never change: `DOCKER_HOST=unix://$HOME/.dory/dory.sock`, a +job-private `KUBECONFIG`, and `HOPS_DORY_DESKTOP=0` (hops skips +`use-context` / docker context switching). No `destroy`/`stop` of +`dory-k8s`. After start/doctor/registry, it path-installs the in-repo +fixture `tests/fixtures/config-smoke` (`hops config install --path …`) and +applies a namespaced ConfigMap XR under `hops-ci`. See the workflow header +for runner setup. + +#### Dory architecture (why troubleshooting is different) + +Two network planes matter. Mixing them up is the usual failure mode. + +| Plane | Who | Where | Role | +|-------|-----|--------|------| +| **Engine Docker** | `docker` CLI via `~/.dory/dory.sock` | Linux VM (dockerd) | build/push images | +| **Cluster** | kubectl / Crossplane pods | k3s inside `dory-k8s` | run control plane + pull packages | + +Package registry model (same idea as colima/kind, adapted to Dory): + +- **Pull (Crossplane):** in-cluster Service + `registry.crossplane-system.svc.cluster.local:5000` over **HTTPS** +- **Push (docker on the engine):** k3s **NodePort** on the docker bridge + `{dory-k8s-ip}:30500` (not Mac `localhost:30500`) +- **TLS:** hops generates a self-signed CA/server cert (`Secret hops-local-registry-tls`) + and patches Crossplane to trust that CA. Plain HTTP fails package unpack + (`http: server gave HTTP response to HTTPS client`). + +Do **not** put `host.dory.internal` in Crossplane package refs. That name exists +on the **node** `/etc/hosts`, not in pod CoreDNS, and is the wrong plane for +Crossplane's package manager. + +#### Dory troubleshooting + +**Engine won't start / thrash / `HV_BAD_ARGUMENT` (big Macs)** + +The failure we hit in practice: on a **large laptop** (e.g. **128 GB** host +RAM), Dory's UI **Recommended** / host-scaled default for guest engine memory +was **~half of host RAM → 64 GB** (`DORYD_MEMORY_MB=65536`). That is not +"you need 64 GB of containers" — it is an aggressive ceiling. On Apple Silicon, +Hypervisor.framework often rejects creating a VM that large +(`HV_BAD_ARGUMENT`), so the engine never stays healthy and the UI looks broken +even though a **4 GB** engine works fine. + +```bash +# Pin sane resources (4 GiB / 4 CPUs is enough for hops local) +defaults write com.pythonxi.Dory dory.engineMemoryMB -int 4096 +defaults write com.pythonxi.Dory dory.engineCPUCount -int 4 + +# Confirm the LaunchAgent is not still on 65536: +launchctl print "gui/$(id -u)/dev.dory.doryd" 2>/dev/null | grep DORYD_MEMORY +# Live engine cmdline should show --mem-mb 4096, not 65536: +ps aux | grep 'dory-hv engine' | grep -v grep +``` + +In the Dory app **Resources** panel: set memory to **~4 GB** (8 GB is usually +fine too). **Do not apply Recommended** if it offers tens of GB on a high-RAM +Mac. + +Then fully restart the engine (quit Dory, ensure no thrashing `doryd`/`dory-hv`, +reopen or `dory engine wake`). Once guest RAM is modest, the rest of hops local +is ordinary. + +**Docker socket path** + +Stock Docker API socket is: + +```text +unix://$HOME/.dory/dory.sock +``` + +There is no `~/.dory/engine.sock` for the host Docker API. hops uses `dory.sock` +only. Prefer the docker context hops creates (`hops-dory`) over manual +`DOCKER_HOST`. + +**Kubernetes missing / `no dory-k8s container`** + +k3s is product-owned. hops does not run `dory k8s enable`. + +1. Install the Kubernetes component if needed: `dory component install kubernetes` +2. Enable Kubernetes in the Dory app UI +3. Wait until a `dory-k8s` container is running: `docker ps` (with context hops-dory) +4. Re-run `hops local start --backend dory` + +**`k ctx` has no dory / hops-dory entry** + +Stock Dory writes `~/.kube/dory-config` (context often named `default`). hops +merges that into `~/.kube/config` as **`hops-dory`** on activate/start. If the +merge is missing: + +```bash +hops local doctor --backend dory +kubectl config get-contexts # expect hops-dory +kubectl config use-context hops-dory +``` + +**Configuration Installed=False / HTTPS error** + +```text +http: server gave HTTP response to HTTPS client +``` + +Crossplane always pulls packages with HTTPS. The local registry must be TLS +(hops sets this up). If you have an old HTTP-only registry Deployment: + +```bash +kubectl -n crossplane-system delete deploy registry +kubectl -n crossplane-system delete secret hops-local-registry-tls +hops local start --backend dory # recreates TLS secret + registry + CA patch +``` + +**docker push to localhost:30500 fails (connection refused / HTTPS to HTTP)** + +With the docker context pointed at Dory, **`localhost` is the engine VM**, not +the Mac. hops pushes to `{dory-k8s container IP}:30500` and marks the engine +docker bridge as an insecure registry for that self-signed TLS endpoint. +Check: + +```bash +docker context show # hops-dory +hops local doctor # "package push registry reachable (…:30500)" +``` + +Do not rely on Mac `kubectl port-forward` for engine-side `docker push`. + +**Engine docker broken after daemon.json / dockerd kill** + +Prefer Dory's own recovery: + +```bash +dory repair dockerd --apply +dory engine wake +dory readiness +``` + +Avoid raw `kill` of dockerd inside the guest unless you are prepared to wait for +`dory repair` / LaunchAgent recovery (`live-restore` helps but is not magic). + +**Useful diagnostics** + +```bash +dory doctor +dory readiness +hops local doctor --backend dory +docker context show +kubectl config current-context +kubectl get configuration,provider -A +kubectl -n crossplane-system get deploy,svc,secret | grep -E 'registry|crossplane|hops-local' +``` + ### Local provider setup and auth `hops local aws`, `hops local github`, and `hops local zitadel` install the provider package and bootstrap auth into a local control plane. The exception is `--refresh`, which updates credentials only. @@ -342,12 +576,13 @@ Notes: - Runs `brew install colima`. - `local reset` - Runs `colima kubernetes reset`. -- `local start` - - Runs `colima start --kubernetes --cpu 8 --memory 16 --disk 60` - - Installs Crossplane from `crossplane-stable/crossplane` - - Applies manifests from `bootstrap/` for runtime config, providers, provider configs, and registry (embedded in the binary at build time) - - Configures Docker in Colima for insecure pulls from `registry.crossplane-system.svc.cluster.local:5000` - - Adds host mapping in Colima VM for the registry service DNS name +- `local start [--bootstrap]` + - Brings up the selected backend cluster (colima / kind / dory) + - If Crossplane, k8s+helm providers, and the package registry are already + Healthy/Available, skips helm repo update/upgrade and bootstrap reapply + (fast resume). Pass `--bootstrap` to force a full helm upgrade + reapply. + - Cold path: installs Crossplane, applies `bootstrap/` DRCs/providers/PCs, + deploys the local HTTPS package registry, wires node/engine registry trust - `local stop` - Runs `colima stop`. - `local destroy` diff --git a/bootstrap/registry/registry.yaml b/bootstrap/registry/registry.yaml index 225857b..02cd067 100644 --- a/bootstrap/registry/registry.yaml +++ b/bootstrap/registry/registry.yaml @@ -1,3 +1,6 @@ +# In-cluster package registry for local source installs. +# TLS is required: Crossplane's package manager always uses HTTPS. +# Cert material lives in Secret hops-local-registry-tls (created by hops). apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -10,8 +13,6 @@ spec: requests: storage: 20Gi # Uses the default StorageClass (local-path in colima/k3s). - # Data lives on the colima VM disk, which persists across computer restarts - # (as long as the colima instance/VM itself is not deleted). --- apiVersion: apps/v1 @@ -34,13 +35,40 @@ spec: image: registry:2 ports: - containerPort: 5000 + name: https + env: + # registry:2 serves HTTPS when both are set (still on :5000). + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /certs/tls.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /certs/tls.key volumeMounts: - name: registry-data mountPath: /var/lib/registry + - name: tls + mountPath: /certs + readOnly: true + readinessProbe: + httpGet: + path: /v2/ + port: 5000 + scheme: HTTPS + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /v2/ + port: 5000 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 volumes: - name: registry-data persistentVolumeClaim: claimName: registry-pvc + - name: tls + secret: + secretName: hops-local-registry-tls --- apiVersion: v1 kind: Service @@ -52,6 +80,7 @@ spec: selector: app: registry ports: - - port: 5000 + - name: https + port: 5000 targetPort: 5000 nodePort: 30500 diff --git a/src/commands/auth/bootstrap.rs b/src/commands/auth/bootstrap.rs index fdc5e35..7368483 100644 --- a/src/commands/auth/bootstrap.rs +++ b/src/commands/auth/bootstrap.rs @@ -66,7 +66,10 @@ pub fn run(args: &BootstrapArgs) -> Result<(), Box> { fn write_secret(path: &PathBuf, value: &str, force: bool) -> Result<(), Box> { if path.exists() && !force { - log::info!(" skip {}: already present (use --force to overwrite)", path.display()); + log::info!( + " skip {}: already present (use --force to overwrite)", + path.display() + ); return Ok(()); } if let Some(parent) = path.parent() { @@ -151,11 +154,7 @@ mod tests { "no lowercase: {}", pwd ); - assert!( - pwd.chars().any(|c| c.is_ascii_digit()), - "no digit: {}", - pwd - ); + assert!(pwd.chars().any(|c| c.is_ascii_digit()), "no digit: {}", pwd); assert!( pwd.chars().any(|c| !c.is_ascii_alphanumeric()), "no symbol: {}", diff --git a/src/commands/config/install.rs b/src/commands/config/install.rs index fb6385f..448eae1 100644 --- a/src/commands/config/install.rs +++ b/src/commands/config/install.rs @@ -1,14 +1,12 @@ +use crate::commands::local::backend::{self, Backend}; +use crate::commands::local::package_install::run_watch; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout, ensure_registry, image_config_name, parse_docker_push_digest, parse_repo_spec, resolve_repo_install_target, rewrite_registry, rewrite_registry_with_tag, sanitize_name_component, short_hash, split_ref, strip_registry, - unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH, -}; -use crate::commands::local::package_install::run_watch; -use crate::commands::local::{ - kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output, sync_registry_hosts_entry, - HOPS_KUBE_CONTEXT_ENV, + unique_suffix, RepoInstallTarget, RepoSpec, registry_pull, registry_push, }; +use crate::commands::local::{kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output}; use clap::Args; use flate2::read::GzDecoder; use serde::Deserialize; @@ -43,6 +41,10 @@ pub struct ConfigArgs { #[arg(long)] pub context: Option, + /// Local cluster backend whose node should be wired for local package pulls. + #[arg(long, value_enum)] + pub backend: Option, + /// Watch the project directory for changes and re-run install automatically #[arg(long, conflicts_with = "repo")] pub watch: bool, @@ -108,17 +110,22 @@ struct PackageResource { } pub fn run(args: &ConfigArgs) -> Result<(), Box> { - if let Some(ctx) = &args.context { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + let backend = backend::activate(args.backend, args.context.as_deref()); match (args.repo.as_deref(), args.version.as_deref()) { (Some(repo), Some(version)) => { apply_repo_version(repo, version, args.skip_dependency_resolution) } - (Some(repo), None) => run_repo_install(repo, args.skip_dependency_resolution), + (Some(repo), None) => run_repo_install( + repo, + args.skip_dependency_resolution, + backend, + args.backend, + args.context.as_deref(), + ), (None, _) => { let path = args.path.as_deref().unwrap_or("."); + prepare_local_registry(backend, args.backend, args.context.as_deref())?; run_local_path(path, args.skip_dependency_resolution)?; if args.watch { @@ -137,10 +144,19 @@ pub fn run(args: &ConfigArgs) -> Result<(), Box> { fn run_repo_install( repo: &str, skip_dependency_resolution: bool, + backend: Backend, + backend_flag: Option, + context: Option<&str>, ) -> Result<(), Box> { let spec = parse_repo_spec(repo)?; match resolve_repo_install_target(&spec)? { - RepoInstallTarget::SourceBuild => run_repo_clone(&spec, skip_dependency_resolution), + RepoInstallTarget::SourceBuild => run_repo_clone( + &spec, + skip_dependency_resolution, + backend, + backend_flag, + context, + ), RepoInstallTarget::PublishedVersion(version) => { apply_repo_version_spec(&spec, &version, skip_dependency_resolution) } @@ -150,11 +166,24 @@ fn run_repo_install( fn run_repo_clone( spec: &RepoSpec, skip_dependency_resolution: bool, + backend: Backend, + backend_flag: Option, + context: Option<&str>, ) -> Result<(), Box> { - let cache_path = ensure_cached_repo_checkout(&spec)?; + let cache_path = ensure_cached_repo_checkout(spec)?; + prepare_local_registry(backend, backend_flag, context)?; run_local_path(&cache_path.to_string_lossy(), skip_dependency_resolution) } +fn prepare_local_registry( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + ensure_registry()?; + backend::wire_local_registry_for_target(backend, backend_flag, context) +} + fn apply_repo_version_spec( spec: &RepoSpec, version: &str, @@ -216,18 +245,12 @@ fn apply_repo_version( apply_repo_version_spec(&spec, version, skip_dependency_resolution) } -fn run_local_path( - path: &str, - skip_dependency_resolution: bool, -) -> Result<(), Box> { +fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Box> { let dir = Path::new(path); if !dir.is_dir() { return Err(format!("{} is not a directory", path).into()); } - ensure_registry()?; - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; - // Build the Crossplane package log::info!("Building Crossplane package in {}...", path); let status = Command::new("up") @@ -246,7 +269,7 @@ fn run_local_path( let packages: Vec<_> = fs::read_dir(&output_dir) .map_err(|e| format!("Failed to read {}: {}", output_dir.display(), e))? .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().extension().map_or(false, |ext| ext == "uppkg")) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "uppkg")) .collect(); if packages.is_empty() { @@ -303,7 +326,7 @@ fn run_local_path( continue; } - let push_ref = rewrite_registry(&img.source, REGISTRY_PUSH); + let push_ref = rewrite_registry(&img.source, ®istry_push()); let (img_path, tag) = split_ref(&img.source); // All non-configuration images are Crossplane Function packages (the @@ -316,7 +339,7 @@ fn run_local_path( if tag == arch { let digest = docker_push_and_get_digest(&push_ref)?; - let target_prefix = format!("{}/{}", REGISTRY_PULL, strip_registry(img_path)); + let target_prefix = format!("{}/{}", registry_pull(), strip_registry(img_path)); render_rewrites.insert( img_path.to_string(), RenderRewrite { @@ -364,8 +387,8 @@ spec: } let dev_tag = dev_tag_for_uppkg(&img.uppkg_path)?; - let push_ref = rewrite_registry_with_tag(&img.source, REGISTRY_PUSH, &dev_tag); - let pull_ref = rewrite_registry_with_tag(&img.source, REGISTRY_PULL, &dev_tag); + let push_ref = rewrite_registry_with_tag(&img.source, ®istry_push(), &dev_tag); + let pull_ref = rewrite_registry_with_tag(&img.source, registry_pull(), &dev_tag); log::info!( "Using local build version '{}' for {}...", dev_tag, @@ -886,6 +909,28 @@ fn dev_tag_for_uppkg(uppkg_path: &Path) -> Result> { Ok(format!("dev-{}", &hex[..12])) } +/// Map an image reference's arch tag (e.g. `:amd64` / `:arm64` from +/// `up project build`) to a Docker `--platform` value. +/// +/// Without an explicit platform, buildx on Apple Silicon warns +/// `InvalidBaseImagePlatform` when rebuilding the non-host arch variant +/// (`FROM …:amd64` on arm64, and the reverse on Intel). +fn platform_for_image_ref(src: &str) -> Option<&'static str> { + let (_, tag) = split_ref(src); + // Prefer the trailing path segment when the tag is a digest (rare for + // function images from up, which use :amd64 / :arm64). + let arch = if tag.starts_with("sha256:") { + src.rsplit('/').next().unwrap_or(tag) + } else { + tag + }; + match arch { + "amd64" | "linux/amd64" => Some("linux/amd64"), + "arm64" | "linux/arm64" | "aarch64" => Some("linux/arm64"), + _ => None, + } +} + /// Rebuild a Docker image with just `FROM ` to produce a valid OCI config. /// This fixes images where rootfs.type is empty (a known issue with `up project build` /// render function images). @@ -895,15 +940,21 @@ fn docker_build_from(src: &str, tag: &str) -> Result<(), Box> { // `build_patched_configuration_image`: without these, buildx wraps the // output in a single-arch manifest list that Crossplane (which fetches // package layers as linux/amd64 regardless of host) cannot navigate. + let mut args: Vec = vec![ + "build".into(), + "--provenance=false".into(), + "--sbom=false".into(), + ]; + if let Some(platform) = platform_for_image_ref(src) { + args.push("--platform".into()); + args.push(platform.into()); + } + args.push("-t".into()); + args.push(tag.into()); + args.push("-".into()); + let mut child = Command::new("docker") - .args([ - "build", - "--provenance=false", - "--sbom=false", - "-t", - tag, - "-", - ]) + .args(&args) .stdin(Stdio::piped()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -946,7 +997,7 @@ fn delete_local_registry_config_revisions(config_name: &str) -> Result<(), Box Result<(), Box< if !rev_name.starts_with(config_name) { continue; } - if !package.contains(REGISTRY_PULL) && state == "Inactive" { + if !package.contains(registry_pull()) && state == "Inactive" { run_cmd( "kubectl", &[ @@ -1047,6 +1098,15 @@ spec: assert!(!without_skip.contains("skipDependencyResolution: true")); } + #[test] + fn local_registry_wiring_skips_foreign_context_without_backend_flag() { + assert!(!backend::should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + #[test] fn package_source_strips_tag_and_digest() { assert_eq!( @@ -1072,4 +1132,20 @@ spec: Some("sha256:abcdef") ); } + + #[test] + fn platform_for_image_ref_from_up_arch_tags() { + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke_render:amd64"), + Some("linux/amd64") + ); + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke_render:arm64"), + Some("linux/arm64") + ); + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke:configuration"), + None + ); + } } diff --git a/src/commands/local/backend/colima.rs b/src/commands/local/backend/colima.rs new file mode 100644 index 0000000..15534eb --- /dev/null +++ b/src/commands/local/backend/colima.rs @@ -0,0 +1,523 @@ +//! Colima backend: VM + dockerd + k3s (docker runtime). + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_HOSTNAME, REGISTRY_PULL}; +use crate::commands::local::{run_cmd, run_cmd_output, wait_for_kubernetes}; +use dialoguer::Confirm; +use serde::Deserialize; +use std::error::Error; +use std::io::{IsTerminal, Write}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +const DEFAULT_CPUS: u32 = 8; +const DEFAULT_MEMORY_GIB: u32 = 16; +const DEFAULT_DISK_GIB: u32 = 60; +const GIB: u64 = 1024 * 1024 * 1024; + +pub fn install() -> Result<(), Box> { + log::info!("Installing Colima via Homebrew..."); + run_cmd("brew", &["install", "colima"])?; + log::info!("Colima installed successfully"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling Colima..."); + run_cmd("brew", &["uninstall", "colima"])?; + log::info!("Colima uninstalled"); + Ok(()) +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping Colima..."); + run_cmd("colima", &["stop"])?; + log::info!("Colima stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Destroying Colima VM..."); + run_cmd("colima", &["delete", "--force"])?; + log::info!("Colima VM destroyed"); + Ok(()) +} + +pub fn reset() -> Result<(), Box> { + log::info!("Resetting Colima Kubernetes..."); + run_cmd("colima", &["kubernetes", "reset"])?; + log::info!("Colima Kubernetes reset complete"); + Ok(()) +} + +pub fn start(size: &SizeArgs, assume_yes: bool) -> Result<(), Box> { + let instance = colima_instance()?; + validate_requested_size(size, instance.as_ref())?; + start_or_resize_colima(size, assume_yes, instance.as_ref()) +} + +pub fn resize(size: &SizeArgs) -> Result<(), Box> { + if !size.any_set() { + return Err("Specify at least one of --cpus, --memory, or --disk".into()); + } + + let instance = colima_instance()?; + let instance = instance + .as_ref() + .ok_or("No Colima instance exists yet; use `hops local start` to create one")?; + + validate_requested_size(size, Some(instance))?; + resize_existing_colima(size, Some(instance)) +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +struct ColimaInstance { + #[serde(default)] + status: String, + #[serde(default)] + cpus: Option, + #[serde(default)] + memory: Option, + #[serde(default)] + disk: Option, +} + +impl ColimaInstance { + fn is_running(&self) -> bool { + self.status.eq_ignore_ascii_case("running") + } + + fn memory_gib(&self) -> Option { + self.memory.map(bytes_to_gib) + } + + fn disk_gib(&self) -> Option { + self.disk.map(bytes_to_gib) + } +} + +fn start_or_resize_colima( + size: &SizeArgs, + assume_yes: bool, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + let is_running = instance.map(ColimaInstance::is_running).unwrap_or(false); + + if is_running && size.any_set() { + let changes = requested_size_changes(size, instance.expect("checked is_running")); + if !changes.is_empty() { + confirm_running_resize(size, assume_yes, &changes)?; + resize_existing_colima(size, instance)?; + return Ok(()); + } + + log::info!("Requested Colima size already matches the running VM"); + } + + log::info!("Starting Colima with Kubernetes..."); + + let start_size = if is_running { + SizeArgs::default() + } else { + size.clone() + }; + let include_defaults = instance.is_none(); + start_colima(&start_size, include_defaults) +} + +fn resize_existing_colima( + size: &SizeArgs, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + if instance.map(ColimaInstance::is_running).unwrap_or(false) { + log::info!("Stopping Colima to apply requested size..."); + run_cmd("colima", &["stop"])?; + } + + log::info!("Starting Colima with requested size..."); + start_colima(size, false) +} + +fn start_colima(size: &SizeArgs, include_defaults: bool) -> Result<(), Box> { + let args = colima_start_args(size, include_defaults); + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + run_cmd("colima", &refs) +} + +fn colima_start_args(size: &SizeArgs, include_defaults: bool) -> Vec { + let mut args = vec!["start".to_string(), "--kubernetes".to_string()]; + + if let Some(cpus) = size.cpus.or(include_defaults.then_some(DEFAULT_CPUS)) { + args.push("--cpus".to_string()); + args.push(cpus.to_string()); + } + if let Some(memory) = size + .memory + .or(include_defaults.then_some(DEFAULT_MEMORY_GIB)) + { + args.push("--memory".to_string()); + args.push(memory.to_string()); + } + if let Some(disk) = size.disk.or(include_defaults.then_some(DEFAULT_DISK_GIB)) { + args.push("--disk".to_string()); + args.push(disk.to_string()); + } + + args +} + +fn confirm_running_resize( + size: &SizeArgs, + assume_yes: bool, + changes: &[String], +) -> Result<(), Box> { + if assume_yes { + return Ok(()); + } + + let change_text = changes.join(", "); + let resize_command = format!("hops local resize{}", size.command_suffix()); + let start_command = format!("hops local start{} --yes", size.command_suffix()); + + if !std::io::stdin().is_terminal() { + return Err(format!( + "Colima is already running with different size ({change_text}). Run `{resize_command}` first, or rerun `{start_command}` to stop and resize automatically." + ) + .into()); + } + + let confirmed = Confirm::new() + .with_prompt(format!( + "Colima is already running with different size ({change_text}). Stop and restart it now?" + )) + .default(false) + .interact()?; + + if confirmed { + Ok(()) + } else { + Err(format!( + "Colima size was not changed. Run `{resize_command}` first, then rerun `hops local start`." + ) + .into()) + } +} + +fn validate_requested_size( + size: &SizeArgs, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + if let (Some(requested), Some(current)) = + (size.disk, instance.and_then(ColimaInstance::disk_gib)) + { + if requested < current { + return Err(format!( + "Colima disk cannot be shrunk from {current}GiB to {requested}GiB. Use --disk {current} or larger, or destroy and recreate the VM." + ) + .into()); + } + } + + Ok(()) +} + +fn requested_size_changes(size: &SizeArgs, instance: &ColimaInstance) -> Vec { + let mut changes = Vec::new(); + + if let Some(requested) = size.cpus { + match instance.cpus { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("cpus {current} -> {requested}")), + None => changes.push(format!("cpus unknown -> {requested}")), + } + } + if let Some(requested) = size.memory { + match instance.memory_gib() { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("memory {current}GiB -> {requested}GiB")), + None => changes.push(format!("memory unknown -> {requested}GiB")), + } + } + if let Some(requested) = size.disk { + match instance.disk_gib() { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("disk {current}GiB -> {requested}GiB")), + None => changes.push(format!("disk unknown -> {requested}GiB")), + } + } + + changes +} + +/// Whether a Colima instance exists (running or stopped). Missing binary or +/// failing command reads as "no instance". +pub fn instance_exists() -> bool { + matches!(colima_instance(), Ok(Some(_))) +} + +fn colima_instance() -> Result, Box> { + let output = match run_cmd_output("colima", &["list", "--json"]) { + Ok(output) => output, + Err(_) => return Ok(None), + }; + + parse_colima_list(&output) +} + +fn parse_colima_list(output: &str) -> Result, Box> { + let trimmed = output.trim(); + if trimmed.is_empty() { + return Ok(None); + } + + if let Ok(instance) = serde_json::from_str::(trimmed) { + return Ok(Some(instance)); + } + + if let Ok(instances) = serde_json::from_str::>(trimmed) { + return Ok(instances.into_iter().next()); + } + + for line in trimmed + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + if let Ok(instance) = serde_json::from_str::(line) { + return Ok(Some(instance)); + } + } + + Err("Unable to parse `colima list --json` output".into()) +} + +fn bytes_to_gib(bytes: u64) -> u32 { + (bytes / GIB) as u32 +} + +/// Add the cluster-internal registry to Docker's insecure-registries list +/// inside the Colima VM. Docker defaults to HTTPS for non-localhost registries; +/// our in-cluster registry speaks plain HTTP. +pub fn configure_docker_insecure_registry() -> Result<(), Box> { + let config = run_cmd_output("colima", &["ssh", "--", "cat", "/etc/docker/daemon.json"])?; + + if config.contains("insecure-registries") { + return Ok(()); + } + + log::info!("Configuring Docker for insecure local registry..."); + + // Insert the insecure-registries key before the final closing brace. + let new_config = if let Some(pos) = config.rfind('}') { + let prefix = config[..pos].trim_end(); + format!( + "{},\n \"insecure-registries\": [\"{}\"]\n}}\n", + prefix, REGISTRY_PULL + ) + } else { + return Err("Invalid daemon.json: no closing brace".into()); + }; + + let mut child = Command::new("colima") + .args(["ssh", "--", "sudo", "tee", "/etc/docker/daemon.json"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(new_config.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err("Failed to write Docker daemon.json".into()); + } + + log::info!("Restarting Docker daemon..."); + run_cmd( + "colima", + &["ssh", "--", "sudo", "systemctl", "restart", "docker"], + )?; + + // Wait for Docker to come back. + for _ in 0..30 { + if run_cmd_output("docker", &["info"]).is_ok() { + // Docker restart can temporarily disrupt the Kubernetes API. + wait_for_kubernetes()?; + return Ok(()); + } + thread::sleep(Duration::from_secs(2)); + } + Err("Docker did not come back after restart".into()) +} + +/// Ensure Colima's /etc/hosts maps the registry hostname to the current +/// ClusterIP so the kubelet's docker daemon can resolve pull refs. +pub fn sync_hosts_entry(cluster_ip: &str) -> Result<(), Box> { + let hostname = REGISTRY_HOSTNAME; + let current_ip = run_cmd_output( + "colima", + &[ + "ssh", + "--", + "sh", + "-c", + &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), + ], + ) + .unwrap_or_default(); + if current_ip.trim() == cluster_ip { + return Ok(()); + } + + log::info!("Updating hosts entry: {} -> {}", hostname, cluster_ip); + + let escaped_host = hostname.replace('.', "\\."); + run_cmd( + "colima", + &[ + "ssh", + "--", + "sudo", + "sed", + "-i", + &format!("/{}/d", escaped_host), + "/etc/hosts", + ], + )?; + run_cmd( + "colima", + &[ + "ssh", + "--", + "sudo", + "sh", + "-c", + &format!("echo '{} {}' >> /etc/hosts", cluster_ip, hostname), + ], + )?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn instance(status: &str, cpus: u32, memory_gib: u32, disk_gib: u32) -> ColimaInstance { + ColimaInstance { + status: status.to_string(), + cpus: Some(cpus), + memory: Some(memory_gib as u64 * GIB), + disk: Some(disk_gib as u64 * GIB), + } + } + + #[test] + fn colima_start_args_use_hops_defaults_for_new_profiles() { + let args = colima_start_args(&SizeArgs::default(), true); + + assert_eq!( + args, + vec![ + "start", + "--kubernetes", + "--cpus", + "8", + "--memory", + "16", + "--disk", + "60" + ] + ); + } + + #[test] + fn colima_start_args_pass_only_requested_size_for_existing_profiles() { + let size = SizeArgs { + cpus: Some(12), + memory: Some(32), + disk: None, + }; + + let args = colima_start_args(&size, false); + + assert_eq!( + args, + vec!["start", "--kubernetes", "--cpus", "12", "--memory", "32"] + ); + } + + #[test] + fn requested_size_changes_compare_only_explicit_fields() { + let current = instance("Running", 8, 16, 60); + let size = SizeArgs { + cpus: None, + memory: Some(32), + disk: None, + }; + + assert_eq!( + requested_size_changes(&size, ¤t), + vec!["memory 16GiB -> 32GiB"] + ); + } + + #[test] + fn requested_size_changes_treat_missing_current_value_as_change() { + let current = ColimaInstance { + status: "Running".to_string(), + cpus: None, + memory: None, + disk: None, + }; + let size = SizeArgs { + cpus: Some(12), + memory: None, + disk: None, + }; + + assert_eq!( + requested_size_changes(&size, ¤t), + vec!["cpus unknown -> 12"] + ); + } + + #[test] + fn parse_colima_list_accepts_single_object() { + let output = r#"{"name":"default","status":"Stopped","arch":"aarch64","cpus":8,"memory":17179869184,"disk":64424509440,"runtime":"docker+k3s"}"#; + + let parsed = parse_colima_list(output).expect("parse").expect("instance"); + + assert_eq!(parsed.status, "Stopped"); + assert_eq!(parsed.cpus, Some(8)); + assert_eq!(parsed.memory_gib(), Some(16)); + assert_eq!(parsed.disk_gib(), Some(60)); + } + + #[test] + fn parse_colima_list_accepts_array_output() { + let output = r#"[{"status":"Running","cpus":12,"memory":34359738368,"disk":107374182400}]"#; + + let parsed = parse_colima_list(output).expect("parse").expect("instance"); + + assert!(parsed.is_running()); + assert_eq!(parsed.cpus, Some(12)); + assert_eq!(parsed.memory_gib(), Some(32)); + assert_eq!(parsed.disk_gib(), Some(100)); + } + + #[test] + fn validate_requested_size_rejects_disk_shrink() { + let current = instance("Stopped", 8, 16, 100); + let size = SizeArgs { + cpus: None, + memory: None, + disk: Some(60), + }; + + let err = validate_requested_size(&size, Some(¤t)).expect_err("disk shrink"); + + assert!(err.to_string().contains("cannot be shrunk")); + } +} diff --git a/src/commands/local/backend/dory.rs b/src/commands/local/backend/dory.rs new file mode 100644 index 0000000..7a81497 --- /dev/null +++ b/src/commands/local/backend/dory.rs @@ -0,0 +1,814 @@ +//! dory backend: product k3s on stock Dory (https://augani.github.io/dory). +//! +//! Design (stock Dory only — no hops fork): +//! - Dory.app owns the engine, dockerd, and k3s lifecycle (enable Kubernetes +//! in the app). hops never calls `dory k8s enable`. +//! - hops talks to the engine via `~/.dory/dory.sock` and to the cluster via +//! `~/.kube/dory-config` (context is usually `default`). +//! +//! Package registry (two network planes on Dory): +//! - **Pull (Crossplane pods):** in-cluster Service DNS +//! `registry.crossplane-system.svc.cluster.local:5000`. +//! - **Push (docker via `~/.dory/dory.sock`):** engine dockerd pushes to the +//! k3s NodePort on the docker bridge (`{dory-k8s-ip}:30500`), with that +//! bridge marked insecure (HTTP). Mac localhost is the wrong plane. +//! - **Node image pulls:** k3s `registries.yaml` mirrors → Service ClusterIP. + +use super::SizeArgs; +use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use crate::commands::local::package_install::{ + REGISTRY_HOSTNAME, REGISTRY_PULL_INCLUSTER, REGISTRY_PUSH, +}; +use std::error::Error; +use std::path::PathBuf; +use std::thread; +use std::time::Duration; + +const NODE_CONTAINER: &str = "dory-k8s"; +const REGISTRY_NODE_PORT: &str = "30500"; + +fn home() -> Result> { + Ok(PathBuf::from(std::env::var("HOME").map_err(|_| { + "HOME is not set; unable to locate dory's state directory" + })?)) +} + +/// Stock Dory's host-facing Docker API socket (`~/.dory/dory.sock`). +fn engine_socket() -> Result> { + Ok(home()?.join(".dory/dory.sock")) +} + +/// Stock Dory writes the cluster kubeconfig here (context is typically `default`). +pub fn kubeconfig_path() -> Option { + home() + .ok() + .map(|h| h.join(".kube/dory-config").to_string_lossy().into_owned()) +} + +fn engine_docker(args: &[&str]) -> Result<(), Box> { + let sock = format!("unix://{}", engine_socket()?.display()); + let mut full = vec!["-H", sock.as_str()]; + full.extend_from_slice(args); + run_cmd("docker", &full) +} + +fn engine_docker_output(args: &[&str]) -> Result> { + let sock = format!("unix://{}", engine_socket()?.display()); + let mut full = vec!["-H", sock.as_str()]; + full.extend_from_slice(args); + run_cmd_output("docker", &full) +} + +pub fn install() -> Result<(), Box> { + log::info!("Installing Dory via Homebrew..."); + run_cmd("brew", &["install", "--cask", "Augani/dory/dory"])?; + log::info!( + "Dory installed; open the app, wait until the engine is healthy, \ + enable Kubernetes, then re-run `hops local start --backend dory`" + ); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling Dory..."); + run_cmd("brew", &["uninstall", "--cask", "dory"])?; + log::info!("Dory uninstalled"); + Ok(()) +} + +pub fn start(size: &SizeArgs) -> Result<(), Box> { + if size.any_set() { + return Err(format!( + "the dory backend's VM is sized by the Dory app, not hops; drop{}", + size.command_suffix() + ) + .into()); + } + + preflight()?; + ensure_k8s_node_running()?; + wait_for_node_ready()?; + Ok(()) +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping dory k8s node '{}'...", NODE_CONTAINER); + if !node_exists() { + log::info!("dory k8s node not present"); + return Ok(()); + } + engine_docker(&["stop", NODE_CONTAINER])?; + log::info!("dory cluster stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!( + "Deleting dory k8s node '{}' (hops does not disable product k8s via CLI)...", + NODE_CONTAINER + ); + // Legacy leftovers from earlier registry experiments. + let _ = engine_docker(&["rm", "-f", "hops-local-registry"]); + let _ = engine_docker(&["rm", "-f", "hops-registry-publish"]); + let _ = engine_docker(&["rm", "-f", NODE_CONTAINER]); + log::info!( + "dory k8s node removed; re-enable Kubernetes in the Dory app if you want the product cluster again" + ); + Ok(()) +} + +/// Recreate is app-owned: remove the node; user re-enables k8s in Dory, then start again. +pub fn reset() -> Result<(), Box> { + preflight()?; + destroy()?; + log::info!( + "Enable Kubernetes in the Dory app, then run `hops local start --backend dory` again" + ); + Ok(()) +} + +pub fn resize(_size: &SizeArgs) -> Result<(), Box> { + Err("the dory backend has no hops-managed VM to resize; \ + adjust resources in the Dory app instead" + .into()) +} + +/// Whether the hops-relevant dory cluster exists (running or stopped). +pub fn cluster_exists() -> bool { + let Ok(sock) = engine_socket() else { + return false; + }; + if !sock.exists() { + return false; + } + node_exists() +} + +fn node_exists() -> bool { + engine_docker_output(&["inspect", "-f", "{{.Id}}", NODE_CONTAINER]).is_ok() +} + +fn node_running() -> bool { + engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]) + .map(|state| state.trim() == "true") + .unwrap_or(false) +} + +fn preflight() -> Result<(), Box> { + if !command_exists("docker") { + return Err("docker CLI not found; install it (Dory provides the daemon)".into()); + } + let sock = engine_socket()?; + if !sock.exists() { + return Err(format!( + "dory's engine socket ({}) is missing.\n\ + Open the Dory app and wait until the engine is healthy (not \"needs attention\"), then retry.", + sock.display() + ) + .into()); + } + if engine_docker_output(&["info"]).is_err() { + return Err(format!( + "cannot talk to Dory's docker at {}.\n\ + Open the Dory app, fix any engine/doryd errors in the menu, then retry.", + sock.display() + ) + .into()); + } + Ok(()) +} + +/// k3s is product-owned: start a stopped node, or tell the user to enable it in the app. +fn ensure_k8s_node_running() -> Result<(), Box> { + if node_running() { + log::info!("dory k8s node '{}' is running", NODE_CONTAINER); + return Ok(()); + } + if node_exists() { + log::info!("Starting stopped dory k8s node '{}'...", NODE_CONTAINER); + engine_docker(&["start", NODE_CONTAINER])?; + return Ok(()); + } + Err( + "Dory Kubernetes is not enabled (no `dory-k8s` container).\n\ + In the Dory app: enable Kubernetes, wait until it is running, then re-run:\n\ + hops local start --backend dory\n\ + (hops uses stock Dory only — it does not create the cluster for you.)" + .into(), + ) +} + +fn wait_for_node_ready() -> Result<(), Box> { + log::info!("Waiting for dory k8s node to become Ready..."); + for i in 0..90 { + if !node_running() { + thread::sleep(Duration::from_secs(2)); + continue; + } + let ready = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "kubectl", + "get", + "nodes", + "--no-headers", + ]) + .map(|out| out.contains(" Ready")) + .unwrap_or(false); + if ready { + ensure_side_kubeconfig_hint(); + // Best-effort: merge context "dory" + docker context (also run on activate). + let _ = ensure_desktop_integration(); + return Ok(()); + } + if i > 0 && i % 15 == 0 { + log::info!("Still waiting for k3s Ready ({}s)...", i * 2); + } + thread::sleep(Duration::from_secs(2)); + } + Err( + "timed out waiting for dory-k8s to become Ready; check Kubernetes status in the Dory app" + .into(), + ) +} + +fn ensure_side_kubeconfig_hint() { + let Some(path) = kubeconfig_path() else { + return; + }; + if std::path::Path::new(&path).is_file() { + return; + } + if let Ok(yaml) = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "cat", + "/etc/rancher/k3s/k3s.yaml", + ]) { + if yaml.contains("server:") { + if let Some(parent) = std::path::Path::new(&path).parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write(&path, yaml); + let _ = std::fs::set_permissions( + &path, + std::os::unix::fs::PermissionsExt::from_mode(0o600), + ); + log::info!("Wrote kubeconfig side file {}", path); + } + } +} + +/// No create-time trust: k3s registries.yaml is written in [`wire_registry`]. +pub fn ensure_registry_trust() -> Result<(), Box> { + Ok(()) +} + +/// After the in-cluster registry Service exists: +/// 1. Point k3s/containerd at the Service ClusterIP over HTTP (function images). +/// 2. Allow engine dockerd to push HTTP to the k3s NodePort on the docker bridge. +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + ensure_k3s_registry_mirrors(cluster_ip)?; + ensure_engine_push_path()?; + Ok(()) +} + +/// Docker push address for package installs when `DOCKER_HOST` is the Dory engine. +/// Reachable on the engine docker bridge (not Mac localhost). +pub fn registry_push_addr() -> Result> { + Ok(format!("{}:{}", node_ip()?, REGISTRY_NODE_PORT)) +} + +fn node_ip() -> Result> { + let ip = engine_docker_output(&[ + "inspect", + "-f", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + NODE_CONTAINER, + ])? + .trim() + .to_string(); + if ip.is_empty() { + return Err("dory-k8s has no docker network IP".into()); + } + Ok(ip) +} + +/// k3s `registries.yaml` so containerd pulls HTTP from the in-cluster registry. +/// Restarts the node only when the file content changes. +fn ensure_k3s_registry_mirrors(cluster_ip: &str) -> Result<(), Box> { + if !node_running() { + return Err( + "dory k8s node is not running; enable Kubernetes in the Dory app first".into(), + ); + } + + let push = registry_push_addr()?; + // Registry serves HTTPS (self-signed); skip verify on the node for function image pulls. + let yaml = format!( + "mirrors:\n\ + \x20 \"{svc}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + \x20 \"{push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + \x20 \"{default_push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + configs:\n\ + \x20 \"{ip}:5000\":\n\ + \x20 tls:\n\ + \x20 insecure_skip_verify: true\n\ + \x20 \"{svc}\":\n\ + \x20 tls:\n\ + \x20 insecure_skip_verify: true\n", + svc = REGISTRY_PULL_INCLUSTER, + push = push, + default_push = REGISTRY_PUSH, + ip = cluster_ip, + ); + + let current = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + "cat /etc/rancher/k3s/registries.yaml 2>/dev/null || true", + ]) + .unwrap_or_default(); + if current.trim() == yaml.trim() { + return Ok(()); + } + + log::info!( + "Configuring k3s registry mirrors for {} -> {}:5000...", + REGISTRY_HOSTNAME, + cluster_ip + ); + engine_docker(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + &format!( + "mkdir -p /etc/rancher/k3s && cat > /etc/rancher/k3s/registries.yaml <<'EOF'\n{yaml}EOF" + ), + ])?; + log::info!("Restarting dory k8s node so k3s reloads registries.yaml..."); + engine_docker(&["restart", NODE_CONTAINER])?; + wait_for_node_ready()?; + for _ in 0..30 { + if run_cmd_output("kubectl", &["get", "--raw", "/readyz"]) + .map(|s| s.contains("ok")) + .unwrap_or(false) + { + break; + } + thread::sleep(Duration::from_secs(2)); + } + Ok(()) +} + +/// Engine dockerd must treat the k3s NodePort as an HTTP registry. Push address +/// is `{dory-k8s-ip}:30500` on the docker bridge (see [`registry_push_addr`]). +fn ensure_engine_push_path() -> Result<(), Box> { + // Best-effort cleanup of earlier experiments (ignore missing). + let _ = engine_docker_output(&["rm", "-f", "hops-local-registry"]); + let _ = engine_docker_output(&["rm", "-f", "hops-registry-publish"]); + + let push = registry_push_addr()?; + ensure_dockerd_insecure_for_push(&push)?; + + // Prove NodePort answers HTTPS from the engine network (self-signed → no-check). + for _ in 0..30 { + if engine_docker_output(&[ + "run", + "--rm", + "alpine:latest", + "wget", + "-qO-", + "--no-check-certificate", + &format!("https://{push}/v2/"), + ]) + .is_ok() + { + log::info!("Engine package push endpoint ready at {} (HTTPS)", push); + return Ok(()); + } + thread::sleep(Duration::from_secs(1)); + } + Err(format!( + "timed out waiting for registry NodePort at https://{push}/v2/ \ + (in-cluster TLS registry Service type NodePort on dory-k8s)" + ) + .into()) +} + +fn ensure_dockerd_insecure_for_push(push_hostport: &str) -> Result<(), Box> { + // Already configured? + if let Ok(info) = engine_docker_output(&["info", "-f", "{{json .RegistryConfig.InsecureRegistryCIDRs}}{{json .RegistryConfig.IndexConfigs}}"]) + { + if info.contains(push_hostport) || info.contains("192.168.215.0/24") { + return Ok(()); + } + } + + log::info!( + "Configuring Dory dockerd insecure-registries for HTTP push to {}...", + push_hostport + ); + + // Persist under the engine's /etc/docker (bind-mounted into a helper). + let daemon_json = format!( + r#"{{ + "builder": {{"gc": {{"enabled": true, "defaultKeepStorage": "2GB"}}}}, + "insecure-registries": [ + "127.0.0.0/8", + "localhost:30500", + "192.168.215.0/24", + "{push}" + ] +}} +"#, + push = push_hostport + ); + + engine_docker(&[ + "run", + "--rm", + "-v", + "/etc/docker:/etc/docker", + "alpine:latest", + "sh", + "-c", + &format!("cat > /etc/docker/daemon.json <<'EOF'\n{daemon_json}EOF"), + ])?; + + // Prefer Dory's repair path (live-restore) over raw kill. + if command_exists("dory") { + let _ = run_cmd("dory", &["repair", "dockerd", "--apply"]); + } + + // If dockerd did not pick up the file, force a live-restore restart. + let info = engine_docker_output(&["info"]).unwrap_or_default(); + if !info.contains("192.168.215.0/24") && !info.contains(push_hostport) { + log::info!("Forcing dockerd restart so insecure-registries take effect..."); + let _ = engine_docker(&[ + "run", + "--rm", + "--privileged", + "--pid=host", + "alpine:latest", + "sh", + "-c", + "kill -TERM $(pidof dockerd) 2>/dev/null || true", + ]); + if command_exists("dory") { + let _ = run_cmd("dory", &["repair", "dockerd", "--apply"]); + } + for _ in 0..40 { + if engine_docker_output(&["info"]).is_ok() { + break; + } + thread::sleep(Duration::from_secs(2)); + } + } + + Ok(()) +} + +/// Default kube + docker context name for Dory desktop integration. +pub const DEFAULT_CONTEXT_NAME: &str = "hops-dory"; +const NAME_FILE: &str = "dory-name"; +const NAME_ENV: &str = "HOPS_DORY_NAME"; + +/// Resolved context name: `--name` (persisted) > `HOPS_DORY_NAME` > file > `hops-dory`. +pub fn context_name() -> String { + if let Ok(v) = std::env::var(NAME_ENV) { + let t = v.trim(); + if !t.is_empty() { + return t.to_string(); + } + } + if let Ok(path) = crate::commands::local::local_state_dir() { + if let Ok(raw) = std::fs::read_to_string(path.join(NAME_FILE)) { + let t = raw.trim(); + if !t.is_empty() { + return t.to_string(); + } + } + } + DEFAULT_CONTEXT_NAME.to_string() +} + +/// Persist a user-chosen name for kube/docker context integration. +pub fn persist_context_name(name: &str) -> Result<(), Box> { + let name = validate_context_name(name)?; + let dir = crate::commands::local::local_state_dir()?; + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join(NAME_FILE), format!("{name}\n"))?; + log::info!("Dory desktop name set to '{name}' (kube + docker context)"); + Ok(()) +} + +fn validate_context_name(name: &str) -> Result> { + let name = name.trim(); + if name.is_empty() { + return Err("--name must not be empty".into()); + } + // kubectl context names: keep it simple (no path separators / whitespace). + if name.contains(['/', '\\', ' ', '\t', '\n', ':']) { + return Err(format!( + "invalid --name '{name}': use a simple token (e.g. hops-dory)" + ) + .into()); + } + Ok(name.to_string()) +} + +/// Env: set `HOPS_DORY_DESKTOP=0` (or `false`/`off`/`no`) to skip mutating the +/// machine's default kube/docker contexts. Callers must drive the session with +/// `DOCKER_HOST` and `KUBECONFIG` (or pass `--context`) instead. +pub const DESKTOP_INTEGRATION_ENV: &str = "HOPS_DORY_DESKTOP"; + +/// Whether hops may rewrite `~/.kube/config` / switch docker contexts. +pub fn desktop_integration_enabled() -> bool { + match std::env::var(DESKTOP_INTEGRATION_ENV) { + Ok(v) => { + let v = v.trim().to_ascii_lowercase(); + !matches!(v.as_str(), "0" | "false" | "no" | "off") + } + Err(_) => true, + } +} + +/// Wire Dory into the normal developer desktop: +/// - merge stock `~/.kube/dory-config` into `~/.kube/config` as context **`hops-dory`** +/// (or `--name` / `HOPS_DORY_NAME`) +/// - `kubectl config use-context ` +/// - ensure/use a docker context of the same name pointing at `~/.dory/dory.sock` +/// +/// Called from backend activate and after local start so you don't need +/// `export KUBECONFIG=...` / `export DOCKER_HOST=...`. +/// +/// No-op for host defaults when [`desktop_integration_enabled`] is false: only +/// fills missing `DOCKER_HOST` / `KUBECONFIG` env for this process tree. +pub fn ensure_desktop_integration() -> Result<(), Box> { + ensure_side_kubeconfig_hint(); + if !desktop_integration_enabled() { + ensure_engine_env_only(); + log::info!( + "Dory desktop integration disabled ({DESKTOP_INTEGRATION_ENV}=0); \ + using DOCKER_HOST / KUBECONFIG only (no use-context, no docker context switch)" + ); + return Ok(()); + } + let name = context_name(); + ensure_user_kubeconfig_context(&name)?; + ensure_docker_context_default(&name)?; + Ok(()) +} + +/// Point this process at the engine without touching desktop defaults. +fn ensure_engine_env_only() { + if std::env::var_os("DOCKER_HOST").is_none() { + if let Ok(sock) = engine_socket() { + if sock.exists() { + std::env::set_var("DOCKER_HOST", format!("unix://{}", sock.display())); + } + } + } + if std::env::var_os("KUBECONFIG").is_none() { + export_kubeconfig_env(); + } +} + +/// Fallback when merge fails: point hops child processes at the side file. +pub fn export_kubeconfig_env() { + let Some(dory_cfg) = kubeconfig_path() else { + return; + }; + if !std::path::Path::new(&dory_cfg).is_file() { + return; + } + let existing = std::env::var("KUBECONFIG").unwrap_or_default(); + if existing.split(':').any(|p| p == dory_cfg) { + return; + } + let rest = if existing.is_empty() { + match home() { + Ok(h) => h.join(".kube/config").to_string_lossy().into_owned(), + Err(_) => String::new(), + } + } else { + existing + }; + if rest.is_empty() { + std::env::set_var("KUBECONFIG", &dory_cfg); + } else { + std::env::set_var("KUBECONFIG", format!("{}:{}", dory_cfg, rest)); + } +} + +/// Merge the stock Dory kubeconfig into `~/.kube/config` as the given context name. +fn ensure_user_kubeconfig_context(name: &str) -> Result<(), Box> { + let side = kubeconfig_path().ok_or("HOME unset")?; + if !std::path::Path::new(&side).is_file() { + return Err(format!( + "missing {}; enable Kubernetes in the Dory app (or let hops start write it)", + side + ) + .into()); + } + + let home = home()?; + let kube_dir = home.join(".kube"); + std::fs::create_dir_all(&kube_dir)?; + let main = kube_dir.join("config"); + + // Work on a temp copy so we can rename stock contexts without mutating + // the product side file. + let tmp = kube_dir.join(format!(".dory-merge-{}.yaml", std::process::id())); + std::fs::copy(&side, &tmp)?; + let names = run_cmd_output( + "kubectl", + &[ + "config", + "--kubeconfig", + tmp.to_str().unwrap(), + "get-contexts", + "-o", + "name", + ], + ) + .unwrap_or_default(); + for old in names.lines().map(str::trim).filter(|n| !n.is_empty()) { + if old != name { + let _ = run_cmd( + "kubectl", + &[ + "config", + "--kubeconfig", + tmp.to_str().unwrap(), + "rename-context", + old, + name, + ], + ); + } + } + + if main.is_file() { + let merged = { + let mut child = std::process::Command::new("kubectl") + .args(["config", "view", "--flatten", "--raw"]) + .env( + "KUBECONFIG", + format!("{}:{}", main.display(), tmp.display()), + ) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + let mut out = String::new(); + use std::io::Read; + if let Some(mut s) = child.stdout.take() { + s.read_to_string(&mut out)?; + } + let status = child.wait()?; + if !status.success() { + let _ = std::fs::remove_file(&tmp); + return Err( + "kubectl config view --flatten failed while merging dory kubeconfig".into(), + ); + } + out + }; + let backup = kube_dir.join("config.hops-dory-backup"); + let _ = std::fs::copy(&main, &backup); + std::fs::write(&main, merged)?; + let _ = std::fs::set_permissions( + &main, + std::os::unix::fs::PermissionsExt::from_mode(0o600), + ); + } else { + std::fs::copy(&tmp, &main)?; + let _ = std::fs::set_permissions( + &main, + std::os::unix::fs::PermissionsExt::from_mode(0o600), + ); + } + let _ = std::fs::remove_file(&tmp); + + // Drop the short-lived "dory" name from an earlier hops revision if present + // and distinct from the configured name. + if name != "dory" { + let _ = run_cmd("kubectl", &["config", "delete-context", "dory"]); + } + + let _ = run_cmd("kubectl", &["config", "use-context", name]); + log::info!( + "Kubernetes context '{}' is ready in ~/.kube/config (also: ~/.kube/dory-config)", + name + ); + Ok(()) +} + +/// Docker context of the same name, pointing at Dory's engine socket. +fn ensure_docker_context_default(name: &str) -> Result<(), Box> { + if !command_exists("docker") { + return Ok(()); + } + let sock = engine_socket()?; + if !sock.exists() { + log::warn!( + "Dory engine socket {} missing; skip docker context '{}'", + sock.display(), + name + ); + return Ok(()); + } + let host = format!("host=unix://{}", sock.display()); + let contexts = run_cmd_output("docker", &["context", "ls", "--format", "{{.Name}}"]) + .unwrap_or_default(); + if !contexts.lines().any(|n| n.trim() == name) { + log::info!( + "Creating docker context '{}' → unix://{}", + name, + sock.display() + ); + // Ignore failure if a stale context exists with different metadata. + let create = run_cmd( + "docker", + &["context", "create", name, "--docker", &host], + ); + if create.is_err() { + // Update in place when possible. + let _ = run_cmd( + "docker", + &["context", "update", name, "--docker", &host], + ); + } + } + run_cmd("docker", &["context", "use", name])?; + log::info!( + "Docker context set to '{}' (unix://{})", + name, + sock.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn package_pull_is_in_cluster_service_not_host_gateway() { + assert_eq!( + crate::commands::local::package_install::REGISTRY_PULL_INCLUSTER, + "registry.crossplane-system.svc.cluster.local:5000" + ); + assert!(!REGISTRY_PULL_INCLUSTER.contains("dory.internal")); + assert_eq!(REGISTRY_PUSH, "localhost:30500"); + } + + #[test] + fn start_rejects_size_flags() { + let size = SizeArgs { + cpus: Some(4), + memory: None, + disk: None, + }; + + let err = start(&size).expect_err("size flags must be rejected"); + + assert!(err.to_string().contains("--cpus 4")); + assert!(err.to_string().contains("Dory app")); + } + + #[test] + fn missing_k8s_error_mentions_app_not_fork() { + let msg = "Dory Kubernetes is not enabled (no `dory-k8s` container).\n\ + In the Dory app: enable Kubernetes, wait until it is running, then re-run:\n\ + hops local start --backend dory\n\ + (hops uses stock Dory only — it does not create the cluster for you.)"; + assert!(msg.contains("Dory app")); + assert!(!msg.contains("feat/scriptable")); + assert!(!msg.contains("dory k8s enable")); + } + + #[test] + fn desktop_integration_env_off_values() { + for off in ["0", "false", "FALSE", "no", "off", " Off "] { + std::env::set_var(DESKTOP_INTEGRATION_ENV, off); + assert!( + !desktop_integration_enabled(), + "expected desktop off for {off:?}" + ); + } + std::env::set_var(DESKTOP_INTEGRATION_ENV, "1"); + assert!(desktop_integration_enabled()); + std::env::remove_var(DESKTOP_INTEGRATION_ENV); + } +} diff --git a/src/commands/local/backend/kind.rs b/src/commands/local/backend/kind.rs new file mode 100644 index 0000000..005172d --- /dev/null +++ b/src/commands/local/backend/kind.rs @@ -0,0 +1,332 @@ +//! kind backend: docker containers as nodes, containerd runtime. +//! +//! Works against any reachable docker daemon — Docker Desktop, colima's +//! dockerd, dory's dockerd, or a CI runner's. Registry trust is wired through +//! containerd's certs.d (`config_path` is enabled by default in kind node +//! images since v0.27.0), written after cluster creation; containerd reads +//! certs.d per-pull, so no restart is needed. + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_PULL, REGISTRY_PUSH}; +use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use std::error::Error; +use std::io::Write; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +pub const CLUSTER_NAME: &str = "hops"; +const NODE_CONTAINER: &str = "hops-control-plane"; + +/// kind node images before v0.27.0 ship containerd 1.x without certs.d +/// `config_path` enabled, so our hosts.toml files would be ignored. +const MIN_KIND_VERSION: (u32, u32) = (0, 27); + +const KIND_CONFIG: &str = r#"kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30500 + hostPort: 30500 + listenAddress: "127.0.0.1" +"#; + +pub fn install() -> Result<(), Box> { + log::info!("Installing kind via Homebrew..."); + run_cmd("brew", &["install", "kind"])?; + log::info!("kind installed successfully"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling kind..."); + run_cmd("brew", &["uninstall", "kind"])?; + log::info!("kind uninstalled"); + Ok(()) +} + +pub fn start(size: &SizeArgs) -> Result<(), Box> { + if size.any_set() { + return Err(format!( + "the kind backend has no VM to size; drop{} (resources are governed by the docker daemon kind runs on)", + size.command_suffix() + ) + .into()); + } + + preflight()?; + + if !cluster_exists() { + return create_cluster(); + } + + if node_running() { + log::info!("kind cluster '{}' is already running", CLUSTER_NAME); + return Ok(()); + } + + // kind has no start/stop; the node is a docker container. Restarting a + // single-node cluster is reliable in practice but not guaranteed by kind. + log::info!("Starting stopped kind node '{}'...", NODE_CONTAINER); + run_cmd("docker", &["start", NODE_CONTAINER])?; + wait_for_api_after_restart() +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping kind node '{}'...", NODE_CONTAINER); + run_cmd("docker", &["stop", NODE_CONTAINER])?; + log::info!("kind cluster stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Deleting kind cluster '{}'...", CLUSTER_NAME); + run_cmd("kind", &["delete", "cluster", "--name", CLUSTER_NAME])?; + log::info!("kind cluster deleted"); + Ok(()) +} + +/// kind's node container IS the cluster, so reset means recreate. +pub fn reset() -> Result<(), Box> { + preflight()?; + if cluster_exists() { + destroy()?; + } + create_cluster() +} + +pub fn resize(_size: &SizeArgs) -> Result<(), Box> { + Err( + "kind clusters have no VM to resize; adjust the docker daemon's resources, \ + or `hops local destroy && hops local start` to recreate the cluster" + .into(), + ) +} + +/// Whether the hops kind cluster exists (running or stopped). Missing binary +/// or failing command reads as "no cluster". +pub fn cluster_exists() -> bool { + if !command_exists("kind") { + return false; + } + run_cmd_output("kind", &["get", "clusters"]) + .map(|out| out.lines().any(|line| line.trim() == CLUSTER_NAME)) + .unwrap_or(false) +} + +fn node_running() -> bool { + run_cmd_output( + "docker", + &["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER], + ) + .map(|out| out.trim() == "true") + .unwrap_or(false) +} + +fn preflight() -> Result<(), Box> { + if !command_exists("kind") { + return Err( + "kind is not installed; run `hops local install --backend kind` or `brew install kind`" + .into(), + ); + } + + let version_output = run_cmd_output("kind", &["version"])?; + match parse_kind_version(&version_output) { + Some(version) if version >= MIN_KIND_VERSION => {} + Some((major, minor)) => { + return Err(format!( + "kind v{major}.{minor} is too old: node images before v{}.{} lack containerd \ + certs.d support needed for the local registry. Upgrade with `brew upgrade kind`.", + MIN_KIND_VERSION.0, MIN_KIND_VERSION.1 + ) + .into()); + } + None => log::warn!( + "Unable to parse `kind version` output ({}); continuing", + version_output.trim() + ), + } + + if run_cmd_output("docker", &["info", "--format", "{{.ServerVersion}}"]).is_err() { + return Err( + "no reachable docker daemon; start Docker Desktop / colima / dory \ + (or point DOCKER_HOST / `docker context use` at one) and retry" + .into(), + ); + } + + Ok(()) +} + +fn create_cluster() -> Result<(), Box> { + log::info!("Creating kind cluster '{}'...", CLUSTER_NAME); + let mut child = Command::new("kind") + .args(["create", "cluster", "--name", CLUSTER_NAME, "--config", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(KIND_CONFIG.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err(format!("kind create cluster exited with {}", status).into()); + } + Ok(()) +} + +fn wait_for_api_after_restart() -> Result<(), Box> { + log::info!("Waiting for Kubernetes API..."); + for _ in 0..24 { + if run_cmd_output("kubectl", &["cluster-info"]).is_ok() { + return Ok(()); + } + thread::sleep(Duration::from_secs(5)); + } + Err( + "Kubernetes API did not come back after restarting the kind node; \ + run `hops local reset` to recreate the cluster" + .into(), + ) +} + +/// Alias both registry pull names to the registry Service's ClusterIP via +/// containerd certs.d files on the node. `localhost:30500` is what provider +/// runtime pods reference; aliasing it here means the name never depends on +/// kube-proxy's localhost-NodePort behavior. Files live on the node's +/// writable layer, so they survive docker stop/start (unlike /etc/hosts, +/// which docker regenerates). +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + for name in [REGISTRY_PULL, REGISTRY_PUSH] { + write_hosts_toml(name, cluster_ip)?; + } + Ok(()) +} + +fn hosts_toml(cluster_ip: &str) -> String { + // Local registry serves HTTPS with a hops-managed self-signed cert. + format!( + "[host.\"https://{}:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n", + cluster_ip + ) +} + +fn write_hosts_toml(registry_name: &str, cluster_ip: &str) -> Result<(), Box> { + let dir = format!("/etc/containerd/certs.d/{}", registry_name); + let path = format!("{}/hosts.toml", dir); + let desired = hosts_toml(cluster_ip); + + let current = + run_cmd_output("docker", &["exec", NODE_CONTAINER, "cat", &path]).unwrap_or_default(); + if current == desired { + return Ok(()); + } + + log::info!( + "Wiring containerd registry alias: {} -> {}:5000", + registry_name, + cluster_ip + ); + + let mut child = Command::new("docker") + .args([ + "exec", + "-i", + NODE_CONTAINER, + "sh", + "-c", + &format!("mkdir -p '{}' && cat > '{}'", dir, path), + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(desired.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err(format!("failed to write {} on kind node", path).into()); + } + Ok(()) +} + +fn parse_kind_version(output: &str) -> Option<(u32, u32)> { + // Typical output: "kind v0.32.0 go1.23.4 darwin/arm64" + output.split_whitespace().find_map(|token| { + let token = token.strip_prefix('v').unwrap_or(token); + let mut parts = token.split('.'); + let major: u32 = parts.next()?.parse().ok()?; + let minor: u32 = parts.next()?.parse().ok()?; + Some((major, minor)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind_config_pins_registry_nodeport_to_localhost() { + assert!(KIND_CONFIG.contains("containerPort: 30500")); + assert!(KIND_CONFIG.contains("hostPort: 30500")); + assert!(KIND_CONFIG.contains("listenAddress: \"127.0.0.1\"")); + assert!(KIND_CONFIG.contains("role: control-plane")); + } + + #[test] + fn hosts_toml_aliases_to_cluster_ip_over_https() { + // Local package registry is HTTPS (self-signed); containerd must use + // https:// + skip_verify, not plain http://. + let toml = hosts_toml("10.43.12.7"); + + assert_eq!( + toml, + "[host.\"https://10.43.12.7:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n" + ); + } + + #[test] + fn parse_kind_version_reads_standard_output() { + assert_eq!( + parse_kind_version("kind v0.32.0 go1.23.4 darwin/arm64"), + Some((0, 32)) + ); + assert_eq!( + parse_kind_version("kind v0.27.0 go1.22 linux/amd64"), + Some((0, 27)) + ); + } + + #[test] + fn parse_kind_version_handles_unexpected_output() { + assert_eq!(parse_kind_version("something unparseable"), None); + assert_eq!(parse_kind_version(""), None); + } + + #[test] + fn old_kind_versions_fail_the_minimum_check() { + let version = parse_kind_version("kind v0.26.0 go1.22 linux/amd64").unwrap(); + assert!(version < MIN_KIND_VERSION); + + let new_enough = parse_kind_version("kind v0.27.0 go1.22 linux/amd64").unwrap(); + assert!(new_enough >= MIN_KIND_VERSION); + } + + #[test] + fn start_rejects_size_flags() { + let size = SizeArgs { + cpus: Some(4), + memory: None, + disk: None, + }; + + let err = start(&size).expect_err("size flags must be rejected"); + + assert!(err.to_string().contains("--cpus 4")); + assert!(err.to_string().contains("no VM to size")); + } +} diff --git a/src/commands/local/backend/mod.rs b/src/commands/local/backend/mod.rs new file mode 100644 index 0000000..e0b2098 --- /dev/null +++ b/src/commands/local/backend/mod.rs @@ -0,0 +1,577 @@ +//! Local-cluster backend abstraction. +//! +//! Abstracts the node/VM-level operations that differ between local cluster +//! providers (lifecycle, sizing, registry trust). Everything kubectl/helm +//! shaped lives outside this module and is backend-agnostic. + +mod colima; +mod dory; +mod kind; + +use super::{local_state_dir, run_cmd_output, HOPS_KUBE_CONTEXT_ENV}; +use clap::Args; +use std::error::Error; +use std::fmt; +use std::process::Command; +use std::str::FromStr; + +/// Sizing flags for backends with a resizable VM. +#[derive(Args, Debug, Clone, Default, PartialEq, Eq)] +pub struct SizeArgs { + /// Number of CPUs to allocate to the cluster VM (colima backend only). + #[arg(long = "cpus", visible_alias = "cpu", value_name = "N")] + pub cpus: Option, + + /// Memory to allocate to the cluster VM, in GiB (colima backend only). + #[arg(long, value_name = "GIB")] + pub memory: Option, + + /// Disk size to allocate to the cluster VM, in GiB (colima backend only). + #[arg(long, value_name = "GIB")] + pub disk: Option, +} + +impl SizeArgs { + pub fn any_set(&self) -> bool { + self.cpus.is_some() || self.memory.is_some() || self.disk.is_some() + } + + pub fn command_suffix(&self) -> String { + let mut parts = Vec::new(); + if let Some(cpus) = self.cpus { + parts.push(format!("--cpus {}", cpus)); + } + if let Some(memory) = self.memory { + parts.push(format!("--memory {}", memory)); + } + if let Some(disk) = self.disk { + parts.push(format!("--disk {}", disk)); + } + + if parts.is_empty() { + String::new() + } else { + format!(" {}", parts.join(" ")) + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum Backend { + /// VM + dockerd + k3s (macOS/Linux) + Colima, + /// docker containers as nodes; works on any docker daemon + /// (Docker Desktop, colima, dory, CI runners) + Kind, + /// k3s on dory's shared-VM engine, via the `dory` CLI + Dory, +} + +impl Backend { + /// Human-facing backend name (also the persisted spelling). + pub fn name(self) -> &'static str { + match self { + Backend::Colima => "colima", + Backend::Kind => "kind", + Backend::Dory => "dory", + } + } + + /// kubeconfig context name this backend's cluster registers under. + pub fn kube_context(self) -> String { + match self { + Backend::Colima => "colima".to_string(), + Backend::Kind => "kind-hops".to_string(), + // Merged into ~/.kube/config (default name hops-dory; see dory::context_name). + Backend::Dory => dory::context_name(), + } + } + + pub fn install(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::install(), + Backend::Kind => kind::install(), + Backend::Dory => dory::install(), + } + } + + pub fn uninstall(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::uninstall(), + Backend::Kind => kind::uninstall(), + Backend::Dory => dory::uninstall(), + } + } + + /// Whether this backend's local cluster/VM exists, running or stopped. + pub fn cluster_exists(self) -> bool { + match self { + Backend::Colima => colima::instance_exists(), + Backend::Kind => kind::cluster_exists(), + Backend::Dory => dory::cluster_exists(), + } + } + + /// Bring the cluster up (create, start, or resize as needed). Does not + /// wait for the Kubernetes API; callers follow with `wait_for_kubernetes`. + pub fn start(self, size: &SizeArgs, assume_yes: bool) -> Result<(), Box> { + match self { + Backend::Colima => colima::start(size, assume_yes), + Backend::Kind => kind::start(size), + Backend::Dory => dory::start(size), + } + } + + pub fn stop(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::stop(), + Backend::Kind => kind::stop(), + Backend::Dory => dory::stop(), + } + } + + pub fn destroy(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::destroy(), + Backend::Kind => kind::destroy(), + Backend::Dory => dory::destroy(), + } + } + + pub fn reset(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::reset(), + Backend::Kind => kind::reset(), + Backend::Dory => dory::reset(), + } + } + + pub fn resize(self, size: &SizeArgs) -> Result<(), Box> { + match self { + Backend::Colima => colima::resize(size), + Backend::Kind => kind::resize(size), + Backend::Dory => dory::resize(size), + } + } + + /// Make the node runtime trust the in-cluster registry over HTTP. + /// Runs before any images exist; must be safe to re-run. + pub fn ensure_registry_trust(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::configure_docker_insecure_registry(), + // containerd trust is per-name via certs.d, written in + // wire_registry once the registry Service's ClusterIP is known. + Backend::Kind => Ok(()), + // k3s registries.yaml is written in wire_registry once ClusterIP is known. + Backend::Dory => dory::ensure_registry_trust(), + } + } + + /// Point the node at the registry Service's current ClusterIP so pulls of + /// both registry names resolve. Idempotent; re-run on every start because + /// the ClusterIP changes if the Service is recreated. + /// + /// Dory also publishes host localhost:30500 → node NodePort (engine proxy). + pub fn wire_registry(self, cluster_ip: &str) -> Result<(), Box> { + match self { + Backend::Colima => colima::sync_hosts_entry(cluster_ip), + Backend::Kind => kind::wire_registry(cluster_ip), + Backend::Dory => dory::wire_registry(cluster_ip), + } + } + + /// Backend-specific package registry for local provider/config installs. + /// All backends use an in-cluster NodePort registry for Crossplane package + /// pulls (pod network). Host push is always localhost:30500. + pub fn ensure_package_registry(self) -> Result<(), Box> { + crate::commands::local::package_install::ensure_incluster_registry() + } + + /// Pull address used in Crossplane package / ImageConfig refs for this backend. + /// Always the in-cluster Service — Crossplane runs in the pod network. + pub fn registry_pull(self) -> &'static str { + crate::commands::local::package_install::REGISTRY_PULL_INCLUSTER + } + + /// Docker push host:port for package installs on this backend. + pub fn registry_push(self) -> String { + match self { + Backend::Dory => dory::registry_push_addr().unwrap_or_else(|_| { + crate::commands::local::package_install::REGISTRY_PUSH.to_string() + }), + Backend::Colima | Backend::Kind => { + crate::commands::local::package_install::REGISTRY_PUSH.to_string() + } + } + } +} + +impl fmt::Display for Backend { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +impl FromStr for Backend { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim() { + "colima" => Ok(Backend::Colima), + "kind" => Ok(Backend::Kind), + "dory" => Ok(Backend::Dory), + other => Err(format!( + "unknown backend '{}' (expected colima, kind, or dory)", + other + )), + } + } +} + +const BACKEND_FILE: &str = "backend"; + +/// Persist the Dory desktop context name (`--name`, default hops-dory). +pub fn persist_dory_context_name(name: &str) -> Result<(), Box> { + dory::persist_context_name(name) +} + +pub fn platform_default() -> Backend { + if cfg!(target_os = "macos") { + Backend::Colima + } else { + Backend::Kind + } +} + +/// Resolve which backend to operate on: explicit flag > preference persisted +/// by the last successful start > detection of an existing cluster (colima +/// wins for back-compat with pre-backend installs) > platform default. +pub fn resolve(flag: Option) -> Backend { + resolve_from( + flag, + persisted(), + colima::instance_exists, + kind::cluster_exists, + dory::cluster_exists, + platform_default() == Backend::Colima, + ) +} + +/// Resolve the backend once and activate the kube-targeting environment for +/// child kubectl/helm processes. +pub fn activate(flag: Option, context: Option<&str>) -> Backend { + let backend = resolve(flag); + + if backend == Backend::Dory { + // Merge ~/.kube/dory-config → named context and prefer matching docker context. + if let Err(e) = dory::ensure_desktop_integration() { + log::warn!("dory desktop integration incomplete: {e}"); + dory::export_kubeconfig_env(); + } + } + + let explicit_context = context.filter(|ctx| !ctx.is_empty()); + let backend_context = backend.kube_context(); + let backend_context_exists = if explicit_context.is_none() { + kube_context_exists(&backend_context) + } else { + false + }; + + match kube_context_export(explicit_context, &backend_context, backend_context_exists) { + KubeContextExport::Set(ctx) => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx), + KubeContextExport::Unset { missing_context } => { + std::env::remove_var(HOPS_KUBE_CONTEXT_ENV); + log::warn!( + "Kubernetes context '{}' for backend '{}' was not found; using kubeconfig current-context. Pass --context to target a specific cluster.", + missing_context, + backend.name() + ); + } + } + + backend +} + +fn resolve_from( + flag: Option, + persisted: Option, + colima_detected: impl FnOnce() -> bool, + kind_detected: impl FnOnce() -> bool, + dory_detected: impl FnOnce() -> bool, + macos: bool, +) -> Backend { + if let Some(backend) = flag { + return backend; + } + if let Some(backend) = persisted { + return backend; + } + if colima_detected() { + return Backend::Colima; + } + if kind_detected() { + return Backend::Kind; + } + if dory_detected() { + return Backend::Dory; + } + if macos { + Backend::Colima + } else { + Backend::Kind + } +} + +fn persisted() -> Option { + let path = local_state_dir().ok()?.join(BACKEND_FILE); + std::fs::read_to_string(path).ok()?.parse().ok() +} + +/// Record the backend so later invocations (stop, destroy, doctor, installs) +/// target the same cluster without re-detection. +pub fn persist(backend: Backend) -> Result<(), Box> { + let dir = local_state_dir()?; + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join(BACKEND_FILE), format!("{}\n", backend.name()))?; + Ok(()) +} + +/// Drop the persisted preference (used by uninstall; destroy keeps it). +pub fn clear_persisted() -> Result<(), Box> { + match std::fs::remove_file(local_state_dir()?.join(BACKEND_FILE)) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err.into()), + } +} + +/// Fetch the in-cluster registry Service's ClusterIP. +fn registry_cluster_ip() -> Result> { + let cluster_ip = run_cmd_output( + "kubectl", + &[ + "get", + "svc", + "registry", + "-n", + "crossplane-system", + "-o", + "jsonpath={.spec.clusterIP}", + ], + )?; + let cluster_ip = cluster_ip.trim().to_string(); + if cluster_ip.is_empty() { + return Err("Service crossplane-system/registry has no ClusterIP".into()); + } + Ok(cluster_ip) +} + +/// Wire node-level registry pulls for a backend: fetch the registry Service +/// ClusterIP and apply the backend-specific trust/aliasing (and Dory host publish). +pub fn wire_local_registry(backend: Backend) -> Result<(), Box> { + backend.wire_registry(®istry_cluster_ip()?) +} + +pub fn should_wire_local_registry( + backend_flag: Option, + context: Option<&str>, + backend: Backend, +) -> bool { + if backend_flag.is_some() { + return true; + } + + match context.filter(|ctx| !ctx.is_empty()) { + Some(ctx) => ctx == backend.kube_context(), + None => true, + } +} + +pub fn wire_local_registry_for_target( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + if should_wire_local_registry(backend_flag, context, backend) { + return wire_local_registry(backend); + } + + log::warn!( + "registry node wiring skipped: explicit --context does not match a selected backend" + ); + Ok(()) +} + +#[derive(Debug, PartialEq, Eq)] +enum KubeContextExport<'a> { + Set(&'a str), + Unset { missing_context: &'a str }, +} + +fn kube_context_export<'a>( + explicit_context: Option<&'a str>, + backend_context: &'a str, + backend_context_exists: bool, +) -> KubeContextExport<'a> { + if let Some(ctx) = explicit_context { + return KubeContextExport::Set(ctx); + } + + if backend_context_exists { + KubeContextExport::Set(backend_context) + } else { + KubeContextExport::Unset { + missing_context: backend_context, + } + } +} + +fn kube_context_exists(context: &str) -> bool { + let output = Command::new("kubectl") + .args(["config", "get-contexts", "-o", "name"]) + .output(); + + let Ok(output) = output else { + return false; + }; + if !output.status.success() { + return false; + } + + String::from_utf8_lossy(&output.stdout) + .lines() + .any(|line| line.trim() == context) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn no_detect() -> bool { + false + } + + #[test] + fn flag_beats_persisted_and_detection() { + let resolved = resolve_from( + Some(Backend::Kind), + Some(Backend::Colima), + || true, + no_detect, + no_detect, + true, + ); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn persisted_beats_detection() { + let resolved = resolve_from( + None, + Some(Backend::Kind), + || true, + no_detect, + no_detect, + true, + ); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn colima_detection_beats_kind_detection() { + let resolved = resolve_from(None, None, || true, || true, || true, false); + + assert_eq!(resolved, Backend::Colima); + } + + #[test] + fn dory_detection_used_when_no_colima_or_kind() { + let resolved = resolve_from(None, None, no_detect, no_detect, || true, true); + + assert_eq!(resolved, Backend::Dory); + } + + #[test] + fn kind_detection_used_when_no_colima() { + let resolved = resolve_from(None, None, no_detect, || true, no_detect, true); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn platform_default_when_nothing_detected() { + assert_eq!( + resolve_from(None, None, no_detect, no_detect, no_detect, true), + Backend::Colima + ); + assert_eq!( + resolve_from(None, None, no_detect, no_detect, no_detect, false), + Backend::Kind + ); + } + + #[test] + fn backend_name_round_trips_through_from_str() { + for backend in [Backend::Colima, Backend::Kind, Backend::Dory] { + assert_eq!(backend.name().parse::().unwrap(), backend); + } + assert!("podman".parse::().is_err()); + } + + #[test] + fn explicit_context_is_exported_even_when_backend_context_is_absent() { + assert_eq!( + kube_context_export(Some("foreign"), "colima", false), + KubeContextExport::Set("foreign") + ); + } + + #[test] + fn backend_context_is_exported_only_when_present() { + assert_eq!( + kube_context_export(None, "kind-hops", true), + KubeContextExport::Set("kind-hops") + ); + assert_eq!( + kube_context_export(None, "kind-hops", false), + KubeContextExport::Unset { + missing_context: "kind-hops" + } + ); + } + + #[test] + fn registry_wiring_allowed_without_explicit_context() { + assert!(should_wire_local_registry(None, None, Backend::Colima)); + } + + #[test] + fn registry_wiring_skips_foreign_explicit_context_without_backend_flag() { + assert!(!should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + + #[test] + fn registry_wiring_allowed_when_context_matches_backend() { + assert!(should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Kind + )); + } + + #[test] + fn registry_wiring_allowed_when_backend_is_explicit() { + assert!(should_wire_local_registry( + Some(Backend::Colima), + Some("foreign"), + Backend::Colima + )); + } +} diff --git a/src/commands/local/destroy.rs b/src/commands/local/destroy.rs index 8638986..c860228 100644 --- a/src/commands/local/destroy.rs +++ b/src/commands/local/destroy.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Destroying Colima VM..."); - run_cmd("colima", &["delete", "--force"])?; - log::info!("Colima VM destroyed"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.destroy() } diff --git a/src/commands/local/doctor.rs b/src/commands/local/doctor.rs index e7f8967..dcbe8d1 100644 --- a/src/commands/local/doctor.rs +++ b/src/commands/local/doctor.rs @@ -72,6 +72,17 @@ pub fn run() -> Result<(), Box> { "registry deployment not Available in crossplane-system".into() }, ); + let push = super::package_install::registry_push(); + let push_ok = package_push_registry_reachable(&push); + d.check( + &format!("package push registry reachable ({push})"), + push_ok, + if push_ok { + String::new() + } else { + format!("expected HTTPS registry at {push} for docker push") + }, + ); d.print(); @@ -220,20 +231,17 @@ fn check_provider(d: &mut Doctor, e: &ProviderExpectation) { }, ); - let pc_ok = exists(&[ - "get", - e.pc_resource, - e.pc_name, - "-n", - e.pc_namespace, - ]); + let pc_ok = exists(&["get", e.pc_resource, e.pc_name, "-n", e.pc_namespace]); d.check( "ProviderConfig present", pc_ok, if pc_ok { String::new() } else { - format!("{}/{} missing in namespace {}", e.pc_resource, e.pc_name, e.pc_namespace) + format!( + "{}/{} missing in namespace {}", + e.pc_resource, e.pc_name, e.pc_namespace + ) }, ); } @@ -244,7 +252,10 @@ fn provider_condition(provider: &str, cond: &str) -> String { "provider.pkg.crossplane.io", provider, "-o", - &format!("jsonpath={{.status.conditions[?(@.type==\"{}\")].status}}", cond), + &format!( + "jsonpath={{.status.conditions[?(@.type==\"{}\")].status}}", + cond + ), ]) .unwrap_or_default() } @@ -253,7 +264,11 @@ fn cond_detail(cond: &str, status: &str) -> String { if status == "True" { String::new() } else { - format!("{}={}", cond, if status.is_empty() { "" } else { status }) + format!( + "{}={}", + cond, + if status.is_empty() { "" } else { status } + ) } } @@ -285,6 +300,69 @@ fn provider_package(provider: &str) -> String { .unwrap_or_default() } +/// Whether `push` is a host-loopback address (kind/colima NodePort). +/// +/// Dory pushes via the engine docker bridge (`{dory-k8s-ip}:30500`); that IP is +/// not reachable from the Mac, so host curl is the wrong plane and can hang on +/// TCP timeout (~75s) before the engine probe runs. +fn push_host_is_loopback(push: &str) -> bool { + // host:port — IPv6 may be bracketed as [::1]:30500 + let host = if let Some(rest) = push.strip_prefix('[') { + rest.split(']').next().unwrap_or(rest) + } else { + push.split(':').next().unwrap_or(push) + }; + host == "localhost" || host == "127.0.0.1" || host == "::1" +} + +/// Probe the package push registry on the plane docker will use. +/// +/// - Loopback (`localhost:30500`): short host curl (kind/colima). +/// - Otherwise (dory bridge IP): engine-network probe only — never host curl +/// (wrong plane; TCP hang ~75s) and never default docker.sock (can block on +/// password / missing daemon on a personal Mac). +fn package_push_registry_reachable(push: &str) -> bool { + let url = format!("https://{push}/v2/"); + if push_host_is_loopback(push) { + return super::run_cmd_output( + "curl", + &["-skf", "--connect-timeout", "2", "--max-time", "3", &url], + ) + .is_ok(); + } + + // Prefer Dory's engine socket explicitly so we don't touch host Docker. + let dory_sock = std::env::var("HOME") + .ok() + .map(|h| std::path::PathBuf::from(h).join(".dory/dory.sock")) + .filter(|p| p.exists()); + + let mut args: Vec = Vec::new(); + if let Some(sock) = dory_sock { + args.push("-H".into()); + args.push(format!("unix://{}", sock.display())); + } else if std::env::var_os("DOCKER_HOST").is_none() { + // No engine socket and no DOCKER_HOST: skip rather than hang on host sock. + return false; + } + args.extend( + [ + "run", + "--rm", + "alpine:latest", + "wget", + "-qO-", + "--no-check-certificate", + "--timeout=5", + &url, + ] + .into_iter() + .map(String::from), + ); + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + super::run_cmd_output("docker", &arg_refs).is_ok() +} + /// True when `kubectl get ` finds the resource. Uses `--ignore-not-found` /// so a missing resource is `false` rather than an error. fn exists(get_args: &[&str]) -> bool { @@ -381,3 +459,21 @@ impl Doctor { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn push_host_loopback_for_kind_colima() { + assert!(push_host_is_loopback("localhost:30500")); + assert!(push_host_is_loopback("127.0.0.1:30500")); + assert!(push_host_is_loopback("[::1]:30500")); + } + + #[test] + fn push_host_not_loopback_for_dory_bridge() { + assert!(!push_host_is_loopback("192.168.215.2:30500")); + assert!(!push_host_is_loopback("10.43.0.1:30500")); + } +} diff --git a/src/commands/local/install.rs b/src/commands/local/install.rs index 0715c89..fabbb65 100644 --- a/src/commands/local/install.rs +++ b/src/commands/local/install.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Installing Colima via Homebrew..."); - run_cmd("brew", &["install", "colima"])?; - log::info!("Colima installed successfully"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.install() } diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index 6adfca6..8cc2751 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -1,4 +1,5 @@ mod aws; +pub mod backend; mod cloudflare; mod destroy; mod doctor; @@ -33,22 +34,53 @@ pub const PROVIDER_INSTALL_MANAGED_BY: &str = "hops-provider-install"; /// Env var checked by kubectl helpers to inject `--context `. pub const HOPS_KUBE_CONTEXT_ENV: &str = "HOPS_KUBE_CONTEXT"; -/// Build the kubectl args prefix. Returns `["--context", ctx]` when the env var -/// is set, or an empty vec otherwise. -fn kubectl_context_args() -> Vec { - match std::env::var(HOPS_KUBE_CONTEXT_ENV) { - Ok(ctx) if !ctx.is_empty() => vec!["--context".to_string(), ctx], - _ => vec![], - } +fn kube_context_from_env() -> Option { + std::env::var(HOPS_KUBE_CONTEXT_ENV) + .ok() + .filter(|ctx| !ctx.is_empty()) } /// Prepend `--context` to a kubectl arg slice when configured. fn with_kube_context(args: &[&str]) -> Vec { - let mut out = kubectl_context_args(); + let ctx = kube_context_from_env(); + with_kube_context_value(args, ctx.as_deref()) +} + +fn with_kube_context_value(args: &[&str], context: Option<&str>) -> Vec { + let mut out = match context { + Some(ctx) if !ctx.is_empty() => vec!["--context".to_string(), ctx.to_string()], + _ => vec![], + }; out.extend(args.iter().map(|s| s.to_string())); out } +fn with_helm_kube_context(args: &[&str]) -> Vec { + let ctx = kube_context_from_env(); + with_helm_kube_context_value(args, ctx.as_deref()) +} + +fn with_helm_kube_context_value(args: &[&str], context: Option<&str>) -> Vec { + let Some(ctx) = context.filter(|ctx| !ctx.is_empty()) else { + return args.iter().map(|s| s.to_string()).collect(); + }; + if args.first() == Some(&"repo") { + return args.iter().map(|s| s.to_string()).collect(); + } + + let mut out = Vec::with_capacity(args.len() + 2); + if let Some((command, rest)) = args.split_first() { + out.push((*command).to_string()); + out.push("--kube-context".to_string()); + out.push(ctx.to_string()); + out.extend(rest.iter().map(|s| s.to_string())); + } else { + out.push("--kube-context".to_string()); + out.push(ctx.to_string()); + } + out +} + /// Build a `Command` for kubectl with `--context` injected when configured. pub fn kubectl_command(args: &[&str]) -> Command { let full = with_kube_context(args); @@ -63,21 +95,35 @@ pub struct LocalArgs { pub command: LocalCommands, /// Kubernetes context to use for all kubectl commands (e.g. "colima"). - /// Global: applies to every `hops local` subcommand and may be given before - /// or after the subcommand. + /// Defaults to the resolved backend's own context. Global: applies to + /// every `hops local` subcommand and may be given before or after the + /// subcommand. #[arg(long, global = true)] pub context: Option, + + /// Local cluster backend to target. Defaults to the backend persisted by + /// the last successful `hops local start`, else an existing cluster if + /// one is detected, else the platform default (macOS: colima, otherwise + /// kind). + #[arg(long, global = true, value_enum)] + pub backend: Option, + + /// Name for Dory desktop integration (kube context + docker context). + /// Defaults to `hops-dory`. Persisted under `~/.hops/local/dory-name`. + /// Only used with `--backend dory` (or a persisted dory backend). + #[arg(long, global = true, value_name = "NAME")] + pub name: Option, } #[derive(Subcommand, Debug)] pub enum LocalCommands { - /// Install Colima via Homebrew + /// Install the local cluster backend (colima or kind) via Homebrew Install, - /// Reset local Colima Kubernetes state + /// Reset local Kubernetes state (colima: k8s reset; kind: recreate cluster) Reset, - /// Start local k8s cluster with Crossplane and providers + /// Start local k8s and ensure Crossplane control plane (skips helm when already healthy) Start(start::StartArgs), - /// Resize the local Colima VM without destroying cluster state + /// Resize the local cluster VM without destroying cluster state (colima only) Resize(resize::ResizeArgs), /// Check what `hops local start` set up and report drift Doctor, @@ -93,34 +139,39 @@ pub enum LocalCommands { Listmonk(listmonk::ListmonkArgs), /// Stop the local cluster Stop, - /// Destroy the local cluster VM + /// Destroy the local cluster Destroy, - /// Uninstall Colima - Uninstall, + /// Uninstall the local cluster backend + Uninstall(uninstall::UninstallArgs), } pub fn run(args: &LocalArgs) -> Result<(), Box> { - // Plumb --context through the same env channel the kubectl helpers read, so - // every subcommand's kubectl calls target the chosen context. - if let Some(ctx) = &args.context { - if !ctx.is_empty() { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + if let Some(name) = args.name.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + backend::persist_dory_context_name(name)?; } + + let explicit_context = args.context.as_deref().filter(|ctx| !ctx.is_empty()); + let install_backend = matches!(&args.command, LocalCommands::Install) + .then(|| args.backend.unwrap_or_else(backend::platform_default)); + let activation_flag = install_backend.or(args.backend); + let install_context = install_backend.map(|b| b.kube_context()); + let activation_context = explicit_context.or(install_context.as_deref()); + let backend = backend::activate(activation_flag, activation_context); + match &args.command { - LocalCommands::Install => install::run(), - LocalCommands::Reset => reset::run(), - LocalCommands::Start(start_args) => start::run(start_args), - LocalCommands::Resize(resize_args) => resize::run(resize_args), + LocalCommands::Install => install::run(backend), + LocalCommands::Reset => reset::run(backend), + LocalCommands::Start(start_args) => start::run(backend, start_args), + LocalCommands::Resize(resize_args) => resize::run(backend, resize_args), LocalCommands::Doctor => doctor::run(), LocalCommands::Aws(aws_args) => aws::run(aws_args), LocalCommands::Cloudflare(cloudflare_args) => cloudflare::run(cloudflare_args), LocalCommands::Github(github_args) => github::run(github_args), LocalCommands::Zitadel(zitadel_args) => zitadel::run(zitadel_args), LocalCommands::Listmonk(listmonk_args) => listmonk::run(listmonk_args), - LocalCommands::Stop => stop::run(), - LocalCommands::Destroy => destroy::run(), - LocalCommands::Uninstall => uninstall::run(), + LocalCommands::Stop => stop::run(backend), + LocalCommands::Destroy => destroy::run(backend), + LocalCommands::Uninstall(uninstall_args) => uninstall::run(backend, uninstall_args), } } @@ -132,11 +183,17 @@ pub fn run_cmd(program: &str, args: &[&str]) -> Result<(), Box> { let refs: Vec<&str> = full.iter().map(|s| s.as_str()).collect(); return run_cmd_with_logged_args(program, &refs, &refs); } + if program == "helm" { + let full = with_helm_kube_context(args); + let refs: Vec<&str> = full.iter().map(|s| s.as_str()).collect(); + return run_cmd_with_logged_args(program, &refs, &refs); + } run_cmd_with_logged_args(program, args, args) } /// Run an external command and capture stdout. -/// For kubectl commands, automatically injects `--context` when configured. +/// For kubectl/helm commands, automatically injects the active context when +/// configured. pub fn run_cmd_output(program: &str, args: &[&str]) -> Result> { if program == "kubectl" { let full = with_kube_context(args); @@ -148,6 +205,16 @@ pub fn run_cmd_output(program: &str, args: &[&str]) -> Result Result> Ok(local_state_dir()?.join(REPO_CACHE_DIR).join(org).join(repo)) } -fn local_state_dir() -> Result> { +pub(crate) fn local_state_dir() -> Result> { let home = std::env::var("HOME") .map_err(|_| "HOME is not set; unable to determine local state directory")?; Ok(Path::new(&home).join(LOCAL_STATE_DIR)) } -fn command_exists(program: &str) -> bool { +pub(crate) fn command_exists(program: &str) -> bool { Command::new("sh") .args(["-c", &format!("command -v {} >/dev/null 2>&1", program)]) .status() @@ -195,94 +262,74 @@ fn command_exists(program: &str) -> bool { .unwrap_or(false) } -/// Ensure Colima's /etc/hosts maps a service hostname to the current ClusterIP. -pub fn sync_registry_hosts_entry( - namespace: &str, - service: &str, - hostname: &str, -) -> Result<(), Box> { - let cluster_ip = run_cmd_output( - "kubectl", - &[ - "get", - "svc", - service, - "-n", - namespace, - "-o", - "jsonpath={.spec.clusterIP}", - ], - )?; - let cluster_ip = cluster_ip.trim(); - if cluster_ip.is_empty() { - return Err(format!("Service {}/{} has no ClusterIP", namespace, service).into()); - } - - let current_ip = run_cmd_output( - "colima", - &[ - "ssh", - "--", - "sh", - "-c", - &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), - ], - ) - .unwrap_or_default(); - if current_ip.trim() == cluster_ip { - return Ok(()); +/// Poll until the Kubernetes API server is reachable. +pub(crate) fn wait_for_kubernetes() -> Result<(), Box> { + log::info!("Waiting for Kubernetes API..."); + // ~10 minutes — nested-virt apiserver can stay overloaded after package install. + for _ in 0..120 { + let result = run_cmd_output("kubectl", &["get", "--raw", "/readyz"]); + if result.is_ok() { + return Ok(()); + } + // Fall back to a cheap list if /readyz is denied on some setups. + if run_cmd_output("kubectl", &["get", "ns", "default"]).is_ok() { + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_secs(5)); } - - log::info!("Updating hosts entry: {} -> {}", hostname, cluster_ip); - - let escaped_host = hostname.replace('.', "\\."); - run_cmd( - "colima", - &[ - "ssh", - "--", - "sudo", - "sed", - "-i", - &format!("/{}/d", escaped_host), - "/etc/hosts", - ], - )?; - run_cmd( - "colima", - &[ - "ssh", - "--", - "sudo", - "sh", - "-c", - &format!("echo '{} {}' >> /etc/hosts", cluster_ip, hostname), - ], - )?; - - Ok(()) + Err("Timed out waiting for Kubernetes API".into()) } /// Pipe a YAML string into `kubectl apply -f -`. /// Automatically injects `--context` when configured. +/// +/// Uses `--validate=false` so a slow/overloaded API server (common on nested +/// virt CI while Crossplane is warming) does not fail the apply solely because +/// OpenAPI schema download timed out. Retries a few times for transient +/// connection errors. pub fn kubectl_apply_stdin(yaml: &str) -> Result<(), Box> { - let full = with_kube_context(&["apply", "-f", "-"]); - let mut child = Command::new("kubectl") - .args(&full) - .stdin(Stdio::piped()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn()?; + let full = with_kube_context(&["apply", "--validate=false", "-f", "-"]); + let mut last_status = None; + // Nested-virt CI (colima/GHA) can lose the apiserver for minutes after + // Crossplane/provider install (TLS handshake timeouts). Retry with backoff + // and re-probe the API between attempts. + const ATTEMPTS: u32 = 12; - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(yaml.as_bytes())?; - } + for attempt in 1..=ATTEMPTS { + let mut child = Command::new("kubectl") + .args(&full) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; - let status = child.wait()?; - if !status.success() { - return Err(format!("kubectl apply exited with {}", status).into()); + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(yaml.as_bytes())?; + } + + let status = child.wait()?; + if status.success() { + return Ok(()); + } + last_status = Some(status); + log::warn!( + "kubectl apply failed (attempt {}/{}, status {}); waiting for API...", + attempt, + ATTEMPTS, + status + ); + // Best-effort API recovery before the next apply. + let _ = wait_for_kubernetes(); + std::thread::sleep(std::time::Duration::from_secs(10)); } - Ok(()) + + Err(format!( + "kubectl apply exited with {} after retries", + last_status + .map(|s| s.to_string()) + .unwrap_or_else(|| "unknown".into()) + ) + .into()) } /// Apply a JSON merge patch with `kubectl patch --type merge`. @@ -313,3 +360,48 @@ pub fn kubectl_patch_merge( let logged_refs: Vec<&str> = full_logged.iter().map(|s| s.as_str()).collect(); run_cmd_with_logged_args("kubectl", &args_refs, &logged_refs) } + +#[cfg(test)] +mod tests { + use super::*; + + fn strings(args: Vec) -> Vec { + args + } + + #[test] + fn kubectl_args_prepend_context() { + assert_eq!( + strings(with_kube_context_value(&["get", "pods"], Some("kind-hops"))), + vec!["--context", "kind-hops", "get", "pods"] + ); + } + + #[test] + fn helm_upgrade_injects_kube_context_after_subcommand() { + assert_eq!( + strings(with_helm_kube_context_value( + &["upgrade", "--install", "crossplane"], + Some("kind-hops") + )), + vec![ + "upgrade", + "--kube-context", + "kind-hops", + "--install", + "crossplane" + ] + ); + } + + #[test] + fn helm_repo_commands_skip_kube_context() { + assert_eq!( + strings(with_helm_kube_context_value( + &["repo", "update", "crossplane-stable"], + Some("kind-hops") + )), + vec!["repo", "update", "crossplane-stable"] + ); + } +} diff --git a/src/commands/local/package_install.rs b/src/commands/local/package_install.rs index 28adf1c..1af9b3b 100644 --- a/src/commands/local/package_install.rs +++ b/src/commands/local/package_install.rs @@ -11,13 +11,29 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const REGISTRY_YAML: &str = include_str!("../../../bootstrap/registry/registry.yaml"); -/// Host address for `docker push` (NodePort exposed by the in-cluster registry) +/// Default docker push host for colima/kind (NodePort on localhost). +/// On Dory use [`registry_push`] — engine dockerd is not the Mac. pub const REGISTRY_PUSH: &str = "localhost:30500"; -/// Cluster-internal address used in Crossplane package references -pub const REGISTRY_PULL: &str = "registry.crossplane-system.svc.cluster.local:5000"; +/// Cluster-internal address used in Crossplane package references (all backends). +pub const REGISTRY_PULL_INCLUSTER: &str = + "registry.crossplane-system.svc.cluster.local:5000"; pub const REGISTRY_HOSTNAME: &str = "registry.crossplane-system.svc.cluster.local"; +/// Back-compat alias — prefer [`registry_pull`] when the backend is known. +pub const REGISTRY_PULL: &str = REGISTRY_PULL_INCLUSTER; + +/// Resolve the package pull address for the active local backend. +pub fn registry_pull() -> &'static str { + super::backend::resolve(None).registry_pull() +} + +/// Docker push registry host:port for the active backend. +/// Colima/kind → `localhost:30500`. Dory → `{dory-k8s-ip}:30500` on the engine bridge. +pub fn registry_push() -> String { + super::backend::resolve(None).registry_push() +} + #[derive(Clone, Debug)] pub struct RepoSpec { pub org: String, @@ -169,31 +185,25 @@ pub fn image_config_name(source: &str) -> String { format!("{prefix}{body}-{hash}") } -/// Ensure the in-cluster registry is deployed and available. +/// TLS Secret for the local package registry (server cert + CA). +pub const REGISTRY_TLS_SECRET: &str = "hops-local-registry-tls"; +const REGISTRY_TLS_SECRET_NS: &str = "crossplane-system"; + +/// Ensure the local package registry for the active backend is available. pub fn ensure_registry() -> Result<(), Box> { - let result = run_cmd_output( - "kubectl", - &[ - "get", - "deployment", - "registry", - "-n", - "crossplane-system", - "-o", - "jsonpath={.status.availableReplicas}", - ], - ); + super::backend::resolve(None).ensure_package_registry() +} - if let Ok(replicas) = result { - if replicas.trim() == "1" { - return Ok(()); - } - } +/// Ensure the in-cluster HTTPS NodePort registry is deployed and Crossplane +/// trusts its CA. Crossplane's package manager always uses HTTPS. +pub fn ensure_incluster_registry() -> Result<(), Box> { + ensure_namespace(REGISTRY_TLS_SECRET_NS)?; + ensure_registry_tls_secret()?; - log::info!("Deploying local package registry..."); + log::info!("Deploying local package registry (HTTPS)..."); kubectl_apply_stdin(REGISTRY_YAML)?; - for _ in 0..60 { + for _ in 0..90 { let out = run_cmd_output( "kubectl", &[ @@ -201,13 +211,14 @@ pub fn ensure_registry() -> Result<(), Box> { "deployment", "registry", "-n", - "crossplane-system", + REGISTRY_TLS_SECRET_NS, "-o", "jsonpath={.status.availableReplicas}", ], ); if let Ok(r) = out { if r.trim() == "1" { + ensure_crossplane_trusts_local_registry_ca()?; return Ok(()); } } @@ -217,6 +228,260 @@ pub fn ensure_registry() -> Result<(), Box> { Err("Timed out waiting for registry deployment".into()) } +fn ensure_namespace(ns: &str) -> Result<(), Box> { + let exists = run_cmd_output("kubectl", &["get", "ns", ns, "-o", "name"]).is_ok(); + if exists { + return Ok(()); + } + run_cmd("kubectl", &["create", "namespace", ns]) +} + +/// Create (or reuse) a self-signed CA + server cert for the registry Service DNS. +fn ensure_registry_tls_secret() -> Result<(), Box> { + let exists = run_cmd_output( + "kubectl", + &[ + "get", + "secret", + REGISTRY_TLS_SECRET, + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "name", + ], + ) + .is_ok(); + if exists { + return Ok(()); + } + + log::info!("Generating self-signed TLS for local package registry..."); + let dir = std::env::temp_dir().join(format!( + "hops-registry-tls-{}", + unique_suffix() + )); + fs::create_dir_all(&dir)?; + let ca_key = dir.join("ca.key"); + let ca_crt = dir.join("ca.crt"); + let tls_key = dir.join("tls.key"); + let tls_csr = dir.join("tls.csr"); + let tls_crt = dir.join("tls.crt"); + let san = dir.join("san.cnf"); + + fs::write( + &san, + "[req]\n\ + distinguished_name = dn\n\ + req_extensions = ext\n\ + prompt = no\n\ + [dn]\n\ + CN = registry.crossplane-system.svc.cluster.local\n\ + [ext]\n\ + subjectAltName = @alt\n\ + basicConstraints = CA:FALSE\n\ + keyUsage = digitalSignature, keyEncipherment\n\ + extendedKeyUsage = serverAuth\n\ + [alt]\n\ + DNS.1 = registry\n\ + DNS.2 = registry.crossplane-system\n\ + DNS.3 = registry.crossplane-system.svc\n\ + DNS.4 = registry.crossplane-system.svc.cluster.local\n\ + DNS.5 = localhost\n\ + IP.1 = 127.0.0.1\n", + )?; + + run_cmd( + "openssl", + &[ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ca_key.to_str().unwrap(), + "-out", + ca_crt.to_str().unwrap(), + "-days", + "3650", + "-subj", + "/CN=hops-local-registry-ca", + ], + )?; + run_cmd( + "openssl", + &[ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + tls_key.to_str().unwrap(), + "-out", + tls_csr.to_str().unwrap(), + "-config", + san.to_str().unwrap(), + ], + )?; + run_cmd( + "openssl", + &[ + "x509", + "-req", + "-in", + tls_csr.to_str().unwrap(), + "-CA", + ca_crt.to_str().unwrap(), + "-CAkey", + ca_key.to_str().unwrap(), + "-CAcreateserial", + "-out", + tls_crt.to_str().unwrap(), + "-days", + "3650", + "-extfile", + san.to_str().unwrap(), + "-extensions", + "ext", + ], + )?; + + run_cmd( + "kubectl", + &[ + "create", + "secret", + "generic", + REGISTRY_TLS_SECRET, + "-n", + REGISTRY_TLS_SECRET_NS, + &format!("--from-file=ca.crt={}", ca_crt.display()), + &format!("--from-file=tls.crt={}", tls_crt.display()), + &format!("--from-file=tls.key={}", tls_key.display()), + ], + )?; + let _ = fs::remove_dir_all(&dir); + Ok(()) +} + +/// Merge the local registry CA into Crossplane's trust store so package pulls +/// over HTTPS succeed without replacing public CAs (xpkg.crossplane.io). +pub fn ensure_crossplane_trusts_local_registry_ca() -> Result<(), Box> { + let xp_ready = run_cmd_output( + "kubectl", + &[ + "get", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "name", + ], + ) + .is_ok(); + if !xp_ready { + log::warn!("crossplane deployment not found yet; skip CA trust patch"); + return Ok(()); + } + + // Already patched? + let has_init = run_cmd_output( + "kubectl", + &[ + "get", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "jsonpath={.spec.template.spec.initContainers[*].name}", + ], + ) + .unwrap_or_default(); + if has_init.split_whitespace().any(|n| n == "hops-merge-registry-ca") { + return Ok(()); + } + + log::info!("Patching Crossplane to trust local registry CA..."); + // Strategic merge: initContainer concatenates system CAs + hops CA; SSL_CERT_FILE points at it. + let patch = r#"{ + "spec": { + "template": { + "spec": { + "initContainers": [ + { + "name": "hops-merge-registry-ca", + "image": "alpine:3.20", + "command": ["sh", "-c"], + "args": [ + "set -e; cat /etc/ssl/certs/ca-certificates.crt /hops-ca/ca.crt > /combined/ca-bundle.crt" + ], + "volumeMounts": [ + {"name": "hops-local-registry-ca", "mountPath": "/hops-ca", "readOnly": true}, + {"name": "hops-combined-ca", "mountPath": "/combined"} + ] + } + ], + "containers": [ + { + "name": "crossplane", + "env": [ + {"name": "SSL_CERT_FILE", "value": "/combined/ca-bundle.crt"} + ], + "volumeMounts": [ + {"name": "hops-combined-ca", "mountPath": "/combined", "readOnly": true} + ] + } + ], + "volumes": [ + { + "name": "hops-local-registry-ca", + "secret": { + "secretName": "hops-local-registry-tls", + "items": [{"key": "ca.crt", "path": "ca.crt"}] + } + }, + { + "name": "hops-combined-ca", + "emptyDir": {} + } + ] + } + } + } +}"#; + + run_cmd( + "kubectl", + &[ + "patch", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "--type", + "strategic", + "-p", + patch, + ], + )?; + + // Wait for rollout so package installs see the new trust store. + let _ = run_cmd( + "kubectl", + &[ + "rollout", + "status", + "deploy/crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "--timeout=180s", + ], + ); + Ok(()) +} + pub fn interactive_stdio_available() -> bool { io::stdin().is_terminal() && io::stdout().is_terminal() } @@ -231,7 +496,10 @@ pub fn parse_repo_install_choice(input: &str) -> Result) -> Option { +pub fn resolve_published_version_input( + input: &str, + default_version: Option<&str>, +) -> Option { let trimmed = input.trim(); if !trimmed.is_empty() { return Some(trimmed.to_string()); @@ -394,7 +662,9 @@ fn clone_repo_into_cache( "Cloning {} into local cache at {}{}...", clone_url, cache_path.display(), - branch.map(|b| format!(" (branch {})", b)).unwrap_or_default() + branch + .map(|b| format!(" (branch {})", b)) + .unwrap_or_default() ); let mut args: Vec<&str> = vec!["clone"]; if let Some(b) = branch { @@ -405,10 +675,7 @@ fn clone_repo_into_cache( Ok(()) } -fn refresh_cached_repo( - cache_path: &Path, - branch: Option<&str>, -) -> Result<(), Box> { +fn refresh_cached_repo(cache_path: &Path, branch: Option<&str>) -> Result<(), Box> { let cache_path_str = cache_path.to_string_lossy().to_string(); run_cmd( "git", @@ -445,11 +712,7 @@ fn should_ignore_path(path: &Path) -> bool { /// Watch `path` for filesystem events and invoke `rebuild` after a debounced /// quiet period. Loops until the watcher channel closes (typically Ctrl+C). -pub fn run_watch( - path: &str, - debounce_secs: u64, - mut rebuild: F, -) -> Result<(), Box> +pub fn run_watch(path: &str, debounce_secs: u64, mut rebuild: F) -> Result<(), Box> where F: FnMut() -> Result<(), Box>, { @@ -457,8 +720,8 @@ where let debounce = Duration::from_secs(debounce_secs); let (tx, rx) = mpsc::channel(); - let mut watcher = notify::recommended_watcher(move |res: notify::Result| { - match res { + let mut watcher = + notify::recommended_watcher(move |res: notify::Result| match res { Ok(event) => { let dominated_by_ignored = event.paths.iter().all(|p| should_ignore_path(p)); log::debug!( @@ -472,8 +735,7 @@ where } } Err(e) => log::debug!("watch error: {:?}", e), - } - })?; + })?; watcher.watch(&dir, RecursiveMode::Recursive)?; log::info!( diff --git a/src/commands/local/reset.rs b/src/commands/local/reset.rs index 6dbf74e..1f4c005 100644 --- a/src/commands/local/reset.rs +++ b/src/commands/local/reset.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Resetting Colima Kubernetes..."); - run_cmd("colima", &["kubernetes", "reset"])?; - log::info!("Colima Kubernetes reset complete"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.reset() } diff --git a/src/commands/local/resize.rs b/src/commands/local/resize.rs index 19a987a..2967efd 100644 --- a/src/commands/local/resize.rs +++ b/src/commands/local/resize.rs @@ -1,15 +1,15 @@ -use super::start::{resize_colima, ColimaSizeArgs}; +use super::backend::{Backend, SizeArgs}; use clap::Args; use std::error::Error; #[derive(Args, Debug, Clone)] pub struct ResizeArgs { #[command(flatten)] - pub size: ColimaSizeArgs, + pub size: SizeArgs, } -pub fn run(args: &ResizeArgs) -> Result<(), Box> { - resize_colima(&args.size)?; - log::info!("Colima resize complete"); +pub fn run(backend: Backend, args: &ResizeArgs) -> Result<(), Box> { + backend.resize(&args.size)?; + log::info!("Resize complete"); Ok(()) } diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 55bac80..a5d5802 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -1,10 +1,7 @@ -use super::{kubectl_apply_stdin, run_cmd, run_cmd_output, sync_registry_hosts_entry}; +use super::backend::{self, SizeArgs}; +use super::{kubectl_apply_stdin, run_cmd, run_cmd_output, wait_for_kubernetes}; use clap::Args; -use dialoguer::Confirm; -use serde::Deserialize; use std::error::Error; -use std::io::{IsTerminal, Write}; -use std::process::{Command, Stdio}; use std::thread; use std::time::Duration; @@ -16,110 +13,122 @@ const PROVIDER_HELM: &str = include_str!("../../../bootstrap/providers/provider- const PROVIDER_K8S: &str = include_str!("../../../bootstrap/providers/provider-kubernetes.yaml"); const PC_HELM: &str = include_str!("../../../bootstrap/helm/pc.yaml"); const PC_K8S: &str = include_str!("../../../bootstrap/k8s/pc.yaml"); -const REGISTRY: &str = include_str!("../../../bootstrap/registry/registry.yaml"); -/// Cluster-internal hostname for the package registry. -const REGISTRY_HOST: &str = "registry.crossplane-system.svc.cluster.local:5000"; -const REGISTRY_HOSTNAME: &str = "registry.crossplane-system.svc.cluster.local"; - -const DEFAULT_CPUS: u32 = 8; -const DEFAULT_MEMORY_GIB: u32 = 16; -const DEFAULT_DISK_GIB: u32 = 60; -const GIB: u64 = 1024 * 1024 * 1024; +const PROVIDER_K8S_NAME: &str = "crossplane-contrib-provider-kubernetes"; +const PROVIDER_HELM_NAME: &str = "crossplane-contrib-provider-helm"; #[derive(Args, Debug, Clone)] pub struct StartArgs { #[command(flatten)] - pub size: ColimaSizeArgs, + pub size: SizeArgs, - /// Stop and restart a running Colima VM without prompting when requested size differs. + /// Stop and restart a running cluster VM without prompting when requested size differs. #[arg(long)] pub yes: bool, + + /// Force helm upgrade and full bootstrap even when the control plane is + /// already healthy. Default is to skip expensive helm/repo work on resume. + #[arg(long)] + pub bootstrap: bool, } -#[derive(Args, Debug, Clone, Default, PartialEq, Eq)] -pub struct ColimaSizeArgs { - /// Number of CPUs to allocate to the Colima VM. - #[arg(long = "cpus", visible_alias = "cpu", value_name = "N")] - pub cpus: Option, +pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box> { + // 1. Bring the backend cluster up + backend.start(&args.size, args.yes)?; - /// Memory to allocate to the Colima VM, in GiB. - #[arg(long, value_name = "GIB")] - pub memory: Option, + // Remember the choice so stop/destroy/doctor and package installs target + // this backend without needing the flag again. + backend::persist(backend)?; - /// Disk size to allocate to the Colima VM, in GiB. - #[arg(long, value_name = "GIB")] - pub disk: Option, -} + // 2. Wait for the Kubernetes API to become reachable. + // The backend may return immediately ("already running") before the + // API server is ready, or a fresh start needs time to initialise. + wait_for_kubernetes()?; -impl ColimaSizeArgs { - pub fn any_set(&self) -> bool { - self.cpus.is_some() || self.memory.is_some() || self.disk.is_some() - } + // 3. Make the node runtime trust the cluster-internal registry over HTTP. + backend.ensure_registry_trust()?; - pub fn command_suffix(&self) -> String { - let mut parts = Vec::new(); - if let Some(cpus) = self.cpus { - parts.push(format!("--cpus {}", cpus)); - } - if let Some(memory) = self.memory { - parts.push(format!("--memory {}", memory)); - } - if let Some(disk) = self.disk { - parts.push(format!("--disk {}", disk)); - } + // After a docker daemon restart (colima path), re-confirm the node is Ready + // so Crossplane pods are not stuck Pending on NotReady. + log::info!("Waiting for Kubernetes nodes to be Ready..."); + run_cmd( + "kubectl", + &[ + "wait", + "--for=condition=Ready", + "nodes", + "--all", + "--timeout=300s", + ], + )?; - if parts.is_empty() { - String::new() - } else { - format!(" {}", parts.join(" ")) - } + // Fast path: cluster already has a healthy hops control plane. + // Skip helm repo update / upgrade and long provider waits — those dominate + // resume latency (CI stop/start, laptop reopen with dory-k8s still up). + if !args.bootstrap && control_plane_healthy() { + log::info!( + "Control plane already healthy; skipping helm upgrade and bootstrap reapply \ + (pass --bootstrap to force)" + ); + ensure_registry_ready(backend)?; + log::info!("Local environment is ready"); + return Ok(()); } -} - -#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] -pub(crate) struct ColimaInstance { - #[serde(default)] - status: String, - #[serde(default)] - cpus: Option, - #[serde(default)] - memory: Option, - #[serde(default)] - disk: Option, -} -impl ColimaInstance { - fn is_running(&self) -> bool { - self.status.eq_ignore_ascii_case("running") + if args.bootstrap { + log::info!("--bootstrap: forcing helm upgrade and full control-plane reapply"); } - fn memory_gib(&self) -> Option { - self.memory.map(bytes_to_gib) - } + bootstrap_control_plane()?; + ensure_registry_ready(backend)?; - fn disk_gib(&self) -> Option { - self.disk.map(bytes_to_gib) - } + log::info!("Local environment is ready"); + Ok(()) } -pub fn run(args: &StartArgs) -> Result<(), Box> { - let instance = colima_instance()?; - validate_requested_size(&args.size, instance.as_ref())?; - - // 1. Start Colima with Kubernetes - start_or_resize_colima(args, instance.as_ref())?; +/// True when Crossplane core, both default providers, and the package registry +/// are already Available/Healthy — the expensive bootstrap steps can be skipped. +fn control_plane_healthy() -> bool { + deployment_available("crossplane-system", "crossplane") + && deployment_available("crossplane-system", "registry") + && provider_healthy(PROVIDER_K8S_NAME) + && provider_healthy(PROVIDER_HELM_NAME) +} - // 2. Wait for the Kubernetes API to become reachable. - // Colima may return immediately ("already running") before the - // API server is ready, or a fresh start needs time to initialise. - wait_for_kubernetes()?; +fn deployment_available(namespace: &str, name: &str) -> bool { + run_cmd_output( + "kubectl", + &[ + "get", + "deployment", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Available\")].status}", + ], + ) + .map(|s| s.trim() == "True") + .unwrap_or(false) +} - // 3. Configure Docker in the VM to allow HTTP pulls from the - // cluster-internal registry. Without this the kubelet's Docker - // daemon defaults to HTTPS and fails. - configure_docker_insecure_registry()?; +fn provider_healthy(provider: &str) -> bool { + run_cmd_output( + "kubectl", + &[ + "get", + "provider.pkg.crossplane.io", + provider, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Healthy\")].status}", + ], + ) + .map(|s| s.trim() == "True") + .unwrap_or(false) +} +/// Helm + Crossplane + providers + ProviderConfigs (the slow cold-start path). +fn bootstrap_control_plane() -> Result<(), Box> { // 4. Add Crossplane Helm repo log::info!("Adding Crossplane Helm repo..."); run_cmd( @@ -131,13 +140,22 @@ pub fn run(args: &StartArgs) -> Result<(), Box> { "https://charts.crossplane.io/stable", ], )?; - run_cmd("helm", &["repo", "update"])?; + // Update only our repo — a bare `helm repo update` fails outright when any + // unrelated repo in the user's helm config has gone stale. + run_cmd("helm", &["repo", "update", "crossplane-stable"])?; // 5. Install Crossplane + // + // Do not use helm --wait: on nested-virt CI (colima/GHA) image pulls and + // scheduling can exceed helm's single wait window, and a failed --wait + // leaves us without structured kubectl diagnostics. Apply the chart, then + // poll deployments ourselves with a longer budget + failure dumps. + // + // After stop/start the k3s API can report nodes Ready while openapi/v2 is + // still timing out (helm validate fails). Retry helm with API re-probes. log::info!("Installing Crossplane..."); - run_cmd( - "helm", - &[ + { + let helm_args = [ "upgrade", "--install", "crossplane", @@ -145,15 +163,43 @@ pub fn run(args: &StartArgs) -> Result<(), Box> { "-n", "crossplane-system", "--create-namespace", - "--wait", "--timeout", "5m", - ], - )?; + ]; + let mut last_err: Option> = None; + for attempt in 1..=6 { + wait_for_kubernetes()?; + match run_cmd("helm", &helm_args) { + Ok(()) => { + last_err = None; + break; + } + Err(e) => { + log::warn!("helm install attempt {attempt}/6 failed: {e}"); + last_err = Some(e); + std::thread::sleep(std::time::Duration::from_secs(20)); + } + } + } + if let Some(e) = last_err { + return Err(e); + } + } - // 6. Wait for Crossplane deployment + // 6. Wait for Crossplane core deployment. + // rbac-manager can flap under nested-virt resource pressure; the core + // controller is what providers need. Best-effort wait for rbac-manager. log::info!("Waiting for Crossplane to be ready..."); - wait_for_deployment("crossplane-system", "crossplane")?; + wait_for_deployment_with_diagnostics("crossplane-system", "crossplane")?; + if let Err(e) = wait_for_deployment_attempts("crossplane-system", "crossplane-rbac-manager", 36) + { + log::warn!( + "crossplane-rbac-manager not Available yet ({e}); continuing — core Crossplane is ready" + ); + } + + // API can be briefly overloaded right after Crossplane becomes leader. + wait_for_kubernetes()?; // 7. Deploy per-provider DRCs (each pins its own cluster-admin SA) log::info!("Applying DeploymentRuntimeConfigs (per-provider)..."); @@ -170,297 +216,64 @@ pub fn run(args: &StartArgs) -> Result<(), Box> { wait_for_crd("providerconfigs.helm.m.crossplane.io")?; wait_for_crd("providerconfigs.kubernetes.m.crossplane.io")?; + // Re-check API before ProviderConfigs (openapi validation can time out if + // the apiserver is still busy — apply also uses --validate=false). + wait_for_kubernetes()?; + // 10. Apply ProviderConfigs log::info!("Applying ProviderConfigs..."); kubectl_apply_stdin(PC_HELM)?; kubectl_apply_stdin(PC_K8S)?; - // 11. Deploy local OCI registry for Crossplane packages - log::info!("Deploying local package registry..."); - kubectl_apply_stdin(REGISTRY)?; - wait_for_deployment("crossplane-system", "registry")?; + // 11. Wait for providers Healthy. + // + // CRDs can become Established while the provider runtime pods are still + // pulling images / rolling. `hops local doctor` requires Healthy=True, and + // the kind smoke path runs doctor immediately after start — without this + // wait, cold CI flakes with "Provider healthy — Healthy=False". + log::info!("Waiting for providers to become Healthy..."); + wait_for_provider_healthy(PROVIDER_K8S_NAME)?; + wait_for_provider_healthy(PROVIDER_HELM_NAME)?; - // 12. Map the registry's cluster-internal hostname to its ClusterIP - // inside the VM so the kubelet can resolve it. - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; - - log::info!("Local environment is ready"); Ok(()) } -fn start_or_resize_colima( - args: &StartArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - let is_running = instance.map(ColimaInstance::is_running).unwrap_or(false); - - if is_running && args.size.any_set() { - let changes = requested_size_changes(&args.size, instance.expect("checked is_running")); - if !changes.is_empty() { - confirm_running_resize(args, &changes)?; - resize_existing_colima(&args.size, instance)?; - return Ok(()); - } - - log::info!("Requested Colima size already matches the running VM"); - } - - log::info!("Starting Colima with Kubernetes..."); - - let size = if is_running { - ColimaSizeArgs::default() - } else { - args.size.clone() - }; - let include_defaults = instance.is_none(); - start_colima(&size, include_defaults) -} - -pub(crate) fn resize_colima(size: &ColimaSizeArgs) -> Result<(), Box> { - if !size.any_set() { - return Err("Specify at least one of --cpus, --memory, or --disk".into()); - } - - let instance = colima_instance()?; - let instance = instance - .as_ref() - .ok_or("No Colima instance exists yet; use `hops local start` to create one")?; - - validate_requested_size(size, Some(instance))?; - resize_existing_colima(size, Some(instance)) -} - -fn resize_existing_colima( - size: &ColimaSizeArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - if instance.map(ColimaInstance::is_running).unwrap_or(false) { - log::info!("Stopping Colima to apply requested size..."); - run_cmd("colima", &["stop"])?; - } - - log::info!("Starting Colima with requested size..."); - start_colima(size, false) -} - -fn start_colima(size: &ColimaSizeArgs, include_defaults: bool) -> Result<(), Box> { - let args = colima_start_args(size, include_defaults); - let refs: Vec<&str> = args.iter().map(String::as_str).collect(); - run_cmd("colima", &refs) -} - -fn colima_start_args(size: &ColimaSizeArgs, include_defaults: bool) -> Vec { - let mut args = vec!["start".to_string(), "--kubernetes".to_string()]; - - if let Some(cpus) = size.cpus.or(include_defaults.then_some(DEFAULT_CPUS)) { - args.push("--cpus".to_string()); - args.push(cpus.to_string()); - } - if let Some(memory) = size - .memory - .or(include_defaults.then_some(DEFAULT_MEMORY_GIB)) - { - args.push("--memory".to_string()); - args.push(memory.to_string()); - } - if let Some(disk) = size.disk.or(include_defaults.then_some(DEFAULT_DISK_GIB)) { - args.push("--disk".to_string()); - args.push(disk.to_string()); - } - - args -} - -fn confirm_running_resize(args: &StartArgs, changes: &[String]) -> Result<(), Box> { - if args.yes { - return Ok(()); - } - - let change_text = changes.join(", "); - let resize_command = format!("hops local resize{}", args.size.command_suffix()); - let start_command = format!("hops local start{} --yes", args.size.command_suffix()); - - if !std::io::stdin().is_terminal() { - return Err(format!( - "Colima is already running with different size ({change_text}). Run `{resize_command}` first, or rerun `{start_command}` to stop and resize automatically." - ) - .into()); - } - - let confirmed = Confirm::new() - .with_prompt(format!( - "Colima is already running with different size ({change_text}). Stop and restart it now?" - )) - .default(false) - .interact()?; - - if confirmed { - Ok(()) - } else { - Err(format!( - "Colima size was not changed. Run `{resize_command}` first, then rerun `hops local start`." - ) - .into()) - } -} - -fn validate_requested_size( - size: &ColimaSizeArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - if let (Some(requested), Some(current)) = - (size.disk, instance.and_then(ColimaInstance::disk_gib)) - { - if requested < current { - return Err(format!( - "Colima disk cannot be shrunk from {current}GiB to {requested}GiB. Use --disk {current} or larger, or destroy and recreate the VM." - ) - .into()); - } - } - +/// In-cluster package registry + backend node/engine wiring. +fn ensure_registry_ready(backend: backend::Backend) -> Result<(), Box> { + // Crossplane package pulls run in the pod network → Service DNS + ClusterIP. + // Docker push: colima/kind → localhost:30500 (host NodePort); dory → + // {dory-k8s-ip}:30500 on the engine docker bridge (daemon is in-engine). + wait_for_kubernetes()?; + log::info!("Pre-pulling registry:2 (best effort)..."); + let _ = run_cmd("docker", &["pull", "registry:2"]); + // TLS secret + registry Deployment + Crossplane CA trust (package manager is HTTPS-only). + log::info!("Deploying local package registry (HTTPS)..."); + backend.ensure_package_registry()?; + wait_for_deployment_with_diagnostics("crossplane-system", "registry")?; + backend::wire_local_registry(backend)?; Ok(()) } -fn requested_size_changes(size: &ColimaSizeArgs, instance: &ColimaInstance) -> Vec { - let mut changes = Vec::new(); - - if let Some(requested) = size.cpus { - match instance.cpus { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("cpus {current} -> {requested}")), - None => changes.push(format!("cpus unknown -> {requested}")), - } - } - if let Some(requested) = size.memory { - match instance.memory_gib() { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("memory {current}GiB -> {requested}GiB")), - None => changes.push(format!("memory unknown -> {requested}GiB")), - } - } - if let Some(requested) = size.disk { - match instance.disk_gib() { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("disk {current}GiB -> {requested}GiB")), - None => changes.push(format!("disk unknown -> {requested}GiB")), - } - } - - changes -} - -pub(crate) fn colima_instance() -> Result, Box> { - let output = match run_cmd_output("colima", &["list", "--json"]) { - Ok(output) => output, - Err(_) => return Ok(None), - }; - - parse_colima_list(&output) -} - -fn parse_colima_list(output: &str) -> Result, Box> { - let trimmed = output.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - if let Ok(instance) = serde_json::from_str::(trimmed) { - return Ok(Some(instance)); - } - - if let Ok(instances) = serde_json::from_str::>(trimmed) { - return Ok(instances.into_iter().next()); - } - - for line in trimmed - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - { - if let Ok(instance) = serde_json::from_str::(line) { - return Ok(Some(instance)); - } - } - - Err("Unable to parse `colima list --json` output".into()) -} - -fn bytes_to_gib(bytes: u64) -> u32 { - (bytes / GIB) as u32 -} - -/// Add the cluster-internal registry to Docker's insecure-registries list -/// inside the Colima VM. Docker defaults to HTTPS for non-localhost registries; -/// our in-cluster registry speaks plain HTTP. -fn configure_docker_insecure_registry() -> Result<(), Box> { - let config = run_cmd_output("colima", &["ssh", "--", "cat", "/etc/docker/daemon.json"])?; - - if config.contains("insecure-registries") { - return Ok(()); - } - - log::info!("Configuring Docker for insecure local registry..."); - - // Insert the insecure-registries key before the final closing brace. - let new_config = if let Some(pos) = config.rfind('}') { - let prefix = config[..pos].trim_end(); - format!( - "{},\n \"insecure-registries\": [\"{}\"]\n}}\n", - prefix, REGISTRY_HOST - ) - } else { - return Err("Invalid daemon.json: no closing brace".into()); - }; - - let mut child = Command::new("colima") - .args(["ssh", "--", "sudo", "tee", "/etc/docker/daemon.json"]) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn()?; - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(new_config.as_bytes())?; - } - let status = child.wait()?; - if !status.success() { - return Err("Failed to write Docker daemon.json".into()); - } - - log::info!("Restarting Docker daemon..."); - run_cmd( - "colima", - &["ssh", "--", "sudo", "systemctl", "restart", "docker"], - )?; - - // Wait for Docker to come back. - for _ in 0..30 { - if run_cmd_output("docker", &["info"]).is_ok() { - // Docker restart can temporarily disrupt the Kubernetes API. - wait_for_kubernetes()?; - return Ok(()); - } - thread::sleep(Duration::from_secs(2)); - } - Err("Docker did not come back after restart".into()) -} - -/// Poll until the Kubernetes API server is reachable. -fn wait_for_kubernetes() -> Result<(), Box> { - log::info!("Waiting for Kubernetes API..."); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["cluster-info"]); - if result.is_ok() { - return Ok(()); +/// Longer wait used for Crossplane on cold nested-virt runners (~15 minutes). +fn wait_for_deployment_with_diagnostics( + namespace: &str, + name: &str, +) -> Result<(), Box> { + match wait_for_deployment_attempts(namespace, name, 180) { + Ok(()) => Ok(()), + Err(e) => { + dump_namespace_diagnostics(namespace); + Err(e) } - thread::sleep(Duration::from_secs(5)); } - Err("Timed out waiting for Kubernetes API".into()) } -/// Poll until a deployment's Available condition is True. -fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box> { - for _ in 0..60 { +fn wait_for_deployment_attempts( + namespace: &str, + name: &str, + attempts: u32, +) -> Result<(), Box> { + for i in 0..attempts { let output = run_cmd_output( "kubectl", &[ @@ -480,142 +293,175 @@ fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box } } + // Periodic progress so CI logs show the wait is alive. + if i > 0 && i % 12 == 0 { + log::info!( + "Still waiting for deployment {}/{} ({}s elapsed)...", + namespace, + name, + i * 5 + ); + let _ = run_cmd( + "kubectl", + &["get", "pods", "-n", namespace, "-o", "wide"], + ); + } + thread::sleep(Duration::from_secs(5)); } Err(format!("Timed out waiting for deployment {}/{}", namespace, name).into()) } -/// Poll until a CRD exists in the cluster. +fn dump_namespace_diagnostics(namespace: &str) { + log::error!("Diagnostics for namespace {} after readiness timeout:", namespace); + let _ = run_cmd("kubectl", &["get", "pods", "-n", namespace, "-o", "wide"]); + let _ = run_cmd("kubectl", &["describe", "pods", "-n", namespace]); + let _ = run_cmd( + "kubectl", + &[ + "get", + "events", + "-n", + namespace, + "--sort-by=.lastTimestamp", + ], + ); + let _ = run_cmd("kubectl", &["get", "nodes", "-o", "wide"]); +} + +/// Poll until a CRD exists **and** is Established (API serves the kind). +/// +/// Merely creating the CRD object is not enough: under load the apiserver can +/// return the CRD while discovery still lacks the kind, causing +/// `no matches for kind "ProviderConfig"` on the next apply. fn wait_for_crd(crd: &str) -> Result<(), Box> { log::info!("Waiting for CRD {}...", crd); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["get", "crd", crd]); - if result.is_ok() { - return Ok(()); + for _ in 0..120 { + let exists = run_cmd_output("kubectl", &["get", "crd", crd]).is_ok(); + if exists { + let established = run_cmd_output( + "kubectl", + &[ + "get", + "crd", + crd, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Established\")].status}", + ], + ) + .unwrap_or_default(); + if established.trim() == "True" { + // Brief settle so discovery caches pick up the new kind. + thread::sleep(Duration::from_secs(2)); + return Ok(()); + } } thread::sleep(Duration::from_secs(5)); } - Err(format!("Timed out waiting for CRD {}", crd).into()) + Err(format!("Timed out waiting for CRD {} to be Established", crd).into()) } -#[cfg(test)] -mod tests { - use super::*; - - fn instance(status: &str, cpus: u32, memory_gib: u32, disk_gib: u32) -> ColimaInstance { - ColimaInstance { - status: status.to_string(), - cpus: Some(cpus), - memory: Some(memory_gib as u64 * GIB), - disk: Some(disk_gib as u64 * GIB), +/// Poll until a Crossplane Provider reports Healthy=True (~10 minutes). +/// +/// Package install can establish CRDs before the runtime Deployment is Ready. +/// Matching doctor's expectation here keeps `hops local start` from returning +/// while the cluster still looks half-bootstrapped. +fn wait_for_provider_healthy(provider: &str) -> Result<(), Box> { + log::info!("Waiting for Provider {} Healthy...", provider); + for i in 0..120 { + let healthy = run_cmd_output( + "kubectl", + &[ + "get", + "provider.pkg.crossplane.io", + provider, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Healthy\")].status}", + ], + ) + .unwrap_or_default(); + if healthy.trim() == "True" { + return Ok(()); } - } - - #[test] - fn colima_start_args_use_hops_defaults_for_new_profiles() { - let args = colima_start_args(&ColimaSizeArgs::default(), true); - - assert_eq!( - args, - vec![ - "start", - "--kubernetes", - "--cpus", - "8", - "--memory", - "16", - "--disk", - "60" - ] - ); - } - - #[test] - fn colima_start_args_pass_only_requested_size_for_existing_profiles() { - let size = ColimaSizeArgs { - cpus: Some(12), - memory: Some(32), - disk: None, - }; - - let args = colima_start_args(&size, false); - - assert_eq!( - args, - vec!["start", "--kubernetes", "--cpus", "12", "--memory", "32"] - ); - } - #[test] - fn requested_size_changes_compare_only_explicit_fields() { - let current = instance("Running", 8, 16, 60); - let size = ColimaSizeArgs { - cpus: None, - memory: Some(32), - disk: None, - }; - - assert_eq!( - requested_size_changes(&size, ¤t), - vec!["memory 16GiB -> 32GiB"] - ); - } - - #[test] - fn requested_size_changes_treat_missing_current_value_as_change() { - let current = ColimaInstance { - status: "Running".to_string(), - cpus: None, - memory: None, - disk: None, - }; - let size = ColimaSizeArgs { - cpus: Some(12), - memory: None, - disk: None, - }; - - assert_eq!( - requested_size_changes(&size, ¤t), - vec!["cpus unknown -> 12"] - ); - } - - #[test] - fn parse_colima_list_accepts_single_object() { - let output = r#"{"name":"default","status":"Stopped","arch":"aarch64","cpus":8,"memory":17179869184,"disk":64424509440,"runtime":"docker+k3s"}"#; - - let parsed = parse_colima_list(output).expect("parse").expect("instance"); + if i > 0 && i % 12 == 0 { + let installed = run_cmd_output( + "kubectl", + &[ + "get", + "provider.pkg.crossplane.io", + provider, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Installed\")].status}", + ], + ) + .unwrap_or_default(); + log::info!( + "Still waiting for Provider {} Healthy ({}s elapsed; Installed={}, Healthy={})...", + provider, + i * 5, + if installed.trim().is_empty() { + "" + } else { + installed.trim() + }, + if healthy.trim().is_empty() { + "" + } else { + healthy.trim() + } + ); + let _ = run_cmd( + "kubectl", + &["get", "pods", "-n", "crossplane-system", "-o", "wide"], + ); + let _ = run_cmd( + "kubectl", + &["get", "provider.pkg.crossplane.io", provider, "-o", "yaml"], + ); + } - assert_eq!(parsed.status, "Stopped"); - assert_eq!(parsed.cpus, Some(8)); - assert_eq!(parsed.memory_gib(), Some(16)); - assert_eq!(parsed.disk_gib(), Some(60)); + thread::sleep(Duration::from_secs(5)); } - #[test] - fn parse_colima_list_accepts_array_output() { - let output = r#"[{"status":"Running","cpus":12,"memory":34359738368,"disk":107374182400}]"#; - - let parsed = parse_colima_list(output).expect("parse").expect("instance"); + let _ = run_cmd( + "kubectl", + &["get", "provider.pkg.crossplane.io", provider, "-o", "yaml"], + ); + dump_namespace_diagnostics("crossplane-system"); + Err(format!( + "Timed out waiting for Provider {} to become Healthy", + provider + ) + .into()) +} - assert!(parsed.is_running()); - assert_eq!(parsed.cpus, Some(12)); - assert_eq!(parsed.memory_gib(), Some(32)); - assert_eq!(parsed.disk_gib(), Some(100)); - } +#[cfg(test)] +mod tests { + use super::*; #[test] - fn validate_requested_size_rejects_disk_shrink() { - let current = instance("Stopped", 8, 16, 100); - let size = ColimaSizeArgs { - cpus: None, - memory: None, - disk: Some(60), - }; - - let err = validate_requested_size(&size, Some(¤t)).expect_err("disk shrink"); - - assert!(err.to_string().contains("cannot be shrunk")); + fn start_args_bootstrap_defaults_false() { + // clap default: bootstrap only when --bootstrap is passed + assert!(!StartArgs { + size: SizeArgs { + cpus: None, + memory: None, + disk: None, + }, + yes: false, + bootstrap: false, + } + .bootstrap); + assert!(StartArgs { + size: SizeArgs { + cpus: None, + memory: None, + disk: None, + }, + yes: false, + bootstrap: true, + } + .bootstrap); } } diff --git a/src/commands/local/stop.rs b/src/commands/local/stop.rs index 16ffa2d..4fa2bf5 100644 --- a/src/commands/local/stop.rs +++ b/src/commands/local/stop.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Stopping Colima..."); - run_cmd("colima", &["stop"])?; - log::info!("Colima stopped"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.stop() } diff --git a/src/commands/local/uninstall.rs b/src/commands/local/uninstall.rs index d84e1b0..6cc4a4e 100644 --- a/src/commands/local/uninstall.rs +++ b/src/commands/local/uninstall.rs @@ -1,21 +1,171 @@ -use super::run_cmd; +use super::backend::Backend; +use clap::Args; use std::error::Error; use std::io::{self, Write}; -pub fn run() -> Result<(), Box> { - print!("Uninstall Colima? This will remove the binary. [y/N] "); - io::stdout().flush()?; +#[derive(Args, Debug)] +pub struct UninstallArgs { + /// Uninstall the backend binary even if its local cluster still exists. + #[arg(long)] + pub force: bool, +} - let mut input = String::new(); - io::stdin().read_line(&mut input)?; +pub fn run(backend: Backend, args: &UninstallArgs) -> Result<(), Box> { + run_inner( + backend, + args.force, + Backend::cluster_exists, + Backend::uninstall, + super::backend::clear_persisted, + confirm_uninstall, + ) +} + +fn run_inner( + backend: Backend, + force: bool, + cluster_exists: ClusterExists, + uninstall: Uninstall, + clear_persisted: ClearPersisted, + confirm: Confirm, +) -> Result<(), Box> +where + ClusterExists: FnOnce(Backend) -> bool, + Uninstall: FnOnce(Backend) -> Result<(), Box>, + ClearPersisted: FnOnce() -> Result<(), Box>, + Confirm: FnOnce(Backend) -> Result>, +{ + if !force && cluster_exists(backend) { + return Err("destroy the cluster first: `hops local destroy` (or pass --force to uninstall the backend binary anyway)".into()); + } - if input.trim().eq_ignore_ascii_case("y") { - log::info!("Uninstalling Colima..."); - run_cmd("brew", &["uninstall", "colima"])?; - log::info!("Colima uninstalled"); + if confirm(backend)? { + uninstall(backend)?; + clear_persisted()?; } else { log::info!("Uninstall cancelled"); } Ok(()) } + +fn confirm_uninstall(backend: Backend) -> Result> { + print!( + "Uninstall {}? This will remove the binary. [y/N] ", + backend.name() + ); + io::stdout().flush()?; + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + + Ok(input.trim().eq_ignore_ascii_case("y")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::rc::Rc; + + #[test] + fn refuses_uninstall_when_cluster_exists_without_force() { + let events = Rc::new(RefCell::new(Vec::new())); + let err = run_inner( + Backend::Kind, + false, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("cluster_exists"); + true + } + }, + |_| { + events.borrow_mut().push("uninstall"); + Ok(()) + }, + || { + events.borrow_mut().push("clear"); + Ok(()) + }, + |_| { + events.borrow_mut().push("confirm"); + Ok(true) + }, + ) + .expect_err("cluster guard should refuse uninstall"); + + assert!(err.to_string().contains("destroy the cluster first")); + assert_eq!(&*events.borrow(), &["cluster_exists"]); + } + + #[test] + fn force_uninstalls_without_cluster_probe_and_clears_after_success() { + let events = Rc::new(RefCell::new(Vec::new())); + run_inner( + Backend::Kind, + true, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("cluster_exists"); + true + } + }, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("uninstall"); + Ok(()) + } + }, + { + let events = Rc::clone(&events); + move || { + events.borrow_mut().push("clear"); + Ok(()) + } + }, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("confirm"); + Ok(true) + } + }, + ) + .expect("force should allow uninstall"); + + assert_eq!(&*events.borrow(), &["confirm", "uninstall", "clear"]); + } + + #[test] + fn failed_uninstall_does_not_clear_persisted_backend() { + let events = Rc::new(RefCell::new(Vec::new())); + let err = run_inner( + Backend::Kind, + false, + |_| false, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("uninstall"); + Err("brew failed".into()) + } + }, + { + let events = Rc::clone(&events); + move || { + events.borrow_mut().push("clear"); + Ok(()) + } + }, + |_| Ok(true), + ) + .expect_err("uninstall failure should propagate"); + + assert_eq!(err.to_string(), "brew failed"); + assert_eq!(&*events.borrow(), &["uninstall"]); + } +} diff --git a/src/commands/provider/install.rs b/src/commands/provider/install.rs index 321b0d6..de0a2da 100644 --- a/src/commands/provider/install.rs +++ b/src/commands/provider/install.rs @@ -1,11 +1,11 @@ +use crate::commands::local::backend::{self, Backend}; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout_at, ensure_registry, parse_repo_spec, resolve_repo_install_target, run_watch, sanitize_name_component, RepoInstallTarget, RepoSpec, - REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH, + registry_pull, registry_push, }; use crate::commands::local::{ - kubectl_apply_stdin, run_cmd, run_cmd_output, sync_registry_hosts_entry, - HOPS_KUBE_CONTEXT_ENV, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, + kubectl_apply_stdin, run_cmd, run_cmd_output, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, }; use clap::Args; use serde::Deserialize; @@ -53,6 +53,10 @@ pub struct ProviderInstallArgs { #[arg(long)] pub context: Option, + /// Local cluster backend whose node should be wired for local package pulls. + #[arg(long, value_enum)] + pub backend: Option, + /// Watch the project directory for changes and re-run install automatically #[arg(long, conflicts_with = "repo")] pub watch: bool, @@ -73,9 +77,7 @@ struct PackageMetadata { } pub fn run(args: &ProviderInstallArgs) -> Result<(), Box> { - if let Some(ctx) = &args.context { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + let backend = backend::activate(args.backend, args.context.as_deref()); match (args.repo.as_deref(), args.version.as_deref()) { (Some(repo), Some(version)) => { @@ -86,10 +88,14 @@ pub fn run(args: &ProviderInstallArgs) -> Result<(), Box> { args.skip_dependency_resolution, args.version_prefix.as_deref(), args.branch.as_deref(), + backend, + args.backend, + args.context.as_deref(), ), (None, _) => { let path = args.path.as_deref().unwrap_or("."); let prefix = args.version_prefix.clone(); + prepare_local_registry(backend, args.backend, args.context.as_deref())?; run_local_path(path, args.skip_dependency_resolution, prefix.as_deref())?; if args.watch { @@ -111,11 +117,15 @@ fn run_repo_install( skip_dependency_resolution: bool, version_prefix: Option<&str>, branch: Option<&str>, + backend: Backend, + backend_flag: Option, + context: Option<&str>, ) -> Result<(), Box> { let spec = parse_repo_spec(repo)?; match resolve_repo_install_target(&spec)? { RepoInstallTarget::SourceBuild => { let cache_path = ensure_cached_repo_checkout_at(&spec, branch)?; + prepare_local_registry(backend, backend_flag, context)?; run_local_path( &cache_path.to_string_lossy(), skip_dependency_resolution, @@ -128,6 +138,15 @@ fn run_repo_install( } } +fn prepare_local_registry( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + ensure_registry()?; + backend::wire_local_registry_for_target(backend, backend_flag, context) +} + fn apply_repo_version( repo: &str, version: &str, @@ -154,16 +173,12 @@ fn apply_repo_version_spec( sanitize_name_component(&spec.repo) ); - log::info!( - "Applying Provider '{}' from {}", - provider_name, - package_ref - ); + log::info!("Applying Provider '{}' from {}", provider_name, package_ref); let providers_json = run_cmd_output( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], )?; - let resolved = resolve_provider_target(&provider_name, &providers_json, REGISTRY_PULL)?; + let resolved = resolve_provider_target(&provider_name, &providers_json, registry_pull())?; // Published install pulls directly from ghcr.io — no ImageConfig rewrite, // no local registry push, no runtime-image override. apply_provider_resources( @@ -189,9 +204,6 @@ fn run_local_path( let provider_name = read_provider_name(dir)?; log::info!("Provider package name: {}", provider_name); - ensure_registry()?; - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; - // Resolve the existing upstream Provider before building so we can: // 1. carry the upstream package URL into the new `spec.package` (Crossplane // records the URL-without-tag as the Lock Source; deps declared as @@ -204,8 +216,8 @@ fn run_local_path( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], )?; - let resolved = resolve_provider_target(&provider_name, &providers_json, REGISTRY_PULL)?; - let upstream_url_prefix = recover_upstream_url_prefix(&resolved, REGISTRY_PULL)?; + let resolved = resolve_provider_target(&provider_name, &providers_json, registry_pull())?; + let upstream_url_prefix = recover_upstream_url_prefix(&resolved, registry_pull())?; let upstream_major = parse_major_version(&resolved.existing_package); ensure_build_submodule(dir)?; @@ -220,7 +232,7 @@ fn run_local_path( let xpkg_path = find_xpkg_for_provider(dir, &provider_name, arch)?; log::info!("Located xpkg: {}", xpkg_path.display()); - let local_image_path_for_tag = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); + let local_image_path_for_tag = format!("{}/hops-ops/{}", registry_pull(), provider_name); let dev_tag = dev_tag_for_file( &xpkg_path, version_prefix, @@ -228,11 +240,8 @@ fn run_local_path( &local_image_path_for_tag, )?; - let push_xpkg_ref = format!( - "{}/hops-ops/{}:{}", - REGISTRY_PUSH, provider_name, dev_tag - ); - let local_pull_xpkg_path = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); + let push_xpkg_ref = format!("{}/hops-ops/{}:{}", registry_push(), provider_name, dev_tag); + let local_pull_xpkg_path = format!("{}/hops-ops/{}", registry_pull(), provider_name); log::info!("Pushing xpkg to {}...", push_xpkg_ref); crossplane_xpkg_push(&xpkg_path, &push_xpkg_ref)?; @@ -277,11 +286,7 @@ fn read_provider_name(dir: &Path) -> Result> { let name = parsed.metadata.name.trim().to_string(); if name.is_empty() { - return Err(format!( - "{} has no metadata.name", - crossplane_yaml.display() - ) - .into()); + return Err(format!("{} has no metadata.name", crossplane_yaml.display()).into()); } Ok(name) } @@ -293,7 +298,10 @@ fn ensure_build_submodule(dir: &Path) -> Result<(), Box> { } if !dir.join(".gitmodules").is_file() { - log::debug!("No .gitmodules in {}; skipping submodule init", dir.display()); + log::debug!( + "No .gitmodules in {}; skipping submodule init", + dir.display() + ); return Ok(()); } @@ -344,28 +352,19 @@ fn find_xpkg_for_provider( .filter_map(|e| e.ok()) .filter(|e| { let p = e.path(); - p.extension().map_or(false, |ext| ext == "xpkg") + p.extension().is_some_and(|ext| ext == "xpkg") && p.file_name() .and_then(|n| n.to_str()) - .map_or(false, |n| n.starts_with(&prefix)) + .is_some_and(|n| n.starts_with(&prefix)) }) .map(|e| e.path()) .collect(); if candidates.is_empty() { - return Err(format!( - "no {}*.xpkg found in {}", - prefix, - xpkg_dir.display() - ) - .into()); + return Err(format!("no {}*.xpkg found in {}", prefix, xpkg_dir.display()).into()); } - candidates.sort_by_key(|p| { - fs::metadata(p) - .and_then(|m| m.modified()) - .ok() - }); + candidates.sort_by_key(|p| fs::metadata(p).and_then(|m| m.modified()).ok()); Ok(candidates.pop().unwrap()) } @@ -395,7 +394,10 @@ fn local_runtime_image_ref(provider_name: &str, arch: &str, tag: &str) -> String // package manager, so they need the node-pullable local registry address. format!( "{}/hops-ops/{}-{}:{}", - REGISTRY_PUSH, provider_name, arch, tag + registry_push(), + provider_name, + arch, + tag ) } @@ -500,9 +502,10 @@ fn next_local_patch_for_major(image_path: &str, major: u64) -> Result bool { let trimmed = s.strip_prefix('v').unwrap_or(s); let parts: Vec<&str> = trimmed.splitn(3, '.').collect(); - parts.len() == 3 && parts.iter().all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) + parts.len() == 3 + && parts + .iter() + .all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) } /// Parse a `vN` (or `N`) bare-major version string, e.g. `v1` -> `Some(1)`. @@ -1104,11 +1110,8 @@ mod tests { #[test] fn build_runtime_config_yaml_overrides_image_when_provided() { - let with_image = build_runtime_config_yaml( - "p-runtime", - "p", - Some("registry.example/p-arm64:dev-abc"), - ); + let with_image = + build_runtime_config_yaml("p-runtime", "p", Some("registry.example/p-arm64:dev-abc")); assert!(with_image.contains("name: package-runtime")); assert!(with_image.contains("image: registry.example/p-arm64:dev-abc")); @@ -1119,7 +1122,6 @@ mod tests { assert!(without.contains("app.kubernetes.io/managed-by: hops-provider-install")); } - #[test] fn local_runtime_image_ref_uses_nodeport_registry() { let image = local_runtime_image_ref("provider-helm", "arm64", "v1.999.3"); @@ -1127,7 +1129,16 @@ mod tests { image, "localhost:30500/hops-ops/provider-helm-arm64:v1.999.3" ); - assert!(!image.contains(REGISTRY_PULL)); + assert!(!image.contains(registry_pull())); + } + + #[test] + fn local_registry_wiring_skips_foreign_context_without_backend_flag() { + assert!(!backend::should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); } #[test] @@ -1165,7 +1176,10 @@ mod tests { fn assert_no_existing_provider_error(err: Box, provider_name: &str) { let msg = err.to_string(); assert!( - msg.contains(&format!("no existing Provider matching '{}'", provider_name)), + msg.contains(&format!( + "no existing Provider matching '{}'", + provider_name + )), "missing 'no existing Provider matching' phrase: {}", msg ); @@ -1183,9 +1197,8 @@ mod tests { #[test] fn resolve_provider_target_errors_when_list_is_empty() { - let err = - resolve_provider_target("provider-helm", "{\"items\":[]}", TEST_LOCAL_REGISTRY) - .expect_err("expected no-existing-Provider error"); + let err = resolve_provider_target("provider-helm", "{\"items\":[]}", TEST_LOCAL_REGISTRY) + .expect_err("expected no-existing-Provider error"); assert_no_existing_provider_error(err, "provider-helm"); } @@ -1245,10 +1258,7 @@ mod tests { // "already_patched" bucket. We must still find it instead of erroring. let json = provider_json(&[( "crossplane-contrib-provider-helm", - &format!( - "{}/hops-ops/provider-helm:dev-abc123", - TEST_LOCAL_REGISTRY - ), + &format!("{}/hops-ops/provider-helm:dev-abc123", TEST_LOCAL_REGISTRY), Some("crossplane-contrib-provider-helm-drc"), )]); @@ -1333,7 +1343,8 @@ mod tests { fn resolve_provider_target_returns_clear_error_on_invalid_json() { let err = resolve_provider_target("provider-helm", "not-json", TEST_LOCAL_REGISTRY) .expect_err("expected parse error"); - assert!(err.to_string().contains("failed to parse providers list JSON")); + assert!(err + .to_string() + .contains("failed to parse providers list JSON")); } } - diff --git a/src/commands/secrets/list.rs b/src/commands/secrets/list.rs index 4ae9878..218ca7f 100644 --- a/src/commands/secrets/list.rs +++ b/src/commands/secrets/list.rs @@ -41,7 +41,8 @@ pub fn run() -> Result<(), Box> { pushed_rows.push(PushedSecretRow { name: secret.name.clone(), owner: owner_label(&secret), - cluster: tag_value(&secret, "hops.ops.com.ai/cluster").unwrap_or("-".to_string()), + cluster: tag_value(&secret, "hops.ops.com.ai/cluster") + .unwrap_or("-".to_string()), namespace: tag_value(&secret, "hops.ops.com.ai/namespace") .unwrap_or("-".to_string()), kms_key: shorten_kms_key(&kms), diff --git a/src/commands/vars/mod.rs b/src/commands/vars/mod.rs index 306d7cb..d2b3370 100644 --- a/src/commands/vars/mod.rs +++ b/src/commands/vars/mod.rs @@ -132,10 +132,7 @@ pub(crate) fn require_command(program: &str) -> Result<(), Box> { } } -pub(crate) fn run_command_output( - program: &str, - args: &[&str], -) -> Result, Box> { +pub(crate) fn run_command_output(program: &str, args: &[&str]) -> Result, Box> { log::debug!("Running: {} {}", program, args.join(" ")); let output = Command::new(program).args(args).output()?; if !output.status.success() { diff --git a/src/commands/vars/sync.rs b/src/commands/vars/sync.rs index 2528330..9c3950d 100644 --- a/src/commands/vars/sync.rs +++ b/src/commands/vars/sync.rs @@ -243,11 +243,6 @@ fn set_github_variable( if !status.success() { return Err(format!("gh variable set exited with {}", status).into()); } - log::info!( - "Set GitHub variable '{}' in '{}/{}'", - var_name, - owner, - repo - ); + log::info!("Set GitHub variable '{}' in '{}/{}'", var_name, owner, repo); Ok(()) } diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs new file mode 100644 index 0000000..5eaef20 --- /dev/null +++ b/tests/colima_smoke_workflow.rs @@ -0,0 +1,99 @@ +//! Structural checks for the shipped colima macOS smoke workflow. +//! +//! These tests drive the real file under `.github/workflows/` so a missing +//! or regressed smoke contract fails `cargo test` without needing a macOS GHA run. + +use std::fs; +use std::path::PathBuf; + +fn workflow_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-colima-smoke.yaml") +} + +fn workflow_text() -> String { + let path = workflow_path(); + assert!( + path.is_file(), + "expected colima smoke workflow at {}", + path.display() + ); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn colima_smoke_workflow_exists_and_pins_nested_virt_runner() { + let text = workflow_text(); + assert!( + text.contains("labeled") && text.contains("test-colima"), + "PR smoke must start only when opted in with the test-colima label" + ); + assert!( + text.contains("github.event_name == 'workflow_dispatch'") + && text.contains("github.event.pull_request.labels.*.name"), + "manual dispatch must remain available while PR runs are label-gated" + ); + assert!( + text.contains("runs-on: macos-15-intel"), + "must pin nested-virt-capable Intel runner" + ); + assert!( + !text + .lines() + .any(|l| l.trim() == "runs-on: macos-latest"), + "must not use bare macos-latest as sole runs-on" + ); + assert!( + text.to_lowercase().contains("nested") && text.to_lowercase().contains("virt"), + "must comment on runner/virt constraints" + ); +} + +#[test] +fn colima_smoke_workflow_kind_parity_sequence() { + let text = workflow_text(); + for needle in [ + "cargo build", + "start --backend colima", + "local doctor", + "localhost:30500", + "registry.crossplane-system.svc.cluster.local:5000", + "--context colima", + "local stop", + "local destroy", + ] { + assert!( + text.contains(needle), + "colima smoke missing kind-parity fragment: {needle}" + ); + } + // stop/start resume: start without --backend after stop + assert!( + text.contains("local start\n") || text.lines().any(|l| l.trim() == "./target/debug/hops-cli local start"), + "must start again without --backend after stop" + ); + for tool in ["colima", "docker", "kubectl", "helm"] { + assert!( + text.contains(tool), + "prereq install must mention {tool}" + ); + } +} + +#[test] +fn colima_smoke_workflow_sizes_vm_for_gha_intel_runner() { + let text = workflow_text(); + // Defaults (8/16/60) exceed macos-15-intel; CI must pass explicit smaller sizes. + assert!( + text.contains("--cpus") && text.contains("--memory") && text.contains("--disk"), + "must pass --cpus/--memory/--disk so the VZ VM fits the runner" + ); + // 8Gi left CoreDNS thrashing; smoke needs headroom above that floor. + assert!( + text.contains("--memory 10") || text.contains("--memory 11") || text.contains("--memory 12"), + "must allocate at least 10Gi to the colima VM on GHA intel runners" + ); + assert!( + text.contains("ha.stderr.log"), + "failure dump must include lima hostagent stderr for nested-virt debug" + ); +} diff --git a/tests/dory_smoke_workflow.rs b/tests/dory_smoke_workflow.rs new file mode 100644 index 0000000..b005d51 --- /dev/null +++ b/tests/dory_smoke_workflow.rs @@ -0,0 +1,182 @@ +//! Structural checks for the shipped dory macOS smoke workflow. +//! +//! Self-hosted stock Dory, env-only session (no desktop context mutation). + +use std::fs; +use std::path::PathBuf; + +fn workflow_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-dory-smoke.yaml") +} + +fn workflow_text() -> String { + let path = workflow_path(); + assert!( + path.is_file(), + "expected dory smoke workflow at {}", + path.display() + ); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn dory_smoke_workflow_self_hosted_stock_contract() { + let text = workflow_text(); + + assert!( + !text.contains("patrickleet/dory"), + "must not clone a hops fork of dory" + ); + assert!( + !text.contains("xcodebuild") && !text.contains("build-dory-ffi"), + "must not build Dory from source" + ); + assert!( + !text.contains("colima start") + && !text.contains("brew install colima") + && !text.contains("colima-surrogate"), + "must not use Colima as a dory.sock surrogate" + ); + assert!( + text.contains("dory.sock"), + "must require real ~/.dory/dory.sock" + ); + assert!( + text.contains("refusing Colima-backed") || text.contains("Colima-backed dory.sock"), + "must refuse a Colima-symlinked dory.sock" + ); + assert!( + text.contains("self-hosted") && text.contains("hops-dory"), + "must target self-hosted runner labeled hops-dory" + ); + assert!( + !text.lines().any(|l| { + let t = l.trim(); + t == "runs-on: macos-15" + || t == "runs-on: macos-latest" + || t == "runs-on: macos-15-intel" + }), + "must not use GitHub-hosted macOS" + ); + assert!( + text.contains("labeled") && text.contains("test-dory"), + "PR smoke must be opt-in via test-dory label" + ); +} + +#[test] +fn dory_smoke_workflow_env_only_no_desktop_mutation() { + let text = workflow_text(); + + // Session entirely via env vars. + assert!( + text.contains("HOPS_DORY_DESKTOP") && text.contains("\"0\""), + "must set HOPS_DORY_DESKTOP=0 so hops does not rewrite desktop defaults" + ); + assert!( + text.contains("DOCKER_HOST") && text.contains("dory.sock"), + "must drive docker via DOCKER_HOST → dory.sock" + ); + assert!( + text.contains("KUBECONFIG") && text.contains("RUNNER_TEMP"), + "must use a job-private KUBECONFIG under RUNNER_TEMP" + ); + + // Never switch/restore machine defaults. + assert!( + !text.contains("kubectl config use-context"), + "must not kubectl config use-context" + ); + assert!( + !text.contains("docker context use"), + "must not docker context use" + ); + assert!( + !text.contains("Restore desktop contexts") && !text.contains("Snapshot desktop"), + "must not snapshot/restore contexts — env-only session needs no restore" + ); + + // Do not destroy product plane. + assert!( + !text.contains("hops-cli local destroy"), + "must not hops local destroy" + ); + assert!( + !text.contains("docker rm -f dory-k8s"), + "must not docker rm dory-k8s" + ); + assert!( + !text.contains("dory engine sleep") && !text.contains("pkill -f"), + "must not sleep engine or kill Dory" + ); + assert!( + !text.contains("hops-cli local stop"), + "must not hops local stop" + ); + + assert!( + text.contains("CARGO_BUILD_JOBS") && text.contains("nice"), + "must limit cargo parallelism and nice the build" + ); +} + +#[test] +fn dory_smoke_workflow_hops_integration_core() { + let text = workflow_text(); + for needle in [ + "cargo build", + "start --backend dory", + "local doctor", + "registry.crossplane-system.svc.cluster.local:5000", + "30500", + "smoke-svc-name", + "smoke-nodeport", + ] { + assert!( + text.contains(needle), + "dory smoke missing hops integration fragment: {needle}" + ); + } + assert!( + text.contains("NODE_IP") || text.contains("NetworkSettings"), + "must push via engine-plane dory-k8s IP" + ); + // Single-platform pull/build avoids multiplatform push warnings. + assert!( + text.contains("docker pull --platform") || text.contains("--platform \"$PLATFORM\""), + "registry smoke must pull busybox with an explicit --platform" + ); + assert!( + text.contains("multiplatform") || text.contains("single-platform"), + "must document why single-platform materialization is used" + ); +} + +#[test] +fn dory_smoke_workflow_path_based_config_install() { + let text = workflow_text(); + assert!( + text.contains("tests/fixtures/config-smoke"), + "must path-install the in-repo config-smoke fixture" + ); + assert!( + text.contains("config install --path") || text.contains("config install --path "), + "must use hops config install --path (no clone)" + ); + assert!( + !text.contains("config install --repo"), + "must not clone a remote config repo for the smoke" + ); + assert!( + text.contains("local/ci-xr.yaml") || text.contains("ci-xr.yaml"), + "must apply the fixture XR" + ); + assert!( + text.contains("hops-ci") && text.contains("configmap"), + "must verify ConfigMap in hops-ci namespace" + ); + assert!( + text.contains("command -v up") || text.contains("up CLI"), + "must require Upbound up CLI for path builds" + ); +} diff --git a/tests/fixtures/config-smoke/.gitignore b/tests/fixtures/config-smoke/.gitignore new file mode 100644 index 0000000..d2e372f --- /dev/null +++ b/tests/fixtures/config-smoke/.gitignore @@ -0,0 +1,4 @@ +_output/ +.up/ +.venv/ +.tmp/ diff --git a/tests/fixtures/config-smoke/README.md b/tests/fixtures/config-smoke/README.md new file mode 100644 index 0000000..0ce3048 --- /dev/null +++ b/tests/fixtures/config-smoke/README.md @@ -0,0 +1,14 @@ +# config-smoke + +Minimal Crossplane configuration fixture for **path-based** hops integration tests. + +```bash +hops local start --backend dory # or kind/colima +hops config install --path tests/fixtures/config-smoke --backend dory +kubectl apply -f tests/fixtures/config-smoke/local/ci-xr.yaml +kubectl -n hops-ci wait --for=condition=Ready configsmoke/hops-ci-smoke --timeout=300s +kubectl -n hops-ci get configmap hops-ci-smoke +``` + +Creates only namespaced resources under `hops-ci` (ConfigMap via provider-kubernetes). +Not a real platform stack — safe on a shared local control plane. diff --git a/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml b/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml new file mode 100644 index 0000000..1c11c03 --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml @@ -0,0 +1,19 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: configsmokes.ci.hops.ops.com.ai +spec: + compositeTypeRef: + apiVersion: ci.hops.ops.com.ai/v1alpha1 + kind: ConfigSmoke + mode: Pipeline + pipeline: + - functionRef: + # Crossplane names the installed Function from the package path + # ghcr.io/hops-ops/config-smoke_render → hops-ops-config-smokerender + # (same convention as gateway-api-stackrender, etc.). + name: hops-ops-config-smokerender + step: render + - functionRef: + name: crossplane-contrib-function-auto-ready + step: crossplane-contrib-function-auto-ready diff --git a/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml b/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml new file mode 100644 index 0000000..4abf95a --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml @@ -0,0 +1,14 @@ +apiVersion: meta.pkg.crossplane.io/v1alpha1 +kind: Configuration +metadata: + name: config-smoke + annotations: + meta.crossplane.io/maintainer: hops-ops CI + meta.crossplane.io/source: github.com/hops-ops/hops-cli + meta.crossplane.io/description: Minimal CI fixture ConfigMap composition for hops path-based config install. +spec: + dependsOn: + - function: xpkg.crossplane.io/crossplane-contrib/function-auto-ready + version: '>=v0.6.0' + - provider: xpkg.crossplane.io/crossplane-contrib/provider-kubernetes + version: '>=v0.15.0' diff --git a/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml b/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml new file mode 100644 index 0000000..14b9559 --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml @@ -0,0 +1,53 @@ +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: configsmokes.ci.hops.ops.com.ai +spec: + group: ci.hops.ops.com.ai + names: + kind: ConfigSmoke + plural: configsmokes + scope: Namespaced + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + description: CI-only composite that materializes a ConfigMap. + type: object + properties: + spec: + type: object + properties: + managementPolicies: + type: array + items: + type: string + default: ["*"] + data: + description: ConfigMap data key/values. + type: object + additionalProperties: + type: string + x-kubernetes-preserve-unknown-fields: true + configMapName: + description: Name of the ConfigMap to create. Defaults to metadata.name. + type: string + providerConfigRef: + type: object + properties: + name: + type: string + kind: + type: string + enum: [ProviderConfig, ClusterProviderConfig] + required: [] + status: + type: object + properties: + ready: + type: boolean + configMapName: + type: string + required: [spec] diff --git a/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml b/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml new file mode 100644 index 0000000..81c7ea3 --- /dev/null +++ b/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml @@ -0,0 +1,13 @@ +apiVersion: ci.hops.ops.com.ai/v1alpha1 +kind: ConfigSmoke +metadata: + name: hops-ci-smoke + namespace: hops-ci +spec: + configMapName: hops-ci-smoke + data: + source: path-based-config-install + smoke: "ok" + providerConfigRef: + name: default + kind: ProviderConfig diff --git a/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl new file mode 100644 index 0000000..9dbb397 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl @@ -0,0 +1,26 @@ +# code: language=yaml +{{- $xr := getCompositeResource . }} +{{- $metadata := $xr.metadata | default dict }} +{{- $spec := $xr.spec | default dict }} + +{{- $name := $metadata.name | default "config-smoke" }} +{{- $namespace := $metadata.namespace | default "default" }} +{{- $configMapName := $spec.configMapName | default $name }} +{{- $managementPolicies := $spec.managementPolicies | default (list "*") }} +{{- $data := $spec.data | default (dict "smoke" "ok") }} +{{- $pc := $spec.providerConfigRef | default dict }} +{{- $providerConfigRef := dict + "name" ($pc.name | default "default") + "kind" ($pc.kind | default "ProviderConfig") +}} + +{{- $state := dict + "name" $name + "namespace" $namespace + "configMapName" $configMapName + "managementPolicies" $managementPolicies + "data" $data + "providerConfigRef" $providerConfigRef + "observed" (dict) + "status" (dict "ready" false "configMapName" $configMapName) +}} diff --git a/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl new file mode 100644 index 0000000..691f1d4 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl @@ -0,0 +1,17 @@ +# code: language=yaml +{{- $raw := $.observed.resources | default dict }} +{{- $entry := get $raw "configmap" | default dict }} +{{- $resource := $entry.resource | default dict }} +{{- $status := $resource.status | default dict }} + +{{- $ready := false }} +{{- range ($status.conditions | default list) }} + {{- if and (eq .type "Ready") (eq .status "True") }} + {{- $ready = true }} + {{- end }} +{{- end }} + +{{- $state = set $state "status" (dict + "ready" $ready + "configMapName" $state.configMapName +) }} diff --git a/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl new file mode 100644 index 0000000..6004660 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl @@ -0,0 +1,28 @@ +# code: language=yaml +{{- $s := $state }} +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: {{ $s.configMapName }} + annotations: + {{ setResourceNameAnnotation "configmap" }} + labels: + hops.ops.com.ai/ci-fixture: "true" + hops.ops.com.ai/config-smoke: {{ $s.name | quote }} +spec: + managementPolicies: {{ $s.managementPolicies | toJson }} + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ $s.configMapName }} + namespace: {{ $s.namespace }} + labels: + hops.ops.com.ai/ci-fixture: "true" + hops.ops.com.ai/config-smoke: {{ $s.name | quote }} + data: {{ $s.data | toJson }} + providerConfigRef: + name: {{ $s.providerConfigRef.name }} + kind: {{ $s.providerConfigRef.kind }} diff --git a/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl new file mode 100644 index 0000000..9e6c392 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl @@ -0,0 +1,8 @@ +# code: language=yaml +{{- $xr := getCompositeResource . }} +--- +apiVersion: {{ $xr.apiVersion }} +kind: {{ $xr.kind }} +status: + ready: {{ $state.status.ready }} + configMapName: {{ $state.status.configMapName | quote }} diff --git a/tests/fixtures/config-smoke/local/ci-xr.yaml b/tests/fixtures/config-smoke/local/ci-xr.yaml new file mode 100644 index 0000000..a611f06 --- /dev/null +++ b/tests/fixtures/config-smoke/local/ci-xr.yaml @@ -0,0 +1,36 @@ +# Applied by dory smoke after `hops config install --path tests/fixtures/config-smoke`. +# Lives in the hops-ci namespace only — safe alongside real workloads. +apiVersion: v1 +kind: Namespace +metadata: + name: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +--- +# Namespaced k8s provider config (mirrors bootstrap/k8s/pc.yaml for default/). +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: ProviderConfig +metadata: + name: default + namespace: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +spec: + credentials: + source: InjectedIdentity +--- +apiVersion: ci.hops.ops.com.ai/v1alpha1 +kind: ConfigSmoke +metadata: + name: hops-ci-smoke + namespace: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +spec: + configMapName: hops-ci-smoke + data: + source: path-based-config-install + smoke: "ok" + providerConfigRef: + name: default + kind: ProviderConfig diff --git a/tests/fixtures/config-smoke/upbound.yaml b/tests/fixtures/config-smoke/upbound.yaml new file mode 100644 index 0000000..fb8db98 --- /dev/null +++ b/tests/fixtures/config-smoke/upbound.yaml @@ -0,0 +1,27 @@ +apiVersion: meta.dev.upbound.io/v2alpha1 +kind: Project +metadata: + name: config-smoke +spec: + dependsOn: + - apiVersion: pkg.crossplane.io/v1 + kind: Function + package: xpkg.crossplane.io/crossplane-contrib/function-auto-ready + version: '>=v0.6.0' + - apiVersion: pkg.crossplane.io/v1 + kind: Provider + package: xpkg.crossplane.io/crossplane-contrib/provider-kubernetes + version: '>=v0.15.0' + description: | + Minimal hops CI fixture. Builds from source via `hops config install --path` + and composes a single Kubernetes ConfigMap through provider-kubernetes. + Safe to install on a shared local control plane (namespaced resources only). + license: Apache-2.0 + maintainer: hops-ops CI + readme: | + # config-smoke + + Test-only configuration for hops local integration. Creates a ConfigMap via + the Kubernetes provider. Not a real platform stack. + repository: ghcr.io/hops-ops/config-smoke + source: github.com/hops-ops/hops-cli