diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml index f1678bc..3696675 100644 --- a/.github/workflows/on-pr-dory-smoke.yaml +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -1,233 +1,256 @@ name: dory backend smoke -# End-to-end smoke for `hops local --backend dory` on public GHA macOS. -# Mirrors on-pr-kind-smoke.yaml: start → doctor → dual registry → stop/start → destroy. +# Real stock-Dory + hops integration on a self-hosted Apple Silicon Mac. # -# Install contract (mandatory — not brew cask / official release): -# 1. Clone patrickleet/dory at a pinned commit of feat/hops-local-integration -# (SHA pin — the integration branch may never merge upstream; bump the ref -# intentionally when adopting a newer known-good tip) -# 2. Build in-pipeline (xcodebuild with deterministic -derivedDataPath) -# 3. Put scripts/dory on PATH -# 4. Provide ~/.dory/engine.sock before hops start (see engine note below) +# Why self-hosted (not macos-15 / Colima): +# - Stock Dory needs dory-hv (Hypervisor.framework). Public GHA VMs cannot nest it. +# - An earlier "green" dory-smoke spent ~10m building a fork, then used Colima as +# ~/.dory/*.sock — that was not product Dory. This workflow refuses that lie. # -# Engine socket on public GHA (important): -# - Dory's native dory-hv engine needs Hypervisor.framework nested virt. -# Dory's own CI documents that GitHub-hosted macOS runners are VMs and -# CANNOT run that engine (see dory benchmark.yml). Observed: Debug Dory.app -# builds, but never creates ~/.dory on macos-15 arm (engine.sock absent). -# - hops requires engine.sock (not only dory.sock). To still exercise the -# hops dory backend (dory k8s enable, context `dory`, registry plumbing) -# on public GHA, we provision a nested-virt Docker daemon with Colima -# (proven on macos-15-intel) and expose it as ~/.dory/engine.sock when -# the native app does not produce one. -# - Prefer trying the built app first (xattr clear + ad-hoc sign + open); -# fall back to Colima-backed engine.sock if sock never appears. +# Trigger: label `test-dory` on the PR, or workflow_dispatch. +# Offline Mac: the job stays queued until the runner is online. # -# Runner: macos-15-intel — nested virt for Colima (same pin as colima smoke). -# Prefer localhost:30500 for registry traffic (macOS 15 Local Network Privacy). -# After cold start, wait for CoreDNS/node Ready before registry pods; long -# Ready timeouts + stop/start rollout restart (colima nested-virt lessons). +# Do-not-interrupt contract (personal Mac while you work): +# - Reuses your already-running Dory engine + dory-k8s +# - Never mutates desktop defaults (no global context switching, no merge into +# ~/.kube/config). Session is entirely env-driven: +# DOCKER_HOST=unix://$HOME/.dory/dory.sock +# KUBECONFIG= +# HOPS_DORY_DESKTOP=0 +# - Never destroy / stop dory-k8s, never kill Dory +# - Caps cargo parallelism and runs the build under nice(1) +# +# Runner setup (one-time on the Mac): +# mkdir -p ~/actions-runners/hops-cli && cd ~/actions-runners/hops-cli +# # download latest macOS arm64 runner from actions/runner releases +# ./config.sh --url https://github.com/hops-ops/hops-cli \ +# --token \ +# --name "$(hostname -s)-dory" \ +# --labels macOS,ARM64,hops-dory \ +# --work _work +# ./run.sh # on-demand; omit svc install so jobs only run when you start it +# +# Prerequisites: Dory engine healthy, Kubernetes enabled (dory-k8s), helm + cargo +# + `up` (Upbound CLI) on the runner user's PATH (path-based config install). on: - # Opt in with the test-dory label. Keep synchronize so later commits rerun - # while labeled; labeled starts the first run on an existing PR. pull_request: types: [opened, reopened, synchronize, labeled] workflow_dispatch: +concurrency: + group: dory-smoke-self-hosted + cancel-in-progress: false + jobs: dory-smoke: if: >- github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'test-dory') - # Nested-virt-capable Intel pin (not bare macos-latest / not arm-only). - # Native dory-hv wants Apple silicon, but public GHA cannot run it; this - # job uses Colima as the docker daemon behind engine.sock when needed. - runs-on: macos-15-intel + runs-on: [self-hosted, macOS, ARM64, hops-dory] timeout-minutes: 90 + env: + # Do not rewrite ~/.kube/config or change the machine docker default. + HOPS_DORY_DESKTOP: "0" + CARGO_BUILD_JOBS: "2" + CARGO_TERM_COLOR: always steps: - name: Checkout hops-cli uses: actions/checkout@v4 - - name: Checkout patrickleet/dory (pinned hops integration commit) - uses: actions/checkout@v4 - with: - repository: patrickleet/dory - # feat/hops-local-integration tip as of 2026-07-29 - # (feat(k8s): headless lifecycle CLI, ports, registries, merged dory context) - ref: f8c61d2fd0fc4d528e5e0da36ffa09b9796b3871 - path: dory-src - - - name: Select Xcode + - name: Require Apple Silicon run: | set -euxo pipefail - newest="$(ls -d /Applications/Xcode*.app | sort -V | tail -1)" - echo "Using Xcode at $newest" - echo "DEVELOPER_DIR=$newest/Contents/Developer" >> "$GITHUB_ENV" - sudo xcode-select -s "$newest/Contents/Developer" - - # Dory's rebased upstream builds its Swift package against a generated - # Rust UniFFI XCFramework. A clean checkout has no binary artifact, so - # raw xcodebuild cannot resolve DoryFFI until this preparation runs. - - name: Install Rust toolchain for Dory FFI - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable - - - name: Install Dory FFI build prerequisites - run: brew install protobuf - - - name: Build Dory from source - working-directory: dory-src - run: | - set -euxo pipefail - # Match dory CI: Metal toolchain download can be flaky; best-effort. - xcodebuild -downloadComponent MetalToolchain || true - # Match dory's scripts/build.sh + scripts/test.sh clean-checkout - # contract: materialize the ignored DoryFFI binary and generated - # Swift bindings before SwiftPM resolves the app dependencies. - scripts/build-dory-ffi-xcframework.sh --if-needed - test -f dory-core-swift/artifacts/DoryFFI.xcframework/macos-arm64_x86_64/libdory_ffi.a - test -f dory-core-swift/Sources/DoryCore/generated/dory_ffi.swift - # Prefer deterministic product path via -derivedDataPath (not scripts/build.sh - # alone, which may use a user-local DerivedData path). - xcodebuild -project Dory.xcodeproj -scheme Dory \ - -destination 'platform=macOS' -configuration Debug \ - -derivedDataPath build \ - build CODE_SIGNING_ALLOWED=NO - test -d build/Build/Products/Debug/Dory.app - # Clear provenance/quarantine that can leave Debug bundles unlaunchable - # (dory scripts/build.sh does the same for DerivedData products). - xattr -cr build/Build/Products/Debug/Dory.app 2>/dev/null || true - xattr -dr com.apple.provenance build/Build/Products/Debug/Dory.app 2>/dev/null || true - xattr -dr com.apple.quarantine build/Build/Products/Debug/Dory.app 2>/dev/null || true - codesign --force --deep --sign - build/Build/Products/Debug/Dory.app 2>/dev/null || true - - - name: Install dory CLI on PATH - run: | - set -euxo pipefail - chmod +x dory-src/scripts/dory - echo "$GITHUB_WORKSPACE/dory-src/scripts" >> "$GITHUB_PATH" - test -x dory-src/scripts/dory - dory-src/scripts/dory version || dory-src/scripts/dory k8s status || true + arch="$(uname -m)" + echo "arch=$arch host=$(hostname -s)" + if [ "$arch" != "arm64" ] && [ "$arch" != "arm64e" ]; then + echo "Stock Dory requires Apple Silicon (got $arch)." >&2 + exit 1 + fi - - name: Install colima, docker, kubectl, helm - run: | - set -euxo pipefail - # colima provides nested-virt docker when native dory-hv cannot run. - brew install colima docker kubectl helm - colima version - docker --version - kubectl version --client - helm version --short - - - name: Ensure engine.sock (app native or Colima surrogate) + - name: Env-only Dory session (private KUBECONFIG + DOCKER_HOST) run: | - set -euxo pipefail - DORY_APP="$GITHUB_WORKSPACE/dory-src/build/Build/Products/Debug/Dory.app" - test -d "$DORY_APP" - mkdir -p "$HOME/.dory" - - ARCH="$(uname -m)" - # dory-hv requires Apple silicon; Intel hosts never produce a native sock. - # Public GHA arm also cannot nested-virt the engine (dory CI docs). Try - # native only briefly on arm; always fall through to Colima if missing. - if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "arm64e" ]; then - open "$DORY_APP" || true - for i in $(seq 1 12); do - if [ -S "$HOME/.dory/engine.sock" ]; then - echo "native engine.sock ready after ${i} attempts" - ls -la "$HOME/.dory" - echo "DORY_ENGINE_SOURCE=native" >> "$GITHUB_ENV" - exit 0 + # No set -x: avoid dumping full PATH / home internals into Actions logs. + set -euo pipefail + sock="$HOME/.dory/dory.sock" + export PATH="$HOME/.dory/bin:/Applications/Dory.app/Contents/Helpers:$PATH" + echo "$HOME/.dory/bin" >> "$GITHUB_PATH" + echo "/Applications/Dory.app/Contents/Helpers" >> "$GITHUB_PATH" + + if [ ! -d /Applications/Dory.app ]; then + echo "Dory.app missing. Once: brew install --cask Augani/dory/dory" >&2 + exit 1 + fi + command -v helm >/dev/null || { echo "helm missing (brew install helm)" >&2; exit 1; } + command -v up >/dev/null || { echo "up CLI missing (needed for hops config install --path)" >&2; exit 1; } + + # Prefer a toolchain cargo that actually runs (shim may be broken). + resolve_cargo_bin() { + if command -v cargo >/dev/null 2>&1 && cargo --version >/dev/null 2>&1; then + command -v cargo + return 0 + fi + local c + for c in \ + "$HOME/.rustup/toolchains/stable-aarch64-apple-darwin/bin/cargo" \ + "$HOME/.rustup/toolchains/stable-x86_64-apple-darwin/bin/cargo" \ + "$HOME/.rustup/toolchains/"*/bin/cargo + do + if [ -x "$c" ] && "$c" --version >/dev/null 2>&1; then + printf '%s\n' "$c" + return 0 fi - sleep 5 done - pkill -f 'Dory.app/Contents/MacOS/Dory' || true - echo "native engine.sock absent; bootstrapping Colima docker as engine.sock" - else - echo "host arch $ARCH: native dory-hv unsupported; Colima surrogate" + return 1 + } + CARGO_BIN="$(resolve_cargo_bin)" || { + echo "cargo missing or broken (install rustup stable toolchain)" >&2 + exit 1 + } + echo "cargo ok ($("$CARGO_BIN" --version))" + echo "$(dirname "$CARGO_BIN")" >> "$GITHUB_PATH" + export PATH="$(dirname "$CARGO_BIN"):$PATH" + + if [ ! -S "$sock" ] || ! docker -H "unix://$sock" info >/dev/null 2>&1; then + echo "waking Dory engine..." + command -v dory >/dev/null 2>&1 && { dory engine wake || dory engine start || true; } >/dev/null 2>&1 || true + if [ ! -S "$sock" ]; then + open -g -j -a Dory 2>/dev/null || true + fi fi - # --- Colima-backed engine.sock (public GHA nested virt for docker) --- - # Sizing fits macos-15-intel (~4 CPU / ~14 GiB). Disk 40 for registry PVC - # when hops later creates the 20Gi registry volume inside k3s-on-docker. - colima start --runtime docker --cpu 3 --memory 8 --disk 40 - DOCKER_SOCK="" - for candidate in \ - "$HOME/.colima/default/docker.sock" \ - "$HOME/.colima/docker.sock" \ - /var/run/docker.sock - do - if [ -S "$candidate" ]; then - DOCKER_SOCK="$candidate" + ready=0 + for i in $(seq 1 36); do + if [ -S "$sock" ] && docker -H "unix://$sock" info >/dev/null 2>&1; then + echo "dory engine ready" + ready=1 break fi + sleep 5 done - if [ -z "$DOCKER_SOCK" ]; then - echo "colima started but no docker.sock found" >&2 - colima status || true - ls -la "$HOME/.colima" || true - ls -la "$HOME/.colima/default" || true + if [ "$ready" -ne 1 ]; then + echo "Dory engine not ready (is Dory.app running?)" >&2 exit 1 fi - # Prefer real path via env (dory's dk() uses DORY_ENGINE_SOCK) and - # keep symlink so hops preflight (`~/.dory/engine.sock`) passes. - ln -sfn "$DOCKER_SOCK" "$HOME/.dory/engine.sock" - ln -sfn "$DOCKER_SOCK" "$HOME/.dory/dory.sock" - test -S "$HOME/.dory/engine.sock" || test -e "$HOME/.dory/engine.sock" - echo "DORY_ENGINE_SOCK=$DOCKER_SOCK" >> "$GITHUB_ENV" - echo "DOCKER_HOST=unix://$DOCKER_SOCK" >> "$GITHUB_ENV" - export DORY_ENGINE_SOCK="$DOCKER_SOCK" - export DOCKER_HOST="unix://$DOCKER_SOCK" - docker info >/dev/null - docker ps -a || true - echo "engine.sock → $DOCKER_SOCK (colima surrogate)" - ls -la "$HOME/.dory" - echo "DORY_ENGINE_SOURCE=colima-surrogate" >> "$GITHUB_ENV" - - - uses: Swatinem/rust-cache@v2 - - - name: Build hops - run: cargo build - - - name: Host resources (for diagnosis) + + if [ -L "$sock" ]; then + target="$(readlink "$sock" || true)" + case "$target" in + *colima*) + echo "refusing Colima-backed dory.sock" >&2 + exit 1 + ;; + esac + fi + + export DOCKER_HOST="unix://$sock" + echo "DOCKER_HOST=unix://$sock" >> "$GITHUB_ENV" + + if ! docker inspect -f '{{.State.Running}}' dory-k8s 2>/dev/null | grep -q true; then + if docker inspect dory-k8s >/dev/null 2>&1; then + echo "starting dory-k8s..." + docker start dory-k8s >/dev/null + else + echo "dory-k8s missing — enable Kubernetes in the Dory app" >&2 + exit 1 + fi + fi + + for i in $(seq 1 60); do + if docker exec dory-k8s kubectl get nodes --no-headers 2>/dev/null | grep -q ' Ready'; then + echo "dory-k8s Ready" + break + fi + if [ "$i" -eq 60 ]; then + echo "dory-k8s did not become Ready" >&2 + exit 1 + fi + sleep 2 + done + + # Job-private kubeconfig only — never touch ~/.kube/config. + kube="$RUNNER_TEMP/hops-dory-ci.kubeconfig" + if [ -f "$HOME/.kube/dory-config" ]; then + cp "$HOME/.kube/dory-config" "$kube" + else + docker exec dory-k8s cat /etc/rancher/k3s/k3s.yaml >"$kube" + if curl -sk --connect-timeout 2 https://127.0.0.1:6443/ >/dev/null 2>&1; then + sed -i.bak 's#server:.*#server: https://127.0.0.1:6443#' "$kube" || true + rm -f "${kube}.bak" + fi + fi + chmod 600 "$kube" + echo "KUBECONFIG=$kube" >> "$GITHUB_ENV" + export KUBECONFIG="$kube" + + # Minimal proof (no docker info / full node dump — personal Mac). + kubectl get nodes --no-headers | awk '{print "node", $1, $2}' + echo "env session ready (DOCKER_HOST + private KUBECONFIG)" + + # No Swatinem/rust-cache: on a personal Mac runner the post-job save + # tars ~100MB+ of target/ and can hang (slow upload, or a password prompt + # with no TTY). Local cargo builds here in ~20s without it. + + - name: Build hops (nice, limited jobs) + run: | + set -euxo pipefail + nice -n 10 cargo build -j "${CARGO_BUILD_JOBS}" + + - name: Host resources run: | set -euxo pipefail sysctl -n hw.ncpu || true sysctl hw.memsize || true uname -m - df -h / || true - echo "DORY_ENGINE_SOURCE=${DORY_ENGINE_SOURCE:-unknown}" + uptime || true + # DOCKER_HOST + KUBECONFIG already in env from prior step + dory readiness || true - name: hops local start --backend dory run: | set -euxo pipefail - export DOCKER_HOST="${DOCKER_HOST:-unix://$HOME/.dory/engine.sock}" - export DORY_ENGINE_SOCK="${DORY_ENGINE_SOCK:-$HOME/.dory/engine.sock}" - # Clean slate: partial dory-k8s can report create-time drift (exit 3). - dory k8s disable || true - docker ps -a || true + # HOPS_DORY_DESKTOP=0 → env session only (no global context changes) ./target/debug/hops-cli local start --backend dory - name: hops local doctor - run: ./target/debug/hops-cli local doctor + run: | + set -euxo pipefail + ./target/debug/hops-cli local doctor - - name: Wait for node + CoreDNS (nested virt settles) + - name: Wait for node + CoreDNS run: | set -euxo pipefail - kubectl --context dory wait --for=condition=Ready nodes --all --timeout=120s - kubectl --context dory -n kube-system wait --for=condition=Ready \ + # Uses job KUBECONFIG only (current-context inside that file). + kubectl wait --for=condition=Ready nodes --all --timeout=120s + kubectl -n kube-system wait --for=condition=Ready \ pod -l k8s-app=kube-dns --timeout=180s || \ - kubectl --context dory -n kube-system wait --for=condition=Ready \ + kubectl -n kube-system wait --for=condition=Ready \ pod -l k8s-app=coredns --timeout=180s || true - kubectl --context dory -n kube-system get pods -o wide || true + kubectl -n kube-system get pods -o wide || true + kubectl -n crossplane-system get pods -o wide || true - - name: Registry round-trip through both pull names + - name: Registry round-trip (engine push + in-cluster pull) run: | set -euxo pipefail - export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + + NODE_IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' dory-k8s)" + test -n "$NODE_IP" + PUSH="${NODE_IP}:30500" + echo "engine push endpoint: $PUSH" + + # Pull + re-materialize as a *single-platform* image for this host. + # A plain `docker pull` of busybox:stable keeps multi-arch index + # metadata; `docker push` then warns: + # Not all multiplatform-content is present and only the available + # single-platform image was pushed + case "$(uname -m)" in + arm64|arm64e|aarch64) PLATFORM=linux/arm64 ;; + x86_64|amd64) PLATFORM=linux/amd64 ;; + *) PLATFORM="linux/$(uname -m)" ;; + esac + SRC=public.ecr.aws/docker/library/busybox:stable for attempt in 1 2 3 4 5; do - if docker pull public.ecr.aws/docker/library/busybox:stable; then + if docker pull --platform "$PLATFORM" "$SRC"; then break fi echo "docker pull failed (attempt $attempt); sleeping..." @@ -236,91 +259,83 @@ jobs: exit 1 fi done - docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci - docker push localhost:30500/smoke/busybox:ci + # Fresh single-platform tag (no incomplete multi-arch index). + docker build --platform "$PLATFORM" --provenance=false --sbom=false \ + -t "${PUSH}/smoke/busybox:ci" - <&1 | head -40 || true - echo "==== cluster ====" - kubectl --context dory get nodes,pods -A -o wide || true - echo "==== smoke pods (default) ====" - kubectl --context dory describe pod smoke-svc-name smoke-localhost 2>/dev/null || true - echo "==== crossplane-system ====" - kubectl --context dory describe pods -n crossplane-system || true - kubectl --context dory get events -A --sort-by=.lastTimestamp | tail -100 || true - ./target/debug/hops-cli local doctor || true - - - name: hops local destroy - if: always() - run: | - set +e - export DOCKER_HOST="unix://$HOME/.dory/engine.sock" - ./target/debug/hops-cli local destroy || true + ./target/debug/hops-cli config install --path "$FIXTURE" --backend dory + + # Wait for Configuration package Healthy. + for i in $(seq 1 90); do + healthy="$(kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Healthy")].status}' 2>/dev/null || true)" + installed="$(kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Installed")].status}' 2>/dev/null || true)" + if [ "$healthy" = "True" ] && [ "$installed" = "True" ]; then + echo "Configuration hops-ops-config-smoke Healthy after ${i} attempts" + break + fi + if [ "$i" -eq 90 ]; then + kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke -o yaml || true + kubectl get configurationrevisions.pkg.crossplane.io -o wide || true + kubectl get functions.pkg.crossplane.io -o wide || true + exit 1 + fi + sleep 5 + done + + # Wait for XRD to be established. + kubectl wait --for=condition=Established \ + crd/configsmokes.ci.hops.ops.com.ai --timeout=180s + + # Apply CI XR into hops-ci only (namespaced ConfigMap). + kubectl apply -f "$FIXTURE/local/ci-xr.yaml" + + # XR Ready (namespaced composite). + kubectl -n hops-ci wait --for=condition=Ready \ + configsmoke.ci.hops.ops.com.ai/hops-ci-smoke --timeout=300s + + # Prove the ConfigMap exists with expected data. + kubectl -n hops-ci get configmap hops-ci-smoke -o yaml + smoke_val="$(kubectl -n hops-ci get configmap hops-ci-smoke \ + -o jsonpath='{.data.smoke}')" + test "$smoke_val" = "ok" - - name: Teardown Dory + Colima + - name: Soft cleanup (CI fixture only) if: always() run: | set +e - dory k8s disable || true - pkill -f 'Dory.app/Contents/MacOS/Dory' || true - colima stop || true - colima delete --force || true - ls -la "$HOME/.dory" || true + kubectl delete pod smoke-svc-name smoke-nodeport --wait=false 2>/dev/null || true + # Remove only hops-ci smoke resources — leave Dory / dory-k8s / Crossplane. + kubectl -n hops-ci delete configsmoke hops-ci-smoke --wait=false 2>/dev/null || true + kubectl -n hops-ci delete configmap hops-ci-smoke --wait=false 2>/dev/null || true + kubectl -n hops-ci delete object hops-ci-smoke --wait=false 2>/dev/null || true + # Leave Configuration package installed (idempotent next run) or drop it: + # kubectl delete configuration.pkg.crossplane.io hops-ops-config-smoke --wait=false diff --git a/.github/workflows/on-pr-kind-smoke.yaml b/.github/workflows/on-pr-kind-smoke.yaml index 354d2e6..9f63322 100644 --- a/.github/workflows/on-pr-kind-smoke.yaml +++ b/.github/workflows/on-pr-kind-smoke.yaml @@ -1,9 +1,7 @@ name: kind backend smoke -# End-to-end check of `hops local` on the kind backend, which is the CI -# path (ubuntu runners ship kind, docker, kubectl, and helm). Exercises the -# registry NodePort mapping and BOTH containerd certs.d trust names, plus the -# stop/start resume path. +# End-to-end check of `hops local` on the kind backend (ubuntu runners). +# Registry round-trip + path-based config install (same fixture as dory smoke). on: pull_request: @@ -17,6 +15,27 @@ jobs: - uses: Swatinem/rust-cache@v2 + - name: Install up CLI + run: | + set -euxo pipefail + # Required for `hops config install --path` (up project build). + curl -sL "https://cli.upbound.io" | sh + echo "$HOME/bin" >> "$GITHUB_PATH" + echo "$HOME/.up/bin" >> "$GITHUB_PATH" + # Installer may place the binary in different locations. + if [ -x "$HOME/bin/up" ]; then + echo "$HOME/bin" >> "$GITHUB_PATH" + elif [ -x /usr/local/bin/up ]; then + true + else + # Fall back: download latest release asset if shell installer differs. + mkdir -p "$HOME/bin" + curl -sL "https://cli.upbound.io/stable/current/linux_amd64/up" -o "$HOME/bin/up" + chmod +x "$HOME/bin/up" + echo "$HOME/bin" >> "$GITHUB_PATH" + fi + up version + - name: Build hops run: cargo build @@ -29,8 +48,14 @@ jobs: - name: Registry round-trip through both pull names run: | set -euxo pipefail - docker pull public.ecr.aws/docker/library/busybox:stable - docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + # Single-platform materialization — avoid multiplatform push warnings. + PLATFORM=linux/amd64 + SRC=public.ecr.aws/docker/library/busybox:stable + docker pull --platform "$PLATFORM" "$SRC" + docker build --platform "$PLATFORM" --provenance=false --sbom=false \ + -t localhost:30500/smoke/busybox:ci - </dev/null || true)" + installed="$(kubectl --context kind-hops get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Installed")].status}' 2>/dev/null || true)" + if [ "$healthy" = "True" ] && [ "$installed" = "True" ]; then + echo "Configuration hops-ops-config-smoke Healthy after ${i} attempts" + break + fi + if [ "$i" -eq 90 ]; then + kubectl --context kind-hops get configuration.pkg.crossplane.io hops-ops-config-smoke -o yaml || true + kubectl --context kind-hops get configurationrevisions.pkg.crossplane.io -o wide || true + kubectl --context kind-hops get functions.pkg.crossplane.io -o wide || true + exit 1 + fi + sleep 5 + done + + kubectl --context kind-hops wait --for=condition=Established \ + crd/configsmokes.ci.hops.ops.com.ai --timeout=180s + + kubectl --context kind-hops apply -f "$FIXTURE/local/ci-xr.yaml" + + kubectl --context kind-hops -n hops-ci wait --for=condition=Ready \ + configsmoke.ci.hops.ops.com.ai/hops-ci-smoke --timeout=300s + + kubectl --context kind-hops -n hops-ci get configmap hops-ci-smoke -o yaml + smoke_val="$(kubectl --context kind-hops -n hops-ci get configmap hops-ci-smoke \ + -o jsonpath='{.data.smoke}')" + test "$smoke_val" = "ok" + + - name: Soft cleanup (CI fixture only) + if: always() + run: | + set +e + kubectl --context kind-hops delete pod smoke-svc-name smoke-localhost --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete configsmoke hops-ci-smoke --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete configmap hops-ci-smoke --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete object hops-ci-smoke --wait=false 2>/dev/null || true - name: hops local destroy - run: ./target/debug/hops-cli local destroy + if: always() + run: ./target/debug/hops-cli local destroy || true diff --git a/README.md b/README.md index a95e79d..22703c9 100644 --- a/README.md +++ b/README.md @@ -222,13 +222,14 @@ hops config install --repo hops-ops/aws-auto-eks-cluster --version v0.11.0 daemon: Docker Desktop, colima's dockerd, or CI runners. No VM of its own, so sizing flags don't apply (size the docker daemon instead); requires kind >= v0.27. -- **dory** — [dory](https://augani.github.io/dory)'s built-in k3s, driven - headlessly through the `dory` CLI (`dory k8s enable/disable/status`). - Requires the Dory app running (it provides the engine and forwards - published ports to localhost) and a `dory` CLI with headless k8s support. - hops writes `~/.dory/k8s/registries.yaml` (k3s' native registry trust) and - publishes the registry NodePort at cluster create. The VM is sized in the - Dory app, so hops sizing flags don't apply. +- **dory** — [dory](https://augani.github.io/dory) stock app: shared Apple + Silicon engine + product k3s. Enable Kubernetes **in the Dory app** (hops + does not fork Dory or call `dory k8s enable`). Package installs use the same + **in-cluster** `registry:2` as other backends (Crossplane pulls + `registry.crossplane-system.svc.cluster.local:5000`). Docker push uses the + k3s **NodePort** on the engine docker bridge (`{dory-k8s-ip}:30500`) because + dockerd runs *inside* the engine — Mac `localhost` is the wrong plane. The + VM is sized in the Dory app, so hops sizing flags don't apply. Select with the global `--backend` flag: @@ -243,31 +244,207 @@ persisted choice > existing cluster detection (colima wins) > platform default (macOS: colima, otherwise kind). Unless `--context` is given, kubectl commands automatically use the backend's -kubeconfig context (`colima`, `kind-hops`, or `dory`), regardless of your -current-context. For dory, hops also prepends `~/.kube/dory-config` (where -dory keeps its kubeconfig) to `KUBECONFIG` for its own kubectl/helm calls. +kubeconfig context (`colima`, `kind-hops`, or `hops-dory`), regardless of your +current-context. #### Using dory -With a `dory` CLI that supports `dory k8s enable` (headless Kubernetes), -use the native backend: +Stock Dory only (brew cask / [Dory.app](https://augani.github.io/dory)). No hops +fork of Dory required. ```bash +# 1. Open Dory.app — engine healthy (not "needs attention") +# 2. Enable Kubernetes in the app; wait until the cluster is running +# (product container is usually named dory-k8s) +# 3. Bootstrap Crossplane + local package registry hops local start --backend dory ``` -For `hops provider install` / `hops config install` builds, point your docker -CLI at dory's engine (`export DOCKER_HOST=unix://$HOME/.dory/engine.sock`) so -image builds/pushes land on the daemon that reaches the registry. +On start/activate, hops: -Without the headless CLI, dory still exposes a real docker socket, so the -kind backend works against it: +- merges stock `~/.kube/dory-config` into `~/.kube/config` as context **`hops-dory`** + (override with `--name ` or `HOPS_DORY_NAME`; persisted in `~/.hops/local/dory-name`) +- runs `kubectl config use-context hops-dory` +- creates/uses a docker context of the same name → `unix://$HOME/.dory/dory.sock` + +So you should **not** need: + +```bash +export KUBECONFIG=$HOME/.kube/dory-config +export DOCKER_HOST=unix://$HOME/.dory/dory.sock +``` ```bash -docker context use dory # or: export DOCKER_HOST=unix://$HOME/.dory/dory.sock +hops local start --backend dory # name defaults to hops-dory +hops local start --backend dory --name mine # custom kube+docker context name + +kubectl get nodes # context hops-dory +docker info # context hops-dory +hops local doctor +hops local github -o hops-ops +hops config install --path … --backend dory +``` + +Alternatively, use kind on Dory's docker socket (no product k3s): + +```bash +docker context use dory # product context from Dory.app hops local start --backend kind ``` +**CI:** `.github/workflows/on-pr-dory-smoke.yaml` runs on a **self-hosted** +Apple Silicon Mac labeled `hops-dory` (opt-in with PR label `test-dory` or +`workflow_dispatch`). Stock Dory only — no fork build, no Colima sock. +Offline runner → job waits. The session is **env-only** so your desktop +defaults never change: `DOCKER_HOST=unix://$HOME/.dory/dory.sock`, a +job-private `KUBECONFIG`, and `HOPS_DORY_DESKTOP=0` (hops skips +`use-context` / docker context switching). No `destroy`/`stop` of +`dory-k8s`. After start/doctor/registry, it path-installs the in-repo +fixture `tests/fixtures/config-smoke` (`hops config install --path …`) and +applies a namespaced ConfigMap XR under `hops-ci`. See the workflow header +for runner setup. + +#### Dory architecture (why troubleshooting is different) + +Two network planes matter. Mixing them up is the usual failure mode. + +| Plane | Who | Where | Role | +|-------|-----|--------|------| +| **Engine Docker** | `docker` CLI via `~/.dory/dory.sock` | Linux VM (dockerd) | build/push images | +| **Cluster** | kubectl / Crossplane pods | k3s inside `dory-k8s` | run control plane + pull packages | + +Package registry model (same idea as colima/kind, adapted to Dory): + +- **Pull (Crossplane):** in-cluster Service + `registry.crossplane-system.svc.cluster.local:5000` over **HTTPS** +- **Push (docker on the engine):** k3s **NodePort** on the docker bridge + `{dory-k8s-ip}:30500` (not Mac `localhost:30500`) +- **TLS:** hops generates a self-signed CA/server cert (`Secret hops-local-registry-tls`) + and patches Crossplane to trust that CA. Plain HTTP fails package unpack + (`http: server gave HTTP response to HTTPS client`). + +Do **not** put `host.dory.internal` in Crossplane package refs. That name exists +on the **node** `/etc/hosts`, not in pod CoreDNS, and is the wrong plane for +Crossplane's package manager. + +#### Dory troubleshooting + +**Engine won't start / thrash / `HV_BAD_ARGUMENT` (big Macs)** + +The failure we hit in practice: on a **large laptop** (e.g. **128 GB** host +RAM), Dory's UI **Recommended** / host-scaled default for guest engine memory +was **~half of host RAM → 64 GB** (`DORYD_MEMORY_MB=65536`). That is not +"you need 64 GB of containers" — it is an aggressive ceiling. On Apple Silicon, +Hypervisor.framework often rejects creating a VM that large +(`HV_BAD_ARGUMENT`), so the engine never stays healthy and the UI looks broken +even though a **4 GB** engine works fine. + +```bash +# Pin sane resources (4 GiB / 4 CPUs is enough for hops local) +defaults write com.pythonxi.Dory dory.engineMemoryMB -int 4096 +defaults write com.pythonxi.Dory dory.engineCPUCount -int 4 + +# Confirm the LaunchAgent is not still on 65536: +launchctl print "gui/$(id -u)/dev.dory.doryd" 2>/dev/null | grep DORYD_MEMORY +# Live engine cmdline should show --mem-mb 4096, not 65536: +ps aux | grep 'dory-hv engine' | grep -v grep +``` + +In the Dory app **Resources** panel: set memory to **~4 GB** (8 GB is usually +fine too). **Do not apply Recommended** if it offers tens of GB on a high-RAM +Mac. + +Then fully restart the engine (quit Dory, ensure no thrashing `doryd`/`dory-hv`, +reopen or `dory engine wake`). Once guest RAM is modest, the rest of hops local +is ordinary. + +**Docker socket path** + +Stock Docker API socket is: + +```text +unix://$HOME/.dory/dory.sock +``` + +There is no `~/.dory/engine.sock` for the host Docker API. hops uses `dory.sock` +only. Prefer the docker context hops creates (`hops-dory`) over manual +`DOCKER_HOST`. + +**Kubernetes missing / `no dory-k8s container`** + +k3s is product-owned. hops does not run `dory k8s enable`. + +1. Install the Kubernetes component if needed: `dory component install kubernetes` +2. Enable Kubernetes in the Dory app UI +3. Wait until a `dory-k8s` container is running: `docker ps` (with context hops-dory) +4. Re-run `hops local start --backend dory` + +**`k ctx` has no dory / hops-dory entry** + +Stock Dory writes `~/.kube/dory-config` (context often named `default`). hops +merges that into `~/.kube/config` as **`hops-dory`** on activate/start. If the +merge is missing: + +```bash +hops local doctor --backend dory +kubectl config get-contexts # expect hops-dory +kubectl config use-context hops-dory +``` + +**Configuration Installed=False / HTTPS error** + +```text +http: server gave HTTP response to HTTPS client +``` + +Crossplane always pulls packages with HTTPS. The local registry must be TLS +(hops sets this up). If you have an old HTTP-only registry Deployment: + +```bash +kubectl -n crossplane-system delete deploy registry +kubectl -n crossplane-system delete secret hops-local-registry-tls +hops local start --backend dory # recreates TLS secret + registry + CA patch +``` + +**docker push to localhost:30500 fails (connection refused / HTTPS to HTTP)** + +With the docker context pointed at Dory, **`localhost` is the engine VM**, not +the Mac. hops pushes to `{dory-k8s container IP}:30500` and marks the engine +docker bridge as an insecure registry for that self-signed TLS endpoint. +Check: + +```bash +docker context show # hops-dory +hops local doctor # "package push registry reachable (…:30500)" +``` + +Do not rely on Mac `kubectl port-forward` for engine-side `docker push`. + +**Engine docker broken after daemon.json / dockerd kill** + +Prefer Dory's own recovery: + +```bash +dory repair dockerd --apply +dory engine wake +dory readiness +``` + +Avoid raw `kill` of dockerd inside the guest unless you are prepared to wait for +`dory repair` / LaunchAgent recovery (`live-restore` helps but is not magic). + +**Useful diagnostics** + +```bash +dory doctor +dory readiness +hops local doctor --backend dory +docker context show +kubectl config current-context +kubectl get configuration,provider -A +kubectl -n crossplane-system get deploy,svc,secret | grep -E 'registry|crossplane|hops-local' +``` + ### Local provider setup and auth `hops local aws`, `hops local github`, and `hops local zitadel` install the provider package and bootstrap auth into a local control plane. The exception is `--refresh`, which updates credentials only. @@ -399,12 +576,13 @@ Notes: - Runs `brew install colima`. - `local reset` - Runs `colima kubernetes reset`. -- `local start` - - Runs `colima start --kubernetes --cpu 8 --memory 16 --disk 60` - - Installs Crossplane from `crossplane-stable/crossplane` - - Applies manifests from `bootstrap/` for runtime config, providers, provider configs, and registry (embedded in the binary at build time) - - Configures Docker in Colima for insecure pulls from `registry.crossplane-system.svc.cluster.local:5000` - - Adds host mapping in Colima VM for the registry service DNS name +- `local start [--bootstrap]` + - Brings up the selected backend cluster (colima / kind / dory) + - If Crossplane, k8s+helm providers, and the package registry are already + Healthy/Available, skips helm repo update/upgrade and bootstrap reapply + (fast resume). Pass `--bootstrap` to force a full helm upgrade + reapply. + - Cold path: installs Crossplane, applies `bootstrap/` DRCs/providers/PCs, + deploys the local HTTPS package registry, wires node/engine registry trust - `local stop` - Runs `colima stop`. - `local destroy` diff --git a/bootstrap/registry/registry.yaml b/bootstrap/registry/registry.yaml index 225857b..02cd067 100644 --- a/bootstrap/registry/registry.yaml +++ b/bootstrap/registry/registry.yaml @@ -1,3 +1,6 @@ +# In-cluster package registry for local source installs. +# TLS is required: Crossplane's package manager always uses HTTPS. +# Cert material lives in Secret hops-local-registry-tls (created by hops). apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -10,8 +13,6 @@ spec: requests: storage: 20Gi # Uses the default StorageClass (local-path in colima/k3s). - # Data lives on the colima VM disk, which persists across computer restarts - # (as long as the colima instance/VM itself is not deleted). --- apiVersion: apps/v1 @@ -34,13 +35,40 @@ spec: image: registry:2 ports: - containerPort: 5000 + name: https + env: + # registry:2 serves HTTPS when both are set (still on :5000). + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /certs/tls.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /certs/tls.key volumeMounts: - name: registry-data mountPath: /var/lib/registry + - name: tls + mountPath: /certs + readOnly: true + readinessProbe: + httpGet: + path: /v2/ + port: 5000 + scheme: HTTPS + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /v2/ + port: 5000 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 volumes: - name: registry-data persistentVolumeClaim: claimName: registry-pvc + - name: tls + secret: + secretName: hops-local-registry-tls --- apiVersion: v1 kind: Service @@ -52,6 +80,7 @@ spec: selector: app: registry ports: - - port: 5000 + - name: https + port: 5000 targetPort: 5000 nodePort: 30500 diff --git a/src/commands/config/install.rs b/src/commands/config/install.rs index dd396c2..448eae1 100644 --- a/src/commands/config/install.rs +++ b/src/commands/config/install.rs @@ -4,7 +4,7 @@ use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout, ensure_registry, image_config_name, parse_docker_push_digest, parse_repo_spec, resolve_repo_install_target, rewrite_registry, rewrite_registry_with_tag, sanitize_name_component, short_hash, split_ref, strip_registry, - unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_PULL, REGISTRY_PUSH, + unique_suffix, RepoInstallTarget, RepoSpec, registry_pull, registry_push, }; use crate::commands::local::{kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output}; use clap::Args; @@ -326,7 +326,7 @@ fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Bo continue; } - let push_ref = rewrite_registry(&img.source, REGISTRY_PUSH); + let push_ref = rewrite_registry(&img.source, ®istry_push()); let (img_path, tag) = split_ref(&img.source); // All non-configuration images are Crossplane Function packages (the @@ -339,7 +339,7 @@ fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Bo if tag == arch { let digest = docker_push_and_get_digest(&push_ref)?; - let target_prefix = format!("{}/{}", REGISTRY_PULL, strip_registry(img_path)); + let target_prefix = format!("{}/{}", registry_pull(), strip_registry(img_path)); render_rewrites.insert( img_path.to_string(), RenderRewrite { @@ -387,8 +387,8 @@ spec: } let dev_tag = dev_tag_for_uppkg(&img.uppkg_path)?; - let push_ref = rewrite_registry_with_tag(&img.source, REGISTRY_PUSH, &dev_tag); - let pull_ref = rewrite_registry_with_tag(&img.source, REGISTRY_PULL, &dev_tag); + let push_ref = rewrite_registry_with_tag(&img.source, ®istry_push(), &dev_tag); + let pull_ref = rewrite_registry_with_tag(&img.source, registry_pull(), &dev_tag); log::info!( "Using local build version '{}' for {}...", dev_tag, @@ -909,6 +909,28 @@ fn dev_tag_for_uppkg(uppkg_path: &Path) -> Result> { Ok(format!("dev-{}", &hex[..12])) } +/// Map an image reference's arch tag (e.g. `:amd64` / `:arm64` from +/// `up project build`) to a Docker `--platform` value. +/// +/// Without an explicit platform, buildx on Apple Silicon warns +/// `InvalidBaseImagePlatform` when rebuilding the non-host arch variant +/// (`FROM …:amd64` on arm64, and the reverse on Intel). +fn platform_for_image_ref(src: &str) -> Option<&'static str> { + let (_, tag) = split_ref(src); + // Prefer the trailing path segment when the tag is a digest (rare for + // function images from up, which use :amd64 / :arm64). + let arch = if tag.starts_with("sha256:") { + src.rsplit('/').next().unwrap_or(tag) + } else { + tag + }; + match arch { + "amd64" | "linux/amd64" => Some("linux/amd64"), + "arm64" | "linux/arm64" | "aarch64" => Some("linux/arm64"), + _ => None, + } +} + /// Rebuild a Docker image with just `FROM ` to produce a valid OCI config. /// This fixes images where rootfs.type is empty (a known issue with `up project build` /// render function images). @@ -918,15 +940,21 @@ fn docker_build_from(src: &str, tag: &str) -> Result<(), Box> { // `build_patched_configuration_image`: without these, buildx wraps the // output in a single-arch manifest list that Crossplane (which fetches // package layers as linux/amd64 regardless of host) cannot navigate. + let mut args: Vec = vec![ + "build".into(), + "--provenance=false".into(), + "--sbom=false".into(), + ]; + if let Some(platform) = platform_for_image_ref(src) { + args.push("--platform".into()); + args.push(platform.into()); + } + args.push("-t".into()); + args.push(tag.into()); + args.push("-".into()); + let mut child = Command::new("docker") - .args([ - "build", - "--provenance=false", - "--sbom=false", - "-t", - tag, - "-", - ]) + .args(&args) .stdin(Stdio::piped()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -969,7 +997,7 @@ fn delete_local_registry_config_revisions(config_name: &str) -> Result<(), Box Result<(), Box< if !rev_name.starts_with(config_name) { continue; } - if !package.contains(REGISTRY_PULL) && state == "Inactive" { + if !package.contains(registry_pull()) && state == "Inactive" { run_cmd( "kubectl", &[ @@ -1104,4 +1132,20 @@ spec: Some("sha256:abcdef") ); } + + #[test] + fn platform_for_image_ref_from_up_arch_tags() { + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke_render:amd64"), + Some("linux/amd64") + ); + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke_render:arm64"), + Some("linux/arm64") + ); + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke:configuration"), + None + ); + } } diff --git a/src/commands/local/backend/dory.rs b/src/commands/local/backend/dory.rs index 00ad26b..7a81497 100644 --- a/src/commands/local/backend/dory.rs +++ b/src/commands/local/backend/dory.rs @@ -1,32 +1,31 @@ -//! dory backend: k3s in a container on dory's shared-VM dockerd -//! (https://augani.github.io/dory), driven headlessly through the `dory` CLI -//! (`dory k8s enable|disable|status`) and the engine docker socket. +//! dory backend: product k3s on stock Dory (https://augani.github.io/dory). //! -//! Registry plumbing differs from colima/kind because the cluster container -//! has create-time config shared with the Dory app: -//! - published NodePorts are static config: hops writes `~/.dory/k8s/ports` -//! BEFORE `dory k8s enable`, so GUI-side recreates keep the same port. -//! - trust is static config: hops writes `~/.dory/k8s/registries.yaml` -//! BEFORE `dory k8s enable`; dory bind-mounts it and k3s reads it at boot, -//! aliasing both pull names to the registry Service hostname over HTTP. -//! - name resolution is dynamic: `wire_registry` syncs the hostname -> -//! ClusterIP in the node container's /etc/hosts on every start (same -//! re-wire-on-start model as the other backends). -//! - the host reaches `localhost:30500` because `dory k8s enable --publish -//! 30500:30500` publishes the NodePort and the Dory app's port forwarder -//! maps published ports to host loopback. +//! Design (stock Dory only — no hops fork): +//! - Dory.app owns the engine, dockerd, and k3s lifecycle (enable Kubernetes +//! in the app). hops never calls `dory k8s enable`. +//! - hops talks to the engine via `~/.dory/dory.sock` and to the cluster via +//! `~/.kube/dory-config` (context is usually `default`). +//! +//! Package registry (two network planes on Dory): +//! - **Pull (Crossplane pods):** in-cluster Service DNS +//! `registry.crossplane-system.svc.cluster.local:5000`. +//! - **Push (docker via `~/.dory/dory.sock`):** engine dockerd pushes to the +//! k3s NodePort on the docker bridge (`{dory-k8s-ip}:30500`), with that +//! bridge marked insecure (HTTP). Mac localhost is the wrong plane. +//! - **Node image pulls:** k3s `registries.yaml` mirrors → Service ClusterIP. use super::SizeArgs; -use crate::commands::local::package_install::{REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH}; use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use crate::commands::local::package_install::{ + REGISTRY_HOSTNAME, REGISTRY_PULL_INCLUSTER, REGISTRY_PUSH, +}; use std::error::Error; use std::path::PathBuf; +use std::thread; +use std::time::Duration; const NODE_CONTAINER: &str = "dory-k8s"; -/// hops' registry NodePort, published on the cluster container at create. -const REGISTRY_PORT_PUBLISH: &str = "30500:30500"; -const REGISTRIES_BEGIN: &str = "# BEGIN hops-managed (do not edit inside)"; -const REGISTRIES_END: &str = "# END hops-managed"; +const REGISTRY_NODE_PORT: &str = "30500"; fn home() -> Result> { Ok(PathBuf::from(std::env::var("HOME").map_err(|_| { @@ -34,77 +33,18 @@ fn home() -> Result> { })?)) } +/// Stock Dory's host-facing Docker API socket (`~/.dory/dory.sock`). fn engine_socket() -> Result> { - Ok(home()?.join(".dory/engine.sock")) + Ok(home()?.join(".dory/dory.sock")) } -/// dory's side-file kubeconfig (context name `dory`). Current dory also -/// merges the context into ~/.kube/config at enable time; the side file is -/// the pre-merge fallback and dory's own `--kubeconfig` input. +/// Stock Dory writes the cluster kubeconfig here (context is typically `default`). pub fn kubeconfig_path() -> Option { home() .ok() .map(|h| h.join(".kube/dory-config").to_string_lossy().into_owned()) } -fn registries_yaml_path() -> Result> { - Ok(home()?.join(".dory/k8s/registries.yaml")) -} - -fn ports_file_path() -> Result> { - Ok(home()?.join(".dory/k8s/ports")) -} - -/// k3s' native registry config: alias both pull names to the registry -/// Service hostname over plain HTTP. The hostname resolves through the -/// node's /etc/hosts, which `wire_registry` keeps pointed at the ClusterIP. -fn registries_yaml() -> String { - format!( - "# Written by `hops local start --backend dory`.\n\ - # k3s reads this at boot; edits require `hops local reset`.\n\ - # Hops manages only the marked block; keep user mirrors outside it.\n\ - mirrors:\n\ - {block}", - block = registries_yaml_block(), - ) -} - -fn registries_yaml_block() -> String { - format!( - " {begin}\n\ - \x20\x20# hops: mirror {pull}\n\ - \x20\x20\"{pull}\":\n\ - \x20\x20 endpoint:\n\ - \x20\x20 - \"http://{pull}\"\n\ - \x20\x20# hops: mirror {push}\n\ - \x20\x20\"{push}\":\n\ - \x20\x20 endpoint:\n\ - \x20\x20 - \"http://{pull}\"\n\ - \x20\x20{end}\n", - begin = REGISTRIES_BEGIN, - end = REGISTRIES_END, - pull = REGISTRY_PULL, - push = REGISTRY_PUSH, - ) -} - -fn legacy_registries_yaml() -> String { - format!( - "# Written by `hops local start --backend dory`.\n\ - # k3s reads this at boot; edits require `hops local reset`.\n\ - mirrors:\n\ - \x20 \"{pull}\":\n\ - \x20 endpoint:\n\ - \x20 - \"http://{pull}\"\n\ - \x20 \"{push}\":\n\ - \x20 endpoint:\n\ - \x20 - \"http://{pull}\"\n", - pull = REGISTRY_PULL, - push = REGISTRY_PUSH, - ) -} - -/// Run docker against dory's engine socket (the daemon the Dory app manages). fn engine_docker(args: &[&str]) -> Result<(), Box> { let sock = format!("unix://{}", engine_socket()?.display()); let mut full = vec!["-H", sock.as_str()]; @@ -122,7 +62,10 @@ fn engine_docker_output(args: &[&str]) -> Result> { pub fn install() -> Result<(), Box> { log::info!("Installing Dory via Homebrew..."); run_cmd("brew", &["install", "--cask", "Augani/dory/dory"])?; - log::info!("Dory installed; launch the Dory app once so it provisions its engine"); + log::info!( + "Dory installed; open the app, wait until the engine is healthy, \ + enable Kubernetes, then re-run `hops local start --backend dory`" + ); Ok(()) } @@ -143,36 +86,45 @@ pub fn start(size: &SizeArgs) -> Result<(), Box> { } preflight()?; - write_ports_file()?; - write_registries_yaml()?; - - // Creates, restarts, or reuses the dory-k8s container as needed. If the - // running container has create-time config drift, dory exits 3 rather than - // destroying state; surface that plus the hops reset path. - run_dory_enable(false) + ensure_k8s_node_running()?; + wait_for_node_ready()?; + Ok(()) } pub fn stop() -> Result<(), Box> { log::info!("Stopping dory k8s node '{}'...", NODE_CONTAINER); + if !node_exists() { + log::info!("dory k8s node not present"); + return Ok(()); + } engine_docker(&["stop", NODE_CONTAINER])?; log::info!("dory cluster stopped"); Ok(()) } pub fn destroy() -> Result<(), Box> { - log::info!("Deleting dory k8s cluster..."); - run_cmd("dory", &["k8s", "disable"])?; - log::info!("dory cluster deleted"); + log::info!( + "Deleting dory k8s node '{}' (hops does not disable product k8s via CLI)...", + NODE_CONTAINER + ); + // Legacy leftovers from earlier registry experiments. + let _ = engine_docker(&["rm", "-f", "hops-local-registry"]); + let _ = engine_docker(&["rm", "-f", "hops-registry-publish"]); + let _ = engine_docker(&["rm", "-f", NODE_CONTAINER]); + log::info!( + "dory k8s node removed; re-enable Kubernetes in the Dory app if you want the product cluster again" + ); Ok(()) } -/// The cluster container IS the cluster, so reset means recreate. +/// Recreate is app-owned: remove the node; user re-enables k8s in Dory, then start again. pub fn reset() -> Result<(), Box> { preflight()?; - run_cmd("dory", &["k8s", "disable"])?; - write_ports_file()?; - write_registries_yaml()?; - run_dory_enable(true) + destroy()?; + log::info!( + "Enable Kubernetes in the Dory app, then run `hops local start --backend dory` again" + ); + Ok(()) } pub fn resize(_size: &SizeArgs) -> Result<(), Box> { @@ -182,7 +134,6 @@ pub fn resize(_size: &SizeArgs) -> Result<(), Box> { } /// Whether the hops-relevant dory cluster exists (running or stopped). -/// Missing app/engine/CLI reads as "no cluster". pub fn cluster_exists() -> bool { let Ok(sock) = engine_socket() else { return false; @@ -190,366 +141,457 @@ pub fn cluster_exists() -> bool { if !sock.exists() { return false; } - engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]).is_ok() + node_exists() +} + +fn node_exists() -> bool { + engine_docker_output(&["inspect", "-f", "{{.Id}}", NODE_CONTAINER]).is_ok() +} + +fn node_running() -> bool { + engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]) + .map(|state| state.trim() == "true") + .unwrap_or(false) } fn preflight() -> Result<(), Box> { - if !command_exists("dory") { - return Err("the `dory` CLI is not on PATH; link it from the dory repo \ - (`ln -sf /scripts/dory /opt/homebrew/bin/dory`)" - .into()); + if !command_exists("docker") { + return Err("docker CLI not found; install it (Dory provides the daemon)".into()); } let sock = engine_socket()?; if !sock.exists() { return Err(format!( - "dory's engine socket ({}) is missing; launch the Dory app and wait for its engine to start", + "dory's engine socket ({}) is missing.\n\ + Open the Dory app and wait until the engine is healthy (not \"needs attention\"), then retry.", sock.display() ) .into()); } - if !command_exists("docker") { - return Err("docker CLI not found; install it (dory provides the daemon)".into()); + if engine_docker_output(&["info"]).is_err() { + return Err(format!( + "cannot talk to Dory's docker at {}.\n\ + Open the Dory app, fix any engine/doryd errors in the menu, then retry.", + sock.display() + ) + .into()); } Ok(()) } -/// Write the static registry trust config read by k3s at boot. Must exist -/// before `dory k8s enable` because dory binds it at container create. -fn write_registries_yaml() -> Result<(), Box> { - let path = registries_yaml_path()?; - let current = match std::fs::read_to_string(&path) { - Ok(content) => Some(content), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, - Err(err) => return Err(err.into()), - }; - let desired = merge_registries_yaml(current.as_deref())?; - if current.as_deref() == Some(desired.as_str()) { +/// k3s is product-owned: start a stopped node, or tell the user to enable it in the app. +fn ensure_k8s_node_running() -> Result<(), Box> { + if node_running() { + log::info!("dory k8s node '{}' is running", NODE_CONTAINER); return Ok(()); } - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - log::info!("Writing k3s registry config: {}", path.display()); - std::fs::write(&path, desired)?; - Ok(()) -} - -fn write_ports_file() -> Result<(), Box> { - let path = ports_file_path()?; - let current = match std::fs::read_to_string(&path) { - Ok(content) => content, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => String::new(), - Err(err) => return Err(err.into()), - }; - let desired = ports_file_with_publish(¤t); - if current == desired { + if node_exists() { + log::info!("Starting stopped dory k8s node '{}'...", NODE_CONTAINER); + engine_docker(&["start", NODE_CONTAINER])?; return Ok(()); } - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - log::info!( - "Ensuring dory k8s port config includes {}: {}", - REGISTRY_PORT_PUBLISH, - path.display() - ); - std::fs::write(&path, desired)?; - Ok(()) -} - -/// Seconds after the Dory app (re)creates its engine socket during which it is -/// still provisioning the engine. A dockerd restart at the end of that window -/// SIGTERMs every container — including a k3s node enabled meanwhile, which -/// reports Ready and then dies under the bootstrap (observed ~90s on Dory -/// 0.2.0; padded for slower machines). -const ENGINE_LAUNCH_WINDOW_SECS: u64 = 180; - -/// Age of the current engine session: the app recreates the engine socket at -/// launch, so its mtime marks when provisioning began. None when unreadable. -fn engine_session_age() -> Option { - let sock = engine_socket().ok()?; - let modified = std::fs::metadata(&sock).ok()?.modified().ok()?; - std::time::SystemTime::now().duration_since(modified).ok() -} - -/// Time left inside the app's provisioning window for a given engine session -/// age; None once the window has passed. -fn launch_window_remaining(age: std::time::Duration) -> Option { - std::time::Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS) - .checked_sub(age) - .filter(|remaining| !remaining.is_zero()) + Err( + "Dory Kubernetes is not enabled (no `dory-k8s` container).\n\ + In the Dory app: enable Kubernetes, wait until it is running, then re-run:\n\ + hops local start --backend dory\n\ + (hops uses stock Dory only — it does not create the cluster for you.)" + .into(), + ) } -fn node_running() -> bool { - engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]) - .map(|state| state.trim() == "true") - .unwrap_or(false) +fn wait_for_node_ready() -> Result<(), Box> { + log::info!("Waiting for dory k8s node to become Ready..."); + for i in 0..90 { + if !node_running() { + thread::sleep(Duration::from_secs(2)); + continue; + } + let ready = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "kubectl", + "get", + "nodes", + "--no-headers", + ]) + .map(|out| out.contains(" Ready")) + .unwrap_or(false); + if ready { + ensure_side_kubeconfig_hint(); + // Best-effort: merge context "dory" + docker context (also run on activate). + let _ = ensure_desktop_integration(); + return Ok(()); + } + if i > 0 && i % 15 == 0 { + log::info!("Still waiting for k3s Ready ({}s)...", i * 2); + } + thread::sleep(Duration::from_secs(2)); + } + Err( + "timed out waiting for dory-k8s to become Ready; check Kubernetes status in the Dory app" + .into(), + ) } -/// Hold a freshly-enabled cluster under observation while the Dory app may -/// still be provisioning its engine, re-enabling if the engine restart takes -/// the node down. Immediate no-op when the window has already passed, so -/// steady-state starts pay one container inspect and nothing more. -fn hold_through_engine_launch_window() -> Result<(), Box> { - let in_window = |age: Option| { - age.map(|a| launch_window_remaining(a).is_some()) - .unwrap_or(false) +fn ensure_side_kubeconfig_hint() { + let Some(path) = kubeconfig_path() else { + return; }; - if in_window(engine_session_age()) { - log::info!( - "Dory engine session is younger than {}s; watching the k8s node through the app's provisioning window...", - ENGINE_LAUNCH_WINDOW_SECS - ); + if std::path::Path::new(&path).is_file() { + return; } - let mut reenables = 0; - loop { - match (node_running(), in_window(engine_session_age())) { - (true, false) => return Ok(()), - (true, true) => {} - (false, _) => { - if reenables >= 3 { - return Err("the dory engine keeps stopping the k8s node during app startup; \ - wait for the Dory app to finish provisioning, then re-run `hops local start --backend dory`" - .into()); - } - reenables += 1; - log::warn!( - "dory engine restart stopped the k8s node; re-enabling ({}/3)...", - reenables - ); - let args = dory_enable_args(false); - run_cmd("dory", &args)?; + if let Ok(yaml) = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "cat", + "/etc/rancher/k3s/k3s.yaml", + ]) { + if yaml.contains("server:") { + if let Some(parent) = std::path::Path::new(&path).parent() { + let _ = std::fs::create_dir_all(parent); } - } - std::thread::sleep(std::time::Duration::from_secs(3)); - } -} - -fn run_dory_enable(recreate: bool) -> Result<(), Box> { - let args = dory_enable_args(recreate); - match run_cmd("dory", &args) { - Ok(()) => hold_through_engine_launch_window(), - // Exit 3 = create-time config drift (ports / registries bind). hops just - // wrote the desired ports+registries files; applying them requires - // recreate. Auto-retry once so `local start` is not a dead end after - // hops updates publish config (reset remains available explicitly). - // - // Exit 1 after "did not become Ready" is common on stop→start: docker - // start of the existing k3s container can leave the node stuck past - // dory's wait window. Recreate once (same as `dory k8s disable && enable`). - Err(err) if !recreate && should_recreate_on_enable_error(&err.to_string()) => { - log::warn!( - "dory k8s enable failed ({err}); recreating cluster once..." + let _ = std::fs::write(&path, yaml); + let _ = std::fs::set_permissions( + &path, + std::os::unix::fs::PermissionsExt::from_mode(0o600), ); - let _ = run_cmd("dory", &["k8s", "disable"]); - run_dory_enable(true) + log::info!("Wrote kubeconfig side file {}", path); } - Err(err) => Err(err), } } -fn should_recreate_on_enable_error(msg: &str) -> bool { - // run_cmd only surfaces exit status (stderr is inherited), so match codes. - // 3 = create-time config drift; 1 = dory k8s_wait_ready / generic enable fail. - msg.contains("exit status: 3") || msg.contains("exit status: 1") +/// No create-time trust: k3s registries.yaml is written in [`wire_registry`]. +pub fn ensure_registry_trust() -> Result<(), Box> { + Ok(()) +} + +/// After the in-cluster registry Service exists: +/// 1. Point k3s/containerd at the Service ClusterIP over HTTP (function images). +/// 2. Allow engine dockerd to push HTTP to the k3s NodePort on the docker bridge. +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + ensure_k3s_registry_mirrors(cluster_ip)?; + ensure_engine_push_path()?; + Ok(()) +} + +/// Docker push address for package installs when `DOCKER_HOST` is the Dory engine. +/// Reachable on the engine docker bridge (not Mac localhost). +pub fn registry_push_addr() -> Result> { + Ok(format!("{}:{}", node_ip()?, REGISTRY_NODE_PORT)) } -fn dory_enable_args(recreate: bool) -> Vec<&'static str> { - let mut args = vec!["k8s", "enable"]; - if recreate { - args.push("--recreate"); +fn node_ip() -> Result> { + let ip = engine_docker_output(&[ + "inspect", + "-f", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + NODE_CONTAINER, + ])? + .trim() + .to_string(); + if ip.is_empty() { + return Err("dory-k8s has no docker network IP".into()); } - args.extend(["--publish", REGISTRY_PORT_PUBLISH]); - args + Ok(ip) } -fn merge_registries_yaml(existing: Option<&str>) -> Result> { - let Some(existing) = existing else { - return Ok(registries_yaml()); - }; - if existing.trim().is_empty() || existing == legacy_registries_yaml() { - return Ok(registries_yaml()); +/// k3s `registries.yaml` so containerd pulls HTTP from the in-cluster registry. +/// Restarts the node only when the file content changes. +fn ensure_k3s_registry_mirrors(cluster_ip: &str) -> Result<(), Box> { + if !node_running() { + return Err( + "dory k8s node is not running; enable Kubernetes in the Dory app first".into(), + ); + } + + let push = registry_push_addr()?; + // Registry serves HTTPS (self-signed); skip verify on the node for function image pulls. + let yaml = format!( + "mirrors:\n\ + \x20 \"{svc}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + \x20 \"{push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + \x20 \"{default_push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + configs:\n\ + \x20 \"{ip}:5000\":\n\ + \x20 tls:\n\ + \x20 insecure_skip_verify: true\n\ + \x20 \"{svc}\":\n\ + \x20 tls:\n\ + \x20 insecure_skip_verify: true\n", + svc = REGISTRY_PULL_INCLUSTER, + push = push, + default_push = REGISTRY_PUSH, + ip = cluster_ip, + ); + + let current = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + "cat /etc/rancher/k3s/registries.yaml 2>/dev/null || true", + ]) + .unwrap_or_default(); + if current.trim() == yaml.trim() { + return Ok(()); } - match ( - existing.find(REGISTRIES_BEGIN), - existing.find(REGISTRIES_END), - ) { - (Some(begin), Some(end)) if begin <= end => replace_registries_block(existing, begin, end), - (Some(_), Some(_)) | (Some(_), None) | (None, Some(_)) => { - Err(format!("malformed dory registries.yaml managed block; expected `{REGISTRIES_BEGIN}` before `{REGISTRIES_END}`").into()) + log::info!( + "Configuring k3s registry mirrors for {} -> {}:5000...", + REGISTRY_HOSTNAME, + cluster_ip + ); + engine_docker(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + &format!( + "mkdir -p /etc/rancher/k3s && cat > /etc/rancher/k3s/registries.yaml <<'EOF'\n{yaml}EOF" + ), + ])?; + log::info!("Restarting dory k8s node so k3s reloads registries.yaml..."); + engine_docker(&["restart", NODE_CONTAINER])?; + wait_for_node_ready()?; + for _ in 0..30 { + if run_cmd_output("kubectl", &["get", "--raw", "/readyz"]) + .map(|s| s.contains("ok")) + .unwrap_or(false) + { + break; } - (None, None) => insert_registries_block(existing), + thread::sleep(Duration::from_secs(2)); } + Ok(()) } -fn replace_registries_block( - existing: &str, - begin: usize, - end: usize, -) -> Result> { - let line_start = existing[..begin].rfind('\n').map_or(0, |idx| idx + 1); - let line_end = existing[end..] - .find('\n') - .map_or(existing.len(), |idx| end + idx + 1); - let mut merged = String::with_capacity(existing.len() + registries_yaml_block().len()); - merged.push_str(&existing[..line_start]); - merged.push_str(®istries_yaml_block()); - merged.push_str(&existing[line_end..]); - Ok(merged) +/// Engine dockerd must treat the k3s NodePort as an HTTP registry. Push address +/// is `{dory-k8s-ip}:30500` on the docker bridge (see [`registry_push_addr`]). +fn ensure_engine_push_path() -> Result<(), Box> { + // Best-effort cleanup of earlier experiments (ignore missing). + let _ = engine_docker_output(&["rm", "-f", "hops-local-registry"]); + let _ = engine_docker_output(&["rm", "-f", "hops-registry-publish"]); + + let push = registry_push_addr()?; + ensure_dockerd_insecure_for_push(&push)?; + + // Prove NodePort answers HTTPS from the engine network (self-signed → no-check). + for _ in 0..30 { + if engine_docker_output(&[ + "run", + "--rm", + "alpine:latest", + "wget", + "-qO-", + "--no-check-certificate", + &format!("https://{push}/v2/"), + ]) + .is_ok() + { + log::info!("Engine package push endpoint ready at {} (HTTPS)", push); + return Ok(()); + } + thread::sleep(Duration::from_secs(1)); + } + Err(format!( + "timed out waiting for registry NodePort at https://{push}/v2/ \ + (in-cluster TLS registry Service type NodePort on dory-k8s)" + ) + .into()) } -fn insert_registries_block(existing: &str) -> Result> { - if contains_hops_mirror_key(existing) { - return Err("dory registries.yaml already contains hops registry mirror entries without managed markers; remove those entries or wrap them in the hops-managed block" - .into()); +fn ensure_dockerd_insecure_for_push(push_hostport: &str) -> Result<(), Box> { + // Already configured? + if let Ok(info) = engine_docker_output(&["info", "-f", "{{json .RegistryConfig.InsecureRegistryCIDRs}}{{json .RegistryConfig.IndexConfigs}}"]) + { + if info.contains(push_hostport) || info.contains("192.168.215.0/24") { + return Ok(()); + } } - let mut merged = ensure_trailing_newline(existing); - if let Some(insert_at) = mirrors_line_insert_position(&merged) { - merged.insert_str(insert_at, ®istries_yaml_block()); - return Ok(merged); - } + log::info!( + "Configuring Dory dockerd insecure-registries for HTTP push to {}...", + push_hostport + ); - merged.push_str("mirrors:\n"); - merged.push_str(®istries_yaml_block()); - Ok(merged) -} + // Persist under the engine's /etc/docker (bind-mounted into a helper). + let daemon_json = format!( + r#"{{ + "builder": {{"gc": {{"enabled": true, "defaultKeepStorage": "2GB"}}}}, + "insecure-registries": [ + "127.0.0.0/8", + "localhost:30500", + "192.168.215.0/24", + "{push}" + ] +}} +"#, + push = push_hostport + ); -fn contains_hops_mirror_key(content: &str) -> bool { - content.contains(&format!("\"{}\":", REGISTRY_PULL)) - || content.contains(&format!("\"{}\":", REGISTRY_PUSH)) -} + engine_docker(&[ + "run", + "--rm", + "-v", + "/etc/docker:/etc/docker", + "alpine:latest", + "sh", + "-c", + &format!("cat > /etc/docker/daemon.json <<'EOF'\n{daemon_json}EOF"), + ])?; -fn mirrors_line_insert_position(content: &str) -> Option { - let mut offset = 0; - for line in content.split_inclusive('\n') { - let without_newline = line.trim_end_matches('\n').trim_end_matches('\r'); - if is_top_level_mirrors_line(without_newline) { - return Some(offset + line.len()); + // Prefer Dory's repair path (live-restore) over raw kill. + if command_exists("dory") { + let _ = run_cmd("dory", &["repair", "dockerd", "--apply"]); + } + + // If dockerd did not pick up the file, force a live-restore restart. + let info = engine_docker_output(&["info"]).unwrap_or_default(); + if !info.contains("192.168.215.0/24") && !info.contains(push_hostport) { + log::info!("Forcing dockerd restart so insecure-registries take effect..."); + let _ = engine_docker(&[ + "run", + "--rm", + "--privileged", + "--pid=host", + "alpine:latest", + "sh", + "-c", + "kill -TERM $(pidof dockerd) 2>/dev/null || true", + ]); + if command_exists("dory") { + let _ = run_cmd("dory", &["repair", "dockerd", "--apply"]); + } + for _ in 0..40 { + if engine_docker_output(&["info"]).is_ok() { + break; + } + thread::sleep(Duration::from_secs(2)); } - offset += line.len(); } - None -} -fn is_top_level_mirrors_line(line: &str) -> bool { - line.starts_with("mirrors:") && line.split('#').next().unwrap_or("").trim_end() == "mirrors:" + Ok(()) } -fn ensure_trailing_newline(content: &str) -> String { - let mut out = content.to_string(); - if !out.is_empty() && !out.ends_with('\n') { - out.push('\n'); +/// Default kube + docker context name for Dory desktop integration. +pub const DEFAULT_CONTEXT_NAME: &str = "hops-dory"; +const NAME_FILE: &str = "dory-name"; +const NAME_ENV: &str = "HOPS_DORY_NAME"; + +/// Resolved context name: `--name` (persisted) > `HOPS_DORY_NAME` > file > `hops-dory`. +pub fn context_name() -> String { + if let Ok(v) = std::env::var(NAME_ENV) { + let t = v.trim(); + if !t.is_empty() { + return t.to_string(); + } } - out + if let Ok(path) = crate::commands::local::local_state_dir() { + if let Ok(raw) = std::fs::read_to_string(path.join(NAME_FILE)) { + let t = raw.trim(); + if !t.is_empty() { + return t.to_string(); + } + } + } + DEFAULT_CONTEXT_NAME.to_string() } -#[derive(Debug, PartialEq, Eq)] -struct PortPublish { - host: u16, - container: u16, - proto: String, +/// Persist a user-chosen name for kube/docker context integration. +pub fn persist_context_name(name: &str) -> Result<(), Box> { + let name = validate_context_name(name)?; + let dir = crate::commands::local::local_state_dir()?; + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join(NAME_FILE), format!("{name}\n"))?; + log::info!("Dory desktop name set to '{name}' (kube + docker context)"); + Ok(()) } -fn ports_file_with_publish(existing: &str) -> String { - if ports_file_contains_publish(existing, REGISTRY_PORT_PUBLISH) { - return existing.to_string(); +fn validate_context_name(name: &str) -> Result> { + let name = name.trim(); + if name.is_empty() { + return Err("--name must not be empty".into()); } - - let mut out = ensure_trailing_newline(existing); - out.push_str(REGISTRY_PORT_PUBLISH); - out.push('\n'); - out + // kubectl context names: keep it simple (no path separators / whitespace). + if name.contains(['/', '\\', ' ', '\t', '\n', ':']) { + return Err(format!( + "invalid --name '{name}': use a simple token (e.g. hops-dory)" + ) + .into()); + } + Ok(name.to_string()) } -fn ports_file_contains_publish(existing: &str, desired: &str) -> bool { - let Some(desired) = parse_port_publish(desired) else { - return false; - }; - existing - .lines() - .filter_map(parse_port_publish) - .any(|port| port == desired) -} - -fn parse_port_publish(line: &str) -> Option { - let cleaned: String = line - .split('#') - .next() - .unwrap_or("") - .chars() - .filter(|c| !c.is_whitespace()) - .collect(); - if cleaned.is_empty() { - return None; - } - let (ports, proto) = cleaned - .split_once('/') - .map_or((cleaned.as_str(), "tcp"), |(ports, proto)| (ports, proto)); - if proto != "tcp" && proto != "udp" { - return None; - } - let (host, container) = ports.split_once(':')?; - let host = host.parse().ok().filter(|port| *port > 0)?; - let container = container.parse().ok().filter(|port| *port > 0)?; - Some(PortPublish { - host, - container, - proto: proto.to_string(), - }) -} - -/// Keep the node container's /etc/hosts pointing the registry hostname at -/// the current ClusterIP (docker regenerates /etc/hosts on restart, and the -/// ClusterIP changes if the Service is recreated — hence re-run per start). -pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { - let hostname = REGISTRY_HOSTNAME; - let current_ip = engine_docker_output(&[ - "exec", - NODE_CONTAINER, - "sh", - "-c", - &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), - ]) - .unwrap_or_default(); - if current_ip.trim() == cluster_ip { +/// Env: set `HOPS_DORY_DESKTOP=0` (or `false`/`off`/`no`) to skip mutating the +/// machine's default kube/docker contexts. Callers must drive the session with +/// `DOCKER_HOST` and `KUBECONFIG` (or pass `--context`) instead. +pub const DESKTOP_INTEGRATION_ENV: &str = "HOPS_DORY_DESKTOP"; + +/// Whether hops may rewrite `~/.kube/config` / switch docker contexts. +pub fn desktop_integration_enabled() -> bool { + match std::env::var(DESKTOP_INTEGRATION_ENV) { + Ok(v) => { + let v = v.trim().to_ascii_lowercase(); + !matches!(v.as_str(), "0" | "false" | "no" | "off") + } + Err(_) => true, + } +} + +/// Wire Dory into the normal developer desktop: +/// - merge stock `~/.kube/dory-config` into `~/.kube/config` as context **`hops-dory`** +/// (or `--name` / `HOPS_DORY_NAME`) +/// - `kubectl config use-context ` +/// - ensure/use a docker context of the same name pointing at `~/.dory/dory.sock` +/// +/// Called from backend activate and after local start so you don't need +/// `export KUBECONFIG=...` / `export DOCKER_HOST=...`. +/// +/// No-op for host defaults when [`desktop_integration_enabled`] is false: only +/// fills missing `DOCKER_HOST` / `KUBECONFIG` env for this process tree. +pub fn ensure_desktop_integration() -> Result<(), Box> { + ensure_side_kubeconfig_hint(); + if !desktop_integration_enabled() { + ensure_engine_env_only(); + log::info!( + "Dory desktop integration disabled ({DESKTOP_INTEGRATION_ENV}=0); \ + using DOCKER_HOST / KUBECONFIG only (no use-context, no docker context switch)" + ); return Ok(()); } - - log::info!("Updating node hosts entry: {} -> {}", hostname, cluster_ip); - - // /etc/hosts is a bind mount inside the container: `sed -i` fails with - // "Resource busy" (rename over a mount point), so rewrite it in place. - engine_docker(&[ - "exec", - NODE_CONTAINER, - "sh", - "-c", - &format!( - "awk '$2 != \"{host}\"' /etc/hosts > /tmp/hosts.new && \ - cat /tmp/hosts.new > /etc/hosts && rm -f /tmp/hosts.new && \ - echo '{ip} {host}' >> /etc/hosts", - host = hostname, - ip = cluster_ip - ), - ])?; - + let name = context_name(); + ensure_user_kubeconfig_context(&name)?; + ensure_docker_context_default(&name)?; Ok(()) } -/// Make `--context dory` resolvable. Current dory merges the context into -/// ~/.kube/config at enable time, so normally there is nothing to do — -/// mutating KUBECONFIG for every child process is then pure noise. Older -/// dory versions only write the side file; for those, prepend it to -/// KUBECONFIG (preserving whatever the user already has). -pub fn export_kubeconfig_env() { - if effective_kubeconfig_has_dory_context() { - return; +/// Point this process at the engine without touching desktop defaults. +fn ensure_engine_env_only() { + if std::env::var_os("DOCKER_HOST").is_none() { + if let Ok(sock) = engine_socket() { + if sock.exists() { + std::env::set_var("DOCKER_HOST", format!("unix://{}", sock.display())); + } + } + } + if std::env::var_os("KUBECONFIG").is_none() { + export_kubeconfig_env(); } +} + +/// Fallback when merge fails: point hops child processes at the side file. +pub fn export_kubeconfig_env() { let Some(dory_cfg) = kubeconfig_path() else { return; }; + if !std::path::Path::new(&dory_cfg).is_file() { + return; + } let existing = std::env::var("KUBECONFIG").unwrap_or_default(); if existing.split(':').any(|p| p == dory_cfg) { return; @@ -557,202 +599,178 @@ pub fn export_kubeconfig_env() { let rest = if existing.is_empty() { match home() { Ok(h) => h.join(".kube/config").to_string_lossy().into_owned(), - Err(_) => return, + Err(_) => String::new(), } } else { existing }; - std::env::set_var("KUBECONFIG", format!("{}:{}", dory_cfg, rest)); -} - -/// Whether the kubeconfig(s) kubectl will read without our help — the -/// $KUBECONFIG chain when set, else ~/.kube/config — already define a -/// `dory` entry (i.e. dory's kubectl-merge ran against a file in scope). -fn effective_kubeconfig_has_dory_context() -> bool { - let paths: Vec = match std::env::var("KUBECONFIG") { - Ok(chain) if !chain.is_empty() => chain.split(':').map(PathBuf::from).collect(), - _ => match home() { - Ok(h) => vec![h.join(".kube/config")], - Err(_) => return false, - }, - }; - paths.iter().any(|path| { - std::fs::read_to_string(path) - .map(|content| has_dory_entry(&content)) - .unwrap_or(false) - }) -} - -/// Line-anchored scan for a kubeconfig entry named `dory` — the mapping -/// form (`name: dory`, as kubectl writes context/cluster names) or the -/// sequence-item form (`- name: dory`, the users list). `name: dory-prod` -/// or `username: dory` must not count. -fn has_dory_entry(kubeconfig: &str) -> bool { - kubeconfig.lines().any(|line| { - let trimmed = line.trim(); - trimmed == "name: dory" || trimmed == "- name: dory" - }) + if rest.is_empty() { + std::env::set_var("KUBECONFIG", &dory_cfg); + } else { + std::env::set_var("KUBECONFIG", format!("{}:{}", dory_cfg, rest)); + } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn launch_window_remaining_covers_only_the_provisioning_window() { - use std::time::Duration; - - assert_eq!( - launch_window_remaining(Duration::ZERO), - Some(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)) - ); - assert_eq!( - launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS - 1)), - Some(Duration::from_secs(1)) - ); - assert_eq!( - launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)), - None - ); - assert_eq!(launch_window_remaining(Duration::from_secs(3600)), None); +/// Merge the stock Dory kubeconfig into `~/.kube/config` as the given context name. +fn ensure_user_kubeconfig_context(name: &str) -> Result<(), Box> { + let side = kubeconfig_path().ok_or("HOME unset")?; + if !std::path::Path::new(&side).is_file() { + return Err(format!( + "missing {}; enable Kubernetes in the Dory app (or let hops start write it)", + side + ) + .into()); } - #[test] - fn has_dory_entry_matches_mapping_and_sequence_forms_only() { - let merged = "contexts:\n- context:\n cluster: dory\n user: dory\n name: dory\n"; - let users_list = "users:\n- name: dory\n user: {}\n"; - let near_misses = "name: dory-prod\nusername: dory\n# name: dory\nfullname: dory\n"; - - assert!(has_dory_entry(merged)); - assert!(has_dory_entry(users_list)); - assert!(!has_dory_entry(near_misses)); - assert!(!has_dory_entry("")); + let home = home()?; + let kube_dir = home.join(".kube"); + std::fs::create_dir_all(&kube_dir)?; + let main = kube_dir.join("config"); + + // Work on a temp copy so we can rename stock contexts without mutating + // the product side file. + let tmp = kube_dir.join(format!(".dory-merge-{}.yaml", std::process::id())); + std::fs::copy(&side, &tmp)?; + let names = run_cmd_output( + "kubectl", + &[ + "config", + "--kubeconfig", + tmp.to_str().unwrap(), + "get-contexts", + "-o", + "name", + ], + ) + .unwrap_or_default(); + for old in names.lines().map(str::trim).filter(|n| !n.is_empty()) { + if old != name { + let _ = run_cmd( + "kubectl", + &[ + "config", + "--kubeconfig", + tmp.to_str().unwrap(), + "rename-context", + old, + name, + ], + ); + } } - #[test] - fn registries_yaml_aliases_both_pull_names_to_the_service_over_http() { - let yaml = registries_yaml(); - - assert!(yaml.contains(REGISTRIES_BEGIN)); - assert!(yaml.contains(REGISTRIES_END)); - assert!(yaml.contains("\"registry.crossplane-system.svc.cluster.local:5000\":")); - assert!(yaml.contains("\"localhost:30500\":")); - // Both mirrors resolve to the same HTTP endpoint on the Service name. - assert_eq!( - yaml.matches("- \"http://registry.crossplane-system.svc.cluster.local:5000\"") - .count(), - 2 + if main.is_file() { + let merged = { + let mut child = std::process::Command::new("kubectl") + .args(["config", "view", "--flatten", "--raw"]) + .env( + "KUBECONFIG", + format!("{}:{}", main.display(), tmp.display()), + ) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + let mut out = String::new(); + use std::io::Read; + if let Some(mut s) = child.stdout.take() { + s.read_to_string(&mut out)?; + } + let status = child.wait()?; + if !status.success() { + let _ = std::fs::remove_file(&tmp); + return Err( + "kubectl config view --flatten failed while merging dory kubeconfig".into(), + ); + } + out + }; + let backup = kube_dir.join("config.hops-dory-backup"); + let _ = std::fs::copy(&main, &backup); + std::fs::write(&main, merged)?; + let _ = std::fs::set_permissions( + &main, + std::os::unix::fs::PermissionsExt::from_mode(0o600), ); - assert!(!yaml.contains("https://")); - } - - #[test] - fn registries_yaml_replaces_only_hops_managed_block() { - let existing = format!( - "configs:\n example: value\nmirrors:\n {begin}\n old: value\n {end}\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\n", - begin = REGISTRIES_BEGIN, - end = REGISTRIES_END, + } else { + std::fs::copy(&tmp, &main)?; + let _ = std::fs::set_permissions( + &main, + std::os::unix::fs::PermissionsExt::from_mode(0o600), ); - - let merged = merge_registries_yaml(Some(&existing)).unwrap(); - - assert!(merged.starts_with("configs:\n example: value\nmirrors:\n")); - assert!(merged.contains(" \"user.local:5000\":\n")); - assert!(!merged.contains("old: value")); - assert!(merged.contains(&format!(" \"{}\":", REGISTRY_PULL))); - assert_eq!(merged.matches(REGISTRIES_BEGIN).count(), 1); - assert_eq!(merged.matches(REGISTRIES_END).count(), 1); } + let _ = std::fs::remove_file(&tmp); - #[test] - fn registries_yaml_inserts_block_under_existing_mirrors_key() { - let existing = "mirrors:\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\nconfigs:\n another: value\n"; - - let merged = merge_registries_yaml(Some(existing)).unwrap(); - - let block_pos = merged.find(REGISTRIES_BEGIN).unwrap(); - let user_pos = merged.find("\"user.local:5000\"").unwrap(); - assert!(block_pos < user_pos); - assert!(merged.contains("configs:\n another: value\n")); + // Drop the short-lived "dory" name from an earlier hops revision if present + // and distinct from the configured name. + if name != "dory" { + let _ = run_cmd("kubectl", &["config", "delete-context", "dory"]); } - #[test] - fn registries_yaml_appends_mirrors_section_when_missing() { - let existing = "configs:\n example: value\n"; - - let merged = merge_registries_yaml(Some(existing)).unwrap(); + let _ = run_cmd("kubectl", &["config", "use-context", name]); + log::info!( + "Kubernetes context '{}' is ready in ~/.kube/config (also: ~/.kube/dory-config)", + name + ); + Ok(()) +} - assert!(merged.starts_with(existing)); - assert!(merged.contains("\nmirrors:\n")); - assert!(merged.contains(REGISTRIES_BEGIN)); +/// Docker context of the same name, pointing at Dory's engine socket. +fn ensure_docker_context_default(name: &str) -> Result<(), Box> { + if !command_exists("docker") { + return Ok(()); } - - #[test] - fn registries_yaml_upgrades_legacy_hops_file_to_managed_block() { - let merged = merge_registries_yaml(Some(&legacy_registries_yaml())).unwrap(); - - assert_eq!(merged, registries_yaml()); - assert_eq!( - merged.matches(&format!("\"{}\":", REGISTRY_PULL)).count(), - 1 - ); - assert_eq!( - merged.matches(&format!("\"{}\":", REGISTRY_PUSH)).count(), - 1 + let sock = engine_socket()?; + if !sock.exists() { + log::warn!( + "Dory engine socket {} missing; skip docker context '{}'", + sock.display(), + name ); + return Ok(()); } - - #[test] - fn ports_file_appends_registry_port_without_touching_existing_lines() { - let existing = "# user port\n8080:80/udp\n"; - - let merged = ports_file_with_publish(existing); - - assert_eq!(merged, "# user port\n8080:80/udp\n30500:30500\n"); - } - - #[test] - fn ports_file_absent_writes_only_registry_port() { - assert_eq!(ports_file_with_publish(""), "30500:30500\n"); + let host = format!("host=unix://{}", sock.display()); + let contexts = run_cmd_output("docker", &["context", "ls", "--format", "{{.Name}}"]) + .unwrap_or_default(); + if !contexts.lines().any(|n| n.trim() == name) { + log::info!( + "Creating docker context '{}' → unix://{}", + name, + sock.display() + ); + // Ignore failure if a stale context exists with different metadata. + let create = run_cmd( + "docker", + &["context", "create", name, "--docker", &host], + ); + if create.is_err() { + // Update in place when possible. + let _ = run_cmd( + "docker", + &["context", "update", name, "--docker", &host], + ); + } } + run_cmd("docker", &["context", "use", name])?; + log::info!( + "Docker context set to '{}' (unix://{})", + name, + sock.display() + ); + Ok(()) +} - #[test] - fn ports_file_treats_tcp_variant_as_already_present() { - let existing = " 30500 : 30500 / tcp # registry\n"; - - assert_eq!(ports_file_with_publish(existing), existing); - } +#[cfg(test)] +mod tests { + use super::*; #[test] - fn dory_enable_args_only_recreate_for_reset_path() { + fn package_pull_is_in_cluster_service_not_host_gateway() { assert_eq!( - dory_enable_args(false), - vec!["k8s", "enable", "--publish", REGISTRY_PORT_PUBLISH] + crate::commands::local::package_install::REGISTRY_PULL_INCLUSTER, + "registry.crossplane-system.svc.cluster.local:5000" ); - assert_eq!( - dory_enable_args(true), - vec![ - "k8s", - "enable", - "--recreate", - "--publish", - REGISTRY_PORT_PUBLISH - ] - ); - } - - #[test] - fn recreate_on_enable_matches_dory_exit_codes() { - assert!(should_recreate_on_enable_error( - "dory exited with exit status: 3" - )); - assert!(should_recreate_on_enable_error( - "dory exited with exit status: 1" - )); - assert!(!should_recreate_on_enable_error( - "dory exited with exit status: 2" - )); - assert!(!should_recreate_on_enable_error("connection refused")); + assert!(!REGISTRY_PULL_INCLUSTER.contains("dory.internal")); + assert_eq!(REGISTRY_PUSH, "localhost:30500"); } #[test] @@ -768,4 +786,29 @@ mod tests { assert!(err.to_string().contains("--cpus 4")); assert!(err.to_string().contains("Dory app")); } + + #[test] + fn missing_k8s_error_mentions_app_not_fork() { + let msg = "Dory Kubernetes is not enabled (no `dory-k8s` container).\n\ + In the Dory app: enable Kubernetes, wait until it is running, then re-run:\n\ + hops local start --backend dory\n\ + (hops uses stock Dory only — it does not create the cluster for you.)"; + assert!(msg.contains("Dory app")); + assert!(!msg.contains("feat/scriptable")); + assert!(!msg.contains("dory k8s enable")); + } + + #[test] + fn desktop_integration_env_off_values() { + for off in ["0", "false", "FALSE", "no", "off", " Off "] { + std::env::set_var(DESKTOP_INTEGRATION_ENV, off); + assert!( + !desktop_integration_enabled(), + "expected desktop off for {off:?}" + ); + } + std::env::set_var(DESKTOP_INTEGRATION_ENV, "1"); + assert!(desktop_integration_enabled()); + std::env::remove_var(DESKTOP_INTEGRATION_ENV); + } } diff --git a/src/commands/local/backend/kind.rs b/src/commands/local/backend/kind.rs index 2a22f62..005172d 100644 --- a/src/commands/local/backend/kind.rs +++ b/src/commands/local/backend/kind.rs @@ -207,8 +207,9 @@ pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { } fn hosts_toml(cluster_ip: &str) -> String { + // Local registry serves HTTPS with a hops-managed self-signed cert. format!( - "[host.\"http://{}:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n", + "[host.\"https://{}:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n", cluster_ip ) } @@ -277,12 +278,14 @@ mod tests { } #[test] - fn hosts_toml_aliases_to_cluster_ip_over_http() { + fn hosts_toml_aliases_to_cluster_ip_over_https() { + // Local package registry is HTTPS (self-signed); containerd must use + // https:// + skip_verify, not plain http://. let toml = hosts_toml("10.43.12.7"); assert_eq!( toml, - "[host.\"http://10.43.12.7:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n" + "[host.\"https://10.43.12.7:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n" ); } diff --git a/src/commands/local/backend/mod.rs b/src/commands/local/backend/mod.rs index 2e2427f..e0b2098 100644 --- a/src/commands/local/backend/mod.rs +++ b/src/commands/local/backend/mod.rs @@ -78,11 +78,12 @@ impl Backend { } /// kubeconfig context name this backend's cluster registers under. - pub fn kube_context(self) -> &'static str { + pub fn kube_context(self) -> String { match self { - Backend::Colima => "colima", - Backend::Kind => "kind-hops", - Backend::Dory => "dory", + Backend::Colima => "colima".to_string(), + Backend::Kind => "kind-hops".to_string(), + // Merged into ~/.kube/config (default name hops-dory; see dory::context_name). + Backend::Dory => dory::context_name(), } } @@ -161,15 +162,16 @@ impl Backend { // containerd trust is per-name via certs.d, written in // wire_registry once the registry Service's ClusterIP is known. Backend::Kind => Ok(()), - // trust is static registries.yaml, written by dory::start before - // the cluster boots (k3s reads it at boot). - Backend::Dory => Ok(()), + // k3s registries.yaml is written in wire_registry once ClusterIP is known. + Backend::Dory => dory::ensure_registry_trust(), } } /// Point the node at the registry Service's current ClusterIP so pulls of /// both registry names resolve. Idempotent; re-run on every start because /// the ClusterIP changes if the Service is recreated. + /// + /// Dory also publishes host localhost:30500 → node NodePort (engine proxy). pub fn wire_registry(self, cluster_ip: &str) -> Result<(), Box> { match self { Backend::Colima => colima::sync_hosts_entry(cluster_ip), @@ -177,6 +179,31 @@ impl Backend { Backend::Dory => dory::wire_registry(cluster_ip), } } + + /// Backend-specific package registry for local provider/config installs. + /// All backends use an in-cluster NodePort registry for Crossplane package + /// pulls (pod network). Host push is always localhost:30500. + pub fn ensure_package_registry(self) -> Result<(), Box> { + crate::commands::local::package_install::ensure_incluster_registry() + } + + /// Pull address used in Crossplane package / ImageConfig refs for this backend. + /// Always the in-cluster Service — Crossplane runs in the pod network. + pub fn registry_pull(self) -> &'static str { + crate::commands::local::package_install::REGISTRY_PULL_INCLUSTER + } + + /// Docker push host:port for package installs on this backend. + pub fn registry_push(self) -> String { + match self { + Backend::Dory => dory::registry_push_addr().unwrap_or_else(|_| { + crate::commands::local::package_install::REGISTRY_PUSH.to_string() + }), + Backend::Colima | Backend::Kind => { + crate::commands::local::package_install::REGISTRY_PUSH.to_string() + } + } + } } impl fmt::Display for Backend { @@ -203,6 +230,11 @@ impl FromStr for Backend { const BACKEND_FILE: &str = "backend"; +/// Persist the Dory desktop context name (`--name`, default hops-dory). +pub fn persist_dory_context_name(name: &str) -> Result<(), Box> { + dory::persist_context_name(name) +} + pub fn platform_default() -> Backend { if cfg!(target_os = "macos") { Backend::Colima @@ -231,17 +263,22 @@ pub fn activate(flag: Option, context: Option<&str>) -> Backend { let backend = resolve(flag); if backend == Backend::Dory { - dory::export_kubeconfig_env(); + // Merge ~/.kube/dory-config → named context and prefer matching docker context. + if let Err(e) = dory::ensure_desktop_integration() { + log::warn!("dory desktop integration incomplete: {e}"); + dory::export_kubeconfig_env(); + } } let explicit_context = context.filter(|ctx| !ctx.is_empty()); + let backend_context = backend.kube_context(); let backend_context_exists = if explicit_context.is_none() { - kube_context_exists(backend.kube_context()) + kube_context_exists(&backend_context) } else { false }; - match kube_context_export(backend, explicit_context, backend_context_exists) { + match kube_context_export(explicit_context, &backend_context, backend_context_exists) { KubeContextExport::Set(ctx) => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx), KubeContextExport::Unset { missing_context } => { std::env::remove_var(HOPS_KUBE_CONTEXT_ENV); @@ -331,7 +368,7 @@ fn registry_cluster_ip() -> Result> { } /// Wire node-level registry pulls for a backend: fetch the registry Service -/// ClusterIP and apply the backend-specific trust/aliasing. +/// ClusterIP and apply the backend-specific trust/aliasing (and Dory host publish). pub fn wire_local_registry(backend: Backend) -> Result<(), Box> { backend.wire_registry(®istry_cluster_ip()?) } @@ -369,19 +406,18 @@ pub fn wire_local_registry_for_target( #[derive(Debug, PartialEq, Eq)] enum KubeContextExport<'a> { Set(&'a str), - Unset { missing_context: &'static str }, + Unset { missing_context: &'a str }, } fn kube_context_export<'a>( - backend: Backend, explicit_context: Option<&'a str>, + backend_context: &'a str, backend_context_exists: bool, ) -> KubeContextExport<'a> { if let Some(ctx) = explicit_context { return KubeContextExport::Set(ctx); } - let backend_context = backend.kube_context(); if backend_context_exists { KubeContextExport::Set(backend_context) } else { @@ -488,7 +524,7 @@ mod tests { #[test] fn explicit_context_is_exported_even_when_backend_context_is_absent() { assert_eq!( - kube_context_export(Backend::Colima, Some("foreign"), false), + kube_context_export(Some("foreign"), "colima", false), KubeContextExport::Set("foreign") ); } @@ -496,11 +532,11 @@ mod tests { #[test] fn backend_context_is_exported_only_when_present() { assert_eq!( - kube_context_export(Backend::Kind, None, true), + kube_context_export(None, "kind-hops", true), KubeContextExport::Set("kind-hops") ); assert_eq!( - kube_context_export(Backend::Kind, None, false), + kube_context_export(None, "kind-hops", false), KubeContextExport::Unset { missing_context: "kind-hops" } diff --git a/src/commands/local/doctor.rs b/src/commands/local/doctor.rs index 248c588..dcbe8d1 100644 --- a/src/commands/local/doctor.rs +++ b/src/commands/local/doctor.rs @@ -72,6 +72,17 @@ pub fn run() -> Result<(), Box> { "registry deployment not Available in crossplane-system".into() }, ); + let push = super::package_install::registry_push(); + let push_ok = package_push_registry_reachable(&push); + d.check( + &format!("package push registry reachable ({push})"), + push_ok, + if push_ok { + String::new() + } else { + format!("expected HTTPS registry at {push} for docker push") + }, + ); d.print(); @@ -289,6 +300,69 @@ fn provider_package(provider: &str) -> String { .unwrap_or_default() } +/// Whether `push` is a host-loopback address (kind/colima NodePort). +/// +/// Dory pushes via the engine docker bridge (`{dory-k8s-ip}:30500`); that IP is +/// not reachable from the Mac, so host curl is the wrong plane and can hang on +/// TCP timeout (~75s) before the engine probe runs. +fn push_host_is_loopback(push: &str) -> bool { + // host:port — IPv6 may be bracketed as [::1]:30500 + let host = if let Some(rest) = push.strip_prefix('[') { + rest.split(']').next().unwrap_or(rest) + } else { + push.split(':').next().unwrap_or(push) + }; + host == "localhost" || host == "127.0.0.1" || host == "::1" +} + +/// Probe the package push registry on the plane docker will use. +/// +/// - Loopback (`localhost:30500`): short host curl (kind/colima). +/// - Otherwise (dory bridge IP): engine-network probe only — never host curl +/// (wrong plane; TCP hang ~75s) and never default docker.sock (can block on +/// password / missing daemon on a personal Mac). +fn package_push_registry_reachable(push: &str) -> bool { + let url = format!("https://{push}/v2/"); + if push_host_is_loopback(push) { + return super::run_cmd_output( + "curl", + &["-skf", "--connect-timeout", "2", "--max-time", "3", &url], + ) + .is_ok(); + } + + // Prefer Dory's engine socket explicitly so we don't touch host Docker. + let dory_sock = std::env::var("HOME") + .ok() + .map(|h| std::path::PathBuf::from(h).join(".dory/dory.sock")) + .filter(|p| p.exists()); + + let mut args: Vec = Vec::new(); + if let Some(sock) = dory_sock { + args.push("-H".into()); + args.push(format!("unix://{}", sock.display())); + } else if std::env::var_os("DOCKER_HOST").is_none() { + // No engine socket and no DOCKER_HOST: skip rather than hang on host sock. + return false; + } + args.extend( + [ + "run", + "--rm", + "alpine:latest", + "wget", + "-qO-", + "--no-check-certificate", + "--timeout=5", + &url, + ] + .into_iter() + .map(String::from), + ); + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + super::run_cmd_output("docker", &arg_refs).is_ok() +} + /// True when `kubectl get ` finds the resource. Uses `--ignore-not-found` /// so a missing resource is `false` rather than an error. fn exists(get_args: &[&str]) -> bool { @@ -385,3 +459,21 @@ impl Doctor { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn push_host_loopback_for_kind_colima() { + assert!(push_host_is_loopback("localhost:30500")); + assert!(push_host_is_loopback("127.0.0.1:30500")); + assert!(push_host_is_loopback("[::1]:30500")); + } + + #[test] + fn push_host_not_loopback_for_dory_bridge() { + assert!(!push_host_is_loopback("192.168.215.2:30500")); + assert!(!push_host_is_loopback("10.43.0.1:30500")); + } +} diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index abb27ac..8cc2751 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -107,6 +107,12 @@ pub struct LocalArgs { /// kind). #[arg(long, global = true, value_enum)] pub backend: Option, + + /// Name for Dory desktop integration (kube context + docker context). + /// Defaults to `hops-dory`. Persisted under `~/.hops/local/dory-name`. + /// Only used with `--backend dory` (or a persisted dory backend). + #[arg(long, global = true, value_name = "NAME")] + pub name: Option, } #[derive(Subcommand, Debug)] @@ -115,7 +121,7 @@ pub enum LocalCommands { Install, /// Reset local Kubernetes state (colima: k8s reset; kind: recreate cluster) Reset, - /// Start local k8s cluster with Crossplane and providers + /// Start local k8s and ensure Crossplane control plane (skips helm when already healthy) Start(start::StartArgs), /// Resize the local cluster VM without destroying cluster state (colima only) Resize(resize::ResizeArgs), @@ -140,12 +146,16 @@ pub enum LocalCommands { } pub fn run(args: &LocalArgs) -> Result<(), Box> { + if let Some(name) = args.name.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + backend::persist_dory_context_name(name)?; + } + let explicit_context = args.context.as_deref().filter(|ctx| !ctx.is_empty()); let install_backend = matches!(&args.command, LocalCommands::Install) .then(|| args.backend.unwrap_or_else(backend::platform_default)); let activation_flag = install_backend.or(args.backend); - let install_context = install_backend.map(backend::Backend::kube_context); - let activation_context = explicit_context.or(install_context); + let install_context = install_backend.map(|b| b.kube_context()); + let activation_context = explicit_context.or(install_context.as_deref()); let backend = backend::activate(activation_flag, activation_context); match &args.command { diff --git a/src/commands/local/package_install.rs b/src/commands/local/package_install.rs index dd25544..1af9b3b 100644 --- a/src/commands/local/package_install.rs +++ b/src/commands/local/package_install.rs @@ -11,13 +11,29 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const REGISTRY_YAML: &str = include_str!("../../../bootstrap/registry/registry.yaml"); -/// Host address for `docker push` (NodePort exposed by the in-cluster registry) +/// Default docker push host for colima/kind (NodePort on localhost). +/// On Dory use [`registry_push`] — engine dockerd is not the Mac. pub const REGISTRY_PUSH: &str = "localhost:30500"; -/// Cluster-internal address used in Crossplane package references -pub const REGISTRY_PULL: &str = "registry.crossplane-system.svc.cluster.local:5000"; +/// Cluster-internal address used in Crossplane package references (all backends). +pub const REGISTRY_PULL_INCLUSTER: &str = + "registry.crossplane-system.svc.cluster.local:5000"; pub const REGISTRY_HOSTNAME: &str = "registry.crossplane-system.svc.cluster.local"; +/// Back-compat alias — prefer [`registry_pull`] when the backend is known. +pub const REGISTRY_PULL: &str = REGISTRY_PULL_INCLUSTER; + +/// Resolve the package pull address for the active local backend. +pub fn registry_pull() -> &'static str { + super::backend::resolve(None).registry_pull() +} + +/// Docker push registry host:port for the active backend. +/// Colima/kind → `localhost:30500`. Dory → `{dory-k8s-ip}:30500` on the engine bridge. +pub fn registry_push() -> String { + super::backend::resolve(None).registry_push() +} + #[derive(Clone, Debug)] pub struct RepoSpec { pub org: String, @@ -169,31 +185,25 @@ pub fn image_config_name(source: &str) -> String { format!("{prefix}{body}-{hash}") } -/// Ensure the in-cluster registry is deployed and available. +/// TLS Secret for the local package registry (server cert + CA). +pub const REGISTRY_TLS_SECRET: &str = "hops-local-registry-tls"; +const REGISTRY_TLS_SECRET_NS: &str = "crossplane-system"; + +/// Ensure the local package registry for the active backend is available. pub fn ensure_registry() -> Result<(), Box> { - let result = run_cmd_output( - "kubectl", - &[ - "get", - "deployment", - "registry", - "-n", - "crossplane-system", - "-o", - "jsonpath={.status.availableReplicas}", - ], - ); + super::backend::resolve(None).ensure_package_registry() +} - if let Ok(replicas) = result { - if replicas.trim() == "1" { - return Ok(()); - } - } +/// Ensure the in-cluster HTTPS NodePort registry is deployed and Crossplane +/// trusts its CA. Crossplane's package manager always uses HTTPS. +pub fn ensure_incluster_registry() -> Result<(), Box> { + ensure_namespace(REGISTRY_TLS_SECRET_NS)?; + ensure_registry_tls_secret()?; - log::info!("Deploying local package registry..."); + log::info!("Deploying local package registry (HTTPS)..."); kubectl_apply_stdin(REGISTRY_YAML)?; - for _ in 0..60 { + for _ in 0..90 { let out = run_cmd_output( "kubectl", &[ @@ -201,13 +211,14 @@ pub fn ensure_registry() -> Result<(), Box> { "deployment", "registry", "-n", - "crossplane-system", + REGISTRY_TLS_SECRET_NS, "-o", "jsonpath={.status.availableReplicas}", ], ); if let Ok(r) = out { if r.trim() == "1" { + ensure_crossplane_trusts_local_registry_ca()?; return Ok(()); } } @@ -217,6 +228,260 @@ pub fn ensure_registry() -> Result<(), Box> { Err("Timed out waiting for registry deployment".into()) } +fn ensure_namespace(ns: &str) -> Result<(), Box> { + let exists = run_cmd_output("kubectl", &["get", "ns", ns, "-o", "name"]).is_ok(); + if exists { + return Ok(()); + } + run_cmd("kubectl", &["create", "namespace", ns]) +} + +/// Create (or reuse) a self-signed CA + server cert for the registry Service DNS. +fn ensure_registry_tls_secret() -> Result<(), Box> { + let exists = run_cmd_output( + "kubectl", + &[ + "get", + "secret", + REGISTRY_TLS_SECRET, + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "name", + ], + ) + .is_ok(); + if exists { + return Ok(()); + } + + log::info!("Generating self-signed TLS for local package registry..."); + let dir = std::env::temp_dir().join(format!( + "hops-registry-tls-{}", + unique_suffix() + )); + fs::create_dir_all(&dir)?; + let ca_key = dir.join("ca.key"); + let ca_crt = dir.join("ca.crt"); + let tls_key = dir.join("tls.key"); + let tls_csr = dir.join("tls.csr"); + let tls_crt = dir.join("tls.crt"); + let san = dir.join("san.cnf"); + + fs::write( + &san, + "[req]\n\ + distinguished_name = dn\n\ + req_extensions = ext\n\ + prompt = no\n\ + [dn]\n\ + CN = registry.crossplane-system.svc.cluster.local\n\ + [ext]\n\ + subjectAltName = @alt\n\ + basicConstraints = CA:FALSE\n\ + keyUsage = digitalSignature, keyEncipherment\n\ + extendedKeyUsage = serverAuth\n\ + [alt]\n\ + DNS.1 = registry\n\ + DNS.2 = registry.crossplane-system\n\ + DNS.3 = registry.crossplane-system.svc\n\ + DNS.4 = registry.crossplane-system.svc.cluster.local\n\ + DNS.5 = localhost\n\ + IP.1 = 127.0.0.1\n", + )?; + + run_cmd( + "openssl", + &[ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ca_key.to_str().unwrap(), + "-out", + ca_crt.to_str().unwrap(), + "-days", + "3650", + "-subj", + "/CN=hops-local-registry-ca", + ], + )?; + run_cmd( + "openssl", + &[ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + tls_key.to_str().unwrap(), + "-out", + tls_csr.to_str().unwrap(), + "-config", + san.to_str().unwrap(), + ], + )?; + run_cmd( + "openssl", + &[ + "x509", + "-req", + "-in", + tls_csr.to_str().unwrap(), + "-CA", + ca_crt.to_str().unwrap(), + "-CAkey", + ca_key.to_str().unwrap(), + "-CAcreateserial", + "-out", + tls_crt.to_str().unwrap(), + "-days", + "3650", + "-extfile", + san.to_str().unwrap(), + "-extensions", + "ext", + ], + )?; + + run_cmd( + "kubectl", + &[ + "create", + "secret", + "generic", + REGISTRY_TLS_SECRET, + "-n", + REGISTRY_TLS_SECRET_NS, + &format!("--from-file=ca.crt={}", ca_crt.display()), + &format!("--from-file=tls.crt={}", tls_crt.display()), + &format!("--from-file=tls.key={}", tls_key.display()), + ], + )?; + let _ = fs::remove_dir_all(&dir); + Ok(()) +} + +/// Merge the local registry CA into Crossplane's trust store so package pulls +/// over HTTPS succeed without replacing public CAs (xpkg.crossplane.io). +pub fn ensure_crossplane_trusts_local_registry_ca() -> Result<(), Box> { + let xp_ready = run_cmd_output( + "kubectl", + &[ + "get", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "name", + ], + ) + .is_ok(); + if !xp_ready { + log::warn!("crossplane deployment not found yet; skip CA trust patch"); + return Ok(()); + } + + // Already patched? + let has_init = run_cmd_output( + "kubectl", + &[ + "get", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "jsonpath={.spec.template.spec.initContainers[*].name}", + ], + ) + .unwrap_or_default(); + if has_init.split_whitespace().any(|n| n == "hops-merge-registry-ca") { + return Ok(()); + } + + log::info!("Patching Crossplane to trust local registry CA..."); + // Strategic merge: initContainer concatenates system CAs + hops CA; SSL_CERT_FILE points at it. + let patch = r#"{ + "spec": { + "template": { + "spec": { + "initContainers": [ + { + "name": "hops-merge-registry-ca", + "image": "alpine:3.20", + "command": ["sh", "-c"], + "args": [ + "set -e; cat /etc/ssl/certs/ca-certificates.crt /hops-ca/ca.crt > /combined/ca-bundle.crt" + ], + "volumeMounts": [ + {"name": "hops-local-registry-ca", "mountPath": "/hops-ca", "readOnly": true}, + {"name": "hops-combined-ca", "mountPath": "/combined"} + ] + } + ], + "containers": [ + { + "name": "crossplane", + "env": [ + {"name": "SSL_CERT_FILE", "value": "/combined/ca-bundle.crt"} + ], + "volumeMounts": [ + {"name": "hops-combined-ca", "mountPath": "/combined", "readOnly": true} + ] + } + ], + "volumes": [ + { + "name": "hops-local-registry-ca", + "secret": { + "secretName": "hops-local-registry-tls", + "items": [{"key": "ca.crt", "path": "ca.crt"}] + } + }, + { + "name": "hops-combined-ca", + "emptyDir": {} + } + ] + } + } + } +}"#; + + run_cmd( + "kubectl", + &[ + "patch", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "--type", + "strategic", + "-p", + patch, + ], + )?; + + // Wait for rollout so package installs see the new trust store. + let _ = run_cmd( + "kubectl", + &[ + "rollout", + "status", + "deploy/crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "--timeout=180s", + ], + ); + Ok(()) +} + pub fn interactive_stdio_available() -> bool { io::stdin().is_terminal() && io::stdout().is_terminal() } diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 4f04760..a5d5802 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -13,7 +13,9 @@ const PROVIDER_HELM: &str = include_str!("../../../bootstrap/providers/provider- const PROVIDER_K8S: &str = include_str!("../../../bootstrap/providers/provider-kubernetes.yaml"); const PC_HELM: &str = include_str!("../../../bootstrap/helm/pc.yaml"); const PC_K8S: &str = include_str!("../../../bootstrap/k8s/pc.yaml"); -const REGISTRY: &str = include_str!("../../../bootstrap/registry/registry.yaml"); + +const PROVIDER_K8S_NAME: &str = "crossplane-contrib-provider-kubernetes"; +const PROVIDER_HELM_NAME: &str = "crossplane-contrib-provider-helm"; #[derive(Args, Debug, Clone)] pub struct StartArgs { @@ -23,6 +25,11 @@ pub struct StartArgs { /// Stop and restart a running cluster VM without prompting when requested size differs. #[arg(long)] pub yes: bool, + + /// Force helm upgrade and full bootstrap even when the control plane is + /// already healthy. Default is to skip expensive helm/repo work on resume. + #[arg(long)] + pub bootstrap: bool, } pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box> { @@ -55,6 +62,73 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box bool { + deployment_available("crossplane-system", "crossplane") + && deployment_available("crossplane-system", "registry") + && provider_healthy(PROVIDER_K8S_NAME) + && provider_healthy(PROVIDER_HELM_NAME) +} + +fn deployment_available(namespace: &str, name: &str) -> bool { + run_cmd_output( + "kubectl", + &[ + "get", + "deployment", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Available\")].status}", + ], + ) + .map(|s| s.trim() == "True") + .unwrap_or(false) +} + +fn provider_healthy(provider: &str) -> bool { + run_cmd_output( + "kubectl", + &[ + "get", + "provider.pkg.crossplane.io", + provider, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Healthy\")].status}", + ], + ) + .map(|s| s.trim() == "True") + .unwrap_or(false) +} + +/// Helm + Crossplane + providers + ProviderConfigs (the slow cold-start path). +fn bootstrap_control_plane() -> Result<(), Box> { // 4. Add Crossplane Helm repo log::info!("Adding Crossplane Helm repo..."); run_cmd( @@ -158,27 +232,25 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box> { + // Crossplane package pulls run in the pod network → Service DNS + ClusterIP. + // Docker push: colima/kind → localhost:30500 (host NodePort); dory → + // {dory-k8s-ip}:30500 on the engine docker bridge (daemon is in-engine). wait_for_kubernetes()?; log::info!("Pre-pulling registry:2 (best effort)..."); let _ = run_cmd("docker", &["pull", "registry:2"]); - log::info!("Deploying local package registry..."); - kubectl_apply_stdin(REGISTRY)?; - // Nested virt: image pull + schedule for the registry can exceed the - // default ~5m wait used for lighter resources. + // TLS secret + registry Deployment + Crossplane CA trust (package manager is HTTPS-only). + log::info!("Deploying local package registry (HTTPS)..."); + backend.ensure_package_registry()?; wait_for_deployment_with_diagnostics("crossplane-system", "registry")?; - - // 13. Point the node at the registry Service's ClusterIP so pulls of the - // cluster-internal registry names resolve. backend::wire_local_registry(backend)?; - - log::info!("Local environment is ready"); Ok(()) } @@ -363,3 +435,33 @@ fn wait_for_provider_healthy(provider: &str) -> Result<(), Box> { ) .into()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_args_bootstrap_defaults_false() { + // clap default: bootstrap only when --bootstrap is passed + assert!(!StartArgs { + size: SizeArgs { + cpus: None, + memory: None, + disk: None, + }, + yes: false, + bootstrap: false, + } + .bootstrap); + assert!(StartArgs { + size: SizeArgs { + cpus: None, + memory: None, + disk: None, + }, + yes: false, + bootstrap: true, + } + .bootstrap); + } +} diff --git a/src/commands/provider/install.rs b/src/commands/provider/install.rs index c45a3e6..de0a2da 100644 --- a/src/commands/provider/install.rs +++ b/src/commands/provider/install.rs @@ -2,7 +2,7 @@ use crate::commands::local::backend::{self, Backend}; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout_at, ensure_registry, parse_repo_spec, resolve_repo_install_target, run_watch, sanitize_name_component, RepoInstallTarget, RepoSpec, - REGISTRY_PULL, REGISTRY_PUSH, + registry_pull, registry_push, }; use crate::commands::local::{ kubectl_apply_stdin, run_cmd, run_cmd_output, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, @@ -178,7 +178,7 @@ fn apply_repo_version_spec( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], )?; - let resolved = resolve_provider_target(&provider_name, &providers_json, REGISTRY_PULL)?; + let resolved = resolve_provider_target(&provider_name, &providers_json, registry_pull())?; // Published install pulls directly from ghcr.io — no ImageConfig rewrite, // no local registry push, no runtime-image override. apply_provider_resources( @@ -216,8 +216,8 @@ fn run_local_path( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], )?; - let resolved = resolve_provider_target(&provider_name, &providers_json, REGISTRY_PULL)?; - let upstream_url_prefix = recover_upstream_url_prefix(&resolved, REGISTRY_PULL)?; + let resolved = resolve_provider_target(&provider_name, &providers_json, registry_pull())?; + let upstream_url_prefix = recover_upstream_url_prefix(&resolved, registry_pull())?; let upstream_major = parse_major_version(&resolved.existing_package); ensure_build_submodule(dir)?; @@ -232,7 +232,7 @@ fn run_local_path( let xpkg_path = find_xpkg_for_provider(dir, &provider_name, arch)?; log::info!("Located xpkg: {}", xpkg_path.display()); - let local_image_path_for_tag = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); + let local_image_path_for_tag = format!("{}/hops-ops/{}", registry_pull(), provider_name); let dev_tag = dev_tag_for_file( &xpkg_path, version_prefix, @@ -240,8 +240,8 @@ fn run_local_path( &local_image_path_for_tag, )?; - let push_xpkg_ref = format!("{}/hops-ops/{}:{}", REGISTRY_PUSH, provider_name, dev_tag); - let local_pull_xpkg_path = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); + let push_xpkg_ref = format!("{}/hops-ops/{}:{}", registry_push(), provider_name, dev_tag); + let local_pull_xpkg_path = format!("{}/hops-ops/{}", registry_pull(), provider_name); log::info!("Pushing xpkg to {}...", push_xpkg_ref); crossplane_xpkg_push(&xpkg_path, &push_xpkg_ref)?; @@ -394,7 +394,10 @@ fn local_runtime_image_ref(provider_name: &str, arch: &str, tag: &str) -> String // package manager, so they need the node-pullable local registry address. format!( "{}/hops-ops/{}-{}:{}", - REGISTRY_PUSH, provider_name, arch, tag + registry_push(), + provider_name, + arch, + tag ) } @@ -499,9 +502,10 @@ fn next_local_patch_for_major(image_path: &str, major: u64) -> Result String { } #[test] -fn dory_smoke_workflow_clone_build_install_contract() { +fn dory_smoke_workflow_self_hosted_stock_contract() { let text = workflow_text(); + assert!( - text.contains("patrickleet/dory"), - "must clone patrickleet/dory" + !text.contains("patrickleet/dory"), + "must not clone a hops fork of dory" ); - // Pin a known-good commit of the hops integration work (not a moving branch - // tip — the upstream branch may never merge). Bump intentionally with dory. assert!( - text.contains("f8c61d2fd0fc4d528e5e0da36ffa09b9796b3871"), - "must pin patrickleet/dory to a specific commit SHA" + !text.contains("xcodebuild") && !text.contains("build-dory-ffi"), + "must not build Dory from source" ); assert!( - !text - .lines() - .any(|l| l.trim() == "ref: feat/hops-local-integration"), - "must not track the moving feat/hops-local-integration branch tip" + !text.contains("colima start") + && !text.contains("brew install colima") + && !text.contains("colima-surrogate"), + "must not use Colima as a dory.sock surrogate" ); assert!( - text.contains("xcodebuild") && text.contains("derivedDataPath"), - "must build in-pipeline with deterministic derivedDataPath" + text.contains("dory.sock"), + "must require real ~/.dory/dory.sock" ); assert!( - text.contains("dtolnay/rust-toolchain") && text.contains("brew install protobuf"), - "must install the Rust and protoc prerequisites used by Dory's FFI builder" + text.contains("refusing Colima-backed") || text.contains("Colima-backed dory.sock"), + "must refuse a Colima-symlinked dory.sock" ); - let ffi_build = text - .find("scripts/build-dory-ffi-xcframework.sh --if-needed") - .expect("must materialize DoryFFI from a clean checkout"); - let app_build = text - .find("xcodebuild -project Dory.xcodeproj") - .expect("must build the Dory app"); assert!( - ffi_build < app_build, - "must generate DoryFFI before SwiftPM resolves the Dory app" + text.contains("self-hosted") && text.contains("hops-dory"), + "must target self-hosted runner labeled hops-dory" ); assert!( - text.contains("DoryFFI.xcframework/macos-arm64_x86_64/libdory_ffi.a") - && text.contains("Sources/DoryCore/generated/dory_ffi.swift"), - "must verify both generated Dory FFI artifacts before xcodebuild" + !text.lines().any(|l| { + let t = l.trim(); + t == "runs-on: macos-15" + || t == "runs-on: macos-latest" + || t == "runs-on: macos-15-intel" + }), + "must not use GitHub-hosted macOS" ); assert!( - text.contains("GITHUB_PATH") && text.contains("scripts"), - "must put scripts/dory on PATH" + text.contains("labeled") && text.contains("test-dory"), + "PR smoke must be opt-in via test-dory label" ); +} + +#[test] +fn dory_smoke_workflow_env_only_no_desktop_mutation() { + let text = workflow_text(); + + // Session entirely via env vars. assert!( - text.contains("engine.sock"), - "must ensure ~/.dory/engine.sock before hops start" + text.contains("HOPS_DORY_DESKTOP") && text.contains("\"0\""), + "must set HOPS_DORY_DESKTOP=0 so hops does not rewrite desktop defaults" ); assert!( - !text.contains("brew install --cask") && !text.contains("homebrew/cask"), - "must not use brew cask as primary install" + text.contains("DOCKER_HOST") && text.contains("dory.sock"), + "must drive docker via DOCKER_HOST → dory.sock" ); assert!( - text.contains("workflow_dispatch") && text.contains("pull_request:"), - "must support pull_request (PR-branch runs) and workflow_dispatch" + text.contains("KUBECONFIG") && text.contains("RUNNER_TEMP"), + "must use a job-private KUBECONFIG under RUNNER_TEMP" ); + + // Never switch/restore machine defaults. assert!( - text.contains("labeled") && text.contains("test-dory"), - "PR smoke must start only when opted in with the test-dory label" + !text.contains("kubectl config use-context"), + "must not kubectl config use-context" + ); + assert!( + !text.contains("docker context use"), + "must not docker context use" + ); + assert!( + !text.contains("Restore desktop contexts") && !text.contains("Snapshot desktop"), + "must not snapshot/restore contexts — env-only session needs no restore" + ); + + // Do not destroy product plane. + assert!( + !text.contains("hops-cli local destroy"), + "must not hops local destroy" + ); + assert!( + !text.contains("docker rm -f dory-k8s"), + "must not docker rm dory-k8s" ); assert!( - text.contains("github.event_name == 'workflow_dispatch'") - && text.contains("github.event.pull_request.labels.*.name"), - "manual dispatch must remain available while PR runs are label-gated" + !text.contains("dory engine sleep") && !text.contains("pkill -f"), + "must not sleep engine or kill Dory" ); - // Nested-virt pin for Colima-backed engine.sock on public GHA. assert!( - text.lines() - .any(|l| l.trim() == "runs-on: macos-15-intel"), - "must pin macos-15-intel for nested virt (not bare macos-latest alone)" + !text.contains("hops-cli local stop"), + "must not hops local stop" + ); + + assert!( + text.contains("CARGO_BUILD_JOBS") && text.contains("nice"), + "must limit cargo parallelism and nice the build" ); } #[test] -fn dory_smoke_workflow_kind_parity_when_engine_boots() { +fn dory_smoke_workflow_hops_integration_core() { let text = workflow_text(); for needle in [ "cargo build", "start --backend dory", "local doctor", - "localhost:30500", "registry.crossplane-system.svc.cluster.local:5000", - "--context dory", - "local stop", - "local destroy", + "30500", + "smoke-svc-name", + "smoke-nodeport", ] { assert!( text.contains(needle), - "dory smoke missing kind-parity fragment: {needle}" + "dory smoke missing hops integration fragment: {needle}" ); } - // stop/start resume: start without --backend after stop - assert!( - text.contains("local start\n") - || text - .lines() - .any(|l| l.trim() == "./target/debug/hops-cli local start"), - "must start again without --backend after stop" - ); -} - -#[test] -fn dory_smoke_workflow_carries_colima_lessons() { - let text = workflow_text(); assert!( - text.contains("kube-dns") || text.contains("coredns"), - "must wait for CoreDNS/kube-dns before registry round-trip" + text.contains("NODE_IP") || text.contains("NetworkSettings"), + "must push via engine-plane dory-k8s IP" ); + // Single-platform pull/build avoids multiplatform push warnings. assert!( - text.contains("--timeout=420s"), - "must use a long Ready/Available timeout for nested-virt lag" + text.contains("docker pull --platform") || text.contains("--platform \"$PLATFORM\""), + "registry smoke must pull busybox with an explicit --platform" ); assert!( - text.contains("rollout restart"), - "stop/start resume must rollout-restart stalled deployments" + text.contains("multiplatform") || text.contains("single-platform"), + "must document why single-platform materialization is used" ); +} + +#[test] +fn dory_smoke_workflow_path_based_config_install() { + let text = workflow_text(); assert!( - text.contains("Debug dump on failure") && text.contains("engine.sock"), - "failure dump must capture dory/engine diagnostics" + text.contains("tests/fixtures/config-smoke"), + "must path-install the in-repo config-smoke fixture" ); assert!( - text.contains("describe pod smoke-svc-name") - || text.contains("describe pod smoke-svc-name smoke-localhost"), - "failure dump must describe smoke pods in default ns" + text.contains("config install --path") || text.contains("config install --path "), + "must use hops config install --path (no clone)" ); assert!( - text.to_lowercase().contains("localhost:30500"), - "must exercise localhost:30500 registry path" + !text.contains("config install --repo"), + "must not clone a remote config repo for the smoke" ); -} - -#[test] -fn dory_smoke_workflow_public_gha_engine_fallback() { - let text = workflow_text(); - // Public GHA cannot run dory-hv; workflow must document and implement a - // Colima-backed engine.sock so hops --backend dory remains testable. assert!( - text.contains("colima start") && text.contains("engine.sock"), - "must bootstrap Colima docker as engine.sock when native sock is absent" + text.contains("local/ci-xr.yaml") || text.contains("ci-xr.yaml"), + "must apply the fixture XR" ); assert!( - text.to_lowercase().contains("surrogate") - || text.contains("colima-surrogate") - || text.contains("Colima-backed"), - "must document Colima engine.sock surrogate for public GHA" + text.contains("hops-ci") && text.contains("configmap"), + "must verify ConfigMap in hops-ci namespace" ); assert!( - text.contains("xattr") || text.contains("codesign"), - "must clear quarantine/sign Debug Dory.app before open" + text.contains("command -v up") || text.contains("up CLI"), + "must require Upbound up CLI for path builds" ); } diff --git a/tests/fixtures/config-smoke/.gitignore b/tests/fixtures/config-smoke/.gitignore new file mode 100644 index 0000000..d2e372f --- /dev/null +++ b/tests/fixtures/config-smoke/.gitignore @@ -0,0 +1,4 @@ +_output/ +.up/ +.venv/ +.tmp/ diff --git a/tests/fixtures/config-smoke/README.md b/tests/fixtures/config-smoke/README.md new file mode 100644 index 0000000..0ce3048 --- /dev/null +++ b/tests/fixtures/config-smoke/README.md @@ -0,0 +1,14 @@ +# config-smoke + +Minimal Crossplane configuration fixture for **path-based** hops integration tests. + +```bash +hops local start --backend dory # or kind/colima +hops config install --path tests/fixtures/config-smoke --backend dory +kubectl apply -f tests/fixtures/config-smoke/local/ci-xr.yaml +kubectl -n hops-ci wait --for=condition=Ready configsmoke/hops-ci-smoke --timeout=300s +kubectl -n hops-ci get configmap hops-ci-smoke +``` + +Creates only namespaced resources under `hops-ci` (ConfigMap via provider-kubernetes). +Not a real platform stack — safe on a shared local control plane. diff --git a/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml b/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml new file mode 100644 index 0000000..1c11c03 --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml @@ -0,0 +1,19 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: configsmokes.ci.hops.ops.com.ai +spec: + compositeTypeRef: + apiVersion: ci.hops.ops.com.ai/v1alpha1 + kind: ConfigSmoke + mode: Pipeline + pipeline: + - functionRef: + # Crossplane names the installed Function from the package path + # ghcr.io/hops-ops/config-smoke_render → hops-ops-config-smokerender + # (same convention as gateway-api-stackrender, etc.). + name: hops-ops-config-smokerender + step: render + - functionRef: + name: crossplane-contrib-function-auto-ready + step: crossplane-contrib-function-auto-ready diff --git a/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml b/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml new file mode 100644 index 0000000..4abf95a --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml @@ -0,0 +1,14 @@ +apiVersion: meta.pkg.crossplane.io/v1alpha1 +kind: Configuration +metadata: + name: config-smoke + annotations: + meta.crossplane.io/maintainer: hops-ops CI + meta.crossplane.io/source: github.com/hops-ops/hops-cli + meta.crossplane.io/description: Minimal CI fixture ConfigMap composition for hops path-based config install. +spec: + dependsOn: + - function: xpkg.crossplane.io/crossplane-contrib/function-auto-ready + version: '>=v0.6.0' + - provider: xpkg.crossplane.io/crossplane-contrib/provider-kubernetes + version: '>=v0.15.0' diff --git a/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml b/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml new file mode 100644 index 0000000..14b9559 --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml @@ -0,0 +1,53 @@ +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: configsmokes.ci.hops.ops.com.ai +spec: + group: ci.hops.ops.com.ai + names: + kind: ConfigSmoke + plural: configsmokes + scope: Namespaced + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + description: CI-only composite that materializes a ConfigMap. + type: object + properties: + spec: + type: object + properties: + managementPolicies: + type: array + items: + type: string + default: ["*"] + data: + description: ConfigMap data key/values. + type: object + additionalProperties: + type: string + x-kubernetes-preserve-unknown-fields: true + configMapName: + description: Name of the ConfigMap to create. Defaults to metadata.name. + type: string + providerConfigRef: + type: object + properties: + name: + type: string + kind: + type: string + enum: [ProviderConfig, ClusterProviderConfig] + required: [] + status: + type: object + properties: + ready: + type: boolean + configMapName: + type: string + required: [spec] diff --git a/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml b/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml new file mode 100644 index 0000000..81c7ea3 --- /dev/null +++ b/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml @@ -0,0 +1,13 @@ +apiVersion: ci.hops.ops.com.ai/v1alpha1 +kind: ConfigSmoke +metadata: + name: hops-ci-smoke + namespace: hops-ci +spec: + configMapName: hops-ci-smoke + data: + source: path-based-config-install + smoke: "ok" + providerConfigRef: + name: default + kind: ProviderConfig diff --git a/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl new file mode 100644 index 0000000..9dbb397 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl @@ -0,0 +1,26 @@ +# code: language=yaml +{{- $xr := getCompositeResource . }} +{{- $metadata := $xr.metadata | default dict }} +{{- $spec := $xr.spec | default dict }} + +{{- $name := $metadata.name | default "config-smoke" }} +{{- $namespace := $metadata.namespace | default "default" }} +{{- $configMapName := $spec.configMapName | default $name }} +{{- $managementPolicies := $spec.managementPolicies | default (list "*") }} +{{- $data := $spec.data | default (dict "smoke" "ok") }} +{{- $pc := $spec.providerConfigRef | default dict }} +{{- $providerConfigRef := dict + "name" ($pc.name | default "default") + "kind" ($pc.kind | default "ProviderConfig") +}} + +{{- $state := dict + "name" $name + "namespace" $namespace + "configMapName" $configMapName + "managementPolicies" $managementPolicies + "data" $data + "providerConfigRef" $providerConfigRef + "observed" (dict) + "status" (dict "ready" false "configMapName" $configMapName) +}} diff --git a/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl new file mode 100644 index 0000000..691f1d4 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl @@ -0,0 +1,17 @@ +# code: language=yaml +{{- $raw := $.observed.resources | default dict }} +{{- $entry := get $raw "configmap" | default dict }} +{{- $resource := $entry.resource | default dict }} +{{- $status := $resource.status | default dict }} + +{{- $ready := false }} +{{- range ($status.conditions | default list) }} + {{- if and (eq .type "Ready") (eq .status "True") }} + {{- $ready = true }} + {{- end }} +{{- end }} + +{{- $state = set $state "status" (dict + "ready" $ready + "configMapName" $state.configMapName +) }} diff --git a/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl new file mode 100644 index 0000000..6004660 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl @@ -0,0 +1,28 @@ +# code: language=yaml +{{- $s := $state }} +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: {{ $s.configMapName }} + annotations: + {{ setResourceNameAnnotation "configmap" }} + labels: + hops.ops.com.ai/ci-fixture: "true" + hops.ops.com.ai/config-smoke: {{ $s.name | quote }} +spec: + managementPolicies: {{ $s.managementPolicies | toJson }} + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ $s.configMapName }} + namespace: {{ $s.namespace }} + labels: + hops.ops.com.ai/ci-fixture: "true" + hops.ops.com.ai/config-smoke: {{ $s.name | quote }} + data: {{ $s.data | toJson }} + providerConfigRef: + name: {{ $s.providerConfigRef.name }} + kind: {{ $s.providerConfigRef.kind }} diff --git a/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl new file mode 100644 index 0000000..9e6c392 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl @@ -0,0 +1,8 @@ +# code: language=yaml +{{- $xr := getCompositeResource . }} +--- +apiVersion: {{ $xr.apiVersion }} +kind: {{ $xr.kind }} +status: + ready: {{ $state.status.ready }} + configMapName: {{ $state.status.configMapName | quote }} diff --git a/tests/fixtures/config-smoke/local/ci-xr.yaml b/tests/fixtures/config-smoke/local/ci-xr.yaml new file mode 100644 index 0000000..a611f06 --- /dev/null +++ b/tests/fixtures/config-smoke/local/ci-xr.yaml @@ -0,0 +1,36 @@ +# Applied by dory smoke after `hops config install --path tests/fixtures/config-smoke`. +# Lives in the hops-ci namespace only — safe alongside real workloads. +apiVersion: v1 +kind: Namespace +metadata: + name: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +--- +# Namespaced k8s provider config (mirrors bootstrap/k8s/pc.yaml for default/). +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: ProviderConfig +metadata: + name: default + namespace: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +spec: + credentials: + source: InjectedIdentity +--- +apiVersion: ci.hops.ops.com.ai/v1alpha1 +kind: ConfigSmoke +metadata: + name: hops-ci-smoke + namespace: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +spec: + configMapName: hops-ci-smoke + data: + source: path-based-config-install + smoke: "ok" + providerConfigRef: + name: default + kind: ProviderConfig diff --git a/tests/fixtures/config-smoke/upbound.yaml b/tests/fixtures/config-smoke/upbound.yaml new file mode 100644 index 0000000..fb8db98 --- /dev/null +++ b/tests/fixtures/config-smoke/upbound.yaml @@ -0,0 +1,27 @@ +apiVersion: meta.dev.upbound.io/v2alpha1 +kind: Project +metadata: + name: config-smoke +spec: + dependsOn: + - apiVersion: pkg.crossplane.io/v1 + kind: Function + package: xpkg.crossplane.io/crossplane-contrib/function-auto-ready + version: '>=v0.6.0' + - apiVersion: pkg.crossplane.io/v1 + kind: Provider + package: xpkg.crossplane.io/crossplane-contrib/provider-kubernetes + version: '>=v0.15.0' + description: | + Minimal hops CI fixture. Builds from source via `hops config install --path` + and composes a single Kubernetes ConfigMap through provider-kubernetes. + Safe to install on a shared local control plane (namespaced resources only). + license: Apache-2.0 + maintainer: hops-ops CI + readme: | + # config-smoke + + Test-only configuration for hops local integration. Creates a ConfigMap via + the Kubernetes provider. Not a real platform stack. + repository: ghcr.io/hops-ops/config-smoke + source: github.com/hops-ops/hops-cli