From a0eb1eb0e5e63d80fd2fbb9c5db9a412bab00228 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 4 Jul 2026 17:22:49 -0500 Subject: [PATCH 01/11] refactor: extract local cluster backend seam (colima-only) Move all colima-specific operations behind a Backend enum in src/commands/local/backend/ and rename ColimaSizeArgs to SizeArgs. Registry wiring (hosts sync) now flows through backend::wire_local_registry so provider/config installs stay backend-agnostic. No behavior change: identical command invocations on the colima path. Implements [[tasks/cluster-backend-abstraction]] (phase 1) --- src/commands/auth/bootstrap.rs | 11 +- src/commands/config/install.rs | 25 +- src/commands/local/backend/colima.rs | 517 ++++++++++++++++++++++++++ src/commands/local/backend/mod.rs | 159 ++++++++ src/commands/local/destroy.rs | 9 +- src/commands/local/doctor.rs | 24 +- src/commands/local/install.rs | 9 +- src/commands/local/mod.rs | 91 +---- src/commands/local/package_install.rs | 27 +- src/commands/local/reset.rs | 9 +- src/commands/local/resize.rs | 10 +- src/commands/local/start.rs | 501 +------------------------ src/commands/local/stop.rs | 9 +- src/commands/local/uninstall.rs | 13 +- src/commands/provider/install.rs | 77 ++-- src/commands/secrets/list.rs | 3 +- src/commands/vars/mod.rs | 5 +- src/commands/vars/sync.rs | 7 +- 18 files changed, 805 insertions(+), 701 deletions(-) create mode 100644 src/commands/local/backend/colima.rs create mode 100644 src/commands/local/backend/mod.rs diff --git a/src/commands/auth/bootstrap.rs b/src/commands/auth/bootstrap.rs index fdc5e35..7368483 100644 --- a/src/commands/auth/bootstrap.rs +++ b/src/commands/auth/bootstrap.rs @@ -66,7 +66,10 @@ pub fn run(args: &BootstrapArgs) -> Result<(), Box> { fn write_secret(path: &PathBuf, value: &str, force: bool) -> Result<(), Box> { if path.exists() && !force { - log::info!(" skip {}: already present (use --force to overwrite)", path.display()); + log::info!( + " skip {}: already present (use --force to overwrite)", + path.display() + ); return Ok(()); } if let Some(parent) = path.parent() { @@ -151,11 +154,7 @@ mod tests { "no lowercase: {}", pwd ); - assert!( - pwd.chars().any(|c| c.is_ascii_digit()), - "no digit: {}", - pwd - ); + assert!(pwd.chars().any(|c| c.is_ascii_digit()), "no digit: {}", pwd); assert!( pwd.chars().any(|c| !c.is_ascii_alphanumeric()), "no symbol: {}", diff --git a/src/commands/config/install.rs b/src/commands/config/install.rs index fb6385f..d3812c9 100644 --- a/src/commands/config/install.rs +++ b/src/commands/config/install.rs @@ -1,13 +1,13 @@ +use crate::commands::local::backend::{self, wire_local_registry}; +use crate::commands::local::package_install::run_watch; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout, ensure_registry, image_config_name, parse_docker_push_digest, parse_repo_spec, resolve_repo_install_target, rewrite_registry, rewrite_registry_with_tag, sanitize_name_component, short_hash, split_ref, strip_registry, - unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH, + unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_PULL, REGISTRY_PUSH, }; -use crate::commands::local::package_install::run_watch; use crate::commands::local::{ - kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output, sync_registry_hosts_entry, - HOPS_KUBE_CONTEXT_ENV, + kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output, HOPS_KUBE_CONTEXT_ENV, }; use clap::Args; use flate2::read::GzDecoder; @@ -134,10 +134,7 @@ pub fn run(args: &ConfigArgs) -> Result<(), Box> { } } -fn run_repo_install( - repo: &str, - skip_dependency_resolution: bool, -) -> Result<(), Box> { +fn run_repo_install(repo: &str, skip_dependency_resolution: bool) -> Result<(), Box> { let spec = parse_repo_spec(repo)?; match resolve_repo_install_target(&spec)? { RepoInstallTarget::SourceBuild => run_repo_clone(&spec, skip_dependency_resolution), @@ -147,10 +144,7 @@ fn run_repo_install( } } -fn run_repo_clone( - spec: &RepoSpec, - skip_dependency_resolution: bool, -) -> Result<(), Box> { +fn run_repo_clone(spec: &RepoSpec, skip_dependency_resolution: bool) -> Result<(), Box> { let cache_path = ensure_cached_repo_checkout(&spec)?; run_local_path(&cache_path.to_string_lossy(), skip_dependency_resolution) } @@ -216,17 +210,14 @@ fn apply_repo_version( apply_repo_version_spec(&spec, version, skip_dependency_resolution) } -fn run_local_path( - path: &str, - skip_dependency_resolution: bool, -) -> Result<(), Box> { +fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Box> { let dir = Path::new(path); if !dir.is_dir() { return Err(format!("{} is not a directory", path).into()); } ensure_registry()?; - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; + wire_local_registry(backend::resolve(None))?; // Build the Crossplane package log::info!("Building Crossplane package in {}...", path); diff --git a/src/commands/local/backend/colima.rs b/src/commands/local/backend/colima.rs new file mode 100644 index 0000000..7e543f6 --- /dev/null +++ b/src/commands/local/backend/colima.rs @@ -0,0 +1,517 @@ +//! Colima backend: VM + dockerd + k3s (docker runtime). + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_HOSTNAME, REGISTRY_PULL}; +use crate::commands::local::{run_cmd, run_cmd_output, wait_for_kubernetes}; +use dialoguer::Confirm; +use serde::Deserialize; +use std::error::Error; +use std::io::{IsTerminal, Write}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +const DEFAULT_CPUS: u32 = 8; +const DEFAULT_MEMORY_GIB: u32 = 16; +const DEFAULT_DISK_GIB: u32 = 60; +const GIB: u64 = 1024 * 1024 * 1024; + +pub fn install() -> Result<(), Box> { + log::info!("Installing Colima via Homebrew..."); + run_cmd("brew", &["install", "colima"])?; + log::info!("Colima installed successfully"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling Colima..."); + run_cmd("brew", &["uninstall", "colima"])?; + log::info!("Colima uninstalled"); + Ok(()) +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping Colima..."); + run_cmd("colima", &["stop"])?; + log::info!("Colima stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Destroying Colima VM..."); + run_cmd("colima", &["delete", "--force"])?; + log::info!("Colima VM destroyed"); + Ok(()) +} + +pub fn reset() -> Result<(), Box> { + log::info!("Resetting Colima Kubernetes..."); + run_cmd("colima", &["kubernetes", "reset"])?; + log::info!("Colima Kubernetes reset complete"); + Ok(()) +} + +pub fn start(size: &SizeArgs, assume_yes: bool) -> Result<(), Box> { + let instance = colima_instance()?; + validate_requested_size(size, instance.as_ref())?; + start_or_resize_colima(size, assume_yes, instance.as_ref()) +} + +pub fn resize(size: &SizeArgs) -> Result<(), Box> { + if !size.any_set() { + return Err("Specify at least one of --cpus, --memory, or --disk".into()); + } + + let instance = colima_instance()?; + let instance = instance + .as_ref() + .ok_or("No Colima instance exists yet; use `hops local start` to create one")?; + + validate_requested_size(size, Some(instance))?; + resize_existing_colima(size, Some(instance)) +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +struct ColimaInstance { + #[serde(default)] + status: String, + #[serde(default)] + cpus: Option, + #[serde(default)] + memory: Option, + #[serde(default)] + disk: Option, +} + +impl ColimaInstance { + fn is_running(&self) -> bool { + self.status.eq_ignore_ascii_case("running") + } + + fn memory_gib(&self) -> Option { + self.memory.map(bytes_to_gib) + } + + fn disk_gib(&self) -> Option { + self.disk.map(bytes_to_gib) + } +} + +fn start_or_resize_colima( + size: &SizeArgs, + assume_yes: bool, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + let is_running = instance.map(ColimaInstance::is_running).unwrap_or(false); + + if is_running && size.any_set() { + let changes = requested_size_changes(size, instance.expect("checked is_running")); + if !changes.is_empty() { + confirm_running_resize(size, assume_yes, &changes)?; + resize_existing_colima(size, instance)?; + return Ok(()); + } + + log::info!("Requested Colima size already matches the running VM"); + } + + log::info!("Starting Colima with Kubernetes..."); + + let start_size = if is_running { + SizeArgs::default() + } else { + size.clone() + }; + let include_defaults = instance.is_none(); + start_colima(&start_size, include_defaults) +} + +fn resize_existing_colima( + size: &SizeArgs, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + if instance.map(ColimaInstance::is_running).unwrap_or(false) { + log::info!("Stopping Colima to apply requested size..."); + run_cmd("colima", &["stop"])?; + } + + log::info!("Starting Colima with requested size..."); + start_colima(size, false) +} + +fn start_colima(size: &SizeArgs, include_defaults: bool) -> Result<(), Box> { + let args = colima_start_args(size, include_defaults); + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + run_cmd("colima", &refs) +} + +fn colima_start_args(size: &SizeArgs, include_defaults: bool) -> Vec { + let mut args = vec!["start".to_string(), "--kubernetes".to_string()]; + + if let Some(cpus) = size.cpus.or(include_defaults.then_some(DEFAULT_CPUS)) { + args.push("--cpus".to_string()); + args.push(cpus.to_string()); + } + if let Some(memory) = size + .memory + .or(include_defaults.then_some(DEFAULT_MEMORY_GIB)) + { + args.push("--memory".to_string()); + args.push(memory.to_string()); + } + if let Some(disk) = size.disk.or(include_defaults.then_some(DEFAULT_DISK_GIB)) { + args.push("--disk".to_string()); + args.push(disk.to_string()); + } + + args +} + +fn confirm_running_resize( + size: &SizeArgs, + assume_yes: bool, + changes: &[String], +) -> Result<(), Box> { + if assume_yes { + return Ok(()); + } + + let change_text = changes.join(", "); + let resize_command = format!("hops local resize{}", size.command_suffix()); + let start_command = format!("hops local start{} --yes", size.command_suffix()); + + if !std::io::stdin().is_terminal() { + return Err(format!( + "Colima is already running with different size ({change_text}). Run `{resize_command}` first, or rerun `{start_command}` to stop and resize automatically." + ) + .into()); + } + + let confirmed = Confirm::new() + .with_prompt(format!( + "Colima is already running with different size ({change_text}). Stop and restart it now?" + )) + .default(false) + .interact()?; + + if confirmed { + Ok(()) + } else { + Err(format!( + "Colima size was not changed. Run `{resize_command}` first, then rerun `hops local start`." + ) + .into()) + } +} + +fn validate_requested_size( + size: &SizeArgs, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + if let (Some(requested), Some(current)) = + (size.disk, instance.and_then(ColimaInstance::disk_gib)) + { + if requested < current { + return Err(format!( + "Colima disk cannot be shrunk from {current}GiB to {requested}GiB. Use --disk {current} or larger, or destroy and recreate the VM." + ) + .into()); + } + } + + Ok(()) +} + +fn requested_size_changes(size: &SizeArgs, instance: &ColimaInstance) -> Vec { + let mut changes = Vec::new(); + + if let Some(requested) = size.cpus { + match instance.cpus { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("cpus {current} -> {requested}")), + None => changes.push(format!("cpus unknown -> {requested}")), + } + } + if let Some(requested) = size.memory { + match instance.memory_gib() { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("memory {current}GiB -> {requested}GiB")), + None => changes.push(format!("memory unknown -> {requested}GiB")), + } + } + if let Some(requested) = size.disk { + match instance.disk_gib() { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("disk {current}GiB -> {requested}GiB")), + None => changes.push(format!("disk unknown -> {requested}GiB")), + } + } + + changes +} + +fn colima_instance() -> Result, Box> { + let output = match run_cmd_output("colima", &["list", "--json"]) { + Ok(output) => output, + Err(_) => return Ok(None), + }; + + parse_colima_list(&output) +} + +fn parse_colima_list(output: &str) -> Result, Box> { + let trimmed = output.trim(); + if trimmed.is_empty() { + return Ok(None); + } + + if let Ok(instance) = serde_json::from_str::(trimmed) { + return Ok(Some(instance)); + } + + if let Ok(instances) = serde_json::from_str::>(trimmed) { + return Ok(instances.into_iter().next()); + } + + for line in trimmed + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + if let Ok(instance) = serde_json::from_str::(line) { + return Ok(Some(instance)); + } + } + + Err("Unable to parse `colima list --json` output".into()) +} + +fn bytes_to_gib(bytes: u64) -> u32 { + (bytes / GIB) as u32 +} + +/// Add the cluster-internal registry to Docker's insecure-registries list +/// inside the Colima VM. Docker defaults to HTTPS for non-localhost registries; +/// our in-cluster registry speaks plain HTTP. +pub fn configure_docker_insecure_registry() -> Result<(), Box> { + let config = run_cmd_output("colima", &["ssh", "--", "cat", "/etc/docker/daemon.json"])?; + + if config.contains("insecure-registries") { + return Ok(()); + } + + log::info!("Configuring Docker for insecure local registry..."); + + // Insert the insecure-registries key before the final closing brace. + let new_config = if let Some(pos) = config.rfind('}') { + let prefix = config[..pos].trim_end(); + format!( + "{},\n \"insecure-registries\": [\"{}\"]\n}}\n", + prefix, REGISTRY_PULL + ) + } else { + return Err("Invalid daemon.json: no closing brace".into()); + }; + + let mut child = Command::new("colima") + .args(["ssh", "--", "sudo", "tee", "/etc/docker/daemon.json"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(new_config.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err("Failed to write Docker daemon.json".into()); + } + + log::info!("Restarting Docker daemon..."); + run_cmd( + "colima", + &["ssh", "--", "sudo", "systemctl", "restart", "docker"], + )?; + + // Wait for Docker to come back. + for _ in 0..30 { + if run_cmd_output("docker", &["info"]).is_ok() { + // Docker restart can temporarily disrupt the Kubernetes API. + wait_for_kubernetes()?; + return Ok(()); + } + thread::sleep(Duration::from_secs(2)); + } + Err("Docker did not come back after restart".into()) +} + +/// Ensure Colima's /etc/hosts maps the registry hostname to the current +/// ClusterIP so the kubelet's docker daemon can resolve pull refs. +pub fn sync_hosts_entry(cluster_ip: &str) -> Result<(), Box> { + let hostname = REGISTRY_HOSTNAME; + let current_ip = run_cmd_output( + "colima", + &[ + "ssh", + "--", + "sh", + "-c", + &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), + ], + ) + .unwrap_or_default(); + if current_ip.trim() == cluster_ip { + return Ok(()); + } + + log::info!("Updating hosts entry: {} -> {}", hostname, cluster_ip); + + let escaped_host = hostname.replace('.', "\\."); + run_cmd( + "colima", + &[ + "ssh", + "--", + "sudo", + "sed", + "-i", + &format!("/{}/d", escaped_host), + "/etc/hosts", + ], + )?; + run_cmd( + "colima", + &[ + "ssh", + "--", + "sudo", + "sh", + "-c", + &format!("echo '{} {}' >> /etc/hosts", cluster_ip, hostname), + ], + )?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn instance(status: &str, cpus: u32, memory_gib: u32, disk_gib: u32) -> ColimaInstance { + ColimaInstance { + status: status.to_string(), + cpus: Some(cpus), + memory: Some(memory_gib as u64 * GIB), + disk: Some(disk_gib as u64 * GIB), + } + } + + #[test] + fn colima_start_args_use_hops_defaults_for_new_profiles() { + let args = colima_start_args(&SizeArgs::default(), true); + + assert_eq!( + args, + vec![ + "start", + "--kubernetes", + "--cpus", + "8", + "--memory", + "16", + "--disk", + "60" + ] + ); + } + + #[test] + fn colima_start_args_pass_only_requested_size_for_existing_profiles() { + let size = SizeArgs { + cpus: Some(12), + memory: Some(32), + disk: None, + }; + + let args = colima_start_args(&size, false); + + assert_eq!( + args, + vec!["start", "--kubernetes", "--cpus", "12", "--memory", "32"] + ); + } + + #[test] + fn requested_size_changes_compare_only_explicit_fields() { + let current = instance("Running", 8, 16, 60); + let size = SizeArgs { + cpus: None, + memory: Some(32), + disk: None, + }; + + assert_eq!( + requested_size_changes(&size, ¤t), + vec!["memory 16GiB -> 32GiB"] + ); + } + + #[test] + fn requested_size_changes_treat_missing_current_value_as_change() { + let current = ColimaInstance { + status: "Running".to_string(), + cpus: None, + memory: None, + disk: None, + }; + let size = SizeArgs { + cpus: Some(12), + memory: None, + disk: None, + }; + + assert_eq!( + requested_size_changes(&size, ¤t), + vec!["cpus unknown -> 12"] + ); + } + + #[test] + fn parse_colima_list_accepts_single_object() { + let output = r#"{"name":"default","status":"Stopped","arch":"aarch64","cpus":8,"memory":17179869184,"disk":64424509440,"runtime":"docker+k3s"}"#; + + let parsed = parse_colima_list(output).expect("parse").expect("instance"); + + assert_eq!(parsed.status, "Stopped"); + assert_eq!(parsed.cpus, Some(8)); + assert_eq!(parsed.memory_gib(), Some(16)); + assert_eq!(parsed.disk_gib(), Some(60)); + } + + #[test] + fn parse_colima_list_accepts_array_output() { + let output = r#"[{"status":"Running","cpus":12,"memory":34359738368,"disk":107374182400}]"#; + + let parsed = parse_colima_list(output).expect("parse").expect("instance"); + + assert!(parsed.is_running()); + assert_eq!(parsed.cpus, Some(12)); + assert_eq!(parsed.memory_gib(), Some(32)); + assert_eq!(parsed.disk_gib(), Some(100)); + } + + #[test] + fn validate_requested_size_rejects_disk_shrink() { + let current = instance("Stopped", 8, 16, 100); + let size = SizeArgs { + cpus: None, + memory: None, + disk: Some(60), + }; + + let err = validate_requested_size(&size, Some(¤t)).expect_err("disk shrink"); + + assert!(err.to_string().contains("cannot be shrunk")); + } +} diff --git a/src/commands/local/backend/mod.rs b/src/commands/local/backend/mod.rs new file mode 100644 index 0000000..49cc0f0 --- /dev/null +++ b/src/commands/local/backend/mod.rs @@ -0,0 +1,159 @@ +//! Local-cluster backend abstraction. +//! +//! Abstracts the node/VM-level operations that differ between local cluster +//! providers (lifecycle, sizing, registry trust). Everything kubectl/helm +//! shaped lives outside this module and is backend-agnostic. + +mod colima; + +use super::run_cmd_output; +use clap::Args; +use std::error::Error; + +/// Sizing flags for backends with a resizable VM. +#[derive(Args, Debug, Clone, Default, PartialEq, Eq)] +pub struct SizeArgs { + /// Number of CPUs to allocate to the cluster VM. + #[arg(long = "cpus", visible_alias = "cpu", value_name = "N")] + pub cpus: Option, + + /// Memory to allocate to the cluster VM, in GiB. + #[arg(long, value_name = "GIB")] + pub memory: Option, + + /// Disk size to allocate to the cluster VM, in GiB. + #[arg(long, value_name = "GIB")] + pub disk: Option, +} + +impl SizeArgs { + pub fn any_set(&self) -> bool { + self.cpus.is_some() || self.memory.is_some() || self.disk.is_some() + } + + pub fn command_suffix(&self) -> String { + let mut parts = Vec::new(); + if let Some(cpus) = self.cpus { + parts.push(format!("--cpus {}", cpus)); + } + if let Some(memory) = self.memory { + parts.push(format!("--memory {}", memory)); + } + if let Some(disk) = self.disk { + parts.push(format!("--disk {}", disk)); + } + + if parts.is_empty() { + String::new() + } else { + format!(" {}", parts.join(" ")) + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Backend { + Colima, +} + +impl Backend { + /// Human-facing backend name (also the persisted spelling). + pub fn name(self) -> &'static str { + match self { + Backend::Colima => "colima", + } + } + + pub fn install(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::install(), + } + } + + pub fn uninstall(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::uninstall(), + } + } + + /// Bring the cluster up (create, start, or resize as needed). Does not + /// wait for the Kubernetes API; callers follow with `wait_for_kubernetes`. + pub fn start(self, size: &SizeArgs, assume_yes: bool) -> Result<(), Box> { + match self { + Backend::Colima => colima::start(size, assume_yes), + } + } + + pub fn stop(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::stop(), + } + } + + pub fn destroy(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::destroy(), + } + } + + pub fn reset(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::reset(), + } + } + + pub fn resize(self, size: &SizeArgs) -> Result<(), Box> { + match self { + Backend::Colima => colima::resize(size), + } + } + + /// Make the node runtime trust the in-cluster registry over HTTP. + /// Runs before any images exist; must be safe to re-run. + pub fn ensure_registry_trust(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::configure_docker_insecure_registry(), + } + } + + /// Point the node at the registry Service's current ClusterIP so pulls of + /// both registry names resolve. Idempotent; re-run on every start because + /// the ClusterIP changes if the Service is recreated. + pub fn wire_registry(self, cluster_ip: &str) -> Result<(), Box> { + match self { + Backend::Colima => colima::sync_hosts_entry(cluster_ip), + } + } +} + +/// Resolve which backend to operate on. +pub fn resolve(flag: Option) -> Backend { + flag.unwrap_or(Backend::Colima) +} + +/// Fetch the in-cluster registry Service's ClusterIP. +pub fn registry_cluster_ip() -> Result> { + let cluster_ip = run_cmd_output( + "kubectl", + &[ + "get", + "svc", + "registry", + "-n", + "crossplane-system", + "-o", + "jsonpath={.spec.clusterIP}", + ], + )?; + let cluster_ip = cluster_ip.trim().to_string(); + if cluster_ip.is_empty() { + return Err("Service crossplane-system/registry has no ClusterIP".into()); + } + Ok(cluster_ip) +} + +/// Wire node-level registry pulls for a backend: fetch the registry Service +/// ClusterIP and apply the backend-specific trust/aliasing. +pub fn wire_local_registry(backend: Backend) -> Result<(), Box> { + backend.wire_registry(®istry_cluster_ip()?) +} diff --git a/src/commands/local/destroy.rs b/src/commands/local/destroy.rs index 8638986..c860228 100644 --- a/src/commands/local/destroy.rs +++ b/src/commands/local/destroy.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Destroying Colima VM..."); - run_cmd("colima", &["delete", "--force"])?; - log::info!("Colima VM destroyed"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.destroy() } diff --git a/src/commands/local/doctor.rs b/src/commands/local/doctor.rs index e7f8967..248c588 100644 --- a/src/commands/local/doctor.rs +++ b/src/commands/local/doctor.rs @@ -220,20 +220,17 @@ fn check_provider(d: &mut Doctor, e: &ProviderExpectation) { }, ); - let pc_ok = exists(&[ - "get", - e.pc_resource, - e.pc_name, - "-n", - e.pc_namespace, - ]); + let pc_ok = exists(&["get", e.pc_resource, e.pc_name, "-n", e.pc_namespace]); d.check( "ProviderConfig present", pc_ok, if pc_ok { String::new() } else { - format!("{}/{} missing in namespace {}", e.pc_resource, e.pc_name, e.pc_namespace) + format!( + "{}/{} missing in namespace {}", + e.pc_resource, e.pc_name, e.pc_namespace + ) }, ); } @@ -244,7 +241,10 @@ fn provider_condition(provider: &str, cond: &str) -> String { "provider.pkg.crossplane.io", provider, "-o", - &format!("jsonpath={{.status.conditions[?(@.type==\"{}\")].status}}", cond), + &format!( + "jsonpath={{.status.conditions[?(@.type==\"{}\")].status}}", + cond + ), ]) .unwrap_or_default() } @@ -253,7 +253,11 @@ fn cond_detail(cond: &str, status: &str) -> String { if status == "True" { String::new() } else { - format!("{}={}", cond, if status.is_empty() { "" } else { status }) + format!( + "{}={}", + cond, + if status.is_empty() { "" } else { status } + ) } } diff --git a/src/commands/local/install.rs b/src/commands/local/install.rs index 0715c89..fabbb65 100644 --- a/src/commands/local/install.rs +++ b/src/commands/local/install.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Installing Colima via Homebrew..."); - run_cmd("brew", &["install", "colima"])?; - log::info!("Colima installed successfully"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.install() } diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index 6adfca6..e399980 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -1,4 +1,5 @@ mod aws; +pub mod backend; mod cloudflare; mod destroy; mod doctor; @@ -107,20 +108,21 @@ pub fn run(args: &LocalArgs) -> Result<(), Box> { std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); } } + let backend = backend::resolve(None); match &args.command { - LocalCommands::Install => install::run(), - LocalCommands::Reset => reset::run(), - LocalCommands::Start(start_args) => start::run(start_args), - LocalCommands::Resize(resize_args) => resize::run(resize_args), + LocalCommands::Install => install::run(backend), + LocalCommands::Reset => reset::run(backend), + LocalCommands::Start(start_args) => start::run(backend, start_args), + LocalCommands::Resize(resize_args) => resize::run(backend, resize_args), LocalCommands::Doctor => doctor::run(), LocalCommands::Aws(aws_args) => aws::run(aws_args), LocalCommands::Cloudflare(cloudflare_args) => cloudflare::run(cloudflare_args), LocalCommands::Github(github_args) => github::run(github_args), LocalCommands::Zitadel(zitadel_args) => zitadel::run(zitadel_args), LocalCommands::Listmonk(listmonk_args) => listmonk::run(listmonk_args), - LocalCommands::Stop => stop::run(), - LocalCommands::Destroy => destroy::run(), - LocalCommands::Uninstall => uninstall::run(), + LocalCommands::Stop => stop::run(backend), + LocalCommands::Destroy => destroy::run(backend), + LocalCommands::Uninstall => uninstall::run(backend), } } @@ -195,72 +197,17 @@ fn command_exists(program: &str) -> bool { .unwrap_or(false) } -/// Ensure Colima's /etc/hosts maps a service hostname to the current ClusterIP. -pub fn sync_registry_hosts_entry( - namespace: &str, - service: &str, - hostname: &str, -) -> Result<(), Box> { - let cluster_ip = run_cmd_output( - "kubectl", - &[ - "get", - "svc", - service, - "-n", - namespace, - "-o", - "jsonpath={.spec.clusterIP}", - ], - )?; - let cluster_ip = cluster_ip.trim(); - if cluster_ip.is_empty() { - return Err(format!("Service {}/{} has no ClusterIP", namespace, service).into()); - } - - let current_ip = run_cmd_output( - "colima", - &[ - "ssh", - "--", - "sh", - "-c", - &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), - ], - ) - .unwrap_or_default(); - if current_ip.trim() == cluster_ip { - return Ok(()); +/// Poll until the Kubernetes API server is reachable. +pub(crate) fn wait_for_kubernetes() -> Result<(), Box> { + log::info!("Waiting for Kubernetes API..."); + for _ in 0..60 { + let result = run_cmd_output("kubectl", &["cluster-info"]); + if result.is_ok() { + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_secs(5)); } - - log::info!("Updating hosts entry: {} -> {}", hostname, cluster_ip); - - let escaped_host = hostname.replace('.', "\\."); - run_cmd( - "colima", - &[ - "ssh", - "--", - "sudo", - "sed", - "-i", - &format!("/{}/d", escaped_host), - "/etc/hosts", - ], - )?; - run_cmd( - "colima", - &[ - "ssh", - "--", - "sudo", - "sh", - "-c", - &format!("echo '{} {}' >> /etc/hosts", cluster_ip, hostname), - ], - )?; - - Ok(()) + Err("Timed out waiting for Kubernetes API".into()) } /// Pipe a YAML string into `kubectl apply -f -`. diff --git a/src/commands/local/package_install.rs b/src/commands/local/package_install.rs index 28adf1c..dd25544 100644 --- a/src/commands/local/package_install.rs +++ b/src/commands/local/package_install.rs @@ -231,7 +231,10 @@ pub fn parse_repo_install_choice(input: &str) -> Result) -> Option { +pub fn resolve_published_version_input( + input: &str, + default_version: Option<&str>, +) -> Option { let trimmed = input.trim(); if !trimmed.is_empty() { return Some(trimmed.to_string()); @@ -394,7 +397,9 @@ fn clone_repo_into_cache( "Cloning {} into local cache at {}{}...", clone_url, cache_path.display(), - branch.map(|b| format!(" (branch {})", b)).unwrap_or_default() + branch + .map(|b| format!(" (branch {})", b)) + .unwrap_or_default() ); let mut args: Vec<&str> = vec!["clone"]; if let Some(b) = branch { @@ -405,10 +410,7 @@ fn clone_repo_into_cache( Ok(()) } -fn refresh_cached_repo( - cache_path: &Path, - branch: Option<&str>, -) -> Result<(), Box> { +fn refresh_cached_repo(cache_path: &Path, branch: Option<&str>) -> Result<(), Box> { let cache_path_str = cache_path.to_string_lossy().to_string(); run_cmd( "git", @@ -445,11 +447,7 @@ fn should_ignore_path(path: &Path) -> bool { /// Watch `path` for filesystem events and invoke `rebuild` after a debounced /// quiet period. Loops until the watcher channel closes (typically Ctrl+C). -pub fn run_watch( - path: &str, - debounce_secs: u64, - mut rebuild: F, -) -> Result<(), Box> +pub fn run_watch(path: &str, debounce_secs: u64, mut rebuild: F) -> Result<(), Box> where F: FnMut() -> Result<(), Box>, { @@ -457,8 +455,8 @@ where let debounce = Duration::from_secs(debounce_secs); let (tx, rx) = mpsc::channel(); - let mut watcher = notify::recommended_watcher(move |res: notify::Result| { - match res { + let mut watcher = + notify::recommended_watcher(move |res: notify::Result| match res { Ok(event) => { let dominated_by_ignored = event.paths.iter().all(|p| should_ignore_path(p)); log::debug!( @@ -472,8 +470,7 @@ where } } Err(e) => log::debug!("watch error: {:?}", e), - } - })?; + })?; watcher.watch(&dir, RecursiveMode::Recursive)?; log::info!( diff --git a/src/commands/local/reset.rs b/src/commands/local/reset.rs index 6dbf74e..1f4c005 100644 --- a/src/commands/local/reset.rs +++ b/src/commands/local/reset.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Resetting Colima Kubernetes..."); - run_cmd("colima", &["kubernetes", "reset"])?; - log::info!("Colima Kubernetes reset complete"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.reset() } diff --git a/src/commands/local/resize.rs b/src/commands/local/resize.rs index 19a987a..2967efd 100644 --- a/src/commands/local/resize.rs +++ b/src/commands/local/resize.rs @@ -1,15 +1,15 @@ -use super::start::{resize_colima, ColimaSizeArgs}; +use super::backend::{Backend, SizeArgs}; use clap::Args; use std::error::Error; #[derive(Args, Debug, Clone)] pub struct ResizeArgs { #[command(flatten)] - pub size: ColimaSizeArgs, + pub size: SizeArgs, } -pub fn run(args: &ResizeArgs) -> Result<(), Box> { - resize_colima(&args.size)?; - log::info!("Colima resize complete"); +pub fn run(backend: Backend, args: &ResizeArgs) -> Result<(), Box> { + backend.resize(&args.size)?; + log::info!("Resize complete"); Ok(()) } diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 55bac80..a7f699b 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -1,10 +1,7 @@ -use super::{kubectl_apply_stdin, run_cmd, run_cmd_output, sync_registry_hosts_entry}; +use super::backend::{self, SizeArgs}; +use super::{kubectl_apply_stdin, run_cmd, run_cmd_output, wait_for_kubernetes}; use clap::Args; -use dialoguer::Confirm; -use serde::Deserialize; use std::error::Error; -use std::io::{IsTerminal, Write}; -use std::process::{Command, Stdio}; use std::thread; use std::time::Duration; @@ -18,107 +15,27 @@ const PC_HELM: &str = include_str!("../../../bootstrap/helm/pc.yaml"); const PC_K8S: &str = include_str!("../../../bootstrap/k8s/pc.yaml"); const REGISTRY: &str = include_str!("../../../bootstrap/registry/registry.yaml"); -/// Cluster-internal hostname for the package registry. -const REGISTRY_HOST: &str = "registry.crossplane-system.svc.cluster.local:5000"; -const REGISTRY_HOSTNAME: &str = "registry.crossplane-system.svc.cluster.local"; - -const DEFAULT_CPUS: u32 = 8; -const DEFAULT_MEMORY_GIB: u32 = 16; -const DEFAULT_DISK_GIB: u32 = 60; -const GIB: u64 = 1024 * 1024 * 1024; - #[derive(Args, Debug, Clone)] pub struct StartArgs { #[command(flatten)] - pub size: ColimaSizeArgs, + pub size: SizeArgs, - /// Stop and restart a running Colima VM without prompting when requested size differs. + /// Stop and restart a running cluster VM without prompting when requested size differs. #[arg(long)] pub yes: bool, } -#[derive(Args, Debug, Clone, Default, PartialEq, Eq)] -pub struct ColimaSizeArgs { - /// Number of CPUs to allocate to the Colima VM. - #[arg(long = "cpus", visible_alias = "cpu", value_name = "N")] - pub cpus: Option, - - /// Memory to allocate to the Colima VM, in GiB. - #[arg(long, value_name = "GIB")] - pub memory: Option, - - /// Disk size to allocate to the Colima VM, in GiB. - #[arg(long, value_name = "GIB")] - pub disk: Option, -} - -impl ColimaSizeArgs { - pub fn any_set(&self) -> bool { - self.cpus.is_some() || self.memory.is_some() || self.disk.is_some() - } - - pub fn command_suffix(&self) -> String { - let mut parts = Vec::new(); - if let Some(cpus) = self.cpus { - parts.push(format!("--cpus {}", cpus)); - } - if let Some(memory) = self.memory { - parts.push(format!("--memory {}", memory)); - } - if let Some(disk) = self.disk { - parts.push(format!("--disk {}", disk)); - } - - if parts.is_empty() { - String::new() - } else { - format!(" {}", parts.join(" ")) - } - } -} - -#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] -pub(crate) struct ColimaInstance { - #[serde(default)] - status: String, - #[serde(default)] - cpus: Option, - #[serde(default)] - memory: Option, - #[serde(default)] - disk: Option, -} - -impl ColimaInstance { - fn is_running(&self) -> bool { - self.status.eq_ignore_ascii_case("running") - } - - fn memory_gib(&self) -> Option { - self.memory.map(bytes_to_gib) - } - - fn disk_gib(&self) -> Option { - self.disk.map(bytes_to_gib) - } -} - -pub fn run(args: &StartArgs) -> Result<(), Box> { - let instance = colima_instance()?; - validate_requested_size(&args.size, instance.as_ref())?; - - // 1. Start Colima with Kubernetes - start_or_resize_colima(args, instance.as_ref())?; +pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box> { + // 1. Bring the backend cluster up + backend.start(&args.size, args.yes)?; // 2. Wait for the Kubernetes API to become reachable. - // Colima may return immediately ("already running") before the + // The backend may return immediately ("already running") before the // API server is ready, or a fresh start needs time to initialise. wait_for_kubernetes()?; - // 3. Configure Docker in the VM to allow HTTP pulls from the - // cluster-internal registry. Without this the kubelet's Docker - // daemon defaults to HTTPS and fails. - configure_docker_insecure_registry()?; + // 3. Make the node runtime trust the cluster-internal registry over HTTP. + backend.ensure_registry_trust()?; // 4. Add Crossplane Helm repo log::info!("Adding Crossplane Helm repo..."); @@ -180,284 +97,14 @@ pub fn run(args: &StartArgs) -> Result<(), Box> { kubectl_apply_stdin(REGISTRY)?; wait_for_deployment("crossplane-system", "registry")?; - // 12. Map the registry's cluster-internal hostname to its ClusterIP - // inside the VM so the kubelet can resolve it. - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; + // 12. Point the node at the registry Service's ClusterIP so pulls of the + // cluster-internal registry names resolve. + backend::wire_local_registry(backend)?; log::info!("Local environment is ready"); Ok(()) } -fn start_or_resize_colima( - args: &StartArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - let is_running = instance.map(ColimaInstance::is_running).unwrap_or(false); - - if is_running && args.size.any_set() { - let changes = requested_size_changes(&args.size, instance.expect("checked is_running")); - if !changes.is_empty() { - confirm_running_resize(args, &changes)?; - resize_existing_colima(&args.size, instance)?; - return Ok(()); - } - - log::info!("Requested Colima size already matches the running VM"); - } - - log::info!("Starting Colima with Kubernetes..."); - - let size = if is_running { - ColimaSizeArgs::default() - } else { - args.size.clone() - }; - let include_defaults = instance.is_none(); - start_colima(&size, include_defaults) -} - -pub(crate) fn resize_colima(size: &ColimaSizeArgs) -> Result<(), Box> { - if !size.any_set() { - return Err("Specify at least one of --cpus, --memory, or --disk".into()); - } - - let instance = colima_instance()?; - let instance = instance - .as_ref() - .ok_or("No Colima instance exists yet; use `hops local start` to create one")?; - - validate_requested_size(size, Some(instance))?; - resize_existing_colima(size, Some(instance)) -} - -fn resize_existing_colima( - size: &ColimaSizeArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - if instance.map(ColimaInstance::is_running).unwrap_or(false) { - log::info!("Stopping Colima to apply requested size..."); - run_cmd("colima", &["stop"])?; - } - - log::info!("Starting Colima with requested size..."); - start_colima(size, false) -} - -fn start_colima(size: &ColimaSizeArgs, include_defaults: bool) -> Result<(), Box> { - let args = colima_start_args(size, include_defaults); - let refs: Vec<&str> = args.iter().map(String::as_str).collect(); - run_cmd("colima", &refs) -} - -fn colima_start_args(size: &ColimaSizeArgs, include_defaults: bool) -> Vec { - let mut args = vec!["start".to_string(), "--kubernetes".to_string()]; - - if let Some(cpus) = size.cpus.or(include_defaults.then_some(DEFAULT_CPUS)) { - args.push("--cpus".to_string()); - args.push(cpus.to_string()); - } - if let Some(memory) = size - .memory - .or(include_defaults.then_some(DEFAULT_MEMORY_GIB)) - { - args.push("--memory".to_string()); - args.push(memory.to_string()); - } - if let Some(disk) = size.disk.or(include_defaults.then_some(DEFAULT_DISK_GIB)) { - args.push("--disk".to_string()); - args.push(disk.to_string()); - } - - args -} - -fn confirm_running_resize(args: &StartArgs, changes: &[String]) -> Result<(), Box> { - if args.yes { - return Ok(()); - } - - let change_text = changes.join(", "); - let resize_command = format!("hops local resize{}", args.size.command_suffix()); - let start_command = format!("hops local start{} --yes", args.size.command_suffix()); - - if !std::io::stdin().is_terminal() { - return Err(format!( - "Colima is already running with different size ({change_text}). Run `{resize_command}` first, or rerun `{start_command}` to stop and resize automatically." - ) - .into()); - } - - let confirmed = Confirm::new() - .with_prompt(format!( - "Colima is already running with different size ({change_text}). Stop and restart it now?" - )) - .default(false) - .interact()?; - - if confirmed { - Ok(()) - } else { - Err(format!( - "Colima size was not changed. Run `{resize_command}` first, then rerun `hops local start`." - ) - .into()) - } -} - -fn validate_requested_size( - size: &ColimaSizeArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - if let (Some(requested), Some(current)) = - (size.disk, instance.and_then(ColimaInstance::disk_gib)) - { - if requested < current { - return Err(format!( - "Colima disk cannot be shrunk from {current}GiB to {requested}GiB. Use --disk {current} or larger, or destroy and recreate the VM." - ) - .into()); - } - } - - Ok(()) -} - -fn requested_size_changes(size: &ColimaSizeArgs, instance: &ColimaInstance) -> Vec { - let mut changes = Vec::new(); - - if let Some(requested) = size.cpus { - match instance.cpus { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("cpus {current} -> {requested}")), - None => changes.push(format!("cpus unknown -> {requested}")), - } - } - if let Some(requested) = size.memory { - match instance.memory_gib() { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("memory {current}GiB -> {requested}GiB")), - None => changes.push(format!("memory unknown -> {requested}GiB")), - } - } - if let Some(requested) = size.disk { - match instance.disk_gib() { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("disk {current}GiB -> {requested}GiB")), - None => changes.push(format!("disk unknown -> {requested}GiB")), - } - } - - changes -} - -pub(crate) fn colima_instance() -> Result, Box> { - let output = match run_cmd_output("colima", &["list", "--json"]) { - Ok(output) => output, - Err(_) => return Ok(None), - }; - - parse_colima_list(&output) -} - -fn parse_colima_list(output: &str) -> Result, Box> { - let trimmed = output.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - if let Ok(instance) = serde_json::from_str::(trimmed) { - return Ok(Some(instance)); - } - - if let Ok(instances) = serde_json::from_str::>(trimmed) { - return Ok(instances.into_iter().next()); - } - - for line in trimmed - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - { - if let Ok(instance) = serde_json::from_str::(line) { - return Ok(Some(instance)); - } - } - - Err("Unable to parse `colima list --json` output".into()) -} - -fn bytes_to_gib(bytes: u64) -> u32 { - (bytes / GIB) as u32 -} - -/// Add the cluster-internal registry to Docker's insecure-registries list -/// inside the Colima VM. Docker defaults to HTTPS for non-localhost registries; -/// our in-cluster registry speaks plain HTTP. -fn configure_docker_insecure_registry() -> Result<(), Box> { - let config = run_cmd_output("colima", &["ssh", "--", "cat", "/etc/docker/daemon.json"])?; - - if config.contains("insecure-registries") { - return Ok(()); - } - - log::info!("Configuring Docker for insecure local registry..."); - - // Insert the insecure-registries key before the final closing brace. - let new_config = if let Some(pos) = config.rfind('}') { - let prefix = config[..pos].trim_end(); - format!( - "{},\n \"insecure-registries\": [\"{}\"]\n}}\n", - prefix, REGISTRY_HOST - ) - } else { - return Err("Invalid daemon.json: no closing brace".into()); - }; - - let mut child = Command::new("colima") - .args(["ssh", "--", "sudo", "tee", "/etc/docker/daemon.json"]) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn()?; - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(new_config.as_bytes())?; - } - let status = child.wait()?; - if !status.success() { - return Err("Failed to write Docker daemon.json".into()); - } - - log::info!("Restarting Docker daemon..."); - run_cmd( - "colima", - &["ssh", "--", "sudo", "systemctl", "restart", "docker"], - )?; - - // Wait for Docker to come back. - for _ in 0..30 { - if run_cmd_output("docker", &["info"]).is_ok() { - // Docker restart can temporarily disrupt the Kubernetes API. - wait_for_kubernetes()?; - return Ok(()); - } - thread::sleep(Duration::from_secs(2)); - } - Err("Docker did not come back after restart".into()) -} - -/// Poll until the Kubernetes API server is reachable. -fn wait_for_kubernetes() -> Result<(), Box> { - log::info!("Waiting for Kubernetes API..."); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["cluster-info"]); - if result.is_ok() { - return Ok(()); - } - thread::sleep(Duration::from_secs(5)); - } - Err("Timed out waiting for Kubernetes API".into()) -} - /// Poll until a deployment's Available condition is True. fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box> { for _ in 0..60 { @@ -497,125 +144,3 @@ fn wait_for_crd(crd: &str) -> Result<(), Box> { } Err(format!("Timed out waiting for CRD {}", crd).into()) } - -#[cfg(test)] -mod tests { - use super::*; - - fn instance(status: &str, cpus: u32, memory_gib: u32, disk_gib: u32) -> ColimaInstance { - ColimaInstance { - status: status.to_string(), - cpus: Some(cpus), - memory: Some(memory_gib as u64 * GIB), - disk: Some(disk_gib as u64 * GIB), - } - } - - #[test] - fn colima_start_args_use_hops_defaults_for_new_profiles() { - let args = colima_start_args(&ColimaSizeArgs::default(), true); - - assert_eq!( - args, - vec![ - "start", - "--kubernetes", - "--cpus", - "8", - "--memory", - "16", - "--disk", - "60" - ] - ); - } - - #[test] - fn colima_start_args_pass_only_requested_size_for_existing_profiles() { - let size = ColimaSizeArgs { - cpus: Some(12), - memory: Some(32), - disk: None, - }; - - let args = colima_start_args(&size, false); - - assert_eq!( - args, - vec!["start", "--kubernetes", "--cpus", "12", "--memory", "32"] - ); - } - - #[test] - fn requested_size_changes_compare_only_explicit_fields() { - let current = instance("Running", 8, 16, 60); - let size = ColimaSizeArgs { - cpus: None, - memory: Some(32), - disk: None, - }; - - assert_eq!( - requested_size_changes(&size, ¤t), - vec!["memory 16GiB -> 32GiB"] - ); - } - - #[test] - fn requested_size_changes_treat_missing_current_value_as_change() { - let current = ColimaInstance { - status: "Running".to_string(), - cpus: None, - memory: None, - disk: None, - }; - let size = ColimaSizeArgs { - cpus: Some(12), - memory: None, - disk: None, - }; - - assert_eq!( - requested_size_changes(&size, ¤t), - vec!["cpus unknown -> 12"] - ); - } - - #[test] - fn parse_colima_list_accepts_single_object() { - let output = r#"{"name":"default","status":"Stopped","arch":"aarch64","cpus":8,"memory":17179869184,"disk":64424509440,"runtime":"docker+k3s"}"#; - - let parsed = parse_colima_list(output).expect("parse").expect("instance"); - - assert_eq!(parsed.status, "Stopped"); - assert_eq!(parsed.cpus, Some(8)); - assert_eq!(parsed.memory_gib(), Some(16)); - assert_eq!(parsed.disk_gib(), Some(60)); - } - - #[test] - fn parse_colima_list_accepts_array_output() { - let output = r#"[{"status":"Running","cpus":12,"memory":34359738368,"disk":107374182400}]"#; - - let parsed = parse_colima_list(output).expect("parse").expect("instance"); - - assert!(parsed.is_running()); - assert_eq!(parsed.cpus, Some(12)); - assert_eq!(parsed.memory_gib(), Some(32)); - assert_eq!(parsed.disk_gib(), Some(100)); - } - - #[test] - fn validate_requested_size_rejects_disk_shrink() { - let current = instance("Stopped", 8, 16, 100); - let size = ColimaSizeArgs { - cpus: None, - memory: None, - disk: Some(60), - }; - - let err = validate_requested_size(&size, Some(¤t)).expect_err("disk shrink"); - - assert!(err.to_string().contains("cannot be shrunk")); - } -} diff --git a/src/commands/local/stop.rs b/src/commands/local/stop.rs index 16ffa2d..4fa2bf5 100644 --- a/src/commands/local/stop.rs +++ b/src/commands/local/stop.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Stopping Colima..."); - run_cmd("colima", &["stop"])?; - log::info!("Colima stopped"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.stop() } diff --git a/src/commands/local/uninstall.rs b/src/commands/local/uninstall.rs index d84e1b0..a3d2570 100644 --- a/src/commands/local/uninstall.rs +++ b/src/commands/local/uninstall.rs @@ -1,18 +1,19 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; use std::io::{self, Write}; -pub fn run() -> Result<(), Box> { - print!("Uninstall Colima? This will remove the binary. [y/N] "); +pub fn run(backend: Backend) -> Result<(), Box> { + print!( + "Uninstall {}? This will remove the binary. [y/N] ", + backend.name() + ); io::stdout().flush()?; let mut input = String::new(); io::stdin().read_line(&mut input)?; if input.trim().eq_ignore_ascii_case("y") { - log::info!("Uninstalling Colima..."); - run_cmd("brew", &["uninstall", "colima"])?; - log::info!("Colima uninstalled"); + backend.uninstall()?; } else { log::info!("Uninstall cancelled"); } diff --git a/src/commands/provider/install.rs b/src/commands/provider/install.rs index 321b0d6..c10a13c 100644 --- a/src/commands/provider/install.rs +++ b/src/commands/provider/install.rs @@ -1,11 +1,12 @@ +use crate::commands::local::backend::{self, wire_local_registry}; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout_at, ensure_registry, parse_repo_spec, resolve_repo_install_target, run_watch, sanitize_name_component, RepoInstallTarget, RepoSpec, - REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH, + REGISTRY_PULL, REGISTRY_PUSH, }; use crate::commands::local::{ - kubectl_apply_stdin, run_cmd, run_cmd_output, sync_registry_hosts_entry, - HOPS_KUBE_CONTEXT_ENV, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, + kubectl_apply_stdin, run_cmd, run_cmd_output, HOPS_KUBE_CONTEXT_ENV, MANAGED_BY_LABEL, + PROVIDER_INSTALL_MANAGED_BY, }; use clap::Args; use serde::Deserialize; @@ -154,11 +155,7 @@ fn apply_repo_version_spec( sanitize_name_component(&spec.repo) ); - log::info!( - "Applying Provider '{}' from {}", - provider_name, - package_ref - ); + log::info!("Applying Provider '{}' from {}", provider_name, package_ref); let providers_json = run_cmd_output( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], @@ -190,7 +187,7 @@ fn run_local_path( log::info!("Provider package name: {}", provider_name); ensure_registry()?; - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; + wire_local_registry(backend::resolve(None))?; // Resolve the existing upstream Provider before building so we can: // 1. carry the upstream package URL into the new `spec.package` (Crossplane @@ -228,10 +225,7 @@ fn run_local_path( &local_image_path_for_tag, )?; - let push_xpkg_ref = format!( - "{}/hops-ops/{}:{}", - REGISTRY_PUSH, provider_name, dev_tag - ); + let push_xpkg_ref = format!("{}/hops-ops/{}:{}", REGISTRY_PUSH, provider_name, dev_tag); let local_pull_xpkg_path = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); log::info!("Pushing xpkg to {}...", push_xpkg_ref); crossplane_xpkg_push(&xpkg_path, &push_xpkg_ref)?; @@ -277,11 +271,7 @@ fn read_provider_name(dir: &Path) -> Result> { let name = parsed.metadata.name.trim().to_string(); if name.is_empty() { - return Err(format!( - "{} has no metadata.name", - crossplane_yaml.display() - ) - .into()); + return Err(format!("{} has no metadata.name", crossplane_yaml.display()).into()); } Ok(name) } @@ -293,7 +283,10 @@ fn ensure_build_submodule(dir: &Path) -> Result<(), Box> { } if !dir.join(".gitmodules").is_file() { - log::debug!("No .gitmodules in {}; skipping submodule init", dir.display()); + log::debug!( + "No .gitmodules in {}; skipping submodule init", + dir.display() + ); return Ok(()); } @@ -353,19 +346,10 @@ fn find_xpkg_for_provider( .collect(); if candidates.is_empty() { - return Err(format!( - "no {}*.xpkg found in {}", - prefix, - xpkg_dir.display() - ) - .into()); + return Err(format!("no {}*.xpkg found in {}", prefix, xpkg_dir.display()).into()); } - candidates.sort_by_key(|p| { - fs::metadata(p) - .and_then(|m| m.modified()) - .ok() - }); + candidates.sort_by_key(|p| fs::metadata(p).and_then(|m| m.modified()).ok()); Ok(candidates.pop().unwrap()) } @@ -544,7 +528,10 @@ struct TagsListResponse { fn is_full_semver(s: &str) -> bool { let trimmed = s.strip_prefix('v').unwrap_or(s); let parts: Vec<&str> = trimmed.splitn(3, '.').collect(); - parts.len() == 3 && parts.iter().all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) + parts.len() == 3 + && parts + .iter() + .all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) } /// Parse a `vN` (or `N`) bare-major version string, e.g. `v1` -> `Some(1)`. @@ -1104,11 +1091,8 @@ mod tests { #[test] fn build_runtime_config_yaml_overrides_image_when_provided() { - let with_image = build_runtime_config_yaml( - "p-runtime", - "p", - Some("registry.example/p-arm64:dev-abc"), - ); + let with_image = + build_runtime_config_yaml("p-runtime", "p", Some("registry.example/p-arm64:dev-abc")); assert!(with_image.contains("name: package-runtime")); assert!(with_image.contains("image: registry.example/p-arm64:dev-abc")); @@ -1119,7 +1103,6 @@ mod tests { assert!(without.contains("app.kubernetes.io/managed-by: hops-provider-install")); } - #[test] fn local_runtime_image_ref_uses_nodeport_registry() { let image = local_runtime_image_ref("provider-helm", "arm64", "v1.999.3"); @@ -1165,7 +1148,10 @@ mod tests { fn assert_no_existing_provider_error(err: Box, provider_name: &str) { let msg = err.to_string(); assert!( - msg.contains(&format!("no existing Provider matching '{}'", provider_name)), + msg.contains(&format!( + "no existing Provider matching '{}'", + provider_name + )), "missing 'no existing Provider matching' phrase: {}", msg ); @@ -1183,9 +1169,8 @@ mod tests { #[test] fn resolve_provider_target_errors_when_list_is_empty() { - let err = - resolve_provider_target("provider-helm", "{\"items\":[]}", TEST_LOCAL_REGISTRY) - .expect_err("expected no-existing-Provider error"); + let err = resolve_provider_target("provider-helm", "{\"items\":[]}", TEST_LOCAL_REGISTRY) + .expect_err("expected no-existing-Provider error"); assert_no_existing_provider_error(err, "provider-helm"); } @@ -1245,10 +1230,7 @@ mod tests { // "already_patched" bucket. We must still find it instead of erroring. let json = provider_json(&[( "crossplane-contrib-provider-helm", - &format!( - "{}/hops-ops/provider-helm:dev-abc123", - TEST_LOCAL_REGISTRY - ), + &format!("{}/hops-ops/provider-helm:dev-abc123", TEST_LOCAL_REGISTRY), Some("crossplane-contrib-provider-helm-drc"), )]); @@ -1333,7 +1315,8 @@ mod tests { fn resolve_provider_target_returns_clear_error_on_invalid_json() { let err = resolve_provider_target("provider-helm", "not-json", TEST_LOCAL_REGISTRY) .expect_err("expected parse error"); - assert!(err.to_string().contains("failed to parse providers list JSON")); + assert!(err + .to_string() + .contains("failed to parse providers list JSON")); } } - diff --git a/src/commands/secrets/list.rs b/src/commands/secrets/list.rs index 4ae9878..218ca7f 100644 --- a/src/commands/secrets/list.rs +++ b/src/commands/secrets/list.rs @@ -41,7 +41,8 @@ pub fn run() -> Result<(), Box> { pushed_rows.push(PushedSecretRow { name: secret.name.clone(), owner: owner_label(&secret), - cluster: tag_value(&secret, "hops.ops.com.ai/cluster").unwrap_or("-".to_string()), + cluster: tag_value(&secret, "hops.ops.com.ai/cluster") + .unwrap_or("-".to_string()), namespace: tag_value(&secret, "hops.ops.com.ai/namespace") .unwrap_or("-".to_string()), kms_key: shorten_kms_key(&kms), diff --git a/src/commands/vars/mod.rs b/src/commands/vars/mod.rs index 306d7cb..d2b3370 100644 --- a/src/commands/vars/mod.rs +++ b/src/commands/vars/mod.rs @@ -132,10 +132,7 @@ pub(crate) fn require_command(program: &str) -> Result<(), Box> { } } -pub(crate) fn run_command_output( - program: &str, - args: &[&str], -) -> Result, Box> { +pub(crate) fn run_command_output(program: &str, args: &[&str]) -> Result, Box> { log::debug!("Running: {} {}", program, args.join(" ")); let output = Command::new(program).args(args).output()?; if !output.status.success() { diff --git a/src/commands/vars/sync.rs b/src/commands/vars/sync.rs index 2528330..9c3950d 100644 --- a/src/commands/vars/sync.rs +++ b/src/commands/vars/sync.rs @@ -243,11 +243,6 @@ fn set_github_variable( if !status.success() { return Err(format!("gh variable set exited with {}", status).into()); } - log::info!( - "Set GitHub variable '{}' in '{}/{}'", - var_name, - owner, - repo - ); + log::info!("Set GitHub variable '{}' in '{}/{}'", var_name, owner, repo); Ok(()) } From a1437c13ba482f7c6e5c6699509f7e94bc93e876 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 4 Jul 2026 17:26:05 -0500 Subject: [PATCH 02/11] feat: kind backend for hops local Adds --backend (global on hops local) with resolution order flag > persisted (~/.hops/local/backend) > detected cluster (colima wins for back-compat) > platform default (macOS colima, else kind). kind backend: create with a pinned 127.0.0.1:30500 port mapping for the in-cluster registry NodePort, docker start/stop of the node container for resume, destroy/reset via kind delete + recreate, and containerd certs.d hosts.toml aliases (both registry names -> registry ClusterIP over HTTP) written idempotently after the registry deploys. Sizing flags error on kind. HOPS_KUBE_CONTEXT is auto-set from the backend when --context is absent. Preflight enforces kind >= 0.27 (certs.d config_path default). Implements [[tasks/cluster-backend-abstraction]] (phase 2) --- src/commands/local/backend/colima.rs | 6 + src/commands/local/backend/kind.rs | 329 +++++++++++++++++++++++++++ src/commands/local/backend/mod.rs | 186 ++++++++++++++- src/commands/local/mod.rs | 41 ++-- src/commands/local/start.rs | 4 + src/commands/local/uninstall.rs | 1 + 6 files changed, 543 insertions(+), 24 deletions(-) create mode 100644 src/commands/local/backend/kind.rs diff --git a/src/commands/local/backend/colima.rs b/src/commands/local/backend/colima.rs index 7e543f6..15534eb 100644 --- a/src/commands/local/backend/colima.rs +++ b/src/commands/local/backend/colima.rs @@ -250,6 +250,12 @@ fn requested_size_changes(size: &SizeArgs, instance: &ColimaInstance) -> Vec bool { + matches!(colima_instance(), Ok(Some(_))) +} + fn colima_instance() -> Result, Box> { let output = match run_cmd_output("colima", &["list", "--json"]) { Ok(output) => output, diff --git a/src/commands/local/backend/kind.rs b/src/commands/local/backend/kind.rs new file mode 100644 index 0000000..2a22f62 --- /dev/null +++ b/src/commands/local/backend/kind.rs @@ -0,0 +1,329 @@ +//! kind backend: docker containers as nodes, containerd runtime. +//! +//! Works against any reachable docker daemon — Docker Desktop, colima's +//! dockerd, dory's dockerd, or a CI runner's. Registry trust is wired through +//! containerd's certs.d (`config_path` is enabled by default in kind node +//! images since v0.27.0), written after cluster creation; containerd reads +//! certs.d per-pull, so no restart is needed. + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_PULL, REGISTRY_PUSH}; +use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use std::error::Error; +use std::io::Write; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +pub const CLUSTER_NAME: &str = "hops"; +const NODE_CONTAINER: &str = "hops-control-plane"; + +/// kind node images before v0.27.0 ship containerd 1.x without certs.d +/// `config_path` enabled, so our hosts.toml files would be ignored. +const MIN_KIND_VERSION: (u32, u32) = (0, 27); + +const KIND_CONFIG: &str = r#"kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30500 + hostPort: 30500 + listenAddress: "127.0.0.1" +"#; + +pub fn install() -> Result<(), Box> { + log::info!("Installing kind via Homebrew..."); + run_cmd("brew", &["install", "kind"])?; + log::info!("kind installed successfully"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling kind..."); + run_cmd("brew", &["uninstall", "kind"])?; + log::info!("kind uninstalled"); + Ok(()) +} + +pub fn start(size: &SizeArgs) -> Result<(), Box> { + if size.any_set() { + return Err(format!( + "the kind backend has no VM to size; drop{} (resources are governed by the docker daemon kind runs on)", + size.command_suffix() + ) + .into()); + } + + preflight()?; + + if !cluster_exists() { + return create_cluster(); + } + + if node_running() { + log::info!("kind cluster '{}' is already running", CLUSTER_NAME); + return Ok(()); + } + + // kind has no start/stop; the node is a docker container. Restarting a + // single-node cluster is reliable in practice but not guaranteed by kind. + log::info!("Starting stopped kind node '{}'...", NODE_CONTAINER); + run_cmd("docker", &["start", NODE_CONTAINER])?; + wait_for_api_after_restart() +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping kind node '{}'...", NODE_CONTAINER); + run_cmd("docker", &["stop", NODE_CONTAINER])?; + log::info!("kind cluster stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Deleting kind cluster '{}'...", CLUSTER_NAME); + run_cmd("kind", &["delete", "cluster", "--name", CLUSTER_NAME])?; + log::info!("kind cluster deleted"); + Ok(()) +} + +/// kind's node container IS the cluster, so reset means recreate. +pub fn reset() -> Result<(), Box> { + preflight()?; + if cluster_exists() { + destroy()?; + } + create_cluster() +} + +pub fn resize(_size: &SizeArgs) -> Result<(), Box> { + Err( + "kind clusters have no VM to resize; adjust the docker daemon's resources, \ + or `hops local destroy && hops local start` to recreate the cluster" + .into(), + ) +} + +/// Whether the hops kind cluster exists (running or stopped). Missing binary +/// or failing command reads as "no cluster". +pub fn cluster_exists() -> bool { + if !command_exists("kind") { + return false; + } + run_cmd_output("kind", &["get", "clusters"]) + .map(|out| out.lines().any(|line| line.trim() == CLUSTER_NAME)) + .unwrap_or(false) +} + +fn node_running() -> bool { + run_cmd_output( + "docker", + &["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER], + ) + .map(|out| out.trim() == "true") + .unwrap_or(false) +} + +fn preflight() -> Result<(), Box> { + if !command_exists("kind") { + return Err( + "kind is not installed; run `hops local install --backend kind` or `brew install kind`" + .into(), + ); + } + + let version_output = run_cmd_output("kind", &["version"])?; + match parse_kind_version(&version_output) { + Some(version) if version >= MIN_KIND_VERSION => {} + Some((major, minor)) => { + return Err(format!( + "kind v{major}.{minor} is too old: node images before v{}.{} lack containerd \ + certs.d support needed for the local registry. Upgrade with `brew upgrade kind`.", + MIN_KIND_VERSION.0, MIN_KIND_VERSION.1 + ) + .into()); + } + None => log::warn!( + "Unable to parse `kind version` output ({}); continuing", + version_output.trim() + ), + } + + if run_cmd_output("docker", &["info", "--format", "{{.ServerVersion}}"]).is_err() { + return Err( + "no reachable docker daemon; start Docker Desktop / colima / dory \ + (or point DOCKER_HOST / `docker context use` at one) and retry" + .into(), + ); + } + + Ok(()) +} + +fn create_cluster() -> Result<(), Box> { + log::info!("Creating kind cluster '{}'...", CLUSTER_NAME); + let mut child = Command::new("kind") + .args(["create", "cluster", "--name", CLUSTER_NAME, "--config", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(KIND_CONFIG.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err(format!("kind create cluster exited with {}", status).into()); + } + Ok(()) +} + +fn wait_for_api_after_restart() -> Result<(), Box> { + log::info!("Waiting for Kubernetes API..."); + for _ in 0..24 { + if run_cmd_output("kubectl", &["cluster-info"]).is_ok() { + return Ok(()); + } + thread::sleep(Duration::from_secs(5)); + } + Err( + "Kubernetes API did not come back after restarting the kind node; \ + run `hops local reset` to recreate the cluster" + .into(), + ) +} + +/// Alias both registry pull names to the registry Service's ClusterIP via +/// containerd certs.d files on the node. `localhost:30500` is what provider +/// runtime pods reference; aliasing it here means the name never depends on +/// kube-proxy's localhost-NodePort behavior. Files live on the node's +/// writable layer, so they survive docker stop/start (unlike /etc/hosts, +/// which docker regenerates). +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + for name in [REGISTRY_PULL, REGISTRY_PUSH] { + write_hosts_toml(name, cluster_ip)?; + } + Ok(()) +} + +fn hosts_toml(cluster_ip: &str) -> String { + format!( + "[host.\"http://{}:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n", + cluster_ip + ) +} + +fn write_hosts_toml(registry_name: &str, cluster_ip: &str) -> Result<(), Box> { + let dir = format!("/etc/containerd/certs.d/{}", registry_name); + let path = format!("{}/hosts.toml", dir); + let desired = hosts_toml(cluster_ip); + + let current = + run_cmd_output("docker", &["exec", NODE_CONTAINER, "cat", &path]).unwrap_or_default(); + if current == desired { + return Ok(()); + } + + log::info!( + "Wiring containerd registry alias: {} -> {}:5000", + registry_name, + cluster_ip + ); + + let mut child = Command::new("docker") + .args([ + "exec", + "-i", + NODE_CONTAINER, + "sh", + "-c", + &format!("mkdir -p '{}' && cat > '{}'", dir, path), + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(desired.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err(format!("failed to write {} on kind node", path).into()); + } + Ok(()) +} + +fn parse_kind_version(output: &str) -> Option<(u32, u32)> { + // Typical output: "kind v0.32.0 go1.23.4 darwin/arm64" + output.split_whitespace().find_map(|token| { + let token = token.strip_prefix('v').unwrap_or(token); + let mut parts = token.split('.'); + let major: u32 = parts.next()?.parse().ok()?; + let minor: u32 = parts.next()?.parse().ok()?; + Some((major, minor)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind_config_pins_registry_nodeport_to_localhost() { + assert!(KIND_CONFIG.contains("containerPort: 30500")); + assert!(KIND_CONFIG.contains("hostPort: 30500")); + assert!(KIND_CONFIG.contains("listenAddress: \"127.0.0.1\"")); + assert!(KIND_CONFIG.contains("role: control-plane")); + } + + #[test] + fn hosts_toml_aliases_to_cluster_ip_over_http() { + let toml = hosts_toml("10.43.12.7"); + + assert_eq!( + toml, + "[host.\"http://10.43.12.7:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n" + ); + } + + #[test] + fn parse_kind_version_reads_standard_output() { + assert_eq!( + parse_kind_version("kind v0.32.0 go1.23.4 darwin/arm64"), + Some((0, 32)) + ); + assert_eq!( + parse_kind_version("kind v0.27.0 go1.22 linux/amd64"), + Some((0, 27)) + ); + } + + #[test] + fn parse_kind_version_handles_unexpected_output() { + assert_eq!(parse_kind_version("something unparseable"), None); + assert_eq!(parse_kind_version(""), None); + } + + #[test] + fn old_kind_versions_fail_the_minimum_check() { + let version = parse_kind_version("kind v0.26.0 go1.22 linux/amd64").unwrap(); + assert!(version < MIN_KIND_VERSION); + + let new_enough = parse_kind_version("kind v0.27.0 go1.22 linux/amd64").unwrap(); + assert!(new_enough >= MIN_KIND_VERSION); + } + + #[test] + fn start_rejects_size_flags() { + let size = SizeArgs { + cpus: Some(4), + memory: None, + disk: None, + }; + + let err = start(&size).expect_err("size flags must be rejected"); + + assert!(err.to_string().contains("--cpus 4")); + assert!(err.to_string().contains("no VM to size")); + } +} diff --git a/src/commands/local/backend/mod.rs b/src/commands/local/backend/mod.rs index 49cc0f0..26541ec 100644 --- a/src/commands/local/backend/mod.rs +++ b/src/commands/local/backend/mod.rs @@ -5,23 +5,26 @@ //! shaped lives outside this module and is backend-agnostic. mod colima; +mod kind; -use super::run_cmd_output; +use super::{local_state_dir, run_cmd_output}; use clap::Args; use std::error::Error; +use std::fmt; +use std::str::FromStr; /// Sizing flags for backends with a resizable VM. #[derive(Args, Debug, Clone, Default, PartialEq, Eq)] pub struct SizeArgs { - /// Number of CPUs to allocate to the cluster VM. + /// Number of CPUs to allocate to the cluster VM (colima backend only). #[arg(long = "cpus", visible_alias = "cpu", value_name = "N")] pub cpus: Option, - /// Memory to allocate to the cluster VM, in GiB. + /// Memory to allocate to the cluster VM, in GiB (colima backend only). #[arg(long, value_name = "GIB")] pub memory: Option, - /// Disk size to allocate to the cluster VM, in GiB. + /// Disk size to allocate to the cluster VM, in GiB (colima backend only). #[arg(long, value_name = "GIB")] pub disk: Option, } @@ -51,9 +54,13 @@ impl SizeArgs { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, clap::ValueEnum)] pub enum Backend { + /// VM + dockerd + k3s (macOS/Linux) Colima, + /// docker containers as nodes; works on any docker daemon + /// (Docker Desktop, colima, dory, CI runners) + Kind, } impl Backend { @@ -61,18 +68,29 @@ impl Backend { pub fn name(self) -> &'static str { match self { Backend::Colima => "colima", + Backend::Kind => "kind", + } + } + + /// kubeconfig context name this backend's cluster registers under. + pub fn kube_context(self) -> &'static str { + match self { + Backend::Colima => "colima", + Backend::Kind => "kind-hops", } } pub fn install(self) -> Result<(), Box> { match self { Backend::Colima => colima::install(), + Backend::Kind => kind::install(), } } pub fn uninstall(self) -> Result<(), Box> { match self { Backend::Colima => colima::uninstall(), + Backend::Kind => kind::uninstall(), } } @@ -81,30 +99,35 @@ impl Backend { pub fn start(self, size: &SizeArgs, assume_yes: bool) -> Result<(), Box> { match self { Backend::Colima => colima::start(size, assume_yes), + Backend::Kind => kind::start(size), } } pub fn stop(self) -> Result<(), Box> { match self { Backend::Colima => colima::stop(), + Backend::Kind => kind::stop(), } } pub fn destroy(self) -> Result<(), Box> { match self { Backend::Colima => colima::destroy(), + Backend::Kind => kind::destroy(), } } pub fn reset(self) -> Result<(), Box> { match self { Backend::Colima => colima::reset(), + Backend::Kind => kind::reset(), } } pub fn resize(self, size: &SizeArgs) -> Result<(), Box> { match self { Backend::Colima => colima::resize(size), + Backend::Kind => kind::resize(size), } } @@ -113,6 +136,9 @@ impl Backend { pub fn ensure_registry_trust(self) -> Result<(), Box> { match self { Backend::Colima => colima::configure_docker_insecure_registry(), + // containerd trust is per-name via certs.d, written in + // wire_registry once the registry Service's ClusterIP is known. + Backend::Kind => Ok(()), } } @@ -122,17 +148,98 @@ impl Backend { pub fn wire_registry(self, cluster_ip: &str) -> Result<(), Box> { match self { Backend::Colima => colima::sync_hosts_entry(cluster_ip), + Backend::Kind => kind::wire_registry(cluster_ip), + } + } +} + +impl fmt::Display for Backend { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +impl FromStr for Backend { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim() { + "colima" => Ok(Backend::Colima), + "kind" => Ok(Backend::Kind), + other => Err(format!( + "unknown backend '{}' (expected colima or kind)", + other + )), } } } -/// Resolve which backend to operate on. +const BACKEND_FILE: &str = "backend"; + +/// Resolve which backend to operate on: explicit flag > preference persisted +/// by the last successful start > detection of an existing cluster (colima +/// wins for back-compat with pre-backend installs) > platform default. pub fn resolve(flag: Option) -> Backend { - flag.unwrap_or(Backend::Colima) + resolve_from( + flag, + persisted(), + colima::instance_exists, + kind::cluster_exists, + cfg!(target_os = "macos"), + ) +} + +fn resolve_from( + flag: Option, + persisted: Option, + colima_detected: impl FnOnce() -> bool, + kind_detected: impl FnOnce() -> bool, + macos: bool, +) -> Backend { + if let Some(backend) = flag { + return backend; + } + if let Some(backend) = persisted { + return backend; + } + if colima_detected() { + return Backend::Colima; + } + if kind_detected() { + return Backend::Kind; + } + if macos { + Backend::Colima + } else { + Backend::Kind + } +} + +fn persisted() -> Option { + let path = local_state_dir().ok()?.join(BACKEND_FILE); + std::fs::read_to_string(path).ok()?.parse().ok() +} + +/// Record the backend so later invocations (stop, destroy, doctor, installs) +/// target the same cluster without re-detection. +pub fn persist(backend: Backend) -> Result<(), Box> { + let dir = local_state_dir()?; + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join(BACKEND_FILE), format!("{}\n", backend.name()))?; + Ok(()) +} + +/// Drop the persisted preference (used by uninstall; destroy keeps it). +pub fn clear_persisted() -> Result<(), Box> { + match std::fs::remove_file(local_state_dir()?.join(BACKEND_FILE)) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err.into()), + } } /// Fetch the in-cluster registry Service's ClusterIP. -pub fn registry_cluster_ip() -> Result> { +fn registry_cluster_ip() -> Result> { let cluster_ip = run_cmd_output( "kubectl", &[ @@ -157,3 +264,66 @@ pub fn registry_cluster_ip() -> Result> { pub fn wire_local_registry(backend: Backend) -> Result<(), Box> { backend.wire_registry(®istry_cluster_ip()?) } + +#[cfg(test)] +mod tests { + use super::*; + + fn no_detect() -> bool { + false + } + + #[test] + fn flag_beats_persisted_and_detection() { + let resolved = resolve_from( + Some(Backend::Kind), + Some(Backend::Colima), + || true, + no_detect, + true, + ); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn persisted_beats_detection() { + let resolved = resolve_from(None, Some(Backend::Kind), || true, no_detect, true); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn colima_detection_beats_kind_detection() { + let resolved = resolve_from(None, None, || true, || true, false); + + assert_eq!(resolved, Backend::Colima); + } + + #[test] + fn kind_detection_used_when_no_colima() { + let resolved = resolve_from(None, None, no_detect, || true, true); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn platform_default_when_nothing_detected() { + assert_eq!( + resolve_from(None, None, no_detect, no_detect, true), + Backend::Colima + ); + assert_eq!( + resolve_from(None, None, no_detect, no_detect, false), + Backend::Kind + ); + } + + #[test] + fn backend_name_round_trips_through_from_str() { + for backend in [Backend::Colima, Backend::Kind] { + assert_eq!(backend.name().parse::().unwrap(), backend); + } + assert!("dory".parse::().is_err()); + } +} diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index e399980..e59fb2a 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -64,21 +64,29 @@ pub struct LocalArgs { pub command: LocalCommands, /// Kubernetes context to use for all kubectl commands (e.g. "colima"). - /// Global: applies to every `hops local` subcommand and may be given before - /// or after the subcommand. + /// Defaults to the resolved backend's own context. Global: applies to + /// every `hops local` subcommand and may be given before or after the + /// subcommand. #[arg(long, global = true)] pub context: Option, + + /// Local cluster backend to target. Defaults to the backend persisted by + /// the last successful `hops local start`, else an existing cluster if + /// one is detected, else the platform default (macOS: colima, otherwise + /// kind). + #[arg(long, global = true, value_enum)] + pub backend: Option, } #[derive(Subcommand, Debug)] pub enum LocalCommands { - /// Install Colima via Homebrew + /// Install the local cluster backend (colima or kind) via Homebrew Install, - /// Reset local Colima Kubernetes state + /// Reset local Kubernetes state (colima: k8s reset; kind: recreate cluster) Reset, /// Start local k8s cluster with Crossplane and providers Start(start::StartArgs), - /// Resize the local Colima VM without destroying cluster state + /// Resize the local cluster VM without destroying cluster state (colima only) Resize(resize::ResizeArgs), /// Check what `hops local start` set up and report drift Doctor, @@ -94,21 +102,22 @@ pub enum LocalCommands { Listmonk(listmonk::ListmonkArgs), /// Stop the local cluster Stop, - /// Destroy the local cluster VM + /// Destroy the local cluster Destroy, - /// Uninstall Colima + /// Uninstall the local cluster backend Uninstall, } pub fn run(args: &LocalArgs) -> Result<(), Box> { - // Plumb --context through the same env channel the kubectl helpers read, so - // every subcommand's kubectl calls target the chosen context. - if let Some(ctx) = &args.context { - if !ctx.is_empty() { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + let backend = backend::resolve(args.backend); + // Plumb the context through the same env channel the kubectl helpers + // read, so every subcommand's kubectl calls target the chosen cluster. + // Without an explicit --context, use the backend's own context so + // commands work regardless of kubeconfig's current-context. + match &args.context { + Some(ctx) if !ctx.is_empty() => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx), + _ => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, backend.kube_context()), } - let backend = backend::resolve(None); match &args.command { LocalCommands::Install => install::run(backend), LocalCommands::Reset => reset::run(backend), @@ -183,13 +192,13 @@ pub fn repo_cache_path(org: &str, repo: &str) -> Result> Ok(local_state_dir()?.join(REPO_CACHE_DIR).join(org).join(repo)) } -fn local_state_dir() -> Result> { +pub(crate) fn local_state_dir() -> Result> { let home = std::env::var("HOME") .map_err(|_| "HOME is not set; unable to determine local state directory")?; Ok(Path::new(&home).join(LOCAL_STATE_DIR)) } -fn command_exists(program: &str) -> bool { +pub(crate) fn command_exists(program: &str) -> bool { Command::new("sh") .args(["-c", &format!("command -v {} >/dev/null 2>&1", program)]) .status() diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index a7f699b..7acd95e 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -29,6 +29,10 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box> { if input.trim().eq_ignore_ascii_case("y") { backend.uninstall()?; + super::backend::clear_persisted()?; } else { log::info!("Uninstall cancelled"); } From 09dd90f2bf9ba932855014218a04ddcf9ee40306 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 4 Jul 2026 17:27:15 -0500 Subject: [PATCH 03/11] ci+docs: kind smoke workflow and backend docs GH Actions workflow exercises the CI acceptance path on ubuntu-latest: hops local start --backend kind, doctor, a registry round-trip through both pull names (Service name and localhost:30500), the stop/start resume path via the persisted backend, and destroy. README documents backend selection, persistence/resolution order, and the dory-via-kind recipe. Implements [[tasks/cluster-backend-abstraction]] (phases 3-4) --- .github/workflows/on-pr-kind-smoke.yaml | 62 +++++++++++++++++++++++++ README.md | 51 ++++++++++++++++++-- 2 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/on-pr-kind-smoke.yaml diff --git a/.github/workflows/on-pr-kind-smoke.yaml b/.github/workflows/on-pr-kind-smoke.yaml new file mode 100644 index 0000000..5664e50 --- /dev/null +++ b/.github/workflows/on-pr-kind-smoke.yaml @@ -0,0 +1,62 @@ +name: kind backend smoke + +# End-to-end check of `hops local` on the kind backend, which is the CI +# path (ubuntu runners ship kind, docker, kubectl, and helm). Exercises the +# registry NodePort mapping and BOTH containerd certs.d trust names, plus the +# stop/start resume path. + +on: + pull_request: + branches: + - main + +jobs: + kind-smoke: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Build hops + run: cargo build + + - name: hops local start --backend kind + run: ./target/debug/hops-cli local start --backend kind + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + docker pull public.ecr.aws/docker/library/busybox:stable + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + # Service-name pull: containerd resolves via its certs.d alias. + kubectl --context kind-hops run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # localhost:30500 pull: what provider runtime pods reference. + kubectl --context kind-hops run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + kubectl --context kind-hops wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=180s + kubectl --context kind-hops delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + ./target/debug/hops-cli local stop + ./target/debug/hops-cli local start + kubectl --context kind-hops -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry --timeout=300s + ./target/debug/hops-cli local doctor + + - name: hops local destroy + run: ./target/debug/hops-cli local destroy diff --git a/README.md b/README.md index c1547e8..17313f2 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ This tool supports three related workflows: -- Local cluster setup on Colima +- Local cluster setup on colima or kind - Configuration package install/uninstall against the connected cluster - XR observe/manage/adopt/orphan workflows for existing infrastructure For local development, it can also: -- Install and manage Colima +- Install and manage a local cluster backend (colima or kind) - Start a local k8s cluster with Crossplane installed via Helm - Install the Kubernetes and Helm Crossplane providers - Deploy an in-cluster OCI registry (`crossplane-system/registry`) @@ -56,7 +56,7 @@ See "Releases" for available versions and changenotes. - `up` (Upbound CLI, used by `up project build`) - `aws` CLI v2 (used by `local aws` to export profile credentials) -Note: `hops-cli local install` installs `colima` through Homebrew. +Note: `hops-cli local install` installs the selected backend (`colima` or `kind`) through Homebrew. ## Build @@ -87,7 +87,7 @@ hops service --help `hops-cli` is organized into a few command groups: - `local` - - Manage a local Colima-based control plane, install providers, and bootstrap AWS or GitHub provider auth. + - Manage a local control plane (colima or kind backend), install providers, and bootstrap AWS or GitHub provider auth. - `config` - Build, install, reload, and uninstall Crossplane configuration packages against the connected cluster. - `secrets` @@ -192,7 +192,8 @@ Examples: ## Create a Local Control Plane ```bash -# 1) Install Colima (via Homebrew) +# 1) Install the backend (via Homebrew). Defaults to colima on macOS; +# pass --backend kind to use kind on any docker daemon. hops local install # 2) Start local k8s + Crossplane + providers + local registry @@ -211,6 +212,46 @@ hops local zitadel --source-context pat-local --domain auth.ops.com.ai hops config install --repo hops-ops/aws-auto-eks-cluster --version v0.11.0 ``` +### Cluster backends + +`hops local` supports two backends behind the same commands: + +- **colima** — a VM running dockerd + k3s. macOS/Linux; supports `--cpus`, + `--memory`, `--disk`, and `hops local resize`. +- **kind** — cluster nodes as docker containers on any reachable docker + daemon: Docker Desktop, colima's dockerd, [dory](https://augani.github.io/dory), + or CI runners. No VM of its own, so sizing flags don't apply (size the + docker daemon instead); requires kind >= v0.27. + +Select with the global `--backend` flag: + +```bash +hops local start --backend kind +``` + +The chosen backend is persisted to `~/.hops/local/backend` on a successful +start, so later commands (`stop`, `destroy`, `doctor`, package installs) +target the same cluster without the flag. Resolution order: `--backend` flag > +persisted choice > existing cluster detection (colima wins) > platform +default (macOS: colima, otherwise kind). + +Unless `--context` is given, kubectl commands automatically use the backend's +kubeconfig context (`colima` or `kind-hops`), regardless of your +current-context. + +#### Using dory + +[dory](https://augani.github.io/dory) exposes a real docker socket, so run the +kind backend against it: + +```bash +docker context use dory # or: export DOCKER_HOST=unix://$HOME/.dory/dory.sock +hops local start --backend kind +``` + +Don't use dory's built-in Kubernetes for hops — it publishes only the API +port, so the local package registry would be unreachable from the host. + ### Local provider setup and auth `hops local aws`, `hops local github`, and `hops local zitadel` install the provider package and bootstrap auth into a local control plane. The exception is `--refresh`, which updates credentials only. From 1a7f7af1d03391149237526cafc47128a458f17c Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 11 Jul 2026 13:20:04 -0500 Subject: [PATCH 04/11] feat: native dory backend for hops local (#73) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: native dory backend for hops local Backend::Dory drives dory's built-in k3s headlessly via the dory CLI (k8s enable/disable/status) and the engine docker socket: - start: writes ~/.dory/k8s/registries.yaml (k3s-native trust aliasing both registry pull names to the Service hostname over HTTP; read at boot via dory's bind mount), then 'dory k8s enable --publish 30500:30500'. The Dory app's port forwarder makes the published NodePort host-reachable. - wire_registry: syncs hostname -> ClusterIP in the node's /etc/hosts per start (in-place rewrite — /etc/hosts is a bind mount, sed -i fails). - stop/resume via docker stop/start of dory-k8s; destroy/reset via dory k8s disable (+ enable); sizing flags error (VM sized in the app). - kube_context 'dory' (dory names its kubeconfig context that); hops prepends ~/.kube/dory-config to KUBECONFIG for its kubectl/helm children. - detection order: colima > kind > dory. Also: 'helm repo update crossplane-stable' instead of bare update — a stale unrelated repo in the user's helm config no longer breaks start. Verified end-to-end locally: start (Crossplane 2.3.3 + providers + registry), doctor all green, registry round-trip through BOTH pull names to pod Ready, stop/start resume, doctor green again. Implements [[tasks/dory-native-backend]] * fix: harden dory integration contract (#74) * fix: harden dory integration contract [[tasks/rr-2-dory-contract]] * fix: skip KUBECONFIG plumbing when dory context is already merged Current dory merges the dory context into ~/.kube/config at enable time, so hops no longer needs to mutate KUBECONFIG for child processes; the side-file prepend remains as a fallback for pre-merge dory versions. Implements [[tasks/dory-kubeconfig-merge]] * fix: hold dory cluster through the app's engine provisioning window Launching Dory.app re-provisions its engine for ~90s and restarts dockerd in the VM at the end, SIGTERMing every container — a k3s node enabled during that window reports Ready and then dies mid-bootstrap. The engine socket's mtime marks the session start, so when it is younger than 180s, start/reset now watch the node until the window passes and re-enable it (up to 3x) if the engine restart takes it down. Steady-state starts pay one container inspect. Implements [[tasks/rr-2-dory-contract]] * fix: unify kube context/backend targeting (#75) [[tasks/rr-1-context-targeting]] --- README.md | 38 +- src/commands/config/install.rs | 73 ++- src/commands/local/backend/dory.rs | 741 +++++++++++++++++++++++++++++ src/commands/local/backend/mod.rs | 232 ++++++++- src/commands/local/mod.rs | 131 ++++- src/commands/local/start.rs | 4 +- src/commands/local/uninstall.rs | 162 ++++++- src/commands/provider/install.rs | 46 +- 8 files changed, 1351 insertions(+), 76 deletions(-) create mode 100644 src/commands/local/backend/dory.rs diff --git a/README.md b/README.md index 17313f2..a95e79d 100644 --- a/README.md +++ b/README.md @@ -214,14 +214,21 @@ hops config install --repo hops-ops/aws-auto-eks-cluster --version v0.11.0 ### Cluster backends -`hops local` supports two backends behind the same commands: +`hops local` supports three backends behind the same commands: - **colima** — a VM running dockerd + k3s. macOS/Linux; supports `--cpus`, `--memory`, `--disk`, and `hops local resize`. - **kind** — cluster nodes as docker containers on any reachable docker - daemon: Docker Desktop, colima's dockerd, [dory](https://augani.github.io/dory), - or CI runners. No VM of its own, so sizing flags don't apply (size the - docker daemon instead); requires kind >= v0.27. + daemon: Docker Desktop, colima's dockerd, or CI runners. No VM of its own, + so sizing flags don't apply (size the docker daemon instead); requires + kind >= v0.27. +- **dory** — [dory](https://augani.github.io/dory)'s built-in k3s, driven + headlessly through the `dory` CLI (`dory k8s enable/disable/status`). + Requires the Dory app running (it provides the engine and forwards + published ports to localhost) and a `dory` CLI with headless k8s support. + hops writes `~/.dory/k8s/registries.yaml` (k3s' native registry trust) and + publishes the registry NodePort at cluster create. The VM is sized in the + Dory app, so hops sizing flags don't apply. Select with the global `--backend` flag: @@ -236,22 +243,31 @@ persisted choice > existing cluster detection (colima wins) > platform default (macOS: colima, otherwise kind). Unless `--context` is given, kubectl commands automatically use the backend's -kubeconfig context (`colima` or `kind-hops`), regardless of your -current-context. +kubeconfig context (`colima`, `kind-hops`, or `dory`), regardless of your +current-context. For dory, hops also prepends `~/.kube/dory-config` (where +dory keeps its kubeconfig) to `KUBECONFIG` for its own kubectl/helm calls. #### Using dory -[dory](https://augani.github.io/dory) exposes a real docker socket, so run the -kind backend against it: +With a `dory` CLI that supports `dory k8s enable` (headless Kubernetes), +use the native backend: + +```bash +hops local start --backend dory +``` + +For `hops provider install` / `hops config install` builds, point your docker +CLI at dory's engine (`export DOCKER_HOST=unix://$HOME/.dory/engine.sock`) so +image builds/pushes land on the daemon that reaches the registry. + +Without the headless CLI, dory still exposes a real docker socket, so the +kind backend works against it: ```bash docker context use dory # or: export DOCKER_HOST=unix://$HOME/.dory/dory.sock hops local start --backend kind ``` -Don't use dory's built-in Kubernetes for hops — it publishes only the API -port, so the local package registry would be unreachable from the host. - ### Local provider setup and auth `hops local aws`, `hops local github`, and `hops local zitadel` install the provider package and bootstrap auth into a local control plane. The exception is `--refresh`, which updates credentials only. diff --git a/src/commands/config/install.rs b/src/commands/config/install.rs index d3812c9..dd396c2 100644 --- a/src/commands/config/install.rs +++ b/src/commands/config/install.rs @@ -1,4 +1,4 @@ -use crate::commands::local::backend::{self, wire_local_registry}; +use crate::commands::local::backend::{self, Backend}; use crate::commands::local::package_install::run_watch; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout, ensure_registry, image_config_name, @@ -6,9 +6,7 @@ use crate::commands::local::package_install::{ rewrite_registry_with_tag, sanitize_name_component, short_hash, split_ref, strip_registry, unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_PULL, REGISTRY_PUSH, }; -use crate::commands::local::{ - kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output, HOPS_KUBE_CONTEXT_ENV, -}; +use crate::commands::local::{kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output}; use clap::Args; use flate2::read::GzDecoder; use serde::Deserialize; @@ -43,6 +41,10 @@ pub struct ConfigArgs { #[arg(long)] pub context: Option, + /// Local cluster backend whose node should be wired for local package pulls. + #[arg(long, value_enum)] + pub backend: Option, + /// Watch the project directory for changes and re-run install automatically #[arg(long, conflicts_with = "repo")] pub watch: bool, @@ -108,17 +110,22 @@ struct PackageResource { } pub fn run(args: &ConfigArgs) -> Result<(), Box> { - if let Some(ctx) = &args.context { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + let backend = backend::activate(args.backend, args.context.as_deref()); match (args.repo.as_deref(), args.version.as_deref()) { (Some(repo), Some(version)) => { apply_repo_version(repo, version, args.skip_dependency_resolution) } - (Some(repo), None) => run_repo_install(repo, args.skip_dependency_resolution), + (Some(repo), None) => run_repo_install( + repo, + args.skip_dependency_resolution, + backend, + args.backend, + args.context.as_deref(), + ), (None, _) => { let path = args.path.as_deref().unwrap_or("."); + prepare_local_registry(backend, args.backend, args.context.as_deref())?; run_local_path(path, args.skip_dependency_resolution)?; if args.watch { @@ -134,21 +141,49 @@ pub fn run(args: &ConfigArgs) -> Result<(), Box> { } } -fn run_repo_install(repo: &str, skip_dependency_resolution: bool) -> Result<(), Box> { +fn run_repo_install( + repo: &str, + skip_dependency_resolution: bool, + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { let spec = parse_repo_spec(repo)?; match resolve_repo_install_target(&spec)? { - RepoInstallTarget::SourceBuild => run_repo_clone(&spec, skip_dependency_resolution), + RepoInstallTarget::SourceBuild => run_repo_clone( + &spec, + skip_dependency_resolution, + backend, + backend_flag, + context, + ), RepoInstallTarget::PublishedVersion(version) => { apply_repo_version_spec(&spec, &version, skip_dependency_resolution) } } } -fn run_repo_clone(spec: &RepoSpec, skip_dependency_resolution: bool) -> Result<(), Box> { - let cache_path = ensure_cached_repo_checkout(&spec)?; +fn run_repo_clone( + spec: &RepoSpec, + skip_dependency_resolution: bool, + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + let cache_path = ensure_cached_repo_checkout(spec)?; + prepare_local_registry(backend, backend_flag, context)?; run_local_path(&cache_path.to_string_lossy(), skip_dependency_resolution) } +fn prepare_local_registry( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + ensure_registry()?; + backend::wire_local_registry_for_target(backend, backend_flag, context) +} + fn apply_repo_version_spec( spec: &RepoSpec, version: &str, @@ -216,9 +251,6 @@ fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Bo return Err(format!("{} is not a directory", path).into()); } - ensure_registry()?; - wire_local_registry(backend::resolve(None))?; - // Build the Crossplane package log::info!("Building Crossplane package in {}...", path); let status = Command::new("up") @@ -237,7 +269,7 @@ fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Bo let packages: Vec<_> = fs::read_dir(&output_dir) .map_err(|e| format!("Failed to read {}: {}", output_dir.display(), e))? .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().extension().map_or(false, |ext| ext == "uppkg")) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "uppkg")) .collect(); if packages.is_empty() { @@ -1038,6 +1070,15 @@ spec: assert!(!without_skip.contains("skipDependencyResolution: true")); } + #[test] + fn local_registry_wiring_skips_foreign_context_without_backend_flag() { + assert!(!backend::should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + #[test] fn package_source_strips_tag_and_digest() { assert_eq!( diff --git a/src/commands/local/backend/dory.rs b/src/commands/local/backend/dory.rs new file mode 100644 index 0000000..1c0dfed --- /dev/null +++ b/src/commands/local/backend/dory.rs @@ -0,0 +1,741 @@ +//! dory backend: k3s in a container on dory's shared-VM dockerd +//! (https://augani.github.io/dory), driven headlessly through the `dory` CLI +//! (`dory k8s enable|disable|status`) and the engine docker socket. +//! +//! Registry plumbing differs from colima/kind because the cluster container +//! has create-time config shared with the Dory app: +//! - published NodePorts are static config: hops writes `~/.dory/k8s/ports` +//! BEFORE `dory k8s enable`, so GUI-side recreates keep the same port. +//! - trust is static config: hops writes `~/.dory/k8s/registries.yaml` +//! BEFORE `dory k8s enable`; dory bind-mounts it and k3s reads it at boot, +//! aliasing both pull names to the registry Service hostname over HTTP. +//! - name resolution is dynamic: `wire_registry` syncs the hostname -> +//! ClusterIP in the node container's /etc/hosts on every start (same +//! re-wire-on-start model as the other backends). +//! - the host reaches `localhost:30500` because `dory k8s enable --publish +//! 30500:30500` publishes the NodePort and the Dory app's port forwarder +//! maps published ports to host loopback. + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH}; +use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use std::error::Error; +use std::path::PathBuf; + +const NODE_CONTAINER: &str = "dory-k8s"; +/// hops' registry NodePort, published on the cluster container at create. +const REGISTRY_PORT_PUBLISH: &str = "30500:30500"; +const REGISTRIES_BEGIN: &str = "# BEGIN hops-managed (do not edit inside)"; +const REGISTRIES_END: &str = "# END hops-managed"; + +fn home() -> Result> { + Ok(PathBuf::from(std::env::var("HOME").map_err(|_| { + "HOME is not set; unable to locate dory's state directory" + })?)) +} + +fn engine_socket() -> Result> { + Ok(home()?.join(".dory/engine.sock")) +} + +/// dory's side-file kubeconfig (context name `dory`). Current dory also +/// merges the context into ~/.kube/config at enable time; the side file is +/// the pre-merge fallback and dory's own `--kubeconfig` input. +pub fn kubeconfig_path() -> Option { + home() + .ok() + .map(|h| h.join(".kube/dory-config").to_string_lossy().into_owned()) +} + +fn registries_yaml_path() -> Result> { + Ok(home()?.join(".dory/k8s/registries.yaml")) +} + +fn ports_file_path() -> Result> { + Ok(home()?.join(".dory/k8s/ports")) +} + +/// k3s' native registry config: alias both pull names to the registry +/// Service hostname over plain HTTP. The hostname resolves through the +/// node's /etc/hosts, which `wire_registry` keeps pointed at the ClusterIP. +fn registries_yaml() -> String { + format!( + "# Written by `hops local start --backend dory`.\n\ + # k3s reads this at boot; edits require `hops local reset`.\n\ + # Hops manages only the marked block; keep user mirrors outside it.\n\ + mirrors:\n\ + {block}", + block = registries_yaml_block(), + ) +} + +fn registries_yaml_block() -> String { + format!( + " {begin}\n\ + \x20\x20# hops: mirror {pull}\n\ + \x20\x20\"{pull}\":\n\ + \x20\x20 endpoint:\n\ + \x20\x20 - \"http://{pull}\"\n\ + \x20\x20# hops: mirror {push}\n\ + \x20\x20\"{push}\":\n\ + \x20\x20 endpoint:\n\ + \x20\x20 - \"http://{pull}\"\n\ + \x20\x20{end}\n", + begin = REGISTRIES_BEGIN, + end = REGISTRIES_END, + pull = REGISTRY_PULL, + push = REGISTRY_PUSH, + ) +} + +fn legacy_registries_yaml() -> String { + format!( + "# Written by `hops local start --backend dory`.\n\ + # k3s reads this at boot; edits require `hops local reset`.\n\ + mirrors:\n\ + \x20 \"{pull}\":\n\ + \x20 endpoint:\n\ + \x20 - \"http://{pull}\"\n\ + \x20 \"{push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"http://{pull}\"\n", + pull = REGISTRY_PULL, + push = REGISTRY_PUSH, + ) +} + +/// Run docker against dory's engine socket (the daemon the Dory app manages). +fn engine_docker(args: &[&str]) -> Result<(), Box> { + let sock = format!("unix://{}", engine_socket()?.display()); + let mut full = vec!["-H", sock.as_str()]; + full.extend_from_slice(args); + run_cmd("docker", &full) +} + +fn engine_docker_output(args: &[&str]) -> Result> { + let sock = format!("unix://{}", engine_socket()?.display()); + let mut full = vec!["-H", sock.as_str()]; + full.extend_from_slice(args); + run_cmd_output("docker", &full) +} + +pub fn install() -> Result<(), Box> { + log::info!("Installing Dory via Homebrew..."); + run_cmd("brew", &["install", "--cask", "Augani/dory/dory"])?; + log::info!("Dory installed; launch the Dory app once so it provisions its engine"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling Dory..."); + run_cmd("brew", &["uninstall", "--cask", "dory"])?; + log::info!("Dory uninstalled"); + Ok(()) +} + +pub fn start(size: &SizeArgs) -> Result<(), Box> { + if size.any_set() { + return Err(format!( + "the dory backend's VM is sized by the Dory app, not hops; drop{}", + size.command_suffix() + ) + .into()); + } + + preflight()?; + write_ports_file()?; + write_registries_yaml()?; + + // Creates, restarts, or reuses the dory-k8s container as needed. If the + // running container has create-time config drift, dory exits 3 rather than + // destroying state; surface that plus the hops reset path. + run_dory_enable(false) +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping dory k8s node '{}'...", NODE_CONTAINER); + engine_docker(&["stop", NODE_CONTAINER])?; + log::info!("dory cluster stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Deleting dory k8s cluster..."); + run_cmd("dory", &["k8s", "disable"])?; + log::info!("dory cluster deleted"); + Ok(()) +} + +/// The cluster container IS the cluster, so reset means recreate. +pub fn reset() -> Result<(), Box> { + preflight()?; + run_cmd("dory", &["k8s", "disable"])?; + write_ports_file()?; + write_registries_yaml()?; + run_dory_enable(true) +} + +pub fn resize(_size: &SizeArgs) -> Result<(), Box> { + Err("the dory backend has no hops-managed VM to resize; \ + adjust resources in the Dory app instead" + .into()) +} + +/// Whether the hops-relevant dory cluster exists (running or stopped). +/// Missing app/engine/CLI reads as "no cluster". +pub fn cluster_exists() -> bool { + let Ok(sock) = engine_socket() else { + return false; + }; + if !sock.exists() { + return false; + } + engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]).is_ok() +} + +fn preflight() -> Result<(), Box> { + if !command_exists("dory") { + return Err("the `dory` CLI is not on PATH; link it from the dory repo \ + (`ln -sf /scripts/dory /opt/homebrew/bin/dory`)" + .into()); + } + let sock = engine_socket()?; + if !sock.exists() { + return Err(format!( + "dory's engine socket ({}) is missing; launch the Dory app and wait for its engine to start", + sock.display() + ) + .into()); + } + if !command_exists("docker") { + return Err("docker CLI not found; install it (dory provides the daemon)".into()); + } + Ok(()) +} + +/// Write the static registry trust config read by k3s at boot. Must exist +/// before `dory k8s enable` because dory binds it at container create. +fn write_registries_yaml() -> Result<(), Box> { + let path = registries_yaml_path()?; + let current = match std::fs::read_to_string(&path) { + Ok(content) => Some(content), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, + Err(err) => return Err(err.into()), + }; + let desired = merge_registries_yaml(current.as_deref())?; + if current.as_deref() == Some(desired.as_str()) { + return Ok(()); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + log::info!("Writing k3s registry config: {}", path.display()); + std::fs::write(&path, desired)?; + Ok(()) +} + +fn write_ports_file() -> Result<(), Box> { + let path = ports_file_path()?; + let current = match std::fs::read_to_string(&path) { + Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(err) => return Err(err.into()), + }; + let desired = ports_file_with_publish(¤t); + if current == desired { + return Ok(()); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + log::info!( + "Ensuring dory k8s port config includes {}: {}", + REGISTRY_PORT_PUBLISH, + path.display() + ); + std::fs::write(&path, desired)?; + Ok(()) +} + +/// Seconds after the Dory app (re)creates its engine socket during which it is +/// still provisioning the engine. A dockerd restart at the end of that window +/// SIGTERMs every container — including a k3s node enabled meanwhile, which +/// reports Ready and then dies under the bootstrap (observed ~90s on Dory +/// 0.2.0; padded for slower machines). +const ENGINE_LAUNCH_WINDOW_SECS: u64 = 180; + +/// Age of the current engine session: the app recreates the engine socket at +/// launch, so its mtime marks when provisioning began. None when unreadable. +fn engine_session_age() -> Option { + let sock = engine_socket().ok()?; + let modified = std::fs::metadata(&sock).ok()?.modified().ok()?; + std::time::SystemTime::now().duration_since(modified).ok() +} + +/// Time left inside the app's provisioning window for a given engine session +/// age; None once the window has passed. +fn launch_window_remaining(age: std::time::Duration) -> Option { + std::time::Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS) + .checked_sub(age) + .filter(|remaining| !remaining.is_zero()) +} + +fn node_running() -> bool { + engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]) + .map(|state| state.trim() == "true") + .unwrap_or(false) +} + +/// Hold a freshly-enabled cluster under observation while the Dory app may +/// still be provisioning its engine, re-enabling if the engine restart takes +/// the node down. Immediate no-op when the window has already passed, so +/// steady-state starts pay one container inspect and nothing more. +fn hold_through_engine_launch_window() -> Result<(), Box> { + let in_window = |age: Option| { + age.map(|a| launch_window_remaining(a).is_some()) + .unwrap_or(false) + }; + if in_window(engine_session_age()) { + log::info!( + "Dory engine session is younger than {}s; watching the k8s node through the app's provisioning window...", + ENGINE_LAUNCH_WINDOW_SECS + ); + } + let mut reenables = 0; + loop { + match (node_running(), in_window(engine_session_age())) { + (true, false) => return Ok(()), + (true, true) => {} + (false, _) => { + if reenables >= 3 { + return Err("the dory engine keeps stopping the k8s node during app startup; \ + wait for the Dory app to finish provisioning, then re-run `hops local start --backend dory`" + .into()); + } + reenables += 1; + log::warn!( + "dory engine restart stopped the k8s node; re-enabling ({}/3)...", + reenables + ); + let args = dory_enable_args(false); + run_cmd("dory", &args)?; + } + } + std::thread::sleep(std::time::Duration::from_secs(3)); + } +} + +fn run_dory_enable(recreate: bool) -> Result<(), Box> { + let args = dory_enable_args(recreate); + match run_cmd("dory", &args) { + Ok(()) => hold_through_engine_launch_window(), + Err(err) if !recreate && err.to_string().contains("exit status: 3") => Err(format!( + "{}\nhint: run `hops local reset --backend dory` to recreate the dory cluster and apply create-time config drift", + err + ) + .into()), + Err(err) => Err(err), + } +} + +fn dory_enable_args(recreate: bool) -> Vec<&'static str> { + let mut args = vec!["k8s", "enable"]; + if recreate { + args.push("--recreate"); + } + args.extend(["--publish", REGISTRY_PORT_PUBLISH]); + args +} + +fn merge_registries_yaml(existing: Option<&str>) -> Result> { + let Some(existing) = existing else { + return Ok(registries_yaml()); + }; + if existing.trim().is_empty() || existing == legacy_registries_yaml() { + return Ok(registries_yaml()); + } + + match ( + existing.find(REGISTRIES_BEGIN), + existing.find(REGISTRIES_END), + ) { + (Some(begin), Some(end)) if begin <= end => replace_registries_block(existing, begin, end), + (Some(_), Some(_)) | (Some(_), None) | (None, Some(_)) => { + Err(format!("malformed dory registries.yaml managed block; expected `{REGISTRIES_BEGIN}` before `{REGISTRIES_END}`").into()) + } + (None, None) => insert_registries_block(existing), + } +} + +fn replace_registries_block( + existing: &str, + begin: usize, + end: usize, +) -> Result> { + let line_start = existing[..begin].rfind('\n').map_or(0, |idx| idx + 1); + let line_end = existing[end..] + .find('\n') + .map_or(existing.len(), |idx| end + idx + 1); + let mut merged = String::with_capacity(existing.len() + registries_yaml_block().len()); + merged.push_str(&existing[..line_start]); + merged.push_str(®istries_yaml_block()); + merged.push_str(&existing[line_end..]); + Ok(merged) +} + +fn insert_registries_block(existing: &str) -> Result> { + if contains_hops_mirror_key(existing) { + return Err("dory registries.yaml already contains hops registry mirror entries without managed markers; remove those entries or wrap them in the hops-managed block" + .into()); + } + + let mut merged = ensure_trailing_newline(existing); + if let Some(insert_at) = mirrors_line_insert_position(&merged) { + merged.insert_str(insert_at, ®istries_yaml_block()); + return Ok(merged); + } + + merged.push_str("mirrors:\n"); + merged.push_str(®istries_yaml_block()); + Ok(merged) +} + +fn contains_hops_mirror_key(content: &str) -> bool { + content.contains(&format!("\"{}\":", REGISTRY_PULL)) + || content.contains(&format!("\"{}\":", REGISTRY_PUSH)) +} + +fn mirrors_line_insert_position(content: &str) -> Option { + let mut offset = 0; + for line in content.split_inclusive('\n') { + let without_newline = line.trim_end_matches('\n').trim_end_matches('\r'); + if is_top_level_mirrors_line(without_newline) { + return Some(offset + line.len()); + } + offset += line.len(); + } + None +} + +fn is_top_level_mirrors_line(line: &str) -> bool { + line.starts_with("mirrors:") && line.split('#').next().unwrap_or("").trim_end() == "mirrors:" +} + +fn ensure_trailing_newline(content: &str) -> String { + let mut out = content.to_string(); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out +} + +#[derive(Debug, PartialEq, Eq)] +struct PortPublish { + host: u16, + container: u16, + proto: String, +} + +fn ports_file_with_publish(existing: &str) -> String { + if ports_file_contains_publish(existing, REGISTRY_PORT_PUBLISH) { + return existing.to_string(); + } + + let mut out = ensure_trailing_newline(existing); + out.push_str(REGISTRY_PORT_PUBLISH); + out.push('\n'); + out +} + +fn ports_file_contains_publish(existing: &str, desired: &str) -> bool { + let Some(desired) = parse_port_publish(desired) else { + return false; + }; + existing + .lines() + .filter_map(parse_port_publish) + .any(|port| port == desired) +} + +fn parse_port_publish(line: &str) -> Option { + let cleaned: String = line + .split('#') + .next() + .unwrap_or("") + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); + if cleaned.is_empty() { + return None; + } + let (ports, proto) = cleaned + .split_once('/') + .map_or((cleaned.as_str(), "tcp"), |(ports, proto)| (ports, proto)); + if proto != "tcp" && proto != "udp" { + return None; + } + let (host, container) = ports.split_once(':')?; + let host = host.parse().ok().filter(|port| *port > 0)?; + let container = container.parse().ok().filter(|port| *port > 0)?; + Some(PortPublish { + host, + container, + proto: proto.to_string(), + }) +} + +/// Keep the node container's /etc/hosts pointing the registry hostname at +/// the current ClusterIP (docker regenerates /etc/hosts on restart, and the +/// ClusterIP changes if the Service is recreated — hence re-run per start). +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + let hostname = REGISTRY_HOSTNAME; + let current_ip = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), + ]) + .unwrap_or_default(); + if current_ip.trim() == cluster_ip { + return Ok(()); + } + + log::info!("Updating node hosts entry: {} -> {}", hostname, cluster_ip); + + // /etc/hosts is a bind mount inside the container: `sed -i` fails with + // "Resource busy" (rename over a mount point), so rewrite it in place. + engine_docker(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + &format!( + "awk '$2 != \"{host}\"' /etc/hosts > /tmp/hosts.new && \ + cat /tmp/hosts.new > /etc/hosts && rm -f /tmp/hosts.new && \ + echo '{ip} {host}' >> /etc/hosts", + host = hostname, + ip = cluster_ip + ), + ])?; + + Ok(()) +} + +/// Make `--context dory` resolvable. Current dory merges the context into +/// ~/.kube/config at enable time, so normally there is nothing to do — +/// mutating KUBECONFIG for every child process is then pure noise. Older +/// dory versions only write the side file; for those, prepend it to +/// KUBECONFIG (preserving whatever the user already has). +pub fn export_kubeconfig_env() { + if effective_kubeconfig_has_dory_context() { + return; + } + let Some(dory_cfg) = kubeconfig_path() else { + return; + }; + let existing = std::env::var("KUBECONFIG").unwrap_or_default(); + if existing.split(':').any(|p| p == dory_cfg) { + return; + } + let rest = if existing.is_empty() { + match home() { + Ok(h) => h.join(".kube/config").to_string_lossy().into_owned(), + Err(_) => return, + } + } else { + existing + }; + std::env::set_var("KUBECONFIG", format!("{}:{}", dory_cfg, rest)); +} + +/// Whether the kubeconfig(s) kubectl will read without our help — the +/// $KUBECONFIG chain when set, else ~/.kube/config — already define a +/// `dory` entry (i.e. dory's kubectl-merge ran against a file in scope). +fn effective_kubeconfig_has_dory_context() -> bool { + let paths: Vec = match std::env::var("KUBECONFIG") { + Ok(chain) if !chain.is_empty() => chain.split(':').map(PathBuf::from).collect(), + _ => match home() { + Ok(h) => vec![h.join(".kube/config")], + Err(_) => return false, + }, + }; + paths.iter().any(|path| { + std::fs::read_to_string(path) + .map(|content| has_dory_entry(&content)) + .unwrap_or(false) + }) +} + +/// Line-anchored scan for a kubeconfig entry named `dory` — the mapping +/// form (`name: dory`, as kubectl writes context/cluster names) or the +/// sequence-item form (`- name: dory`, the users list). `name: dory-prod` +/// or `username: dory` must not count. +fn has_dory_entry(kubeconfig: &str) -> bool { + kubeconfig.lines().any(|line| { + let trimmed = line.trim(); + trimmed == "name: dory" || trimmed == "- name: dory" + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn launch_window_remaining_covers_only_the_provisioning_window() { + use std::time::Duration; + + assert_eq!( + launch_window_remaining(Duration::ZERO), + Some(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)) + ); + assert_eq!( + launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS - 1)), + Some(Duration::from_secs(1)) + ); + assert_eq!( + launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)), + None + ); + assert_eq!(launch_window_remaining(Duration::from_secs(3600)), None); + } + + #[test] + fn has_dory_entry_matches_mapping_and_sequence_forms_only() { + let merged = "contexts:\n- context:\n cluster: dory\n user: dory\n name: dory\n"; + let users_list = "users:\n- name: dory\n user: {}\n"; + let near_misses = "name: dory-prod\nusername: dory\n# name: dory\nfullname: dory\n"; + + assert!(has_dory_entry(merged)); + assert!(has_dory_entry(users_list)); + assert!(!has_dory_entry(near_misses)); + assert!(!has_dory_entry("")); + } + + #[test] + fn registries_yaml_aliases_both_pull_names_to_the_service_over_http() { + let yaml = registries_yaml(); + + assert!(yaml.contains(REGISTRIES_BEGIN)); + assert!(yaml.contains(REGISTRIES_END)); + assert!(yaml.contains("\"registry.crossplane-system.svc.cluster.local:5000\":")); + assert!(yaml.contains("\"localhost:30500\":")); + // Both mirrors resolve to the same HTTP endpoint on the Service name. + assert_eq!( + yaml.matches("- \"http://registry.crossplane-system.svc.cluster.local:5000\"") + .count(), + 2 + ); + assert!(!yaml.contains("https://")); + } + + #[test] + fn registries_yaml_replaces_only_hops_managed_block() { + let existing = format!( + "configs:\n example: value\nmirrors:\n {begin}\n old: value\n {end}\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\n", + begin = REGISTRIES_BEGIN, + end = REGISTRIES_END, + ); + + let merged = merge_registries_yaml(Some(&existing)).unwrap(); + + assert!(merged.starts_with("configs:\n example: value\nmirrors:\n")); + assert!(merged.contains(" \"user.local:5000\":\n")); + assert!(!merged.contains("old: value")); + assert!(merged.contains(&format!(" \"{}\":", REGISTRY_PULL))); + assert_eq!(merged.matches(REGISTRIES_BEGIN).count(), 1); + assert_eq!(merged.matches(REGISTRIES_END).count(), 1); + } + + #[test] + fn registries_yaml_inserts_block_under_existing_mirrors_key() { + let existing = "mirrors:\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\nconfigs:\n another: value\n"; + + let merged = merge_registries_yaml(Some(existing)).unwrap(); + + let block_pos = merged.find(REGISTRIES_BEGIN).unwrap(); + let user_pos = merged.find("\"user.local:5000\"").unwrap(); + assert!(block_pos < user_pos); + assert!(merged.contains("configs:\n another: value\n")); + } + + #[test] + fn registries_yaml_appends_mirrors_section_when_missing() { + let existing = "configs:\n example: value\n"; + + let merged = merge_registries_yaml(Some(existing)).unwrap(); + + assert!(merged.starts_with(existing)); + assert!(merged.contains("\nmirrors:\n")); + assert!(merged.contains(REGISTRIES_BEGIN)); + } + + #[test] + fn registries_yaml_upgrades_legacy_hops_file_to_managed_block() { + let merged = merge_registries_yaml(Some(&legacy_registries_yaml())).unwrap(); + + assert_eq!(merged, registries_yaml()); + assert_eq!( + merged.matches(&format!("\"{}\":", REGISTRY_PULL)).count(), + 1 + ); + assert_eq!( + merged.matches(&format!("\"{}\":", REGISTRY_PUSH)).count(), + 1 + ); + } + + #[test] + fn ports_file_appends_registry_port_without_touching_existing_lines() { + let existing = "# user port\n8080:80/udp\n"; + + let merged = ports_file_with_publish(existing); + + assert_eq!(merged, "# user port\n8080:80/udp\n30500:30500\n"); + } + + #[test] + fn ports_file_absent_writes_only_registry_port() { + assert_eq!(ports_file_with_publish(""), "30500:30500\n"); + } + + #[test] + fn ports_file_treats_tcp_variant_as_already_present() { + let existing = " 30500 : 30500 / tcp # registry\n"; + + assert_eq!(ports_file_with_publish(existing), existing); + } + + #[test] + fn dory_enable_args_only_recreate_for_reset_path() { + assert_eq!( + dory_enable_args(false), + vec!["k8s", "enable", "--publish", REGISTRY_PORT_PUBLISH] + ); + assert_eq!( + dory_enable_args(true), + vec![ + "k8s", + "enable", + "--recreate", + "--publish", + REGISTRY_PORT_PUBLISH + ] + ); + } + + #[test] + fn start_rejects_size_flags() { + let size = SizeArgs { + cpus: Some(4), + memory: None, + disk: None, + }; + + let err = start(&size).expect_err("size flags must be rejected"); + + assert!(err.to_string().contains("--cpus 4")); + assert!(err.to_string().contains("Dory app")); + } +} diff --git a/src/commands/local/backend/mod.rs b/src/commands/local/backend/mod.rs index 26541ec..2e2427f 100644 --- a/src/commands/local/backend/mod.rs +++ b/src/commands/local/backend/mod.rs @@ -5,12 +5,14 @@ //! shaped lives outside this module and is backend-agnostic. mod colima; +mod dory; mod kind; -use super::{local_state_dir, run_cmd_output}; +use super::{local_state_dir, run_cmd_output, HOPS_KUBE_CONTEXT_ENV}; use clap::Args; use std::error::Error; use std::fmt; +use std::process::Command; use std::str::FromStr; /// Sizing flags for backends with a resizable VM. @@ -61,6 +63,8 @@ pub enum Backend { /// docker containers as nodes; works on any docker daemon /// (Docker Desktop, colima, dory, CI runners) Kind, + /// k3s on dory's shared-VM engine, via the `dory` CLI + Dory, } impl Backend { @@ -69,6 +73,7 @@ impl Backend { match self { Backend::Colima => "colima", Backend::Kind => "kind", + Backend::Dory => "dory", } } @@ -77,6 +82,7 @@ impl Backend { match self { Backend::Colima => "colima", Backend::Kind => "kind-hops", + Backend::Dory => "dory", } } @@ -84,6 +90,7 @@ impl Backend { match self { Backend::Colima => colima::install(), Backend::Kind => kind::install(), + Backend::Dory => dory::install(), } } @@ -91,6 +98,16 @@ impl Backend { match self { Backend::Colima => colima::uninstall(), Backend::Kind => kind::uninstall(), + Backend::Dory => dory::uninstall(), + } + } + + /// Whether this backend's local cluster/VM exists, running or stopped. + pub fn cluster_exists(self) -> bool { + match self { + Backend::Colima => colima::instance_exists(), + Backend::Kind => kind::cluster_exists(), + Backend::Dory => dory::cluster_exists(), } } @@ -100,6 +117,7 @@ impl Backend { match self { Backend::Colima => colima::start(size, assume_yes), Backend::Kind => kind::start(size), + Backend::Dory => dory::start(size), } } @@ -107,6 +125,7 @@ impl Backend { match self { Backend::Colima => colima::stop(), Backend::Kind => kind::stop(), + Backend::Dory => dory::stop(), } } @@ -114,6 +133,7 @@ impl Backend { match self { Backend::Colima => colima::destroy(), Backend::Kind => kind::destroy(), + Backend::Dory => dory::destroy(), } } @@ -121,6 +141,7 @@ impl Backend { match self { Backend::Colima => colima::reset(), Backend::Kind => kind::reset(), + Backend::Dory => dory::reset(), } } @@ -128,6 +149,7 @@ impl Backend { match self { Backend::Colima => colima::resize(size), Backend::Kind => kind::resize(size), + Backend::Dory => dory::resize(size), } } @@ -139,6 +161,9 @@ impl Backend { // containerd trust is per-name via certs.d, written in // wire_registry once the registry Service's ClusterIP is known. Backend::Kind => Ok(()), + // trust is static registries.yaml, written by dory::start before + // the cluster boots (k3s reads it at boot). + Backend::Dory => Ok(()), } } @@ -149,6 +174,7 @@ impl Backend { match self { Backend::Colima => colima::sync_hosts_entry(cluster_ip), Backend::Kind => kind::wire_registry(cluster_ip), + Backend::Dory => dory::wire_registry(cluster_ip), } } } @@ -166,8 +192,9 @@ impl FromStr for Backend { match s.trim() { "colima" => Ok(Backend::Colima), "kind" => Ok(Backend::Kind), + "dory" => Ok(Backend::Dory), other => Err(format!( - "unknown backend '{}' (expected colima or kind)", + "unknown backend '{}' (expected colima, kind, or dory)", other )), } @@ -176,6 +203,14 @@ impl FromStr for Backend { const BACKEND_FILE: &str = "backend"; +pub fn platform_default() -> Backend { + if cfg!(target_os = "macos") { + Backend::Colima + } else { + Backend::Kind + } +} + /// Resolve which backend to operate on: explicit flag > preference persisted /// by the last successful start > detection of an existing cluster (colima /// wins for back-compat with pre-backend installs) > platform default. @@ -185,15 +220,48 @@ pub fn resolve(flag: Option) -> Backend { persisted(), colima::instance_exists, kind::cluster_exists, - cfg!(target_os = "macos"), + dory::cluster_exists, + platform_default() == Backend::Colima, ) } +/// Resolve the backend once and activate the kube-targeting environment for +/// child kubectl/helm processes. +pub fn activate(flag: Option, context: Option<&str>) -> Backend { + let backend = resolve(flag); + + if backend == Backend::Dory { + dory::export_kubeconfig_env(); + } + + let explicit_context = context.filter(|ctx| !ctx.is_empty()); + let backend_context_exists = if explicit_context.is_none() { + kube_context_exists(backend.kube_context()) + } else { + false + }; + + match kube_context_export(backend, explicit_context, backend_context_exists) { + KubeContextExport::Set(ctx) => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx), + KubeContextExport::Unset { missing_context } => { + std::env::remove_var(HOPS_KUBE_CONTEXT_ENV); + log::warn!( + "Kubernetes context '{}' for backend '{}' was not found; using kubeconfig current-context. Pass --context to target a specific cluster.", + missing_context, + backend.name() + ); + } + } + + backend +} + fn resolve_from( flag: Option, persisted: Option, colima_detected: impl FnOnce() -> bool, kind_detected: impl FnOnce() -> bool, + dory_detected: impl FnOnce() -> bool, macos: bool, ) -> Backend { if let Some(backend) = flag { @@ -208,6 +276,9 @@ fn resolve_from( if kind_detected() { return Backend::Kind; } + if dory_detected() { + return Backend::Dory; + } if macos { Backend::Colima } else { @@ -265,6 +336,78 @@ pub fn wire_local_registry(backend: Backend) -> Result<(), Box> { backend.wire_registry(®istry_cluster_ip()?) } +pub fn should_wire_local_registry( + backend_flag: Option, + context: Option<&str>, + backend: Backend, +) -> bool { + if backend_flag.is_some() { + return true; + } + + match context.filter(|ctx| !ctx.is_empty()) { + Some(ctx) => ctx == backend.kube_context(), + None => true, + } +} + +pub fn wire_local_registry_for_target( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + if should_wire_local_registry(backend_flag, context, backend) { + return wire_local_registry(backend); + } + + log::warn!( + "registry node wiring skipped: explicit --context does not match a selected backend" + ); + Ok(()) +} + +#[derive(Debug, PartialEq, Eq)] +enum KubeContextExport<'a> { + Set(&'a str), + Unset { missing_context: &'static str }, +} + +fn kube_context_export<'a>( + backend: Backend, + explicit_context: Option<&'a str>, + backend_context_exists: bool, +) -> KubeContextExport<'a> { + if let Some(ctx) = explicit_context { + return KubeContextExport::Set(ctx); + } + + let backend_context = backend.kube_context(); + if backend_context_exists { + KubeContextExport::Set(backend_context) + } else { + KubeContextExport::Unset { + missing_context: backend_context, + } + } +} + +fn kube_context_exists(context: &str) -> bool { + let output = Command::new("kubectl") + .args(["config", "get-contexts", "-o", "name"]) + .output(); + + let Ok(output) = output else { + return false; + }; + if !output.status.success() { + return false; + } + + String::from_utf8_lossy(&output.stdout) + .lines() + .any(|line| line.trim() == context) +} + #[cfg(test)] mod tests { use super::*; @@ -280,6 +423,7 @@ mod tests { Some(Backend::Colima), || true, no_detect, + no_detect, true, ); @@ -288,21 +432,35 @@ mod tests { #[test] fn persisted_beats_detection() { - let resolved = resolve_from(None, Some(Backend::Kind), || true, no_detect, true); + let resolved = resolve_from( + None, + Some(Backend::Kind), + || true, + no_detect, + no_detect, + true, + ); assert_eq!(resolved, Backend::Kind); } #[test] fn colima_detection_beats_kind_detection() { - let resolved = resolve_from(None, None, || true, || true, false); + let resolved = resolve_from(None, None, || true, || true, || true, false); assert_eq!(resolved, Backend::Colima); } + #[test] + fn dory_detection_used_when_no_colima_or_kind() { + let resolved = resolve_from(None, None, no_detect, no_detect, || true, true); + + assert_eq!(resolved, Backend::Dory); + } + #[test] fn kind_detection_used_when_no_colima() { - let resolved = resolve_from(None, None, no_detect, || true, true); + let resolved = resolve_from(None, None, no_detect, || true, no_detect, true); assert_eq!(resolved, Backend::Kind); } @@ -310,20 +468,74 @@ mod tests { #[test] fn platform_default_when_nothing_detected() { assert_eq!( - resolve_from(None, None, no_detect, no_detect, true), + resolve_from(None, None, no_detect, no_detect, no_detect, true), Backend::Colima ); assert_eq!( - resolve_from(None, None, no_detect, no_detect, false), + resolve_from(None, None, no_detect, no_detect, no_detect, false), Backend::Kind ); } #[test] fn backend_name_round_trips_through_from_str() { - for backend in [Backend::Colima, Backend::Kind] { + for backend in [Backend::Colima, Backend::Kind, Backend::Dory] { assert_eq!(backend.name().parse::().unwrap(), backend); } - assert!("dory".parse::().is_err()); + assert!("podman".parse::().is_err()); + } + + #[test] + fn explicit_context_is_exported_even_when_backend_context_is_absent() { + assert_eq!( + kube_context_export(Backend::Colima, Some("foreign"), false), + KubeContextExport::Set("foreign") + ); + } + + #[test] + fn backend_context_is_exported_only_when_present() { + assert_eq!( + kube_context_export(Backend::Kind, None, true), + KubeContextExport::Set("kind-hops") + ); + assert_eq!( + kube_context_export(Backend::Kind, None, false), + KubeContextExport::Unset { + missing_context: "kind-hops" + } + ); + } + + #[test] + fn registry_wiring_allowed_without_explicit_context() { + assert!(should_wire_local_registry(None, None, Backend::Colima)); + } + + #[test] + fn registry_wiring_skips_foreign_explicit_context_without_backend_flag() { + assert!(!should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + + #[test] + fn registry_wiring_allowed_when_context_matches_backend() { + assert!(should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Kind + )); + } + + #[test] + fn registry_wiring_allowed_when_backend_is_explicit() { + assert!(should_wire_local_registry( + Some(Backend::Colima), + Some("foreign"), + Backend::Colima + )); } } diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index e59fb2a..0db9db0 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -34,22 +34,53 @@ pub const PROVIDER_INSTALL_MANAGED_BY: &str = "hops-provider-install"; /// Env var checked by kubectl helpers to inject `--context `. pub const HOPS_KUBE_CONTEXT_ENV: &str = "HOPS_KUBE_CONTEXT"; -/// Build the kubectl args prefix. Returns `["--context", ctx]` when the env var -/// is set, or an empty vec otherwise. -fn kubectl_context_args() -> Vec { - match std::env::var(HOPS_KUBE_CONTEXT_ENV) { - Ok(ctx) if !ctx.is_empty() => vec!["--context".to_string(), ctx], - _ => vec![], - } +fn kube_context_from_env() -> Option { + std::env::var(HOPS_KUBE_CONTEXT_ENV) + .ok() + .filter(|ctx| !ctx.is_empty()) } /// Prepend `--context` to a kubectl arg slice when configured. fn with_kube_context(args: &[&str]) -> Vec { - let mut out = kubectl_context_args(); + let ctx = kube_context_from_env(); + with_kube_context_value(args, ctx.as_deref()) +} + +fn with_kube_context_value(args: &[&str], context: Option<&str>) -> Vec { + let mut out = match context { + Some(ctx) if !ctx.is_empty() => vec!["--context".to_string(), ctx.to_string()], + _ => vec![], + }; out.extend(args.iter().map(|s| s.to_string())); out } +fn with_helm_kube_context(args: &[&str]) -> Vec { + let ctx = kube_context_from_env(); + with_helm_kube_context_value(args, ctx.as_deref()) +} + +fn with_helm_kube_context_value(args: &[&str], context: Option<&str>) -> Vec { + let Some(ctx) = context.filter(|ctx| !ctx.is_empty()) else { + return args.iter().map(|s| s.to_string()).collect(); + }; + if args.first() == Some(&"repo") { + return args.iter().map(|s| s.to_string()).collect(); + } + + let mut out = Vec::with_capacity(args.len() + 2); + if let Some((command, rest)) = args.split_first() { + out.push((*command).to_string()); + out.push("--kube-context".to_string()); + out.push(ctx.to_string()); + out.extend(rest.iter().map(|s| s.to_string())); + } else { + out.push("--kube-context".to_string()); + out.push(ctx.to_string()); + } + out +} + /// Build a `Command` for kubectl with `--context` injected when configured. pub fn kubectl_command(args: &[&str]) -> Command { let full = with_kube_context(args); @@ -105,19 +136,18 @@ pub enum LocalCommands { /// Destroy the local cluster Destroy, /// Uninstall the local cluster backend - Uninstall, + Uninstall(uninstall::UninstallArgs), } pub fn run(args: &LocalArgs) -> Result<(), Box> { - let backend = backend::resolve(args.backend); - // Plumb the context through the same env channel the kubectl helpers - // read, so every subcommand's kubectl calls target the chosen cluster. - // Without an explicit --context, use the backend's own context so - // commands work regardless of kubeconfig's current-context. - match &args.context { - Some(ctx) if !ctx.is_empty() => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx), - _ => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, backend.kube_context()), - } + let explicit_context = args.context.as_deref().filter(|ctx| !ctx.is_empty()); + let install_backend = matches!(&args.command, LocalCommands::Install) + .then(|| args.backend.unwrap_or_else(backend::platform_default)); + let activation_flag = install_backend.or(args.backend); + let install_context = install_backend.map(backend::Backend::kube_context); + let activation_context = explicit_context.or(install_context); + let backend = backend::activate(activation_flag, activation_context); + match &args.command { LocalCommands::Install => install::run(backend), LocalCommands::Reset => reset::run(backend), @@ -131,7 +161,7 @@ pub fn run(args: &LocalArgs) -> Result<(), Box> { LocalCommands::Listmonk(listmonk_args) => listmonk::run(listmonk_args), LocalCommands::Stop => stop::run(backend), LocalCommands::Destroy => destroy::run(backend), - LocalCommands::Uninstall => uninstall::run(backend), + LocalCommands::Uninstall(uninstall_args) => uninstall::run(backend, uninstall_args), } } @@ -143,11 +173,17 @@ pub fn run_cmd(program: &str, args: &[&str]) -> Result<(), Box> { let refs: Vec<&str> = full.iter().map(|s| s.as_str()).collect(); return run_cmd_with_logged_args(program, &refs, &refs); } + if program == "helm" { + let full = with_helm_kube_context(args); + let refs: Vec<&str> = full.iter().map(|s| s.as_str()).collect(); + return run_cmd_with_logged_args(program, &refs, &refs); + } run_cmd_with_logged_args(program, args, args) } /// Run an external command and capture stdout. -/// For kubectl commands, automatically injects `--context` when configured. +/// For kubectl/helm commands, automatically injects the active context when +/// configured. pub fn run_cmd_output(program: &str, args: &[&str]) -> Result> { if program == "kubectl" { let full = with_kube_context(args); @@ -159,6 +195,16 @@ pub fn run_cmd_output(program: &str, args: &[&str]) -> Result = full_logged.iter().map(|s| s.as_str()).collect(); run_cmd_with_logged_args("kubectl", &args_refs, &logged_refs) } + +#[cfg(test)] +mod tests { + use super::*; + + fn strings(args: Vec) -> Vec { + args + } + + #[test] + fn kubectl_args_prepend_context() { + assert_eq!( + strings(with_kube_context_value(&["get", "pods"], Some("kind-hops"))), + vec!["--context", "kind-hops", "get", "pods"] + ); + } + + #[test] + fn helm_upgrade_injects_kube_context_after_subcommand() { + assert_eq!( + strings(with_helm_kube_context_value( + &["upgrade", "--install", "crossplane"], + Some("kind-hops") + )), + vec![ + "upgrade", + "--kube-context", + "kind-hops", + "--install", + "crossplane" + ] + ); + } + + #[test] + fn helm_repo_commands_skip_kube_context() { + assert_eq!( + strings(with_helm_kube_context_value( + &["repo", "update", "crossplane-stable"], + Some("kind-hops") + )), + vec!["repo", "update", "crossplane-stable"] + ); + } +} diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 7acd95e..136d760 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -52,7 +52,9 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box> { +#[derive(Args, Debug)] +pub struct UninstallArgs { + /// Uninstall the backend binary even if its local cluster still exists. + #[arg(long)] + pub force: bool, +} + +pub fn run(backend: Backend, args: &UninstallArgs) -> Result<(), Box> { + run_inner( + backend, + args.force, + Backend::cluster_exists, + Backend::uninstall, + super::backend::clear_persisted, + confirm_uninstall, + ) +} + +fn run_inner( + backend: Backend, + force: bool, + cluster_exists: ClusterExists, + uninstall: Uninstall, + clear_persisted: ClearPersisted, + confirm: Confirm, +) -> Result<(), Box> +where + ClusterExists: FnOnce(Backend) -> bool, + Uninstall: FnOnce(Backend) -> Result<(), Box>, + ClearPersisted: FnOnce() -> Result<(), Box>, + Confirm: FnOnce(Backend) -> Result>, +{ + if !force && cluster_exists(backend) { + return Err("destroy the cluster first: `hops local destroy` (or pass --force to uninstall the backend binary anyway)".into()); + } + + if confirm(backend)? { + uninstall(backend)?; + clear_persisted()?; + } else { + log::info!("Uninstall cancelled"); + } + + Ok(()) +} + +fn confirm_uninstall(backend: Backend) -> Result> { print!( "Uninstall {}? This will remove the binary. [y/N] ", backend.name() @@ -12,12 +59,113 @@ pub fn run(backend: Backend) -> Result<(), Box> { let mut input = String::new(); io::stdin().read_line(&mut input)?; - if input.trim().eq_ignore_ascii_case("y") { - backend.uninstall()?; - super::backend::clear_persisted()?; - } else { - log::info!("Uninstall cancelled"); + Ok(input.trim().eq_ignore_ascii_case("y")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::rc::Rc; + + #[test] + fn refuses_uninstall_when_cluster_exists_without_force() { + let events = Rc::new(RefCell::new(Vec::new())); + let err = run_inner( + Backend::Kind, + false, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("cluster_exists"); + true + } + }, + |_| { + events.borrow_mut().push("uninstall"); + Ok(()) + }, + || { + events.borrow_mut().push("clear"); + Ok(()) + }, + |_| { + events.borrow_mut().push("confirm"); + Ok(true) + }, + ) + .expect_err("cluster guard should refuse uninstall"); + + assert!(err.to_string().contains("destroy the cluster first")); + assert_eq!(&*events.borrow(), &["cluster_exists"]); } - Ok(()) + #[test] + fn force_uninstalls_without_cluster_probe_and_clears_after_success() { + let events = Rc::new(RefCell::new(Vec::new())); + run_inner( + Backend::Kind, + true, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("cluster_exists"); + true + } + }, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("uninstall"); + Ok(()) + } + }, + { + let events = Rc::clone(&events); + move || { + events.borrow_mut().push("clear"); + Ok(()) + } + }, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("confirm"); + Ok(true) + } + }, + ) + .expect("force should allow uninstall"); + + assert_eq!(&*events.borrow(), &["confirm", "uninstall", "clear"]); + } + + #[test] + fn failed_uninstall_does_not_clear_persisted_backend() { + let events = Rc::new(RefCell::new(Vec::new())); + let err = run_inner( + Backend::Kind, + false, + |_| false, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("uninstall"); + Err("brew failed".into()) + } + }, + { + let events = Rc::clone(&events); + move || { + events.borrow_mut().push("clear"); + Ok(()) + } + }, + |_| Ok(true), + ) + .expect_err("uninstall failure should propagate"); + + assert_eq!(err.to_string(), "brew failed"); + assert_eq!(&*events.borrow(), &["uninstall"]); + } } diff --git a/src/commands/provider/install.rs b/src/commands/provider/install.rs index c10a13c..c45a3e6 100644 --- a/src/commands/provider/install.rs +++ b/src/commands/provider/install.rs @@ -1,12 +1,11 @@ -use crate::commands::local::backend::{self, wire_local_registry}; +use crate::commands::local::backend::{self, Backend}; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout_at, ensure_registry, parse_repo_spec, resolve_repo_install_target, run_watch, sanitize_name_component, RepoInstallTarget, RepoSpec, REGISTRY_PULL, REGISTRY_PUSH, }; use crate::commands::local::{ - kubectl_apply_stdin, run_cmd, run_cmd_output, HOPS_KUBE_CONTEXT_ENV, MANAGED_BY_LABEL, - PROVIDER_INSTALL_MANAGED_BY, + kubectl_apply_stdin, run_cmd, run_cmd_output, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, }; use clap::Args; use serde::Deserialize; @@ -54,6 +53,10 @@ pub struct ProviderInstallArgs { #[arg(long)] pub context: Option, + /// Local cluster backend whose node should be wired for local package pulls. + #[arg(long, value_enum)] + pub backend: Option, + /// Watch the project directory for changes and re-run install automatically #[arg(long, conflicts_with = "repo")] pub watch: bool, @@ -74,9 +77,7 @@ struct PackageMetadata { } pub fn run(args: &ProviderInstallArgs) -> Result<(), Box> { - if let Some(ctx) = &args.context { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + let backend = backend::activate(args.backend, args.context.as_deref()); match (args.repo.as_deref(), args.version.as_deref()) { (Some(repo), Some(version)) => { @@ -87,10 +88,14 @@ pub fn run(args: &ProviderInstallArgs) -> Result<(), Box> { args.skip_dependency_resolution, args.version_prefix.as_deref(), args.branch.as_deref(), + backend, + args.backend, + args.context.as_deref(), ), (None, _) => { let path = args.path.as_deref().unwrap_or("."); let prefix = args.version_prefix.clone(); + prepare_local_registry(backend, args.backend, args.context.as_deref())?; run_local_path(path, args.skip_dependency_resolution, prefix.as_deref())?; if args.watch { @@ -112,11 +117,15 @@ fn run_repo_install( skip_dependency_resolution: bool, version_prefix: Option<&str>, branch: Option<&str>, + backend: Backend, + backend_flag: Option, + context: Option<&str>, ) -> Result<(), Box> { let spec = parse_repo_spec(repo)?; match resolve_repo_install_target(&spec)? { RepoInstallTarget::SourceBuild => { let cache_path = ensure_cached_repo_checkout_at(&spec, branch)?; + prepare_local_registry(backend, backend_flag, context)?; run_local_path( &cache_path.to_string_lossy(), skip_dependency_resolution, @@ -129,6 +138,15 @@ fn run_repo_install( } } +fn prepare_local_registry( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + ensure_registry()?; + backend::wire_local_registry_for_target(backend, backend_flag, context) +} + fn apply_repo_version( repo: &str, version: &str, @@ -186,9 +204,6 @@ fn run_local_path( let provider_name = read_provider_name(dir)?; log::info!("Provider package name: {}", provider_name); - ensure_registry()?; - wire_local_registry(backend::resolve(None))?; - // Resolve the existing upstream Provider before building so we can: // 1. carry the upstream package URL into the new `spec.package` (Crossplane // records the URL-without-tag as the Lock Source; deps declared as @@ -337,10 +352,10 @@ fn find_xpkg_for_provider( .filter_map(|e| e.ok()) .filter(|e| { let p = e.path(); - p.extension().map_or(false, |ext| ext == "xpkg") + p.extension().is_some_and(|ext| ext == "xpkg") && p.file_name() .and_then(|n| n.to_str()) - .map_or(false, |n| n.starts_with(&prefix)) + .is_some_and(|n| n.starts_with(&prefix)) }) .map(|e| e.path()) .collect(); @@ -1113,6 +1128,15 @@ mod tests { assert!(!image.contains(REGISTRY_PULL)); } + #[test] + fn local_registry_wiring_skips_foreign_context_without_backend_flag() { + assert!(!backend::should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + #[test] fn build_cluster_role_binding_yaml_targets_service_account() { let yaml = build_cluster_role_binding_yaml("p-cluster-admin", "p"); From 29b3db518493099c8bf2de6ee757d79ceca32c95 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 11 Jul 2026 14:04:05 -0500 Subject: [PATCH 05/11] ci: run kind smoke and quality for all PR base branches Match main's on-pr-quality trigger so stacked PRs into this branch also get kind-smoke and quality. --- .github/workflows/on-pr-kind-smoke.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/on-pr-kind-smoke.yaml b/.github/workflows/on-pr-kind-smoke.yaml index 5664e50..354d2e6 100644 --- a/.github/workflows/on-pr-kind-smoke.yaml +++ b/.github/workflows/on-pr-kind-smoke.yaml @@ -7,8 +7,6 @@ name: kind backend smoke on: pull_request: - branches: - - main jobs: kind-smoke: From cdf9145ca708724f416ad4895cabeec1e42d99d5 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 11 Jul 2026 22:54:09 -0500 Subject: [PATCH 06/11] ci: macOS colima backend smoke workflow (#76) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: add macOS colima backend smoke workflow Mirror kind smoke on macos-15-intel with nested-virt pin, --backend colima, and kubectl --context colima. Implements [[tasks/hops-cli-colima-macos-smoke]] under [[tasks/hops-cli-macos-backend-smoke]]. * test: structural checks for colima macOS smoke workflow Assert the shipped on-pr-colima-smoke.yaml pins macos-15-intel and mirrors kind smoke without a full GHA run. * ci: run colima smoke for all PR base branches Drop pull_request.branches: [main] so this job runs on stacked PRs into feat/cluster-backend-abstraction. * ci: size colima smoke VM for macos-15-intel runners Hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard GHA intel runners (~4 / ~14), so VZ hostagent exits during start. Pass explicit smaller sizes and dump lima ha.stderr on failure. * ci: give colima smoke enough memory for Crossplane 2 CPU / 4 GiB booted Colima but helm --wait for Crossplane hit 5m with Available: 0/1. Use 3/8 on macos-15-intel (~14 GiB host) and raise Crossplane helm --wait timeout to 10m for nested virt. * fix: wait longer for Crossplane on nested-virt colima Helm --wait hit 10m with Available:0/1 on GHA macos-15-intel. Apply the chart without --wait, poll deployments for ~15m with periodic pod dumps, wait for nodes Ready after docker restart, and capture cluster state before destroy on CI failure. * fix: harden kubectl apply under nested-virt API load Crossplane came Ready on GHA colima but ProviderConfigs apply failed with openapi schema download timeouts. Apply with --validate=false, retry transient failures, and re-wait for the API after Crossplane becomes leader. Soft-wait rbac-manager when core is healthy. * fix: wait for Established CRDs and room for registry PVC ProviderConfig apply raced CRD discovery; wait for Established. Registry PVC requests 20Gi — give colima smoke a 40Gi VM disk so local-path can bind, and wait longer with diagnostics for registry. * ci: recover colima stop/start resume after VM container churn After colima stop/start, pods Error on missing docker containers. Wait longer for Available and rollout-restart crossplane/registry if the first wait stalls. * fix: survive apiserver overload when applying registry Nested-virt colima CI loses the apiserver (TLS timeouts) right after Crossplane/provider install. Wait longer for /readyz, retry kubectl apply with API re-probes, and pre-pull registry:2 before deploy. * fix(ci): give colima smoke more RAM and settle time for registry pulls At 3/8/40, dual-name registry smoke pods sat in ContainerCreating without IPs while CoreDNS and metrics-server thrashed. Bump VM memory to 10Gi, wait for node/CoreDNS before the push, extend Ready timeout, and dump smoke pod describe on failure. --- .github/workflows/on-pr-colima-smoke.yaml | 166 ++++++++++++++++++++++ src/commands/local/mod.rs | 67 ++++++--- src/commands/local/start.rs | 137 ++++++++++++++++-- tests/colima_smoke_workflow.rs | 90 ++++++++++++ 4 files changed, 432 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/on-pr-colima-smoke.yaml create mode 100644 tests/colima_smoke_workflow.rs diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml new file mode 100644 index 0000000..b1bfa70 --- /dev/null +++ b/.github/workflows/on-pr-colima-smoke.yaml @@ -0,0 +1,166 @@ +name: colima backend smoke + +# End-to-end check of `hops local` on the colima backend on macOS. +# Mirrors on-pr-kind-smoke.yaml with --backend colima and kubectl --context colima. +# +# Runner constraints: +# - Colima needs nested virtualization (Lima VM). Pin macos-15-intel — that +# image is known to support nested virt for Colima/Lima on GHA. +# - Do NOT use bare macos-latest alone (tracks arm images; arm64 GHA macOS +# historically fails with HV_UNSUPPORTED for nested virt). +# - Prefer localhost:30500 for registry traffic, not the Colima VM IP +# (macOS 15 Local Network Privacy can block non-root VM-IP access). +# +# Install: brew provides colima/docker/kubectl/helm only. Do not pre-start +# colima here — hops local start --backend colima owns cluster bring-up. +# +# Sizing: hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard +# macos-15-intel runners (~4 CPU / ~14 GiB). Pass explicit smaller sizes +# so the VZ VM can allocate; stop/start resume uses the persisted profile. + +on: + pull_request: + workflow_dispatch: + +jobs: + colima-smoke: + # Nested-virt-capable Intel pin (not bare macos-latest). + runs-on: macos-15-intel + # Nested virt + cold Crossplane pulls can take well over 30m. + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Install colima, docker, kubectl, helm + run: | + set -euxo pipefail + # Tools only — do not `colima start` here so the smoke exercises hops. + brew install colima docker kubectl helm + colima version + docker version --format '{{.Client.Version}}' || docker --version + kubectl version --client + helm version --short + + - name: Build hops + run: cargo build + + - name: Host resources (for sizing) + run: | + set -euxo pipefail + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + + - name: hops local start --backend colima + run: | + set -euxo pipefail + # Fit macos-15-intel (~4 CPU / ~14 GiB RAM). Defaults (8/16/60) OOM VZ. + # Memory 10 (not 8): at 8Gi CoreDNS/metrics-server thrash and smoke pods + # sit in ContainerCreating without IPs even after images pull. Leave ~4Gi + # for host macOS + VZ. Disk 40: registry PVC requests 20Gi. + ./target/debug/hops-cli local start --backend colima \ + --cpus 3 --memory 10 --disk 40 + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Wait for node + CoreDNS (nested virt settles) + run: | + set -euxo pipefail + # After cold start the control plane is still soft: probes time out and + # new pods stick in ContainerCreating. Wait for basics before registry. + kubectl --context colima wait --for=condition=Ready nodes --all --timeout=120s + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=kube-dns --timeout=180s || \ + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=coredns --timeout=180s || true + kubectl --context colima -n kube-system get pods -o wide || true + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + docker pull public.ecr.aws/docker/library/busybox:stable + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + # Service-name pull: containerd resolves via its certs.d alias. + kubectl --context colima run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # localhost:30500 pull: what provider runtime pods reference. + kubectl --context colima run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # Nested virt: image pull can succeed while CNI/IP assignment lags. + if ! kubectl --context colima wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then + echo "==== smoke pods not Ready ====" + kubectl --context colima get pods smoke-svc-name smoke-localhost -o wide || true + kubectl --context colima describe pod smoke-svc-name smoke-localhost || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-svc-name || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-localhost || true + exit 1 + fi + kubectl --context colima delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + ./target/debug/hops-cli local stop + ./target/debug/hops-cli local start + # After colima VM stop/start, docker container IDs are gone and pods + # often sit in Error until kubelet recreates them. Wait generously and + # force a rollout restart if Available still stalls. + if ! kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s; then + echo "deployments not Available after resume; restarting..." + kubectl --context colima -n crossplane-system get pods -o wide || true + kubectl --context colima -n crossplane-system rollout restart \ + deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true + kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s + fi + ./target/debug/hops-cli local doctor + + # Capture live cluster state BEFORE destroy so failures are diagnosable. + - name: Debug dump on failure + if: failure() + run: | + set +e + echo "==== host ====" + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + echo "==== colima ====" + colima status || true + colima list || true + echo "==== cluster ====" + kubectl --context colima get nodes,pods -A -o wide || true + echo "==== smoke pods (default) ====" + kubectl --context colima describe pod smoke-svc-name smoke-localhost 2>/dev/null || true + echo "==== crossplane-system ====" + kubectl --context colima describe pods -n crossplane-system || true + kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -100 || true + echo "==== lima ha.stderr / serial ====" + for f in \ + "$HOME/.colima/_lima/colima/ha.stderr.log" \ + "$HOME/.colima/_lima/colima/ha.stdout.log" \ + "$HOME/.colima/_lima/colima/serial.log" \ + "$HOME/.colima/_lima/colima/serialv.log" + do + if [ -f "$f" ]; then + echo "----- $f -----" + tail -n 200 "$f" || true + fi + done + ./target/debug/hops-cli local doctor || true + + - name: hops local destroy + if: always() + run: ./target/debug/hops-cli local destroy || true diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index 0db9db0..abb27ac 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -255,11 +255,16 @@ pub(crate) fn command_exists(program: &str) -> bool { /// Poll until the Kubernetes API server is reachable. pub(crate) fn wait_for_kubernetes() -> Result<(), Box> { log::info!("Waiting for Kubernetes API..."); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["cluster-info"]); + // ~10 minutes — nested-virt apiserver can stay overloaded after package install. + for _ in 0..120 { + let result = run_cmd_output("kubectl", &["get", "--raw", "/readyz"]); if result.is_ok() { return Ok(()); } + // Fall back to a cheap list if /readyz is denied on some setups. + if run_cmd_output("kubectl", &["get", "ns", "default"]).is_ok() { + return Ok(()); + } std::thread::sleep(std::time::Duration::from_secs(5)); } Err("Timed out waiting for Kubernetes API".into()) @@ -267,24 +272,54 @@ pub(crate) fn wait_for_kubernetes() -> Result<(), Box> { /// Pipe a YAML string into `kubectl apply -f -`. /// Automatically injects `--context` when configured. +/// +/// Uses `--validate=false` so a slow/overloaded API server (common on nested +/// virt CI while Crossplane is warming) does not fail the apply solely because +/// OpenAPI schema download timed out. Retries a few times for transient +/// connection errors. pub fn kubectl_apply_stdin(yaml: &str) -> Result<(), Box> { - let full = with_kube_context(&["apply", "-f", "-"]); - let mut child = Command::new("kubectl") - .args(&full) - .stdin(Stdio::piped()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn()?; + let full = with_kube_context(&["apply", "--validate=false", "-f", "-"]); + let mut last_status = None; + // Nested-virt CI (colima/GHA) can lose the apiserver for minutes after + // Crossplane/provider install (TLS handshake timeouts). Retry with backoff + // and re-probe the API between attempts. + const ATTEMPTS: u32 = 12; + + for attempt in 1..=ATTEMPTS { + let mut child = Command::new("kubectl") + .args(&full) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(yaml.as_bytes())?; + } - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(yaml.as_bytes())?; + let status = child.wait()?; + if status.success() { + return Ok(()); + } + last_status = Some(status); + log::warn!( + "kubectl apply failed (attempt {}/{}, status {}); waiting for API...", + attempt, + ATTEMPTS, + status + ); + // Best-effort API recovery before the next apply. + let _ = wait_for_kubernetes(); + std::thread::sleep(std::time::Duration::from_secs(10)); } - let status = child.wait()?; - if !status.success() { - return Err(format!("kubectl apply exited with {}", status).into()); - } - Ok(()) + Err(format!( + "kubectl apply exited with {} after retries", + last_status + .map(|s| s.to_string()) + .unwrap_or_else(|| "unknown".into()) + ) + .into()) } /// Apply a JSON merge patch with `kubectl patch --type merge`. diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 136d760..b151675 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -41,6 +41,20 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { - for _ in 0..60 { + wait_for_deployment_attempts(namespace, name, 60) +} + +/// Longer wait used for Crossplane on cold nested-virt runners (~15 minutes). +fn wait_for_deployment_with_diagnostics( + namespace: &str, + name: &str, +) -> Result<(), Box> { + match wait_for_deployment_attempts(namespace, name, 180) { + Ok(()) => Ok(()), + Err(e) => { + dump_namespace_diagnostics(namespace); + Err(e) + } + } +} + +fn wait_for_deployment_attempts( + namespace: &str, + name: &str, + attempts: u32, +) -> Result<(), Box> { + for i in 0..attempts { let output = run_cmd_output( "kubectl", &[ @@ -133,20 +196,70 @@ fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box } } + // Periodic progress so CI logs show the wait is alive. + if i > 0 && i % 12 == 0 { + log::info!( + "Still waiting for deployment {}/{} ({}s elapsed)...", + namespace, + name, + i * 5 + ); + let _ = run_cmd( + "kubectl", + &["get", "pods", "-n", namespace, "-o", "wide"], + ); + } + thread::sleep(Duration::from_secs(5)); } Err(format!("Timed out waiting for deployment {}/{}", namespace, name).into()) } -/// Poll until a CRD exists in the cluster. +fn dump_namespace_diagnostics(namespace: &str) { + log::error!("Diagnostics for namespace {} after readiness timeout:", namespace); + let _ = run_cmd("kubectl", &["get", "pods", "-n", namespace, "-o", "wide"]); + let _ = run_cmd("kubectl", &["describe", "pods", "-n", namespace]); + let _ = run_cmd( + "kubectl", + &[ + "get", + "events", + "-n", + namespace, + "--sort-by=.lastTimestamp", + ], + ); + let _ = run_cmd("kubectl", &["get", "nodes", "-o", "wide"]); +} + +/// Poll until a CRD exists **and** is Established (API serves the kind). +/// +/// Merely creating the CRD object is not enough: under load the apiserver can +/// return the CRD while discovery still lacks the kind, causing +/// `no matches for kind "ProviderConfig"` on the next apply. fn wait_for_crd(crd: &str) -> Result<(), Box> { log::info!("Waiting for CRD {}...", crd); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["get", "crd", crd]); - if result.is_ok() { - return Ok(()); + for _ in 0..120 { + let exists = run_cmd_output("kubectl", &["get", "crd", crd]).is_ok(); + if exists { + let established = run_cmd_output( + "kubectl", + &[ + "get", + "crd", + crd, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Established\")].status}", + ], + ) + .unwrap_or_default(); + if established.trim() == "True" { + // Brief settle so discovery caches pick up the new kind. + thread::sleep(Duration::from_secs(2)); + return Ok(()); + } } thread::sleep(Duration::from_secs(5)); } - Err(format!("Timed out waiting for CRD {}", crd).into()) + Err(format!("Timed out waiting for CRD {} to be Established", crd).into()) } diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs new file mode 100644 index 0000000..b2882cb --- /dev/null +++ b/tests/colima_smoke_workflow.rs @@ -0,0 +1,90 @@ +//! Structural checks for the shipped colima macOS smoke workflow. +//! +//! These tests drive the real file under `.github/workflows/` so a missing +//! or regressed smoke contract fails `cargo test` without needing a macOS GHA run. + +use std::fs; +use std::path::PathBuf; + +fn workflow_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-colima-smoke.yaml") +} + +fn workflow_text() -> String { + let path = workflow_path(); + assert!( + path.is_file(), + "expected colima smoke workflow at {}", + path.display() + ); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn colima_smoke_workflow_exists_and_pins_nested_virt_runner() { + let text = workflow_text(); + assert!( + text.contains("runs-on: macos-15-intel"), + "must pin nested-virt-capable Intel runner" + ); + assert!( + !text + .lines() + .any(|l| l.trim() == "runs-on: macos-latest"), + "must not use bare macos-latest as sole runs-on" + ); + assert!( + text.to_lowercase().contains("nested") && text.to_lowercase().contains("virt"), + "must comment on runner/virt constraints" + ); +} + +#[test] +fn colima_smoke_workflow_kind_parity_sequence() { + let text = workflow_text(); + for needle in [ + "cargo build", + "start --backend colima", + "local doctor", + "localhost:30500", + "registry.crossplane-system.svc.cluster.local:5000", + "--context colima", + "local stop", + "local destroy", + ] { + assert!( + text.contains(needle), + "colima smoke missing kind-parity fragment: {needle}" + ); + } + // stop/start resume: start without --backend after stop + assert!( + text.contains("local start\n") || text.lines().any(|l| l.trim() == "./target/debug/hops-cli local start"), + "must start again without --backend after stop" + ); + for tool in ["colima", "docker", "kubectl", "helm"] { + assert!( + text.contains(tool), + "prereq install must mention {tool}" + ); + } +} + +#[test] +fn colima_smoke_workflow_sizes_vm_for_gha_intel_runner() { + let text = workflow_text(); + // Defaults (8/16/60) exceed macos-15-intel; CI must pass explicit smaller sizes. + assert!( + text.contains("--cpus") && text.contains("--memory") && text.contains("--disk"), + "must pass --cpus/--memory/--disk so the VZ VM fits the runner" + ); + // 8Gi left CoreDNS thrashing; smoke needs headroom above that floor. + assert!( + text.contains("--memory 10") || text.contains("--memory 11") || text.contains("--memory 12"), + "must allocate at least 10Gi to the colima VM on GHA intel runners" + ); + assert!( + text.contains("ha.stderr.log"), + "failure dump must include lima hostagent stderr for nested-virt debug" + ); +} From 48c560ddc8b46d0961298a1615c69e6937576243 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sun, 12 Jul 2026 11:52:45 -0500 Subject: [PATCH 07/11] ci: macOS dory backend smoke spike workflow (#77) * ci: add macOS dory backend smoke spike workflow Clone+build patrickleet/dory @ feat/hops-local-integration in CI, wait for engine.sock, then kind-parity hops smoke with --backend dory. workflow_dispatch / non-required until public GHA can boot the engine. Implements [[tasks/hops-cli-dory-macos-smoke]] under [[tasks/hops-cli-macos-backend-smoke]]. * test: structural checks for dory macOS smoke workflow Assert clone+build install contract and kind-parity steps in the shipped on-pr-dory-smoke.yaml. * fix(ci): port colima nested-virt lessons into dory smoke Bring shared start hardness (CRD Established waits, apply retries, longer Crossplane/registry waits) from the green colima path, and harden the dory workflow: settle CoreDNS before registry pods, 420s Ready, stop/start rollout restart, failure dump, and ECR pull retries. Structural tests lock the contract in. * ci: run dory smoke on pull_request so PR branches can execute it workflow_dispatch only works once the file exists on the default branch; enable pull_request (still non-required) so #77 can actually run dory-smoke. * fix(ci): colima-backed engine.sock when dory-hv cannot run on GHA Public GHA cannot run dory-hv (no nested virt for native engine; app never creates ~/.dory). Pin macos-15-intel, still clone+build dory for scripts/, try the app briefly, then expose Colima docker as ~/.dory/engine.sock so hops local --backend dory can complete kind-parity smoke. * fix(local): recreate dory k8s on create-time config drift dory enable exits 3 when ports/registry binds drift. hops just wrote the desired config; auto-retry once with --recreate so local start applies it. CI: skip native engine wait on Intel, set DORY_ENGINE_SOCK, disable k8s first. * fix(local): retry helm install when apiserver openapi is still soft After dory stop/start, nodes can be Ready while openapi/v2 still times out and helm upgrade fails. Retry helm with wait_for_kubernetes between attempts; CI pause briefly after stop before start. * fix(local): recreate dory k8s when enable fails after stop/start docker stop of dory-k8s often leaves k3s stuck past dory's Ready wait on resume. On enable exit 1/3, disable and re-enable with --recreate once. --- .github/workflows/on-pr-dory-smoke.yaml | 303 ++++++++++++++++++++++++ src/commands/local/backend/dory.rs | 40 +++- src/commands/local/start.rs | 30 ++- tests/dory_smoke_workflow.rs | 139 +++++++++++ 4 files changed, 502 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/on-pr-dory-smoke.yaml create mode 100644 tests/dory_smoke_workflow.rs diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml new file mode 100644 index 0000000..3d75908 --- /dev/null +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -0,0 +1,303 @@ +name: dory backend smoke + +# End-to-end smoke for `hops local --backend dory` on public GHA macOS. +# Mirrors on-pr-kind-smoke.yaml: start → doctor → dual registry → stop/start → destroy. +# +# Install contract (mandatory — not brew cask / official release): +# 1. Clone patrickleet/dory @ feat/hops-local-integration +# 2. Build in-pipeline (xcodebuild with deterministic -derivedDataPath) +# 3. Put scripts/dory on PATH +# 4. Provide ~/.dory/engine.sock before hops start (see engine note below) +# +# Engine socket on public GHA (important): +# - Dory's native dory-hv engine needs Hypervisor.framework nested virt. +# Dory's own CI documents that GitHub-hosted macOS runners are VMs and +# CANNOT run that engine (see dory benchmark.yml). Observed: Debug Dory.app +# builds, but never creates ~/.dory on macos-15 arm (engine.sock absent). +# - hops requires engine.sock (not only dory.sock). To still exercise the +# hops dory backend (dory k8s enable, context `dory`, registry plumbing) +# on public GHA, we provision a nested-virt Docker daemon with Colima +# (proven on macos-15-intel) and expose it as ~/.dory/engine.sock when +# the native app does not produce one. +# - Prefer trying the built app first (xattr clear + ad-hoc sign + open); +# fall back to Colima-backed engine.sock if sock never appears. +# +# Runner: macos-15-intel — nested virt for Colima (same pin as colima smoke). +# Prefer localhost:30500 for registry traffic (macOS 15 Local Network Privacy). +# After cold start, wait for CoreDNS/node Ready before registry pods; long +# Ready timeouts + stop/start rollout restart (colima nested-virt lessons). + +on: + # PR trigger so the workflow runs from the PR branch (workflow_dispatch only + # works once the file exists on the default branch). + pull_request: + workflow_dispatch: + +jobs: + dory-smoke: + # Nested-virt-capable Intel pin (not bare macos-latest / not arm-only). + # Native dory-hv wants Apple silicon, but public GHA cannot run it; this + # job uses Colima as the docker daemon behind engine.sock when needed. + runs-on: macos-15-intel + timeout-minutes: 90 + steps: + - name: Checkout hops-cli + uses: actions/checkout@v4 + + - name: Checkout patrickleet/dory (hops integration branch) + uses: actions/checkout@v4 + with: + repository: patrickleet/dory + ref: feat/hops-local-integration + path: dory-src + + - name: Select Xcode + run: | + set -euxo pipefail + newest="$(ls -d /Applications/Xcode*.app | sort -V | tail -1)" + echo "Using Xcode at $newest" + echo "DEVELOPER_DIR=$newest/Contents/Developer" >> "$GITHUB_ENV" + sudo xcode-select -s "$newest/Contents/Developer" + + - name: Build Dory from source + working-directory: dory-src + run: | + set -euxo pipefail + # Match dory CI: Metal toolchain download can be flaky; best-effort. + xcodebuild -downloadComponent MetalToolchain || true + # Prefer deterministic product path via -derivedDataPath (not scripts/build.sh + # alone, which may use a user-local DerivedData path). + xcodebuild -project Dory.xcodeproj -scheme Dory \ + -destination 'platform=macOS' -configuration Debug \ + -derivedDataPath build \ + build CODE_SIGNING_ALLOWED=NO + test -d build/Build/Products/Debug/Dory.app + # Clear provenance/quarantine that can leave Debug bundles unlaunchable + # (dory scripts/build.sh does the same for DerivedData products). + xattr -cr build/Build/Products/Debug/Dory.app 2>/dev/null || true + xattr -dr com.apple.provenance build/Build/Products/Debug/Dory.app 2>/dev/null || true + xattr -dr com.apple.quarantine build/Build/Products/Debug/Dory.app 2>/dev/null || true + codesign --force --deep --sign - build/Build/Products/Debug/Dory.app 2>/dev/null || true + + - name: Install dory CLI on PATH + run: | + set -euxo pipefail + chmod +x dory-src/scripts/dory + echo "$GITHUB_WORKSPACE/dory-src/scripts" >> "$GITHUB_PATH" + test -x dory-src/scripts/dory + dory-src/scripts/dory version || dory-src/scripts/dory k8s status || true + + - name: Install colima, docker, kubectl, helm + run: | + set -euxo pipefail + # colima provides nested-virt docker when native dory-hv cannot run. + brew install colima docker kubectl helm + colima version + docker --version + kubectl version --client + helm version --short + + - name: Ensure engine.sock (app native or Colima surrogate) + run: | + set -euxo pipefail + DORY_APP="$GITHUB_WORKSPACE/dory-src/build/Build/Products/Debug/Dory.app" + test -d "$DORY_APP" + mkdir -p "$HOME/.dory" + + ARCH="$(uname -m)" + # dory-hv requires Apple silicon; Intel hosts never produce a native sock. + # Public GHA arm also cannot nested-virt the engine (dory CI docs). Try + # native only briefly on arm; always fall through to Colima if missing. + if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "arm64e" ]; then + open "$DORY_APP" || true + for i in $(seq 1 12); do + if [ -S "$HOME/.dory/engine.sock" ]; then + echo "native engine.sock ready after ${i} attempts" + ls -la "$HOME/.dory" + echo "DORY_ENGINE_SOURCE=native" >> "$GITHUB_ENV" + exit 0 + fi + sleep 5 + done + pkill -f 'Dory.app/Contents/MacOS/Dory' || true + echo "native engine.sock absent; bootstrapping Colima docker as engine.sock" + else + echo "host arch $ARCH: native dory-hv unsupported; Colima surrogate" + fi + + # --- Colima-backed engine.sock (public GHA nested virt for docker) --- + # Sizing fits macos-15-intel (~4 CPU / ~14 GiB). Disk 40 for registry PVC + # when hops later creates the 20Gi registry volume inside k3s-on-docker. + colima start --runtime docker --cpu 3 --memory 8 --disk 40 + DOCKER_SOCK="" + for candidate in \ + "$HOME/.colima/default/docker.sock" \ + "$HOME/.colima/docker.sock" \ + /var/run/docker.sock + do + if [ -S "$candidate" ]; then + DOCKER_SOCK="$candidate" + break + fi + done + if [ -z "$DOCKER_SOCK" ]; then + echo "colima started but no docker.sock found" >&2 + colima status || true + ls -la "$HOME/.colima" || true + ls -la "$HOME/.colima/default" || true + exit 1 + fi + # Prefer real path via env (dory's dk() uses DORY_ENGINE_SOCK) and + # keep symlink so hops preflight (`~/.dory/engine.sock`) passes. + ln -sfn "$DOCKER_SOCK" "$HOME/.dory/engine.sock" + ln -sfn "$DOCKER_SOCK" "$HOME/.dory/dory.sock" + test -S "$HOME/.dory/engine.sock" || test -e "$HOME/.dory/engine.sock" + echo "DORY_ENGINE_SOCK=$DOCKER_SOCK" >> "$GITHUB_ENV" + echo "DOCKER_HOST=unix://$DOCKER_SOCK" >> "$GITHUB_ENV" + export DORY_ENGINE_SOCK="$DOCKER_SOCK" + export DOCKER_HOST="unix://$DOCKER_SOCK" + docker info >/dev/null + docker ps -a || true + echo "engine.sock → $DOCKER_SOCK (colima surrogate)" + ls -la "$HOME/.dory" + echo "DORY_ENGINE_SOURCE=colima-surrogate" >> "$GITHUB_ENV" + + - uses: Swatinem/rust-cache@v2 + + - name: Build hops + run: cargo build + + - name: Host resources (for diagnosis) + run: | + set -euxo pipefail + sysctl -n hw.ncpu || true + sysctl hw.memsize || true + uname -m + df -h / || true + echo "DORY_ENGINE_SOURCE=${DORY_ENGINE_SOURCE:-unknown}" + + - name: hops local start --backend dory + run: | + set -euxo pipefail + export DOCKER_HOST="${DOCKER_HOST:-unix://$HOME/.dory/engine.sock}" + export DORY_ENGINE_SOCK="${DORY_ENGINE_SOCK:-$HOME/.dory/engine.sock}" + # Clean slate: partial dory-k8s can report create-time drift (exit 3). + dory k8s disable || true + docker ps -a || true + ./target/debug/hops-cli local start --backend dory + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Wait for node + CoreDNS (nested virt settles) + run: | + set -euxo pipefail + kubectl --context dory wait --for=condition=Ready nodes --all --timeout=120s + kubectl --context dory -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=kube-dns --timeout=180s || \ + kubectl --context dory -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=coredns --timeout=180s || true + kubectl --context dory -n kube-system get pods -o wide || true + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + for attempt in 1 2 3 4 5; do + if docker pull public.ecr.aws/docker/library/busybox:stable; then + break + fi + echo "docker pull failed (attempt $attempt); sleeping..." + sleep $((attempt * 15)) + if [ "$attempt" -eq 5 ]; then + exit 1 + fi + done + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + kubectl --context dory run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + kubectl --context dory run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + if ! kubectl --context dory wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then + echo "==== smoke pods not Ready ====" + kubectl --context dory get pods smoke-svc-name smoke-localhost -o wide || true + kubectl --context dory describe pod smoke-svc-name smoke-localhost || true + kubectl --context dory get events -A --field-selector involvedObject.name=smoke-svc-name || true + kubectl --context dory get events -A --field-selector involvedObject.name=smoke-localhost || true + exit 1 + fi + kubectl --context dory delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + export DOCKER_HOST="${DOCKER_HOST:-unix://$HOME/.dory/engine.sock}" + export DORY_ENGINE_SOCK="${DORY_ENGINE_SOCK:-$HOME/.dory/engine.sock}" + ./target/debug/hops-cli local stop + # After docker stop/start of dory-k8s, give the apiserver a beat before + # hops re-runs helm (openapi can time out immediately after node Ready). + sleep 20 + ./target/debug/hops-cli local start + if ! kubectl --context dory -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s; then + echo "deployments not Available after resume; restarting..." + kubectl --context dory -n crossplane-system get pods -o wide || true + kubectl --context dory -n crossplane-system rollout restart \ + deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true + kubectl --context dory -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s + fi + ./target/debug/hops-cli local doctor + + - name: Debug dump on failure + if: failure() + run: | + set +e + echo "==== host ====" + sysctl -n hw.ncpu + sysctl hw.memsize + uname -m + df -h / + echo "DORY_ENGINE_SOURCE=${DORY_ENGINE_SOURCE:-unknown}" + echo "==== dory / colima ====" + ls -la "$HOME/.dory" || true + dory version || true + dory k8s status || true + colima status || true + colima list || true + export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + docker info 2>&1 | head -40 || true + echo "==== cluster ====" + kubectl --context dory get nodes,pods -A -o wide || true + echo "==== smoke pods (default) ====" + kubectl --context dory describe pod smoke-svc-name smoke-localhost 2>/dev/null || true + echo "==== crossplane-system ====" + kubectl --context dory describe pods -n crossplane-system || true + kubectl --context dory get events -A --sort-by=.lastTimestamp | tail -100 || true + ./target/debug/hops-cli local doctor || true + + - name: hops local destroy + if: always() + run: | + set +e + export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + ./target/debug/hops-cli local destroy || true + + - name: Teardown Dory + Colima + if: always() + run: | + set +e + dory k8s disable || true + pkill -f 'Dory.app/Contents/MacOS/Dory' || true + colima stop || true + colima delete --force || true + ls -la "$HOME/.dory" || true diff --git a/src/commands/local/backend/dory.rs b/src/commands/local/backend/dory.rs index 1c0dfed..00ad26b 100644 --- a/src/commands/local/backend/dory.rs +++ b/src/commands/local/backend/dory.rs @@ -329,15 +329,31 @@ fn run_dory_enable(recreate: bool) -> Result<(), Box> { let args = dory_enable_args(recreate); match run_cmd("dory", &args) { Ok(()) => hold_through_engine_launch_window(), - Err(err) if !recreate && err.to_string().contains("exit status: 3") => Err(format!( - "{}\nhint: run `hops local reset --backend dory` to recreate the dory cluster and apply create-time config drift", - err - ) - .into()), + // Exit 3 = create-time config drift (ports / registries bind). hops just + // wrote the desired ports+registries files; applying them requires + // recreate. Auto-retry once so `local start` is not a dead end after + // hops updates publish config (reset remains available explicitly). + // + // Exit 1 after "did not become Ready" is common on stop→start: docker + // start of the existing k3s container can leave the node stuck past + // dory's wait window. Recreate once (same as `dory k8s disable && enable`). + Err(err) if !recreate && should_recreate_on_enable_error(&err.to_string()) => { + log::warn!( + "dory k8s enable failed ({err}); recreating cluster once..." + ); + let _ = run_cmd("dory", &["k8s", "disable"]); + run_dory_enable(true) + } Err(err) => Err(err), } } +fn should_recreate_on_enable_error(msg: &str) -> bool { + // run_cmd only surfaces exit status (stderr is inherited), so match codes. + // 3 = create-time config drift; 1 = dory k8s_wait_ready / generic enable fail. + msg.contains("exit status: 3") || msg.contains("exit status: 1") +} + fn dory_enable_args(recreate: bool) -> Vec<&'static str> { let mut args = vec!["k8s", "enable"]; if recreate { @@ -725,6 +741,20 @@ mod tests { ); } + #[test] + fn recreate_on_enable_matches_dory_exit_codes() { + assert!(should_recreate_on_enable_error( + "dory exited with exit status: 3" + )); + assert!(should_recreate_on_enable_error( + "dory exited with exit status: 1" + )); + assert!(!should_recreate_on_enable_error( + "dory exited with exit status: 2" + )); + assert!(!should_recreate_on_enable_error("connection refused")); + } + #[test] fn start_rejects_size_flags() { let size = SizeArgs { diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index b151675..19d4e42 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -76,10 +76,12 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box> = None; + for attempt in 1..=6 { + wait_for_kubernetes()?; + match run_cmd("helm", &helm_args) { + Ok(()) => { + last_err = None; + break; + } + Err(e) => { + log::warn!("helm install attempt {attempt}/6 failed: {e}"); + last_err = Some(e); + std::thread::sleep(std::time::Duration::from_secs(20)); + } + } + } + if let Some(e) = last_err { + return Err(e); + } + } // 6. Wait for Crossplane core deployment. // rbac-manager can flap under nested-virt resource pressure; the core diff --git a/tests/dory_smoke_workflow.rs b/tests/dory_smoke_workflow.rs new file mode 100644 index 0000000..98cca83 --- /dev/null +++ b/tests/dory_smoke_workflow.rs @@ -0,0 +1,139 @@ +//! Structural checks for the shipped dory macOS smoke workflow. +//! +//! Drives the real `.github/workflows/on-pr-dory-smoke.yaml` so the clone+build +//! install contract and kind-parity smoke steps cannot silently regress. + +use std::fs; +use std::path::PathBuf; + +fn workflow_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-dory-smoke.yaml") +} + +fn workflow_text() -> String { + let path = workflow_path(); + assert!( + path.is_file(), + "expected dory smoke workflow at {}", + path.display() + ); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn dory_smoke_workflow_clone_build_install_contract() { + let text = workflow_text(); + assert!( + text.contains("patrickleet/dory"), + "must clone patrickleet/dory" + ); + assert!( + text.contains("feat/hops-local-integration"), + "must pin hops integration branch" + ); + assert!( + text.contains("xcodebuild") && text.contains("derivedDataPath"), + "must build in-pipeline with deterministic derivedDataPath" + ); + assert!( + text.contains("GITHUB_PATH") && text.contains("scripts"), + "must put scripts/dory on PATH" + ); + assert!( + text.contains("engine.sock"), + "must ensure ~/.dory/engine.sock before hops start" + ); + assert!( + !text.contains("brew install --cask") && !text.contains("homebrew/cask"), + "must not use brew cask as primary install" + ); + assert!( + text.contains("workflow_dispatch") && text.contains("pull_request:"), + "must support pull_request (PR-branch runs) and workflow_dispatch" + ); + // Nested-virt pin for Colima-backed engine.sock on public GHA. + assert!( + text.lines() + .any(|l| l.trim() == "runs-on: macos-15-intel"), + "must pin macos-15-intel for nested virt (not bare macos-latest alone)" + ); +} + +#[test] +fn dory_smoke_workflow_kind_parity_when_engine_boots() { + let text = workflow_text(); + for needle in [ + "cargo build", + "start --backend dory", + "local doctor", + "localhost:30500", + "registry.crossplane-system.svc.cluster.local:5000", + "--context dory", + "local stop", + "local destroy", + ] { + assert!( + text.contains(needle), + "dory smoke missing kind-parity fragment: {needle}" + ); + } + // stop/start resume: start without --backend after stop + assert!( + text.contains("local start\n") + || text + .lines() + .any(|l| l.trim() == "./target/debug/hops-cli local start"), + "must start again without --backend after stop" + ); +} + +#[test] +fn dory_smoke_workflow_carries_colima_lessons() { + let text = workflow_text(); + assert!( + text.contains("kube-dns") || text.contains("coredns"), + "must wait for CoreDNS/kube-dns before registry round-trip" + ); + assert!( + text.contains("--timeout=420s"), + "must use a long Ready/Available timeout for nested-virt lag" + ); + assert!( + text.contains("rollout restart"), + "stop/start resume must rollout-restart stalled deployments" + ); + assert!( + text.contains("Debug dump on failure") && text.contains("engine.sock"), + "failure dump must capture dory/engine diagnostics" + ); + assert!( + text.contains("describe pod smoke-svc-name") + || text.contains("describe pod smoke-svc-name smoke-localhost"), + "failure dump must describe smoke pods in default ns" + ); + assert!( + text.to_lowercase().contains("localhost:30500"), + "must exercise localhost:30500 registry path" + ); +} + +#[test] +fn dory_smoke_workflow_public_gha_engine_fallback() { + let text = workflow_text(); + // Public GHA cannot run dory-hv; workflow must document and implement a + // Colima-backed engine.sock so hops --backend dory remains testable. + assert!( + text.contains("colima start") && text.contains("engine.sock"), + "must bootstrap Colima docker as engine.sock when native sock is absent" + ); + assert!( + text.to_lowercase().contains("surrogate") + || text.contains("colima-surrogate") + || text.contains("Colima-backed"), + "must document Colima engine.sock surrogate for public GHA" + ); + assert!( + text.contains("xattr") || text.contains("codesign"), + "must clear quarantine/sign Debug Dory.app before open" + ); +} From 9c2596aa386447d749f4a739a6127d1c9537b737 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 16 Jul 2026 15:49:36 -0500 Subject: [PATCH 08/11] ci: gate macOS backend smoke with labels --- .github/workflows/on-pr-colima-smoke.yaml | 4 ++++ .github/workflows/on-pr-dory-smoke.yaml | 8 ++++++-- tests/colima_smoke_workflow.rs | 9 +++++++++ tests/dory_smoke_workflow.rs | 9 +++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml index b1bfa70..52d4d16 100644 --- a/.github/workflows/on-pr-colima-smoke.yaml +++ b/.github/workflows/on-pr-colima-smoke.yaml @@ -20,10 +20,14 @@ name: colima backend smoke on: pull_request: + types: [opened, reopened, synchronize, labeled] workflow_dispatch: jobs: colima-smoke: + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test-colima') # Nested-virt-capable Intel pin (not bare macos-latest). runs-on: macos-15-intel # Nested virt + cold Crossplane pulls can take well over 30m. diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml index 3d75908..11d06c4 100644 --- a/.github/workflows/on-pr-dory-smoke.yaml +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -28,13 +28,17 @@ name: dory backend smoke # Ready timeouts + stop/start rollout restart (colima nested-virt lessons). on: - # PR trigger so the workflow runs from the PR branch (workflow_dispatch only - # works once the file exists on the default branch). + # Opt in with the test-dory label. Keep synchronize so later commits rerun + # while labeled; labeled starts the first run on an existing PR. pull_request: + types: [opened, reopened, synchronize, labeled] workflow_dispatch: jobs: dory-smoke: + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test-dory') # Nested-virt-capable Intel pin (not bare macos-latest / not arm-only). # Native dory-hv wants Apple silicon, but public GHA cannot run it; this # job uses Colima as the docker daemon behind engine.sock when needed. diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs index b2882cb..5eaef20 100644 --- a/tests/colima_smoke_workflow.rs +++ b/tests/colima_smoke_workflow.rs @@ -23,6 +23,15 @@ fn workflow_text() -> String { #[test] fn colima_smoke_workflow_exists_and_pins_nested_virt_runner() { let text = workflow_text(); + assert!( + text.contains("labeled") && text.contains("test-colima"), + "PR smoke must start only when opted in with the test-colima label" + ); + assert!( + text.contains("github.event_name == 'workflow_dispatch'") + && text.contains("github.event.pull_request.labels.*.name"), + "manual dispatch must remain available while PR runs are label-gated" + ); assert!( text.contains("runs-on: macos-15-intel"), "must pin nested-virt-capable Intel runner" diff --git a/tests/dory_smoke_workflow.rs b/tests/dory_smoke_workflow.rs index 98cca83..ac56ed6 100644 --- a/tests/dory_smoke_workflow.rs +++ b/tests/dory_smoke_workflow.rs @@ -51,6 +51,15 @@ fn dory_smoke_workflow_clone_build_install_contract() { text.contains("workflow_dispatch") && text.contains("pull_request:"), "must support pull_request (PR-branch runs) and workflow_dispatch" ); + assert!( + text.contains("labeled") && text.contains("test-dory"), + "PR smoke must start only when opted in with the test-dory label" + ); + assert!( + text.contains("github.event_name == 'workflow_dispatch'") + && text.contains("github.event.pull_request.labels.*.name"), + "manual dispatch must remain available while PR runs are label-gated" + ); // Nested-virt pin for Colima-backed engine.sock on public GHA. assert!( text.lines() From a6a1df46f3e53fa81342a0a52c60bfd6b8a3ff87 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Mon, 20 Jul 2026 11:56:19 -0500 Subject: [PATCH 09/11] fix(ci): build Dory FFI before xcodebuild (#81) Dory's rebased upstream requires a generated Rust UniFFI XCFramework before SwiftPM can resolve the app package. Install rust/protobuf, materialize and verify the ignored artifacts, then build the app.\n\nImplements [[tasks/hops-cli-dory-macos-smoke]] --- .github/workflows/on-pr-dory-smoke.yaml | 15 +++++++++++++++ tests/dory_smoke_workflow.rs | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml index 11d06c4..532a010 100644 --- a/.github/workflows/on-pr-dory-smoke.yaml +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -63,12 +63,27 @@ jobs: echo "DEVELOPER_DIR=$newest/Contents/Developer" >> "$GITHUB_ENV" sudo xcode-select -s "$newest/Contents/Developer" + # Dory's rebased upstream builds its Swift package against a generated + # Rust UniFFI XCFramework. A clean checkout has no binary artifact, so + # raw xcodebuild cannot resolve DoryFFI until this preparation runs. + - name: Install Rust toolchain for Dory FFI + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + + - name: Install Dory FFI build prerequisites + run: brew install protobuf + - name: Build Dory from source working-directory: dory-src run: | set -euxo pipefail # Match dory CI: Metal toolchain download can be flaky; best-effort. xcodebuild -downloadComponent MetalToolchain || true + # Match dory's scripts/build.sh + scripts/test.sh clean-checkout + # contract: materialize the ignored DoryFFI binary and generated + # Swift bindings before SwiftPM resolves the app dependencies. + scripts/build-dory-ffi-xcframework.sh --if-needed + test -f dory-core-swift/artifacts/DoryFFI.xcframework/macos-arm64_x86_64/libdory_ffi.a + test -f dory-core-swift/Sources/DoryCore/generated/dory_ffi.swift # Prefer deterministic product path via -derivedDataPath (not scripts/build.sh # alone, which may use a user-local DerivedData path). xcodebuild -project Dory.xcodeproj -scheme Dory \ diff --git a/tests/dory_smoke_workflow.rs b/tests/dory_smoke_workflow.rs index ac56ed6..3c85164 100644 --- a/tests/dory_smoke_workflow.rs +++ b/tests/dory_smoke_workflow.rs @@ -35,6 +35,25 @@ fn dory_smoke_workflow_clone_build_install_contract() { text.contains("xcodebuild") && text.contains("derivedDataPath"), "must build in-pipeline with deterministic derivedDataPath" ); + assert!( + text.contains("dtolnay/rust-toolchain") && text.contains("brew install protobuf"), + "must install the Rust and protoc prerequisites used by Dory's FFI builder" + ); + let ffi_build = text + .find("scripts/build-dory-ffi-xcframework.sh --if-needed") + .expect("must materialize DoryFFI from a clean checkout"); + let app_build = text + .find("xcodebuild -project Dory.xcodeproj") + .expect("must build the Dory app"); + assert!( + ffi_build < app_build, + "must generate DoryFFI before SwiftPM resolves the Dory app" + ); + assert!( + text.contains("DoryFFI.xcframework/macos-arm64_x86_64/libdory_ffi.a") + && text.contains("Sources/DoryCore/generated/dory_ffi.swift"), + "must verify both generated Dory FFI artifacts before xcodebuild" + ); assert!( text.contains("GITHUB_PATH") && text.contains("scripts"), "must put scripts/dory on PATH" From 62d135d82dcf047029efea915b3713cf88b84500 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 29 Jul 2026 13:53:14 -0500 Subject: [PATCH 10/11] fix(local): wait for provider Healthy and pin dory smoke SHA Doctor requires Healthy=True, but start only waited for CRDs, so kind-smoke could race on cold CI. Poll both bootstrap providers after ProviderConfigs. Pin dory smoke to a known-good commit of the hops integration work so rebases do not move CI without an intentional bump. --- .github/workflows/on-pr-dory-smoke.yaml | 10 ++- src/commands/local/start.rs | 94 +++++++++++++++++++++++-- tests/dory_smoke_workflow.rs | 12 +++- 3 files changed, 104 insertions(+), 12 deletions(-) diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml index 532a010..f1678bc 100644 --- a/.github/workflows/on-pr-dory-smoke.yaml +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -4,7 +4,9 @@ name: dory backend smoke # Mirrors on-pr-kind-smoke.yaml: start → doctor → dual registry → stop/start → destroy. # # Install contract (mandatory — not brew cask / official release): -# 1. Clone patrickleet/dory @ feat/hops-local-integration +# 1. Clone patrickleet/dory at a pinned commit of feat/hops-local-integration +# (SHA pin — the integration branch may never merge upstream; bump the ref +# intentionally when adopting a newer known-good tip) # 2. Build in-pipeline (xcodebuild with deterministic -derivedDataPath) # 3. Put scripts/dory on PATH # 4. Provide ~/.dory/engine.sock before hops start (see engine note below) @@ -48,11 +50,13 @@ jobs: - name: Checkout hops-cli uses: actions/checkout@v4 - - name: Checkout patrickleet/dory (hops integration branch) + - name: Checkout patrickleet/dory (pinned hops integration commit) uses: actions/checkout@v4 with: repository: patrickleet/dory - ref: feat/hops-local-integration + # feat/hops-local-integration tip as of 2026-07-29 + # (feat(k8s): headless lifecycle CLI, ports, registries, merged dory context) + ref: f8c61d2fd0fc4d528e5e0da36ffa09b9796b3871 path: dory-src - name: Select Xcode diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 19d4e42..4f04760 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -151,7 +151,17 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { - wait_for_deployment_attempts(namespace, name, 60) -} - /// Longer wait used for Crossplane on cold nested-virt runners (~15 minutes). fn wait_for_deployment_with_diagnostics( namespace: &str, @@ -283,3 +288,78 @@ fn wait_for_crd(crd: &str) -> Result<(), Box> { } Err(format!("Timed out waiting for CRD {} to be Established", crd).into()) } + +/// Poll until a Crossplane Provider reports Healthy=True (~10 minutes). +/// +/// Package install can establish CRDs before the runtime Deployment is Ready. +/// Matching doctor's expectation here keeps `hops local start` from returning +/// while the cluster still looks half-bootstrapped. +fn wait_for_provider_healthy(provider: &str) -> Result<(), Box> { + log::info!("Waiting for Provider {} Healthy...", provider); + for i in 0..120 { + let healthy = run_cmd_output( + "kubectl", + &[ + "get", + "provider.pkg.crossplane.io", + provider, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Healthy\")].status}", + ], + ) + .unwrap_or_default(); + if healthy.trim() == "True" { + return Ok(()); + } + + if i > 0 && i % 12 == 0 { + let installed = run_cmd_output( + "kubectl", + &[ + "get", + "provider.pkg.crossplane.io", + provider, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Installed\")].status}", + ], + ) + .unwrap_or_default(); + log::info!( + "Still waiting for Provider {} Healthy ({}s elapsed; Installed={}, Healthy={})...", + provider, + i * 5, + if installed.trim().is_empty() { + "" + } else { + installed.trim() + }, + if healthy.trim().is_empty() { + "" + } else { + healthy.trim() + } + ); + let _ = run_cmd( + "kubectl", + &["get", "pods", "-n", "crossplane-system", "-o", "wide"], + ); + let _ = run_cmd( + "kubectl", + &["get", "provider.pkg.crossplane.io", provider, "-o", "yaml"], + ); + } + + thread::sleep(Duration::from_secs(5)); + } + + let _ = run_cmd( + "kubectl", + &["get", "provider.pkg.crossplane.io", provider, "-o", "yaml"], + ); + dump_namespace_diagnostics("crossplane-system"); + Err(format!( + "Timed out waiting for Provider {} to become Healthy", + provider + ) + .into()) +} diff --git a/tests/dory_smoke_workflow.rs b/tests/dory_smoke_workflow.rs index 3c85164..eb80654 100644 --- a/tests/dory_smoke_workflow.rs +++ b/tests/dory_smoke_workflow.rs @@ -27,9 +27,17 @@ fn dory_smoke_workflow_clone_build_install_contract() { text.contains("patrickleet/dory"), "must clone patrickleet/dory" ); + // Pin a known-good commit of the hops integration work (not a moving branch + // tip — the upstream branch may never merge). Bump intentionally with dory. assert!( - text.contains("feat/hops-local-integration"), - "must pin hops integration branch" + text.contains("f8c61d2fd0fc4d528e5e0da36ffa09b9796b3871"), + "must pin patrickleet/dory to a specific commit SHA" + ); + assert!( + !text + .lines() + .any(|l| l.trim() == "ref: feat/hops-local-integration"), + "must not track the moving feat/hops-local-integration branch tip" ); assert!( text.contains("xcodebuild") && text.contains("derivedDataPath"), From 7cca6744e8c3c3caf3e54689a9641e350e5ab0ad Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Fri, 31 Jul 2026 10:30:17 -0500 Subject: [PATCH 11/11] feat(local): stock Dory integration + self-hosted smoke (#84) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(local): dory engine package bridge, no NodePort registry Rebuild the dory backend around product k3s + an engine-side registry: host push localhost:30500, cluster pull host.dory.internal:30500. Drop ports/registries create-time files and in-cluster NodePort on dory. Crossplane still installs the same; package pull address is backend-aware. * refactor(local): stock Dory only — no k8s enable fork hops no longer calls dory k8s enable/disable or requires a scriptable CLI. Users enable Kubernetes in the Dory app; hops starts a stopped dory-k8s node, waits for Ready, and uses the engine package bridge. Destroy/reset only remove the container via the engine docker socket. * feat(local): stock Dory backend, self-hosted smoke, path config fixture Use product Dory only (dory.sock, hops-dory desktop, TLS registry push via engine bridge). Add HOPS_DORY_DESKTOP=0 for env-only sessions that do not mutate host kube/docker defaults. Rewrite dory CI for self-hosted Apple Silicon (label hops-dory / test-dory): no fork build, no Colima sock surrogate, soft cleanup only. Add tests/fixtures/config-smoke and path-based hops config install in smoke (namespaced ConfigMap under hops-ci). * fix(config-install): pass --platform when rebuilding arch-tagged render images Avoid Docker buildx InvalidBaseImagePlatform when `up project build` emits both :amd64 and :arm64 function images and hops rebuilds the non-host arch on Apple Silicon (or the reverse on Intel). * fix(ci): single-platform busybox for dory registry smoke push Pull with --platform for the host arch and re-materialize via docker build so push does not warn about incomplete multiplatform content from the upstream busybox:stable index. * feat(local): fast start when control plane healthy; kind config smoke Skip helm repo update/upgrade and bootstrap reapply when Crossplane, k8s+helm providers, and the package registry are already Available. Pass --bootstrap to force a full reconcile. Kind smoke: drop stop/start resume; install up CLI and run the same path-based config-smoke fixture as dory (namespaced ConfigMap). * fix(local): expect HTTPS in kind hosts.toml registry unit test The local package registry serves TLS; hosts.toml uses https:// + skip_verify. Update the unit test that still asserted http:// after the TLS migration. * ci(dory): drop failure debug dump on personal Mac runner Avoid dumping host-wide docker/pods/doctor state that includes unrelated workloads on a shared self-hosted machine. * fix(local/ci): fast dory doctor probe; drop rust-cache on self-hosted smoke Doctor: for non-loopback push hosts (dory bridge IP) skip host curl and probe via the engine docker network only — host curl hangs on TCP timeout. Dory smoke: remove Swatinem/rust-cache; post-job cache save is slow and can block on password with no TTY on a personal Mac runner. * fix(local/ci): skip host curl for dory doctor; drop rust-cache hang Doctor probes non-loopback push hosts only via engine docker (explicit ~/.dory/dory.sock), avoiding ~75s host TCP timeout and host docker password prompts. Remove Swatinem/rust-cache from dory smoke so post-job target/ upload no longer hangs the personal Mac runner. * ci(dory): resolve working cargo via rustup toolchains Self-hosted Mac runners often have a dangling ~/.cargo/bin/cargo → rustup shim. Prefer a real toolchain cargo so preflight and build succeed. * ci(dory): quiet env-session step logs on personal Mac runner Drop set -x and host dumps (full PATH, docker info, readiness) so Actions logs only show short status lines for the Dory preflight. --- .github/workflows/on-pr-dory-smoke.yaml | 509 ++++---- .github/workflows/on-pr-kind-smoke.yaml | 92 +- README.md | 226 +++- bootstrap/registry/registry.yaml | 35 +- src/commands/config/install.rs | 74 +- src/commands/local/backend/dory.rs | 1161 +++++++++-------- src/commands/local/backend/kind.rs | 9 +- src/commands/local/backend/mod.rs | 70 +- src/commands/local/doctor.rs | 92 ++ src/commands/local/mod.rs | 16 +- src/commands/local/package_install.rs | 313 ++++- src/commands/local/start.rs | 134 +- src/commands/provider/install.rs | 26 +- tests/dory_smoke_workflow.rs | 191 +-- tests/fixtures/config-smoke/.gitignore | 4 + tests/fixtures/config-smoke/README.md | 14 + .../apis/configsmokes/composition.yaml | 19 + .../apis/configsmokes/configuration.yaml | 14 + .../apis/configsmokes/definition.yaml | 53 + .../examples/configsmokes/ci.yaml | 13 + .../render/000-state-init.yaml.gotmpl | 26 + .../render/010-state-status.yaml.gotmpl | 17 + .../render/200-configmap.yaml.gotmpl | 28 + .../functions/render/999-status.yaml.gotmpl | 8 + tests/fixtures/config-smoke/local/ci-xr.yaml | 36 + tests/fixtures/config-smoke/upbound.yaml | 27 + 26 files changed, 2180 insertions(+), 1027 deletions(-) create mode 100644 tests/fixtures/config-smoke/.gitignore create mode 100644 tests/fixtures/config-smoke/README.md create mode 100644 tests/fixtures/config-smoke/apis/configsmokes/composition.yaml create mode 100644 tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml create mode 100644 tests/fixtures/config-smoke/apis/configsmokes/definition.yaml create mode 100644 tests/fixtures/config-smoke/examples/configsmokes/ci.yaml create mode 100644 tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl create mode 100644 tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl create mode 100644 tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl create mode 100644 tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl create mode 100644 tests/fixtures/config-smoke/local/ci-xr.yaml create mode 100644 tests/fixtures/config-smoke/upbound.yaml diff --git a/.github/workflows/on-pr-dory-smoke.yaml b/.github/workflows/on-pr-dory-smoke.yaml index f1678bc..3696675 100644 --- a/.github/workflows/on-pr-dory-smoke.yaml +++ b/.github/workflows/on-pr-dory-smoke.yaml @@ -1,233 +1,256 @@ name: dory backend smoke -# End-to-end smoke for `hops local --backend dory` on public GHA macOS. -# Mirrors on-pr-kind-smoke.yaml: start → doctor → dual registry → stop/start → destroy. +# Real stock-Dory + hops integration on a self-hosted Apple Silicon Mac. # -# Install contract (mandatory — not brew cask / official release): -# 1. Clone patrickleet/dory at a pinned commit of feat/hops-local-integration -# (SHA pin — the integration branch may never merge upstream; bump the ref -# intentionally when adopting a newer known-good tip) -# 2. Build in-pipeline (xcodebuild with deterministic -derivedDataPath) -# 3. Put scripts/dory on PATH -# 4. Provide ~/.dory/engine.sock before hops start (see engine note below) +# Why self-hosted (not macos-15 / Colima): +# - Stock Dory needs dory-hv (Hypervisor.framework). Public GHA VMs cannot nest it. +# - An earlier "green" dory-smoke spent ~10m building a fork, then used Colima as +# ~/.dory/*.sock — that was not product Dory. This workflow refuses that lie. # -# Engine socket on public GHA (important): -# - Dory's native dory-hv engine needs Hypervisor.framework nested virt. -# Dory's own CI documents that GitHub-hosted macOS runners are VMs and -# CANNOT run that engine (see dory benchmark.yml). Observed: Debug Dory.app -# builds, but never creates ~/.dory on macos-15 arm (engine.sock absent). -# - hops requires engine.sock (not only dory.sock). To still exercise the -# hops dory backend (dory k8s enable, context `dory`, registry plumbing) -# on public GHA, we provision a nested-virt Docker daemon with Colima -# (proven on macos-15-intel) and expose it as ~/.dory/engine.sock when -# the native app does not produce one. -# - Prefer trying the built app first (xattr clear + ad-hoc sign + open); -# fall back to Colima-backed engine.sock if sock never appears. +# Trigger: label `test-dory` on the PR, or workflow_dispatch. +# Offline Mac: the job stays queued until the runner is online. # -# Runner: macos-15-intel — nested virt for Colima (same pin as colima smoke). -# Prefer localhost:30500 for registry traffic (macOS 15 Local Network Privacy). -# After cold start, wait for CoreDNS/node Ready before registry pods; long -# Ready timeouts + stop/start rollout restart (colima nested-virt lessons). +# Do-not-interrupt contract (personal Mac while you work): +# - Reuses your already-running Dory engine + dory-k8s +# - Never mutates desktop defaults (no global context switching, no merge into +# ~/.kube/config). Session is entirely env-driven: +# DOCKER_HOST=unix://$HOME/.dory/dory.sock +# KUBECONFIG= +# HOPS_DORY_DESKTOP=0 +# - Never destroy / stop dory-k8s, never kill Dory +# - Caps cargo parallelism and runs the build under nice(1) +# +# Runner setup (one-time on the Mac): +# mkdir -p ~/actions-runners/hops-cli && cd ~/actions-runners/hops-cli +# # download latest macOS arm64 runner from actions/runner releases +# ./config.sh --url https://github.com/hops-ops/hops-cli \ +# --token \ +# --name "$(hostname -s)-dory" \ +# --labels macOS,ARM64,hops-dory \ +# --work _work +# ./run.sh # on-demand; omit svc install so jobs only run when you start it +# +# Prerequisites: Dory engine healthy, Kubernetes enabled (dory-k8s), helm + cargo +# + `up` (Upbound CLI) on the runner user's PATH (path-based config install). on: - # Opt in with the test-dory label. Keep synchronize so later commits rerun - # while labeled; labeled starts the first run on an existing PR. pull_request: types: [opened, reopened, synchronize, labeled] workflow_dispatch: +concurrency: + group: dory-smoke-self-hosted + cancel-in-progress: false + jobs: dory-smoke: if: >- github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'test-dory') - # Nested-virt-capable Intel pin (not bare macos-latest / not arm-only). - # Native dory-hv wants Apple silicon, but public GHA cannot run it; this - # job uses Colima as the docker daemon behind engine.sock when needed. - runs-on: macos-15-intel + runs-on: [self-hosted, macOS, ARM64, hops-dory] timeout-minutes: 90 + env: + # Do not rewrite ~/.kube/config or change the machine docker default. + HOPS_DORY_DESKTOP: "0" + CARGO_BUILD_JOBS: "2" + CARGO_TERM_COLOR: always steps: - name: Checkout hops-cli uses: actions/checkout@v4 - - name: Checkout patrickleet/dory (pinned hops integration commit) - uses: actions/checkout@v4 - with: - repository: patrickleet/dory - # feat/hops-local-integration tip as of 2026-07-29 - # (feat(k8s): headless lifecycle CLI, ports, registries, merged dory context) - ref: f8c61d2fd0fc4d528e5e0da36ffa09b9796b3871 - path: dory-src - - - name: Select Xcode + - name: Require Apple Silicon run: | set -euxo pipefail - newest="$(ls -d /Applications/Xcode*.app | sort -V | tail -1)" - echo "Using Xcode at $newest" - echo "DEVELOPER_DIR=$newest/Contents/Developer" >> "$GITHUB_ENV" - sudo xcode-select -s "$newest/Contents/Developer" - - # Dory's rebased upstream builds its Swift package against a generated - # Rust UniFFI XCFramework. A clean checkout has no binary artifact, so - # raw xcodebuild cannot resolve DoryFFI until this preparation runs. - - name: Install Rust toolchain for Dory FFI - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable - - - name: Install Dory FFI build prerequisites - run: brew install protobuf - - - name: Build Dory from source - working-directory: dory-src - run: | - set -euxo pipefail - # Match dory CI: Metal toolchain download can be flaky; best-effort. - xcodebuild -downloadComponent MetalToolchain || true - # Match dory's scripts/build.sh + scripts/test.sh clean-checkout - # contract: materialize the ignored DoryFFI binary and generated - # Swift bindings before SwiftPM resolves the app dependencies. - scripts/build-dory-ffi-xcframework.sh --if-needed - test -f dory-core-swift/artifacts/DoryFFI.xcframework/macos-arm64_x86_64/libdory_ffi.a - test -f dory-core-swift/Sources/DoryCore/generated/dory_ffi.swift - # Prefer deterministic product path via -derivedDataPath (not scripts/build.sh - # alone, which may use a user-local DerivedData path). - xcodebuild -project Dory.xcodeproj -scheme Dory \ - -destination 'platform=macOS' -configuration Debug \ - -derivedDataPath build \ - build CODE_SIGNING_ALLOWED=NO - test -d build/Build/Products/Debug/Dory.app - # Clear provenance/quarantine that can leave Debug bundles unlaunchable - # (dory scripts/build.sh does the same for DerivedData products). - xattr -cr build/Build/Products/Debug/Dory.app 2>/dev/null || true - xattr -dr com.apple.provenance build/Build/Products/Debug/Dory.app 2>/dev/null || true - xattr -dr com.apple.quarantine build/Build/Products/Debug/Dory.app 2>/dev/null || true - codesign --force --deep --sign - build/Build/Products/Debug/Dory.app 2>/dev/null || true - - - name: Install dory CLI on PATH - run: | - set -euxo pipefail - chmod +x dory-src/scripts/dory - echo "$GITHUB_WORKSPACE/dory-src/scripts" >> "$GITHUB_PATH" - test -x dory-src/scripts/dory - dory-src/scripts/dory version || dory-src/scripts/dory k8s status || true + arch="$(uname -m)" + echo "arch=$arch host=$(hostname -s)" + if [ "$arch" != "arm64" ] && [ "$arch" != "arm64e" ]; then + echo "Stock Dory requires Apple Silicon (got $arch)." >&2 + exit 1 + fi - - name: Install colima, docker, kubectl, helm - run: | - set -euxo pipefail - # colima provides nested-virt docker when native dory-hv cannot run. - brew install colima docker kubectl helm - colima version - docker --version - kubectl version --client - helm version --short - - - name: Ensure engine.sock (app native or Colima surrogate) + - name: Env-only Dory session (private KUBECONFIG + DOCKER_HOST) run: | - set -euxo pipefail - DORY_APP="$GITHUB_WORKSPACE/dory-src/build/Build/Products/Debug/Dory.app" - test -d "$DORY_APP" - mkdir -p "$HOME/.dory" - - ARCH="$(uname -m)" - # dory-hv requires Apple silicon; Intel hosts never produce a native sock. - # Public GHA arm also cannot nested-virt the engine (dory CI docs). Try - # native only briefly on arm; always fall through to Colima if missing. - if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "arm64e" ]; then - open "$DORY_APP" || true - for i in $(seq 1 12); do - if [ -S "$HOME/.dory/engine.sock" ]; then - echo "native engine.sock ready after ${i} attempts" - ls -la "$HOME/.dory" - echo "DORY_ENGINE_SOURCE=native" >> "$GITHUB_ENV" - exit 0 + # No set -x: avoid dumping full PATH / home internals into Actions logs. + set -euo pipefail + sock="$HOME/.dory/dory.sock" + export PATH="$HOME/.dory/bin:/Applications/Dory.app/Contents/Helpers:$PATH" + echo "$HOME/.dory/bin" >> "$GITHUB_PATH" + echo "/Applications/Dory.app/Contents/Helpers" >> "$GITHUB_PATH" + + if [ ! -d /Applications/Dory.app ]; then + echo "Dory.app missing. Once: brew install --cask Augani/dory/dory" >&2 + exit 1 + fi + command -v helm >/dev/null || { echo "helm missing (brew install helm)" >&2; exit 1; } + command -v up >/dev/null || { echo "up CLI missing (needed for hops config install --path)" >&2; exit 1; } + + # Prefer a toolchain cargo that actually runs (shim may be broken). + resolve_cargo_bin() { + if command -v cargo >/dev/null 2>&1 && cargo --version >/dev/null 2>&1; then + command -v cargo + return 0 + fi + local c + for c in \ + "$HOME/.rustup/toolchains/stable-aarch64-apple-darwin/bin/cargo" \ + "$HOME/.rustup/toolchains/stable-x86_64-apple-darwin/bin/cargo" \ + "$HOME/.rustup/toolchains/"*/bin/cargo + do + if [ -x "$c" ] && "$c" --version >/dev/null 2>&1; then + printf '%s\n' "$c" + return 0 fi - sleep 5 done - pkill -f 'Dory.app/Contents/MacOS/Dory' || true - echo "native engine.sock absent; bootstrapping Colima docker as engine.sock" - else - echo "host arch $ARCH: native dory-hv unsupported; Colima surrogate" + return 1 + } + CARGO_BIN="$(resolve_cargo_bin)" || { + echo "cargo missing or broken (install rustup stable toolchain)" >&2 + exit 1 + } + echo "cargo ok ($("$CARGO_BIN" --version))" + echo "$(dirname "$CARGO_BIN")" >> "$GITHUB_PATH" + export PATH="$(dirname "$CARGO_BIN"):$PATH" + + if [ ! -S "$sock" ] || ! docker -H "unix://$sock" info >/dev/null 2>&1; then + echo "waking Dory engine..." + command -v dory >/dev/null 2>&1 && { dory engine wake || dory engine start || true; } >/dev/null 2>&1 || true + if [ ! -S "$sock" ]; then + open -g -j -a Dory 2>/dev/null || true + fi fi - # --- Colima-backed engine.sock (public GHA nested virt for docker) --- - # Sizing fits macos-15-intel (~4 CPU / ~14 GiB). Disk 40 for registry PVC - # when hops later creates the 20Gi registry volume inside k3s-on-docker. - colima start --runtime docker --cpu 3 --memory 8 --disk 40 - DOCKER_SOCK="" - for candidate in \ - "$HOME/.colima/default/docker.sock" \ - "$HOME/.colima/docker.sock" \ - /var/run/docker.sock - do - if [ -S "$candidate" ]; then - DOCKER_SOCK="$candidate" + ready=0 + for i in $(seq 1 36); do + if [ -S "$sock" ] && docker -H "unix://$sock" info >/dev/null 2>&1; then + echo "dory engine ready" + ready=1 break fi + sleep 5 done - if [ -z "$DOCKER_SOCK" ]; then - echo "colima started but no docker.sock found" >&2 - colima status || true - ls -la "$HOME/.colima" || true - ls -la "$HOME/.colima/default" || true + if [ "$ready" -ne 1 ]; then + echo "Dory engine not ready (is Dory.app running?)" >&2 exit 1 fi - # Prefer real path via env (dory's dk() uses DORY_ENGINE_SOCK) and - # keep symlink so hops preflight (`~/.dory/engine.sock`) passes. - ln -sfn "$DOCKER_SOCK" "$HOME/.dory/engine.sock" - ln -sfn "$DOCKER_SOCK" "$HOME/.dory/dory.sock" - test -S "$HOME/.dory/engine.sock" || test -e "$HOME/.dory/engine.sock" - echo "DORY_ENGINE_SOCK=$DOCKER_SOCK" >> "$GITHUB_ENV" - echo "DOCKER_HOST=unix://$DOCKER_SOCK" >> "$GITHUB_ENV" - export DORY_ENGINE_SOCK="$DOCKER_SOCK" - export DOCKER_HOST="unix://$DOCKER_SOCK" - docker info >/dev/null - docker ps -a || true - echo "engine.sock → $DOCKER_SOCK (colima surrogate)" - ls -la "$HOME/.dory" - echo "DORY_ENGINE_SOURCE=colima-surrogate" >> "$GITHUB_ENV" - - - uses: Swatinem/rust-cache@v2 - - - name: Build hops - run: cargo build - - - name: Host resources (for diagnosis) + + if [ -L "$sock" ]; then + target="$(readlink "$sock" || true)" + case "$target" in + *colima*) + echo "refusing Colima-backed dory.sock" >&2 + exit 1 + ;; + esac + fi + + export DOCKER_HOST="unix://$sock" + echo "DOCKER_HOST=unix://$sock" >> "$GITHUB_ENV" + + if ! docker inspect -f '{{.State.Running}}' dory-k8s 2>/dev/null | grep -q true; then + if docker inspect dory-k8s >/dev/null 2>&1; then + echo "starting dory-k8s..." + docker start dory-k8s >/dev/null + else + echo "dory-k8s missing — enable Kubernetes in the Dory app" >&2 + exit 1 + fi + fi + + for i in $(seq 1 60); do + if docker exec dory-k8s kubectl get nodes --no-headers 2>/dev/null | grep -q ' Ready'; then + echo "dory-k8s Ready" + break + fi + if [ "$i" -eq 60 ]; then + echo "dory-k8s did not become Ready" >&2 + exit 1 + fi + sleep 2 + done + + # Job-private kubeconfig only — never touch ~/.kube/config. + kube="$RUNNER_TEMP/hops-dory-ci.kubeconfig" + if [ -f "$HOME/.kube/dory-config" ]; then + cp "$HOME/.kube/dory-config" "$kube" + else + docker exec dory-k8s cat /etc/rancher/k3s/k3s.yaml >"$kube" + if curl -sk --connect-timeout 2 https://127.0.0.1:6443/ >/dev/null 2>&1; then + sed -i.bak 's#server:.*#server: https://127.0.0.1:6443#' "$kube" || true + rm -f "${kube}.bak" + fi + fi + chmod 600 "$kube" + echo "KUBECONFIG=$kube" >> "$GITHUB_ENV" + export KUBECONFIG="$kube" + + # Minimal proof (no docker info / full node dump — personal Mac). + kubectl get nodes --no-headers | awk '{print "node", $1, $2}' + echo "env session ready (DOCKER_HOST + private KUBECONFIG)" + + # No Swatinem/rust-cache: on a personal Mac runner the post-job save + # tars ~100MB+ of target/ and can hang (slow upload, or a password prompt + # with no TTY). Local cargo builds here in ~20s without it. + + - name: Build hops (nice, limited jobs) + run: | + set -euxo pipefail + nice -n 10 cargo build -j "${CARGO_BUILD_JOBS}" + + - name: Host resources run: | set -euxo pipefail sysctl -n hw.ncpu || true sysctl hw.memsize || true uname -m - df -h / || true - echo "DORY_ENGINE_SOURCE=${DORY_ENGINE_SOURCE:-unknown}" + uptime || true + # DOCKER_HOST + KUBECONFIG already in env from prior step + dory readiness || true - name: hops local start --backend dory run: | set -euxo pipefail - export DOCKER_HOST="${DOCKER_HOST:-unix://$HOME/.dory/engine.sock}" - export DORY_ENGINE_SOCK="${DORY_ENGINE_SOCK:-$HOME/.dory/engine.sock}" - # Clean slate: partial dory-k8s can report create-time drift (exit 3). - dory k8s disable || true - docker ps -a || true + # HOPS_DORY_DESKTOP=0 → env session only (no global context changes) ./target/debug/hops-cli local start --backend dory - name: hops local doctor - run: ./target/debug/hops-cli local doctor + run: | + set -euxo pipefail + ./target/debug/hops-cli local doctor - - name: Wait for node + CoreDNS (nested virt settles) + - name: Wait for node + CoreDNS run: | set -euxo pipefail - kubectl --context dory wait --for=condition=Ready nodes --all --timeout=120s - kubectl --context dory -n kube-system wait --for=condition=Ready \ + # Uses job KUBECONFIG only (current-context inside that file). + kubectl wait --for=condition=Ready nodes --all --timeout=120s + kubectl -n kube-system wait --for=condition=Ready \ pod -l k8s-app=kube-dns --timeout=180s || \ - kubectl --context dory -n kube-system wait --for=condition=Ready \ + kubectl -n kube-system wait --for=condition=Ready \ pod -l k8s-app=coredns --timeout=180s || true - kubectl --context dory -n kube-system get pods -o wide || true + kubectl -n kube-system get pods -o wide || true + kubectl -n crossplane-system get pods -o wide || true - - name: Registry round-trip through both pull names + - name: Registry round-trip (engine push + in-cluster pull) run: | set -euxo pipefail - export DOCKER_HOST="unix://$HOME/.dory/engine.sock" + + NODE_IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' dory-k8s)" + test -n "$NODE_IP" + PUSH="${NODE_IP}:30500" + echo "engine push endpoint: $PUSH" + + # Pull + re-materialize as a *single-platform* image for this host. + # A plain `docker pull` of busybox:stable keeps multi-arch index + # metadata; `docker push` then warns: + # Not all multiplatform-content is present and only the available + # single-platform image was pushed + case "$(uname -m)" in + arm64|arm64e|aarch64) PLATFORM=linux/arm64 ;; + x86_64|amd64) PLATFORM=linux/amd64 ;; + *) PLATFORM="linux/$(uname -m)" ;; + esac + SRC=public.ecr.aws/docker/library/busybox:stable for attempt in 1 2 3 4 5; do - if docker pull public.ecr.aws/docker/library/busybox:stable; then + if docker pull --platform "$PLATFORM" "$SRC"; then break fi echo "docker pull failed (attempt $attempt); sleeping..." @@ -236,91 +259,83 @@ jobs: exit 1 fi done - docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci - docker push localhost:30500/smoke/busybox:ci + # Fresh single-platform tag (no incomplete multi-arch index). + docker build --platform "$PLATFORM" --provenance=false --sbom=false \ + -t "${PUSH}/smoke/busybox:ci" - <&1 | head -40 || true - echo "==== cluster ====" - kubectl --context dory get nodes,pods -A -o wide || true - echo "==== smoke pods (default) ====" - kubectl --context dory describe pod smoke-svc-name smoke-localhost 2>/dev/null || true - echo "==== crossplane-system ====" - kubectl --context dory describe pods -n crossplane-system || true - kubectl --context dory get events -A --sort-by=.lastTimestamp | tail -100 || true - ./target/debug/hops-cli local doctor || true - - - name: hops local destroy - if: always() - run: | - set +e - export DOCKER_HOST="unix://$HOME/.dory/engine.sock" - ./target/debug/hops-cli local destroy || true + ./target/debug/hops-cli config install --path "$FIXTURE" --backend dory + + # Wait for Configuration package Healthy. + for i in $(seq 1 90); do + healthy="$(kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Healthy")].status}' 2>/dev/null || true)" + installed="$(kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Installed")].status}' 2>/dev/null || true)" + if [ "$healthy" = "True" ] && [ "$installed" = "True" ]; then + echo "Configuration hops-ops-config-smoke Healthy after ${i} attempts" + break + fi + if [ "$i" -eq 90 ]; then + kubectl get configuration.pkg.crossplane.io hops-ops-config-smoke -o yaml || true + kubectl get configurationrevisions.pkg.crossplane.io -o wide || true + kubectl get functions.pkg.crossplane.io -o wide || true + exit 1 + fi + sleep 5 + done + + # Wait for XRD to be established. + kubectl wait --for=condition=Established \ + crd/configsmokes.ci.hops.ops.com.ai --timeout=180s + + # Apply CI XR into hops-ci only (namespaced ConfigMap). + kubectl apply -f "$FIXTURE/local/ci-xr.yaml" + + # XR Ready (namespaced composite). + kubectl -n hops-ci wait --for=condition=Ready \ + configsmoke.ci.hops.ops.com.ai/hops-ci-smoke --timeout=300s + + # Prove the ConfigMap exists with expected data. + kubectl -n hops-ci get configmap hops-ci-smoke -o yaml + smoke_val="$(kubectl -n hops-ci get configmap hops-ci-smoke \ + -o jsonpath='{.data.smoke}')" + test "$smoke_val" = "ok" - - name: Teardown Dory + Colima + - name: Soft cleanup (CI fixture only) if: always() run: | set +e - dory k8s disable || true - pkill -f 'Dory.app/Contents/MacOS/Dory' || true - colima stop || true - colima delete --force || true - ls -la "$HOME/.dory" || true + kubectl delete pod smoke-svc-name smoke-nodeport --wait=false 2>/dev/null || true + # Remove only hops-ci smoke resources — leave Dory / dory-k8s / Crossplane. + kubectl -n hops-ci delete configsmoke hops-ci-smoke --wait=false 2>/dev/null || true + kubectl -n hops-ci delete configmap hops-ci-smoke --wait=false 2>/dev/null || true + kubectl -n hops-ci delete object hops-ci-smoke --wait=false 2>/dev/null || true + # Leave Configuration package installed (idempotent next run) or drop it: + # kubectl delete configuration.pkg.crossplane.io hops-ops-config-smoke --wait=false diff --git a/.github/workflows/on-pr-kind-smoke.yaml b/.github/workflows/on-pr-kind-smoke.yaml index 354d2e6..9f63322 100644 --- a/.github/workflows/on-pr-kind-smoke.yaml +++ b/.github/workflows/on-pr-kind-smoke.yaml @@ -1,9 +1,7 @@ name: kind backend smoke -# End-to-end check of `hops local` on the kind backend, which is the CI -# path (ubuntu runners ship kind, docker, kubectl, and helm). Exercises the -# registry NodePort mapping and BOTH containerd certs.d trust names, plus the -# stop/start resume path. +# End-to-end check of `hops local` on the kind backend (ubuntu runners). +# Registry round-trip + path-based config install (same fixture as dory smoke). on: pull_request: @@ -17,6 +15,27 @@ jobs: - uses: Swatinem/rust-cache@v2 + - name: Install up CLI + run: | + set -euxo pipefail + # Required for `hops config install --path` (up project build). + curl -sL "https://cli.upbound.io" | sh + echo "$HOME/bin" >> "$GITHUB_PATH" + echo "$HOME/.up/bin" >> "$GITHUB_PATH" + # Installer may place the binary in different locations. + if [ -x "$HOME/bin/up" ]; then + echo "$HOME/bin" >> "$GITHUB_PATH" + elif [ -x /usr/local/bin/up ]; then + true + else + # Fall back: download latest release asset if shell installer differs. + mkdir -p "$HOME/bin" + curl -sL "https://cli.upbound.io/stable/current/linux_amd64/up" -o "$HOME/bin/up" + chmod +x "$HOME/bin/up" + echo "$HOME/bin" >> "$GITHUB_PATH" + fi + up version + - name: Build hops run: cargo build @@ -29,8 +48,14 @@ jobs: - name: Registry round-trip through both pull names run: | set -euxo pipefail - docker pull public.ecr.aws/docker/library/busybox:stable - docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + # Single-platform materialization — avoid multiplatform push warnings. + PLATFORM=linux/amd64 + SRC=public.ecr.aws/docker/library/busybox:stable + docker pull --platform "$PLATFORM" "$SRC" + docker build --platform "$PLATFORM" --provenance=false --sbom=false \ + -t localhost:30500/smoke/busybox:ci - </dev/null || true)" + installed="$(kubectl --context kind-hops get configuration.pkg.crossplane.io hops-ops-config-smoke \ + -o jsonpath='{.status.conditions[?(@.type=="Installed")].status}' 2>/dev/null || true)" + if [ "$healthy" = "True" ] && [ "$installed" = "True" ]; then + echo "Configuration hops-ops-config-smoke Healthy after ${i} attempts" + break + fi + if [ "$i" -eq 90 ]; then + kubectl --context kind-hops get configuration.pkg.crossplane.io hops-ops-config-smoke -o yaml || true + kubectl --context kind-hops get configurationrevisions.pkg.crossplane.io -o wide || true + kubectl --context kind-hops get functions.pkg.crossplane.io -o wide || true + exit 1 + fi + sleep 5 + done + + kubectl --context kind-hops wait --for=condition=Established \ + crd/configsmokes.ci.hops.ops.com.ai --timeout=180s + + kubectl --context kind-hops apply -f "$FIXTURE/local/ci-xr.yaml" + + kubectl --context kind-hops -n hops-ci wait --for=condition=Ready \ + configsmoke.ci.hops.ops.com.ai/hops-ci-smoke --timeout=300s + + kubectl --context kind-hops -n hops-ci get configmap hops-ci-smoke -o yaml + smoke_val="$(kubectl --context kind-hops -n hops-ci get configmap hops-ci-smoke \ + -o jsonpath='{.data.smoke}')" + test "$smoke_val" = "ok" + + - name: Soft cleanup (CI fixture only) + if: always() + run: | + set +e + kubectl --context kind-hops delete pod smoke-svc-name smoke-localhost --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete configsmoke hops-ci-smoke --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete configmap hops-ci-smoke --wait=false 2>/dev/null || true + kubectl --context kind-hops -n hops-ci delete object hops-ci-smoke --wait=false 2>/dev/null || true - name: hops local destroy - run: ./target/debug/hops-cli local destroy + if: always() + run: ./target/debug/hops-cli local destroy || true diff --git a/README.md b/README.md index a95e79d..22703c9 100644 --- a/README.md +++ b/README.md @@ -222,13 +222,14 @@ hops config install --repo hops-ops/aws-auto-eks-cluster --version v0.11.0 daemon: Docker Desktop, colima's dockerd, or CI runners. No VM of its own, so sizing flags don't apply (size the docker daemon instead); requires kind >= v0.27. -- **dory** — [dory](https://augani.github.io/dory)'s built-in k3s, driven - headlessly through the `dory` CLI (`dory k8s enable/disable/status`). - Requires the Dory app running (it provides the engine and forwards - published ports to localhost) and a `dory` CLI with headless k8s support. - hops writes `~/.dory/k8s/registries.yaml` (k3s' native registry trust) and - publishes the registry NodePort at cluster create. The VM is sized in the - Dory app, so hops sizing flags don't apply. +- **dory** — [dory](https://augani.github.io/dory) stock app: shared Apple + Silicon engine + product k3s. Enable Kubernetes **in the Dory app** (hops + does not fork Dory or call `dory k8s enable`). Package installs use the same + **in-cluster** `registry:2` as other backends (Crossplane pulls + `registry.crossplane-system.svc.cluster.local:5000`). Docker push uses the + k3s **NodePort** on the engine docker bridge (`{dory-k8s-ip}:30500`) because + dockerd runs *inside* the engine — Mac `localhost` is the wrong plane. The + VM is sized in the Dory app, so hops sizing flags don't apply. Select with the global `--backend` flag: @@ -243,31 +244,207 @@ persisted choice > existing cluster detection (colima wins) > platform default (macOS: colima, otherwise kind). Unless `--context` is given, kubectl commands automatically use the backend's -kubeconfig context (`colima`, `kind-hops`, or `dory`), regardless of your -current-context. For dory, hops also prepends `~/.kube/dory-config` (where -dory keeps its kubeconfig) to `KUBECONFIG` for its own kubectl/helm calls. +kubeconfig context (`colima`, `kind-hops`, or `hops-dory`), regardless of your +current-context. #### Using dory -With a `dory` CLI that supports `dory k8s enable` (headless Kubernetes), -use the native backend: +Stock Dory only (brew cask / [Dory.app](https://augani.github.io/dory)). No hops +fork of Dory required. ```bash +# 1. Open Dory.app — engine healthy (not "needs attention") +# 2. Enable Kubernetes in the app; wait until the cluster is running +# (product container is usually named dory-k8s) +# 3. Bootstrap Crossplane + local package registry hops local start --backend dory ``` -For `hops provider install` / `hops config install` builds, point your docker -CLI at dory's engine (`export DOCKER_HOST=unix://$HOME/.dory/engine.sock`) so -image builds/pushes land on the daemon that reaches the registry. +On start/activate, hops: -Without the headless CLI, dory still exposes a real docker socket, so the -kind backend works against it: +- merges stock `~/.kube/dory-config` into `~/.kube/config` as context **`hops-dory`** + (override with `--name ` or `HOPS_DORY_NAME`; persisted in `~/.hops/local/dory-name`) +- runs `kubectl config use-context hops-dory` +- creates/uses a docker context of the same name → `unix://$HOME/.dory/dory.sock` + +So you should **not** need: + +```bash +export KUBECONFIG=$HOME/.kube/dory-config +export DOCKER_HOST=unix://$HOME/.dory/dory.sock +``` ```bash -docker context use dory # or: export DOCKER_HOST=unix://$HOME/.dory/dory.sock +hops local start --backend dory # name defaults to hops-dory +hops local start --backend dory --name mine # custom kube+docker context name + +kubectl get nodes # context hops-dory +docker info # context hops-dory +hops local doctor +hops local github -o hops-ops +hops config install --path … --backend dory +``` + +Alternatively, use kind on Dory's docker socket (no product k3s): + +```bash +docker context use dory # product context from Dory.app hops local start --backend kind ``` +**CI:** `.github/workflows/on-pr-dory-smoke.yaml` runs on a **self-hosted** +Apple Silicon Mac labeled `hops-dory` (opt-in with PR label `test-dory` or +`workflow_dispatch`). Stock Dory only — no fork build, no Colima sock. +Offline runner → job waits. The session is **env-only** so your desktop +defaults never change: `DOCKER_HOST=unix://$HOME/.dory/dory.sock`, a +job-private `KUBECONFIG`, and `HOPS_DORY_DESKTOP=0` (hops skips +`use-context` / docker context switching). No `destroy`/`stop` of +`dory-k8s`. After start/doctor/registry, it path-installs the in-repo +fixture `tests/fixtures/config-smoke` (`hops config install --path …`) and +applies a namespaced ConfigMap XR under `hops-ci`. See the workflow header +for runner setup. + +#### Dory architecture (why troubleshooting is different) + +Two network planes matter. Mixing them up is the usual failure mode. + +| Plane | Who | Where | Role | +|-------|-----|--------|------| +| **Engine Docker** | `docker` CLI via `~/.dory/dory.sock` | Linux VM (dockerd) | build/push images | +| **Cluster** | kubectl / Crossplane pods | k3s inside `dory-k8s` | run control plane + pull packages | + +Package registry model (same idea as colima/kind, adapted to Dory): + +- **Pull (Crossplane):** in-cluster Service + `registry.crossplane-system.svc.cluster.local:5000` over **HTTPS** +- **Push (docker on the engine):** k3s **NodePort** on the docker bridge + `{dory-k8s-ip}:30500` (not Mac `localhost:30500`) +- **TLS:** hops generates a self-signed CA/server cert (`Secret hops-local-registry-tls`) + and patches Crossplane to trust that CA. Plain HTTP fails package unpack + (`http: server gave HTTP response to HTTPS client`). + +Do **not** put `host.dory.internal` in Crossplane package refs. That name exists +on the **node** `/etc/hosts`, not in pod CoreDNS, and is the wrong plane for +Crossplane's package manager. + +#### Dory troubleshooting + +**Engine won't start / thrash / `HV_BAD_ARGUMENT` (big Macs)** + +The failure we hit in practice: on a **large laptop** (e.g. **128 GB** host +RAM), Dory's UI **Recommended** / host-scaled default for guest engine memory +was **~half of host RAM → 64 GB** (`DORYD_MEMORY_MB=65536`). That is not +"you need 64 GB of containers" — it is an aggressive ceiling. On Apple Silicon, +Hypervisor.framework often rejects creating a VM that large +(`HV_BAD_ARGUMENT`), so the engine never stays healthy and the UI looks broken +even though a **4 GB** engine works fine. + +```bash +# Pin sane resources (4 GiB / 4 CPUs is enough for hops local) +defaults write com.pythonxi.Dory dory.engineMemoryMB -int 4096 +defaults write com.pythonxi.Dory dory.engineCPUCount -int 4 + +# Confirm the LaunchAgent is not still on 65536: +launchctl print "gui/$(id -u)/dev.dory.doryd" 2>/dev/null | grep DORYD_MEMORY +# Live engine cmdline should show --mem-mb 4096, not 65536: +ps aux | grep 'dory-hv engine' | grep -v grep +``` + +In the Dory app **Resources** panel: set memory to **~4 GB** (8 GB is usually +fine too). **Do not apply Recommended** if it offers tens of GB on a high-RAM +Mac. + +Then fully restart the engine (quit Dory, ensure no thrashing `doryd`/`dory-hv`, +reopen or `dory engine wake`). Once guest RAM is modest, the rest of hops local +is ordinary. + +**Docker socket path** + +Stock Docker API socket is: + +```text +unix://$HOME/.dory/dory.sock +``` + +There is no `~/.dory/engine.sock` for the host Docker API. hops uses `dory.sock` +only. Prefer the docker context hops creates (`hops-dory`) over manual +`DOCKER_HOST`. + +**Kubernetes missing / `no dory-k8s container`** + +k3s is product-owned. hops does not run `dory k8s enable`. + +1. Install the Kubernetes component if needed: `dory component install kubernetes` +2. Enable Kubernetes in the Dory app UI +3. Wait until a `dory-k8s` container is running: `docker ps` (with context hops-dory) +4. Re-run `hops local start --backend dory` + +**`k ctx` has no dory / hops-dory entry** + +Stock Dory writes `~/.kube/dory-config` (context often named `default`). hops +merges that into `~/.kube/config` as **`hops-dory`** on activate/start. If the +merge is missing: + +```bash +hops local doctor --backend dory +kubectl config get-contexts # expect hops-dory +kubectl config use-context hops-dory +``` + +**Configuration Installed=False / HTTPS error** + +```text +http: server gave HTTP response to HTTPS client +``` + +Crossplane always pulls packages with HTTPS. The local registry must be TLS +(hops sets this up). If you have an old HTTP-only registry Deployment: + +```bash +kubectl -n crossplane-system delete deploy registry +kubectl -n crossplane-system delete secret hops-local-registry-tls +hops local start --backend dory # recreates TLS secret + registry + CA patch +``` + +**docker push to localhost:30500 fails (connection refused / HTTPS to HTTP)** + +With the docker context pointed at Dory, **`localhost` is the engine VM**, not +the Mac. hops pushes to `{dory-k8s container IP}:30500` and marks the engine +docker bridge as an insecure registry for that self-signed TLS endpoint. +Check: + +```bash +docker context show # hops-dory +hops local doctor # "package push registry reachable (…:30500)" +``` + +Do not rely on Mac `kubectl port-forward` for engine-side `docker push`. + +**Engine docker broken after daemon.json / dockerd kill** + +Prefer Dory's own recovery: + +```bash +dory repair dockerd --apply +dory engine wake +dory readiness +``` + +Avoid raw `kill` of dockerd inside the guest unless you are prepared to wait for +`dory repair` / LaunchAgent recovery (`live-restore` helps but is not magic). + +**Useful diagnostics** + +```bash +dory doctor +dory readiness +hops local doctor --backend dory +docker context show +kubectl config current-context +kubectl get configuration,provider -A +kubectl -n crossplane-system get deploy,svc,secret | grep -E 'registry|crossplane|hops-local' +``` + ### Local provider setup and auth `hops local aws`, `hops local github`, and `hops local zitadel` install the provider package and bootstrap auth into a local control plane. The exception is `--refresh`, which updates credentials only. @@ -399,12 +576,13 @@ Notes: - Runs `brew install colima`. - `local reset` - Runs `colima kubernetes reset`. -- `local start` - - Runs `colima start --kubernetes --cpu 8 --memory 16 --disk 60` - - Installs Crossplane from `crossplane-stable/crossplane` - - Applies manifests from `bootstrap/` for runtime config, providers, provider configs, and registry (embedded in the binary at build time) - - Configures Docker in Colima for insecure pulls from `registry.crossplane-system.svc.cluster.local:5000` - - Adds host mapping in Colima VM for the registry service DNS name +- `local start [--bootstrap]` + - Brings up the selected backend cluster (colima / kind / dory) + - If Crossplane, k8s+helm providers, and the package registry are already + Healthy/Available, skips helm repo update/upgrade and bootstrap reapply + (fast resume). Pass `--bootstrap` to force a full helm upgrade + reapply. + - Cold path: installs Crossplane, applies `bootstrap/` DRCs/providers/PCs, + deploys the local HTTPS package registry, wires node/engine registry trust - `local stop` - Runs `colima stop`. - `local destroy` diff --git a/bootstrap/registry/registry.yaml b/bootstrap/registry/registry.yaml index 225857b..02cd067 100644 --- a/bootstrap/registry/registry.yaml +++ b/bootstrap/registry/registry.yaml @@ -1,3 +1,6 @@ +# In-cluster package registry for local source installs. +# TLS is required: Crossplane's package manager always uses HTTPS. +# Cert material lives in Secret hops-local-registry-tls (created by hops). apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -10,8 +13,6 @@ spec: requests: storage: 20Gi # Uses the default StorageClass (local-path in colima/k3s). - # Data lives on the colima VM disk, which persists across computer restarts - # (as long as the colima instance/VM itself is not deleted). --- apiVersion: apps/v1 @@ -34,13 +35,40 @@ spec: image: registry:2 ports: - containerPort: 5000 + name: https + env: + # registry:2 serves HTTPS when both are set (still on :5000). + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /certs/tls.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /certs/tls.key volumeMounts: - name: registry-data mountPath: /var/lib/registry + - name: tls + mountPath: /certs + readOnly: true + readinessProbe: + httpGet: + path: /v2/ + port: 5000 + scheme: HTTPS + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /v2/ + port: 5000 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 volumes: - name: registry-data persistentVolumeClaim: claimName: registry-pvc + - name: tls + secret: + secretName: hops-local-registry-tls --- apiVersion: v1 kind: Service @@ -52,6 +80,7 @@ spec: selector: app: registry ports: - - port: 5000 + - name: https + port: 5000 targetPort: 5000 nodePort: 30500 diff --git a/src/commands/config/install.rs b/src/commands/config/install.rs index dd396c2..448eae1 100644 --- a/src/commands/config/install.rs +++ b/src/commands/config/install.rs @@ -4,7 +4,7 @@ use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout, ensure_registry, image_config_name, parse_docker_push_digest, parse_repo_spec, resolve_repo_install_target, rewrite_registry, rewrite_registry_with_tag, sanitize_name_component, short_hash, split_ref, strip_registry, - unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_PULL, REGISTRY_PUSH, + unique_suffix, RepoInstallTarget, RepoSpec, registry_pull, registry_push, }; use crate::commands::local::{kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output}; use clap::Args; @@ -326,7 +326,7 @@ fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Bo continue; } - let push_ref = rewrite_registry(&img.source, REGISTRY_PUSH); + let push_ref = rewrite_registry(&img.source, ®istry_push()); let (img_path, tag) = split_ref(&img.source); // All non-configuration images are Crossplane Function packages (the @@ -339,7 +339,7 @@ fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Bo if tag == arch { let digest = docker_push_and_get_digest(&push_ref)?; - let target_prefix = format!("{}/{}", REGISTRY_PULL, strip_registry(img_path)); + let target_prefix = format!("{}/{}", registry_pull(), strip_registry(img_path)); render_rewrites.insert( img_path.to_string(), RenderRewrite { @@ -387,8 +387,8 @@ spec: } let dev_tag = dev_tag_for_uppkg(&img.uppkg_path)?; - let push_ref = rewrite_registry_with_tag(&img.source, REGISTRY_PUSH, &dev_tag); - let pull_ref = rewrite_registry_with_tag(&img.source, REGISTRY_PULL, &dev_tag); + let push_ref = rewrite_registry_with_tag(&img.source, ®istry_push(), &dev_tag); + let pull_ref = rewrite_registry_with_tag(&img.source, registry_pull(), &dev_tag); log::info!( "Using local build version '{}' for {}...", dev_tag, @@ -909,6 +909,28 @@ fn dev_tag_for_uppkg(uppkg_path: &Path) -> Result> { Ok(format!("dev-{}", &hex[..12])) } +/// Map an image reference's arch tag (e.g. `:amd64` / `:arm64` from +/// `up project build`) to a Docker `--platform` value. +/// +/// Without an explicit platform, buildx on Apple Silicon warns +/// `InvalidBaseImagePlatform` when rebuilding the non-host arch variant +/// (`FROM …:amd64` on arm64, and the reverse on Intel). +fn platform_for_image_ref(src: &str) -> Option<&'static str> { + let (_, tag) = split_ref(src); + // Prefer the trailing path segment when the tag is a digest (rare for + // function images from up, which use :amd64 / :arm64). + let arch = if tag.starts_with("sha256:") { + src.rsplit('/').next().unwrap_or(tag) + } else { + tag + }; + match arch { + "amd64" | "linux/amd64" => Some("linux/amd64"), + "arm64" | "linux/arm64" | "aarch64" => Some("linux/arm64"), + _ => None, + } +} + /// Rebuild a Docker image with just `FROM ` to produce a valid OCI config. /// This fixes images where rootfs.type is empty (a known issue with `up project build` /// render function images). @@ -918,15 +940,21 @@ fn docker_build_from(src: &str, tag: &str) -> Result<(), Box> { // `build_patched_configuration_image`: without these, buildx wraps the // output in a single-arch manifest list that Crossplane (which fetches // package layers as linux/amd64 regardless of host) cannot navigate. + let mut args: Vec = vec![ + "build".into(), + "--provenance=false".into(), + "--sbom=false".into(), + ]; + if let Some(platform) = platform_for_image_ref(src) { + args.push("--platform".into()); + args.push(platform.into()); + } + args.push("-t".into()); + args.push(tag.into()); + args.push("-".into()); + let mut child = Command::new("docker") - .args([ - "build", - "--provenance=false", - "--sbom=false", - "-t", - tag, - "-", - ]) + .args(&args) .stdin(Stdio::piped()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) @@ -969,7 +997,7 @@ fn delete_local_registry_config_revisions(config_name: &str) -> Result<(), Box Result<(), Box< if !rev_name.starts_with(config_name) { continue; } - if !package.contains(REGISTRY_PULL) && state == "Inactive" { + if !package.contains(registry_pull()) && state == "Inactive" { run_cmd( "kubectl", &[ @@ -1104,4 +1132,20 @@ spec: Some("sha256:abcdef") ); } + + #[test] + fn platform_for_image_ref_from_up_arch_tags() { + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke_render:amd64"), + Some("linux/amd64") + ); + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke_render:arm64"), + Some("linux/arm64") + ); + assert_eq!( + platform_for_image_ref("ghcr.io/hops-ops/config-smoke:configuration"), + None + ); + } } diff --git a/src/commands/local/backend/dory.rs b/src/commands/local/backend/dory.rs index 00ad26b..7a81497 100644 --- a/src/commands/local/backend/dory.rs +++ b/src/commands/local/backend/dory.rs @@ -1,32 +1,31 @@ -//! dory backend: k3s in a container on dory's shared-VM dockerd -//! (https://augani.github.io/dory), driven headlessly through the `dory` CLI -//! (`dory k8s enable|disable|status`) and the engine docker socket. +//! dory backend: product k3s on stock Dory (https://augani.github.io/dory). //! -//! Registry plumbing differs from colima/kind because the cluster container -//! has create-time config shared with the Dory app: -//! - published NodePorts are static config: hops writes `~/.dory/k8s/ports` -//! BEFORE `dory k8s enable`, so GUI-side recreates keep the same port. -//! - trust is static config: hops writes `~/.dory/k8s/registries.yaml` -//! BEFORE `dory k8s enable`; dory bind-mounts it and k3s reads it at boot, -//! aliasing both pull names to the registry Service hostname over HTTP. -//! - name resolution is dynamic: `wire_registry` syncs the hostname -> -//! ClusterIP in the node container's /etc/hosts on every start (same -//! re-wire-on-start model as the other backends). -//! - the host reaches `localhost:30500` because `dory k8s enable --publish -//! 30500:30500` publishes the NodePort and the Dory app's port forwarder -//! maps published ports to host loopback. +//! Design (stock Dory only — no hops fork): +//! - Dory.app owns the engine, dockerd, and k3s lifecycle (enable Kubernetes +//! in the app). hops never calls `dory k8s enable`. +//! - hops talks to the engine via `~/.dory/dory.sock` and to the cluster via +//! `~/.kube/dory-config` (context is usually `default`). +//! +//! Package registry (two network planes on Dory): +//! - **Pull (Crossplane pods):** in-cluster Service DNS +//! `registry.crossplane-system.svc.cluster.local:5000`. +//! - **Push (docker via `~/.dory/dory.sock`):** engine dockerd pushes to the +//! k3s NodePort on the docker bridge (`{dory-k8s-ip}:30500`), with that +//! bridge marked insecure (HTTP). Mac localhost is the wrong plane. +//! - **Node image pulls:** k3s `registries.yaml` mirrors → Service ClusterIP. use super::SizeArgs; -use crate::commands::local::package_install::{REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH}; use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use crate::commands::local::package_install::{ + REGISTRY_HOSTNAME, REGISTRY_PULL_INCLUSTER, REGISTRY_PUSH, +}; use std::error::Error; use std::path::PathBuf; +use std::thread; +use std::time::Duration; const NODE_CONTAINER: &str = "dory-k8s"; -/// hops' registry NodePort, published on the cluster container at create. -const REGISTRY_PORT_PUBLISH: &str = "30500:30500"; -const REGISTRIES_BEGIN: &str = "# BEGIN hops-managed (do not edit inside)"; -const REGISTRIES_END: &str = "# END hops-managed"; +const REGISTRY_NODE_PORT: &str = "30500"; fn home() -> Result> { Ok(PathBuf::from(std::env::var("HOME").map_err(|_| { @@ -34,77 +33,18 @@ fn home() -> Result> { })?)) } +/// Stock Dory's host-facing Docker API socket (`~/.dory/dory.sock`). fn engine_socket() -> Result> { - Ok(home()?.join(".dory/engine.sock")) + Ok(home()?.join(".dory/dory.sock")) } -/// dory's side-file kubeconfig (context name `dory`). Current dory also -/// merges the context into ~/.kube/config at enable time; the side file is -/// the pre-merge fallback and dory's own `--kubeconfig` input. +/// Stock Dory writes the cluster kubeconfig here (context is typically `default`). pub fn kubeconfig_path() -> Option { home() .ok() .map(|h| h.join(".kube/dory-config").to_string_lossy().into_owned()) } -fn registries_yaml_path() -> Result> { - Ok(home()?.join(".dory/k8s/registries.yaml")) -} - -fn ports_file_path() -> Result> { - Ok(home()?.join(".dory/k8s/ports")) -} - -/// k3s' native registry config: alias both pull names to the registry -/// Service hostname over plain HTTP. The hostname resolves through the -/// node's /etc/hosts, which `wire_registry` keeps pointed at the ClusterIP. -fn registries_yaml() -> String { - format!( - "# Written by `hops local start --backend dory`.\n\ - # k3s reads this at boot; edits require `hops local reset`.\n\ - # Hops manages only the marked block; keep user mirrors outside it.\n\ - mirrors:\n\ - {block}", - block = registries_yaml_block(), - ) -} - -fn registries_yaml_block() -> String { - format!( - " {begin}\n\ - \x20\x20# hops: mirror {pull}\n\ - \x20\x20\"{pull}\":\n\ - \x20\x20 endpoint:\n\ - \x20\x20 - \"http://{pull}\"\n\ - \x20\x20# hops: mirror {push}\n\ - \x20\x20\"{push}\":\n\ - \x20\x20 endpoint:\n\ - \x20\x20 - \"http://{pull}\"\n\ - \x20\x20{end}\n", - begin = REGISTRIES_BEGIN, - end = REGISTRIES_END, - pull = REGISTRY_PULL, - push = REGISTRY_PUSH, - ) -} - -fn legacy_registries_yaml() -> String { - format!( - "# Written by `hops local start --backend dory`.\n\ - # k3s reads this at boot; edits require `hops local reset`.\n\ - mirrors:\n\ - \x20 \"{pull}\":\n\ - \x20 endpoint:\n\ - \x20 - \"http://{pull}\"\n\ - \x20 \"{push}\":\n\ - \x20 endpoint:\n\ - \x20 - \"http://{pull}\"\n", - pull = REGISTRY_PULL, - push = REGISTRY_PUSH, - ) -} - -/// Run docker against dory's engine socket (the daemon the Dory app manages). fn engine_docker(args: &[&str]) -> Result<(), Box> { let sock = format!("unix://{}", engine_socket()?.display()); let mut full = vec!["-H", sock.as_str()]; @@ -122,7 +62,10 @@ fn engine_docker_output(args: &[&str]) -> Result> { pub fn install() -> Result<(), Box> { log::info!("Installing Dory via Homebrew..."); run_cmd("brew", &["install", "--cask", "Augani/dory/dory"])?; - log::info!("Dory installed; launch the Dory app once so it provisions its engine"); + log::info!( + "Dory installed; open the app, wait until the engine is healthy, \ + enable Kubernetes, then re-run `hops local start --backend dory`" + ); Ok(()) } @@ -143,36 +86,45 @@ pub fn start(size: &SizeArgs) -> Result<(), Box> { } preflight()?; - write_ports_file()?; - write_registries_yaml()?; - - // Creates, restarts, or reuses the dory-k8s container as needed. If the - // running container has create-time config drift, dory exits 3 rather than - // destroying state; surface that plus the hops reset path. - run_dory_enable(false) + ensure_k8s_node_running()?; + wait_for_node_ready()?; + Ok(()) } pub fn stop() -> Result<(), Box> { log::info!("Stopping dory k8s node '{}'...", NODE_CONTAINER); + if !node_exists() { + log::info!("dory k8s node not present"); + return Ok(()); + } engine_docker(&["stop", NODE_CONTAINER])?; log::info!("dory cluster stopped"); Ok(()) } pub fn destroy() -> Result<(), Box> { - log::info!("Deleting dory k8s cluster..."); - run_cmd("dory", &["k8s", "disable"])?; - log::info!("dory cluster deleted"); + log::info!( + "Deleting dory k8s node '{}' (hops does not disable product k8s via CLI)...", + NODE_CONTAINER + ); + // Legacy leftovers from earlier registry experiments. + let _ = engine_docker(&["rm", "-f", "hops-local-registry"]); + let _ = engine_docker(&["rm", "-f", "hops-registry-publish"]); + let _ = engine_docker(&["rm", "-f", NODE_CONTAINER]); + log::info!( + "dory k8s node removed; re-enable Kubernetes in the Dory app if you want the product cluster again" + ); Ok(()) } -/// The cluster container IS the cluster, so reset means recreate. +/// Recreate is app-owned: remove the node; user re-enables k8s in Dory, then start again. pub fn reset() -> Result<(), Box> { preflight()?; - run_cmd("dory", &["k8s", "disable"])?; - write_ports_file()?; - write_registries_yaml()?; - run_dory_enable(true) + destroy()?; + log::info!( + "Enable Kubernetes in the Dory app, then run `hops local start --backend dory` again" + ); + Ok(()) } pub fn resize(_size: &SizeArgs) -> Result<(), Box> { @@ -182,7 +134,6 @@ pub fn resize(_size: &SizeArgs) -> Result<(), Box> { } /// Whether the hops-relevant dory cluster exists (running or stopped). -/// Missing app/engine/CLI reads as "no cluster". pub fn cluster_exists() -> bool { let Ok(sock) = engine_socket() else { return false; @@ -190,366 +141,457 @@ pub fn cluster_exists() -> bool { if !sock.exists() { return false; } - engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]).is_ok() + node_exists() +} + +fn node_exists() -> bool { + engine_docker_output(&["inspect", "-f", "{{.Id}}", NODE_CONTAINER]).is_ok() +} + +fn node_running() -> bool { + engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]) + .map(|state| state.trim() == "true") + .unwrap_or(false) } fn preflight() -> Result<(), Box> { - if !command_exists("dory") { - return Err("the `dory` CLI is not on PATH; link it from the dory repo \ - (`ln -sf /scripts/dory /opt/homebrew/bin/dory`)" - .into()); + if !command_exists("docker") { + return Err("docker CLI not found; install it (Dory provides the daemon)".into()); } let sock = engine_socket()?; if !sock.exists() { return Err(format!( - "dory's engine socket ({}) is missing; launch the Dory app and wait for its engine to start", + "dory's engine socket ({}) is missing.\n\ + Open the Dory app and wait until the engine is healthy (not \"needs attention\"), then retry.", sock.display() ) .into()); } - if !command_exists("docker") { - return Err("docker CLI not found; install it (dory provides the daemon)".into()); + if engine_docker_output(&["info"]).is_err() { + return Err(format!( + "cannot talk to Dory's docker at {}.\n\ + Open the Dory app, fix any engine/doryd errors in the menu, then retry.", + sock.display() + ) + .into()); } Ok(()) } -/// Write the static registry trust config read by k3s at boot. Must exist -/// before `dory k8s enable` because dory binds it at container create. -fn write_registries_yaml() -> Result<(), Box> { - let path = registries_yaml_path()?; - let current = match std::fs::read_to_string(&path) { - Ok(content) => Some(content), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, - Err(err) => return Err(err.into()), - }; - let desired = merge_registries_yaml(current.as_deref())?; - if current.as_deref() == Some(desired.as_str()) { +/// k3s is product-owned: start a stopped node, or tell the user to enable it in the app. +fn ensure_k8s_node_running() -> Result<(), Box> { + if node_running() { + log::info!("dory k8s node '{}' is running", NODE_CONTAINER); return Ok(()); } - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - log::info!("Writing k3s registry config: {}", path.display()); - std::fs::write(&path, desired)?; - Ok(()) -} - -fn write_ports_file() -> Result<(), Box> { - let path = ports_file_path()?; - let current = match std::fs::read_to_string(&path) { - Ok(content) => content, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => String::new(), - Err(err) => return Err(err.into()), - }; - let desired = ports_file_with_publish(¤t); - if current == desired { + if node_exists() { + log::info!("Starting stopped dory k8s node '{}'...", NODE_CONTAINER); + engine_docker(&["start", NODE_CONTAINER])?; return Ok(()); } - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - log::info!( - "Ensuring dory k8s port config includes {}: {}", - REGISTRY_PORT_PUBLISH, - path.display() - ); - std::fs::write(&path, desired)?; - Ok(()) -} - -/// Seconds after the Dory app (re)creates its engine socket during which it is -/// still provisioning the engine. A dockerd restart at the end of that window -/// SIGTERMs every container — including a k3s node enabled meanwhile, which -/// reports Ready and then dies under the bootstrap (observed ~90s on Dory -/// 0.2.0; padded for slower machines). -const ENGINE_LAUNCH_WINDOW_SECS: u64 = 180; - -/// Age of the current engine session: the app recreates the engine socket at -/// launch, so its mtime marks when provisioning began. None when unreadable. -fn engine_session_age() -> Option { - let sock = engine_socket().ok()?; - let modified = std::fs::metadata(&sock).ok()?.modified().ok()?; - std::time::SystemTime::now().duration_since(modified).ok() -} - -/// Time left inside the app's provisioning window for a given engine session -/// age; None once the window has passed. -fn launch_window_remaining(age: std::time::Duration) -> Option { - std::time::Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS) - .checked_sub(age) - .filter(|remaining| !remaining.is_zero()) + Err( + "Dory Kubernetes is not enabled (no `dory-k8s` container).\n\ + In the Dory app: enable Kubernetes, wait until it is running, then re-run:\n\ + hops local start --backend dory\n\ + (hops uses stock Dory only — it does not create the cluster for you.)" + .into(), + ) } -fn node_running() -> bool { - engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]) - .map(|state| state.trim() == "true") - .unwrap_or(false) +fn wait_for_node_ready() -> Result<(), Box> { + log::info!("Waiting for dory k8s node to become Ready..."); + for i in 0..90 { + if !node_running() { + thread::sleep(Duration::from_secs(2)); + continue; + } + let ready = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "kubectl", + "get", + "nodes", + "--no-headers", + ]) + .map(|out| out.contains(" Ready")) + .unwrap_or(false); + if ready { + ensure_side_kubeconfig_hint(); + // Best-effort: merge context "dory" + docker context (also run on activate). + let _ = ensure_desktop_integration(); + return Ok(()); + } + if i > 0 && i % 15 == 0 { + log::info!("Still waiting for k3s Ready ({}s)...", i * 2); + } + thread::sleep(Duration::from_secs(2)); + } + Err( + "timed out waiting for dory-k8s to become Ready; check Kubernetes status in the Dory app" + .into(), + ) } -/// Hold a freshly-enabled cluster under observation while the Dory app may -/// still be provisioning its engine, re-enabling if the engine restart takes -/// the node down. Immediate no-op when the window has already passed, so -/// steady-state starts pay one container inspect and nothing more. -fn hold_through_engine_launch_window() -> Result<(), Box> { - let in_window = |age: Option| { - age.map(|a| launch_window_remaining(a).is_some()) - .unwrap_or(false) +fn ensure_side_kubeconfig_hint() { + let Some(path) = kubeconfig_path() else { + return; }; - if in_window(engine_session_age()) { - log::info!( - "Dory engine session is younger than {}s; watching the k8s node through the app's provisioning window...", - ENGINE_LAUNCH_WINDOW_SECS - ); + if std::path::Path::new(&path).is_file() { + return; } - let mut reenables = 0; - loop { - match (node_running(), in_window(engine_session_age())) { - (true, false) => return Ok(()), - (true, true) => {} - (false, _) => { - if reenables >= 3 { - return Err("the dory engine keeps stopping the k8s node during app startup; \ - wait for the Dory app to finish provisioning, then re-run `hops local start --backend dory`" - .into()); - } - reenables += 1; - log::warn!( - "dory engine restart stopped the k8s node; re-enabling ({}/3)...", - reenables - ); - let args = dory_enable_args(false); - run_cmd("dory", &args)?; + if let Ok(yaml) = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "cat", + "/etc/rancher/k3s/k3s.yaml", + ]) { + if yaml.contains("server:") { + if let Some(parent) = std::path::Path::new(&path).parent() { + let _ = std::fs::create_dir_all(parent); } - } - std::thread::sleep(std::time::Duration::from_secs(3)); - } -} - -fn run_dory_enable(recreate: bool) -> Result<(), Box> { - let args = dory_enable_args(recreate); - match run_cmd("dory", &args) { - Ok(()) => hold_through_engine_launch_window(), - // Exit 3 = create-time config drift (ports / registries bind). hops just - // wrote the desired ports+registries files; applying them requires - // recreate. Auto-retry once so `local start` is not a dead end after - // hops updates publish config (reset remains available explicitly). - // - // Exit 1 after "did not become Ready" is common on stop→start: docker - // start of the existing k3s container can leave the node stuck past - // dory's wait window. Recreate once (same as `dory k8s disable && enable`). - Err(err) if !recreate && should_recreate_on_enable_error(&err.to_string()) => { - log::warn!( - "dory k8s enable failed ({err}); recreating cluster once..." + let _ = std::fs::write(&path, yaml); + let _ = std::fs::set_permissions( + &path, + std::os::unix::fs::PermissionsExt::from_mode(0o600), ); - let _ = run_cmd("dory", &["k8s", "disable"]); - run_dory_enable(true) + log::info!("Wrote kubeconfig side file {}", path); } - Err(err) => Err(err), } } -fn should_recreate_on_enable_error(msg: &str) -> bool { - // run_cmd only surfaces exit status (stderr is inherited), so match codes. - // 3 = create-time config drift; 1 = dory k8s_wait_ready / generic enable fail. - msg.contains("exit status: 3") || msg.contains("exit status: 1") +/// No create-time trust: k3s registries.yaml is written in [`wire_registry`]. +pub fn ensure_registry_trust() -> Result<(), Box> { + Ok(()) +} + +/// After the in-cluster registry Service exists: +/// 1. Point k3s/containerd at the Service ClusterIP over HTTP (function images). +/// 2. Allow engine dockerd to push HTTP to the k3s NodePort on the docker bridge. +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + ensure_k3s_registry_mirrors(cluster_ip)?; + ensure_engine_push_path()?; + Ok(()) +} + +/// Docker push address for package installs when `DOCKER_HOST` is the Dory engine. +/// Reachable on the engine docker bridge (not Mac localhost). +pub fn registry_push_addr() -> Result> { + Ok(format!("{}:{}", node_ip()?, REGISTRY_NODE_PORT)) } -fn dory_enable_args(recreate: bool) -> Vec<&'static str> { - let mut args = vec!["k8s", "enable"]; - if recreate { - args.push("--recreate"); +fn node_ip() -> Result> { + let ip = engine_docker_output(&[ + "inspect", + "-f", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + NODE_CONTAINER, + ])? + .trim() + .to_string(); + if ip.is_empty() { + return Err("dory-k8s has no docker network IP".into()); } - args.extend(["--publish", REGISTRY_PORT_PUBLISH]); - args + Ok(ip) } -fn merge_registries_yaml(existing: Option<&str>) -> Result> { - let Some(existing) = existing else { - return Ok(registries_yaml()); - }; - if existing.trim().is_empty() || existing == legacy_registries_yaml() { - return Ok(registries_yaml()); +/// k3s `registries.yaml` so containerd pulls HTTP from the in-cluster registry. +/// Restarts the node only when the file content changes. +fn ensure_k3s_registry_mirrors(cluster_ip: &str) -> Result<(), Box> { + if !node_running() { + return Err( + "dory k8s node is not running; enable Kubernetes in the Dory app first".into(), + ); + } + + let push = registry_push_addr()?; + // Registry serves HTTPS (self-signed); skip verify on the node for function image pulls. + let yaml = format!( + "mirrors:\n\ + \x20 \"{svc}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + \x20 \"{push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + \x20 \"{default_push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"https://{ip}:5000\"\n\ + configs:\n\ + \x20 \"{ip}:5000\":\n\ + \x20 tls:\n\ + \x20 insecure_skip_verify: true\n\ + \x20 \"{svc}\":\n\ + \x20 tls:\n\ + \x20 insecure_skip_verify: true\n", + svc = REGISTRY_PULL_INCLUSTER, + push = push, + default_push = REGISTRY_PUSH, + ip = cluster_ip, + ); + + let current = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + "cat /etc/rancher/k3s/registries.yaml 2>/dev/null || true", + ]) + .unwrap_or_default(); + if current.trim() == yaml.trim() { + return Ok(()); } - match ( - existing.find(REGISTRIES_BEGIN), - existing.find(REGISTRIES_END), - ) { - (Some(begin), Some(end)) if begin <= end => replace_registries_block(existing, begin, end), - (Some(_), Some(_)) | (Some(_), None) | (None, Some(_)) => { - Err(format!("malformed dory registries.yaml managed block; expected `{REGISTRIES_BEGIN}` before `{REGISTRIES_END}`").into()) + log::info!( + "Configuring k3s registry mirrors for {} -> {}:5000...", + REGISTRY_HOSTNAME, + cluster_ip + ); + engine_docker(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + &format!( + "mkdir -p /etc/rancher/k3s && cat > /etc/rancher/k3s/registries.yaml <<'EOF'\n{yaml}EOF" + ), + ])?; + log::info!("Restarting dory k8s node so k3s reloads registries.yaml..."); + engine_docker(&["restart", NODE_CONTAINER])?; + wait_for_node_ready()?; + for _ in 0..30 { + if run_cmd_output("kubectl", &["get", "--raw", "/readyz"]) + .map(|s| s.contains("ok")) + .unwrap_or(false) + { + break; } - (None, None) => insert_registries_block(existing), + thread::sleep(Duration::from_secs(2)); } + Ok(()) } -fn replace_registries_block( - existing: &str, - begin: usize, - end: usize, -) -> Result> { - let line_start = existing[..begin].rfind('\n').map_or(0, |idx| idx + 1); - let line_end = existing[end..] - .find('\n') - .map_or(existing.len(), |idx| end + idx + 1); - let mut merged = String::with_capacity(existing.len() + registries_yaml_block().len()); - merged.push_str(&existing[..line_start]); - merged.push_str(®istries_yaml_block()); - merged.push_str(&existing[line_end..]); - Ok(merged) +/// Engine dockerd must treat the k3s NodePort as an HTTP registry. Push address +/// is `{dory-k8s-ip}:30500` on the docker bridge (see [`registry_push_addr`]). +fn ensure_engine_push_path() -> Result<(), Box> { + // Best-effort cleanup of earlier experiments (ignore missing). + let _ = engine_docker_output(&["rm", "-f", "hops-local-registry"]); + let _ = engine_docker_output(&["rm", "-f", "hops-registry-publish"]); + + let push = registry_push_addr()?; + ensure_dockerd_insecure_for_push(&push)?; + + // Prove NodePort answers HTTPS from the engine network (self-signed → no-check). + for _ in 0..30 { + if engine_docker_output(&[ + "run", + "--rm", + "alpine:latest", + "wget", + "-qO-", + "--no-check-certificate", + &format!("https://{push}/v2/"), + ]) + .is_ok() + { + log::info!("Engine package push endpoint ready at {} (HTTPS)", push); + return Ok(()); + } + thread::sleep(Duration::from_secs(1)); + } + Err(format!( + "timed out waiting for registry NodePort at https://{push}/v2/ \ + (in-cluster TLS registry Service type NodePort on dory-k8s)" + ) + .into()) } -fn insert_registries_block(existing: &str) -> Result> { - if contains_hops_mirror_key(existing) { - return Err("dory registries.yaml already contains hops registry mirror entries without managed markers; remove those entries or wrap them in the hops-managed block" - .into()); +fn ensure_dockerd_insecure_for_push(push_hostport: &str) -> Result<(), Box> { + // Already configured? + if let Ok(info) = engine_docker_output(&["info", "-f", "{{json .RegistryConfig.InsecureRegistryCIDRs}}{{json .RegistryConfig.IndexConfigs}}"]) + { + if info.contains(push_hostport) || info.contains("192.168.215.0/24") { + return Ok(()); + } } - let mut merged = ensure_trailing_newline(existing); - if let Some(insert_at) = mirrors_line_insert_position(&merged) { - merged.insert_str(insert_at, ®istries_yaml_block()); - return Ok(merged); - } + log::info!( + "Configuring Dory dockerd insecure-registries for HTTP push to {}...", + push_hostport + ); - merged.push_str("mirrors:\n"); - merged.push_str(®istries_yaml_block()); - Ok(merged) -} + // Persist under the engine's /etc/docker (bind-mounted into a helper). + let daemon_json = format!( + r#"{{ + "builder": {{"gc": {{"enabled": true, "defaultKeepStorage": "2GB"}}}}, + "insecure-registries": [ + "127.0.0.0/8", + "localhost:30500", + "192.168.215.0/24", + "{push}" + ] +}} +"#, + push = push_hostport + ); -fn contains_hops_mirror_key(content: &str) -> bool { - content.contains(&format!("\"{}\":", REGISTRY_PULL)) - || content.contains(&format!("\"{}\":", REGISTRY_PUSH)) -} + engine_docker(&[ + "run", + "--rm", + "-v", + "/etc/docker:/etc/docker", + "alpine:latest", + "sh", + "-c", + &format!("cat > /etc/docker/daemon.json <<'EOF'\n{daemon_json}EOF"), + ])?; -fn mirrors_line_insert_position(content: &str) -> Option { - let mut offset = 0; - for line in content.split_inclusive('\n') { - let without_newline = line.trim_end_matches('\n').trim_end_matches('\r'); - if is_top_level_mirrors_line(without_newline) { - return Some(offset + line.len()); + // Prefer Dory's repair path (live-restore) over raw kill. + if command_exists("dory") { + let _ = run_cmd("dory", &["repair", "dockerd", "--apply"]); + } + + // If dockerd did not pick up the file, force a live-restore restart. + let info = engine_docker_output(&["info"]).unwrap_or_default(); + if !info.contains("192.168.215.0/24") && !info.contains(push_hostport) { + log::info!("Forcing dockerd restart so insecure-registries take effect..."); + let _ = engine_docker(&[ + "run", + "--rm", + "--privileged", + "--pid=host", + "alpine:latest", + "sh", + "-c", + "kill -TERM $(pidof dockerd) 2>/dev/null || true", + ]); + if command_exists("dory") { + let _ = run_cmd("dory", &["repair", "dockerd", "--apply"]); + } + for _ in 0..40 { + if engine_docker_output(&["info"]).is_ok() { + break; + } + thread::sleep(Duration::from_secs(2)); } - offset += line.len(); } - None -} -fn is_top_level_mirrors_line(line: &str) -> bool { - line.starts_with("mirrors:") && line.split('#').next().unwrap_or("").trim_end() == "mirrors:" + Ok(()) } -fn ensure_trailing_newline(content: &str) -> String { - let mut out = content.to_string(); - if !out.is_empty() && !out.ends_with('\n') { - out.push('\n'); +/// Default kube + docker context name for Dory desktop integration. +pub const DEFAULT_CONTEXT_NAME: &str = "hops-dory"; +const NAME_FILE: &str = "dory-name"; +const NAME_ENV: &str = "HOPS_DORY_NAME"; + +/// Resolved context name: `--name` (persisted) > `HOPS_DORY_NAME` > file > `hops-dory`. +pub fn context_name() -> String { + if let Ok(v) = std::env::var(NAME_ENV) { + let t = v.trim(); + if !t.is_empty() { + return t.to_string(); + } } - out + if let Ok(path) = crate::commands::local::local_state_dir() { + if let Ok(raw) = std::fs::read_to_string(path.join(NAME_FILE)) { + let t = raw.trim(); + if !t.is_empty() { + return t.to_string(); + } + } + } + DEFAULT_CONTEXT_NAME.to_string() } -#[derive(Debug, PartialEq, Eq)] -struct PortPublish { - host: u16, - container: u16, - proto: String, +/// Persist a user-chosen name for kube/docker context integration. +pub fn persist_context_name(name: &str) -> Result<(), Box> { + let name = validate_context_name(name)?; + let dir = crate::commands::local::local_state_dir()?; + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join(NAME_FILE), format!("{name}\n"))?; + log::info!("Dory desktop name set to '{name}' (kube + docker context)"); + Ok(()) } -fn ports_file_with_publish(existing: &str) -> String { - if ports_file_contains_publish(existing, REGISTRY_PORT_PUBLISH) { - return existing.to_string(); +fn validate_context_name(name: &str) -> Result> { + let name = name.trim(); + if name.is_empty() { + return Err("--name must not be empty".into()); } - - let mut out = ensure_trailing_newline(existing); - out.push_str(REGISTRY_PORT_PUBLISH); - out.push('\n'); - out + // kubectl context names: keep it simple (no path separators / whitespace). + if name.contains(['/', '\\', ' ', '\t', '\n', ':']) { + return Err(format!( + "invalid --name '{name}': use a simple token (e.g. hops-dory)" + ) + .into()); + } + Ok(name.to_string()) } -fn ports_file_contains_publish(existing: &str, desired: &str) -> bool { - let Some(desired) = parse_port_publish(desired) else { - return false; - }; - existing - .lines() - .filter_map(parse_port_publish) - .any(|port| port == desired) -} - -fn parse_port_publish(line: &str) -> Option { - let cleaned: String = line - .split('#') - .next() - .unwrap_or("") - .chars() - .filter(|c| !c.is_whitespace()) - .collect(); - if cleaned.is_empty() { - return None; - } - let (ports, proto) = cleaned - .split_once('/') - .map_or((cleaned.as_str(), "tcp"), |(ports, proto)| (ports, proto)); - if proto != "tcp" && proto != "udp" { - return None; - } - let (host, container) = ports.split_once(':')?; - let host = host.parse().ok().filter(|port| *port > 0)?; - let container = container.parse().ok().filter(|port| *port > 0)?; - Some(PortPublish { - host, - container, - proto: proto.to_string(), - }) -} - -/// Keep the node container's /etc/hosts pointing the registry hostname at -/// the current ClusterIP (docker regenerates /etc/hosts on restart, and the -/// ClusterIP changes if the Service is recreated — hence re-run per start). -pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { - let hostname = REGISTRY_HOSTNAME; - let current_ip = engine_docker_output(&[ - "exec", - NODE_CONTAINER, - "sh", - "-c", - &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), - ]) - .unwrap_or_default(); - if current_ip.trim() == cluster_ip { +/// Env: set `HOPS_DORY_DESKTOP=0` (or `false`/`off`/`no`) to skip mutating the +/// machine's default kube/docker contexts. Callers must drive the session with +/// `DOCKER_HOST` and `KUBECONFIG` (or pass `--context`) instead. +pub const DESKTOP_INTEGRATION_ENV: &str = "HOPS_DORY_DESKTOP"; + +/// Whether hops may rewrite `~/.kube/config` / switch docker contexts. +pub fn desktop_integration_enabled() -> bool { + match std::env::var(DESKTOP_INTEGRATION_ENV) { + Ok(v) => { + let v = v.trim().to_ascii_lowercase(); + !matches!(v.as_str(), "0" | "false" | "no" | "off") + } + Err(_) => true, + } +} + +/// Wire Dory into the normal developer desktop: +/// - merge stock `~/.kube/dory-config` into `~/.kube/config` as context **`hops-dory`** +/// (or `--name` / `HOPS_DORY_NAME`) +/// - `kubectl config use-context ` +/// - ensure/use a docker context of the same name pointing at `~/.dory/dory.sock` +/// +/// Called from backend activate and after local start so you don't need +/// `export KUBECONFIG=...` / `export DOCKER_HOST=...`. +/// +/// No-op for host defaults when [`desktop_integration_enabled`] is false: only +/// fills missing `DOCKER_HOST` / `KUBECONFIG` env for this process tree. +pub fn ensure_desktop_integration() -> Result<(), Box> { + ensure_side_kubeconfig_hint(); + if !desktop_integration_enabled() { + ensure_engine_env_only(); + log::info!( + "Dory desktop integration disabled ({DESKTOP_INTEGRATION_ENV}=0); \ + using DOCKER_HOST / KUBECONFIG only (no use-context, no docker context switch)" + ); return Ok(()); } - - log::info!("Updating node hosts entry: {} -> {}", hostname, cluster_ip); - - // /etc/hosts is a bind mount inside the container: `sed -i` fails with - // "Resource busy" (rename over a mount point), so rewrite it in place. - engine_docker(&[ - "exec", - NODE_CONTAINER, - "sh", - "-c", - &format!( - "awk '$2 != \"{host}\"' /etc/hosts > /tmp/hosts.new && \ - cat /tmp/hosts.new > /etc/hosts && rm -f /tmp/hosts.new && \ - echo '{ip} {host}' >> /etc/hosts", - host = hostname, - ip = cluster_ip - ), - ])?; - + let name = context_name(); + ensure_user_kubeconfig_context(&name)?; + ensure_docker_context_default(&name)?; Ok(()) } -/// Make `--context dory` resolvable. Current dory merges the context into -/// ~/.kube/config at enable time, so normally there is nothing to do — -/// mutating KUBECONFIG for every child process is then pure noise. Older -/// dory versions only write the side file; for those, prepend it to -/// KUBECONFIG (preserving whatever the user already has). -pub fn export_kubeconfig_env() { - if effective_kubeconfig_has_dory_context() { - return; +/// Point this process at the engine without touching desktop defaults. +fn ensure_engine_env_only() { + if std::env::var_os("DOCKER_HOST").is_none() { + if let Ok(sock) = engine_socket() { + if sock.exists() { + std::env::set_var("DOCKER_HOST", format!("unix://{}", sock.display())); + } + } + } + if std::env::var_os("KUBECONFIG").is_none() { + export_kubeconfig_env(); } +} + +/// Fallback when merge fails: point hops child processes at the side file. +pub fn export_kubeconfig_env() { let Some(dory_cfg) = kubeconfig_path() else { return; }; + if !std::path::Path::new(&dory_cfg).is_file() { + return; + } let existing = std::env::var("KUBECONFIG").unwrap_or_default(); if existing.split(':').any(|p| p == dory_cfg) { return; @@ -557,202 +599,178 @@ pub fn export_kubeconfig_env() { let rest = if existing.is_empty() { match home() { Ok(h) => h.join(".kube/config").to_string_lossy().into_owned(), - Err(_) => return, + Err(_) => String::new(), } } else { existing }; - std::env::set_var("KUBECONFIG", format!("{}:{}", dory_cfg, rest)); -} - -/// Whether the kubeconfig(s) kubectl will read without our help — the -/// $KUBECONFIG chain when set, else ~/.kube/config — already define a -/// `dory` entry (i.e. dory's kubectl-merge ran against a file in scope). -fn effective_kubeconfig_has_dory_context() -> bool { - let paths: Vec = match std::env::var("KUBECONFIG") { - Ok(chain) if !chain.is_empty() => chain.split(':').map(PathBuf::from).collect(), - _ => match home() { - Ok(h) => vec![h.join(".kube/config")], - Err(_) => return false, - }, - }; - paths.iter().any(|path| { - std::fs::read_to_string(path) - .map(|content| has_dory_entry(&content)) - .unwrap_or(false) - }) -} - -/// Line-anchored scan for a kubeconfig entry named `dory` — the mapping -/// form (`name: dory`, as kubectl writes context/cluster names) or the -/// sequence-item form (`- name: dory`, the users list). `name: dory-prod` -/// or `username: dory` must not count. -fn has_dory_entry(kubeconfig: &str) -> bool { - kubeconfig.lines().any(|line| { - let trimmed = line.trim(); - trimmed == "name: dory" || trimmed == "- name: dory" - }) + if rest.is_empty() { + std::env::set_var("KUBECONFIG", &dory_cfg); + } else { + std::env::set_var("KUBECONFIG", format!("{}:{}", dory_cfg, rest)); + } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn launch_window_remaining_covers_only_the_provisioning_window() { - use std::time::Duration; - - assert_eq!( - launch_window_remaining(Duration::ZERO), - Some(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)) - ); - assert_eq!( - launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS - 1)), - Some(Duration::from_secs(1)) - ); - assert_eq!( - launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)), - None - ); - assert_eq!(launch_window_remaining(Duration::from_secs(3600)), None); +/// Merge the stock Dory kubeconfig into `~/.kube/config` as the given context name. +fn ensure_user_kubeconfig_context(name: &str) -> Result<(), Box> { + let side = kubeconfig_path().ok_or("HOME unset")?; + if !std::path::Path::new(&side).is_file() { + return Err(format!( + "missing {}; enable Kubernetes in the Dory app (or let hops start write it)", + side + ) + .into()); } - #[test] - fn has_dory_entry_matches_mapping_and_sequence_forms_only() { - let merged = "contexts:\n- context:\n cluster: dory\n user: dory\n name: dory\n"; - let users_list = "users:\n- name: dory\n user: {}\n"; - let near_misses = "name: dory-prod\nusername: dory\n# name: dory\nfullname: dory\n"; - - assert!(has_dory_entry(merged)); - assert!(has_dory_entry(users_list)); - assert!(!has_dory_entry(near_misses)); - assert!(!has_dory_entry("")); + let home = home()?; + let kube_dir = home.join(".kube"); + std::fs::create_dir_all(&kube_dir)?; + let main = kube_dir.join("config"); + + // Work on a temp copy so we can rename stock contexts without mutating + // the product side file. + let tmp = kube_dir.join(format!(".dory-merge-{}.yaml", std::process::id())); + std::fs::copy(&side, &tmp)?; + let names = run_cmd_output( + "kubectl", + &[ + "config", + "--kubeconfig", + tmp.to_str().unwrap(), + "get-contexts", + "-o", + "name", + ], + ) + .unwrap_or_default(); + for old in names.lines().map(str::trim).filter(|n| !n.is_empty()) { + if old != name { + let _ = run_cmd( + "kubectl", + &[ + "config", + "--kubeconfig", + tmp.to_str().unwrap(), + "rename-context", + old, + name, + ], + ); + } } - #[test] - fn registries_yaml_aliases_both_pull_names_to_the_service_over_http() { - let yaml = registries_yaml(); - - assert!(yaml.contains(REGISTRIES_BEGIN)); - assert!(yaml.contains(REGISTRIES_END)); - assert!(yaml.contains("\"registry.crossplane-system.svc.cluster.local:5000\":")); - assert!(yaml.contains("\"localhost:30500\":")); - // Both mirrors resolve to the same HTTP endpoint on the Service name. - assert_eq!( - yaml.matches("- \"http://registry.crossplane-system.svc.cluster.local:5000\"") - .count(), - 2 + if main.is_file() { + let merged = { + let mut child = std::process::Command::new("kubectl") + .args(["config", "view", "--flatten", "--raw"]) + .env( + "KUBECONFIG", + format!("{}:{}", main.display(), tmp.display()), + ) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + let mut out = String::new(); + use std::io::Read; + if let Some(mut s) = child.stdout.take() { + s.read_to_string(&mut out)?; + } + let status = child.wait()?; + if !status.success() { + let _ = std::fs::remove_file(&tmp); + return Err( + "kubectl config view --flatten failed while merging dory kubeconfig".into(), + ); + } + out + }; + let backup = kube_dir.join("config.hops-dory-backup"); + let _ = std::fs::copy(&main, &backup); + std::fs::write(&main, merged)?; + let _ = std::fs::set_permissions( + &main, + std::os::unix::fs::PermissionsExt::from_mode(0o600), ); - assert!(!yaml.contains("https://")); - } - - #[test] - fn registries_yaml_replaces_only_hops_managed_block() { - let existing = format!( - "configs:\n example: value\nmirrors:\n {begin}\n old: value\n {end}\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\n", - begin = REGISTRIES_BEGIN, - end = REGISTRIES_END, + } else { + std::fs::copy(&tmp, &main)?; + let _ = std::fs::set_permissions( + &main, + std::os::unix::fs::PermissionsExt::from_mode(0o600), ); - - let merged = merge_registries_yaml(Some(&existing)).unwrap(); - - assert!(merged.starts_with("configs:\n example: value\nmirrors:\n")); - assert!(merged.contains(" \"user.local:5000\":\n")); - assert!(!merged.contains("old: value")); - assert!(merged.contains(&format!(" \"{}\":", REGISTRY_PULL))); - assert_eq!(merged.matches(REGISTRIES_BEGIN).count(), 1); - assert_eq!(merged.matches(REGISTRIES_END).count(), 1); } + let _ = std::fs::remove_file(&tmp); - #[test] - fn registries_yaml_inserts_block_under_existing_mirrors_key() { - let existing = "mirrors:\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\nconfigs:\n another: value\n"; - - let merged = merge_registries_yaml(Some(existing)).unwrap(); - - let block_pos = merged.find(REGISTRIES_BEGIN).unwrap(); - let user_pos = merged.find("\"user.local:5000\"").unwrap(); - assert!(block_pos < user_pos); - assert!(merged.contains("configs:\n another: value\n")); + // Drop the short-lived "dory" name from an earlier hops revision if present + // and distinct from the configured name. + if name != "dory" { + let _ = run_cmd("kubectl", &["config", "delete-context", "dory"]); } - #[test] - fn registries_yaml_appends_mirrors_section_when_missing() { - let existing = "configs:\n example: value\n"; - - let merged = merge_registries_yaml(Some(existing)).unwrap(); + let _ = run_cmd("kubectl", &["config", "use-context", name]); + log::info!( + "Kubernetes context '{}' is ready in ~/.kube/config (also: ~/.kube/dory-config)", + name + ); + Ok(()) +} - assert!(merged.starts_with(existing)); - assert!(merged.contains("\nmirrors:\n")); - assert!(merged.contains(REGISTRIES_BEGIN)); +/// Docker context of the same name, pointing at Dory's engine socket. +fn ensure_docker_context_default(name: &str) -> Result<(), Box> { + if !command_exists("docker") { + return Ok(()); } - - #[test] - fn registries_yaml_upgrades_legacy_hops_file_to_managed_block() { - let merged = merge_registries_yaml(Some(&legacy_registries_yaml())).unwrap(); - - assert_eq!(merged, registries_yaml()); - assert_eq!( - merged.matches(&format!("\"{}\":", REGISTRY_PULL)).count(), - 1 - ); - assert_eq!( - merged.matches(&format!("\"{}\":", REGISTRY_PUSH)).count(), - 1 + let sock = engine_socket()?; + if !sock.exists() { + log::warn!( + "Dory engine socket {} missing; skip docker context '{}'", + sock.display(), + name ); + return Ok(()); } - - #[test] - fn ports_file_appends_registry_port_without_touching_existing_lines() { - let existing = "# user port\n8080:80/udp\n"; - - let merged = ports_file_with_publish(existing); - - assert_eq!(merged, "# user port\n8080:80/udp\n30500:30500\n"); - } - - #[test] - fn ports_file_absent_writes_only_registry_port() { - assert_eq!(ports_file_with_publish(""), "30500:30500\n"); + let host = format!("host=unix://{}", sock.display()); + let contexts = run_cmd_output("docker", &["context", "ls", "--format", "{{.Name}}"]) + .unwrap_or_default(); + if !contexts.lines().any(|n| n.trim() == name) { + log::info!( + "Creating docker context '{}' → unix://{}", + name, + sock.display() + ); + // Ignore failure if a stale context exists with different metadata. + let create = run_cmd( + "docker", + &["context", "create", name, "--docker", &host], + ); + if create.is_err() { + // Update in place when possible. + let _ = run_cmd( + "docker", + &["context", "update", name, "--docker", &host], + ); + } } + run_cmd("docker", &["context", "use", name])?; + log::info!( + "Docker context set to '{}' (unix://{})", + name, + sock.display() + ); + Ok(()) +} - #[test] - fn ports_file_treats_tcp_variant_as_already_present() { - let existing = " 30500 : 30500 / tcp # registry\n"; - - assert_eq!(ports_file_with_publish(existing), existing); - } +#[cfg(test)] +mod tests { + use super::*; #[test] - fn dory_enable_args_only_recreate_for_reset_path() { + fn package_pull_is_in_cluster_service_not_host_gateway() { assert_eq!( - dory_enable_args(false), - vec!["k8s", "enable", "--publish", REGISTRY_PORT_PUBLISH] + crate::commands::local::package_install::REGISTRY_PULL_INCLUSTER, + "registry.crossplane-system.svc.cluster.local:5000" ); - assert_eq!( - dory_enable_args(true), - vec![ - "k8s", - "enable", - "--recreate", - "--publish", - REGISTRY_PORT_PUBLISH - ] - ); - } - - #[test] - fn recreate_on_enable_matches_dory_exit_codes() { - assert!(should_recreate_on_enable_error( - "dory exited with exit status: 3" - )); - assert!(should_recreate_on_enable_error( - "dory exited with exit status: 1" - )); - assert!(!should_recreate_on_enable_error( - "dory exited with exit status: 2" - )); - assert!(!should_recreate_on_enable_error("connection refused")); + assert!(!REGISTRY_PULL_INCLUSTER.contains("dory.internal")); + assert_eq!(REGISTRY_PUSH, "localhost:30500"); } #[test] @@ -768,4 +786,29 @@ mod tests { assert!(err.to_string().contains("--cpus 4")); assert!(err.to_string().contains("Dory app")); } + + #[test] + fn missing_k8s_error_mentions_app_not_fork() { + let msg = "Dory Kubernetes is not enabled (no `dory-k8s` container).\n\ + In the Dory app: enable Kubernetes, wait until it is running, then re-run:\n\ + hops local start --backend dory\n\ + (hops uses stock Dory only — it does not create the cluster for you.)"; + assert!(msg.contains("Dory app")); + assert!(!msg.contains("feat/scriptable")); + assert!(!msg.contains("dory k8s enable")); + } + + #[test] + fn desktop_integration_env_off_values() { + for off in ["0", "false", "FALSE", "no", "off", " Off "] { + std::env::set_var(DESKTOP_INTEGRATION_ENV, off); + assert!( + !desktop_integration_enabled(), + "expected desktop off for {off:?}" + ); + } + std::env::set_var(DESKTOP_INTEGRATION_ENV, "1"); + assert!(desktop_integration_enabled()); + std::env::remove_var(DESKTOP_INTEGRATION_ENV); + } } diff --git a/src/commands/local/backend/kind.rs b/src/commands/local/backend/kind.rs index 2a22f62..005172d 100644 --- a/src/commands/local/backend/kind.rs +++ b/src/commands/local/backend/kind.rs @@ -207,8 +207,9 @@ pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { } fn hosts_toml(cluster_ip: &str) -> String { + // Local registry serves HTTPS with a hops-managed self-signed cert. format!( - "[host.\"http://{}:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n", + "[host.\"https://{}:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n", cluster_ip ) } @@ -277,12 +278,14 @@ mod tests { } #[test] - fn hosts_toml_aliases_to_cluster_ip_over_http() { + fn hosts_toml_aliases_to_cluster_ip_over_https() { + // Local package registry is HTTPS (self-signed); containerd must use + // https:// + skip_verify, not plain http://. let toml = hosts_toml("10.43.12.7"); assert_eq!( toml, - "[host.\"http://10.43.12.7:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n" + "[host.\"https://10.43.12.7:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n" ); } diff --git a/src/commands/local/backend/mod.rs b/src/commands/local/backend/mod.rs index 2e2427f..e0b2098 100644 --- a/src/commands/local/backend/mod.rs +++ b/src/commands/local/backend/mod.rs @@ -78,11 +78,12 @@ impl Backend { } /// kubeconfig context name this backend's cluster registers under. - pub fn kube_context(self) -> &'static str { + pub fn kube_context(self) -> String { match self { - Backend::Colima => "colima", - Backend::Kind => "kind-hops", - Backend::Dory => "dory", + Backend::Colima => "colima".to_string(), + Backend::Kind => "kind-hops".to_string(), + // Merged into ~/.kube/config (default name hops-dory; see dory::context_name). + Backend::Dory => dory::context_name(), } } @@ -161,15 +162,16 @@ impl Backend { // containerd trust is per-name via certs.d, written in // wire_registry once the registry Service's ClusterIP is known. Backend::Kind => Ok(()), - // trust is static registries.yaml, written by dory::start before - // the cluster boots (k3s reads it at boot). - Backend::Dory => Ok(()), + // k3s registries.yaml is written in wire_registry once ClusterIP is known. + Backend::Dory => dory::ensure_registry_trust(), } } /// Point the node at the registry Service's current ClusterIP so pulls of /// both registry names resolve. Idempotent; re-run on every start because /// the ClusterIP changes if the Service is recreated. + /// + /// Dory also publishes host localhost:30500 → node NodePort (engine proxy). pub fn wire_registry(self, cluster_ip: &str) -> Result<(), Box> { match self { Backend::Colima => colima::sync_hosts_entry(cluster_ip), @@ -177,6 +179,31 @@ impl Backend { Backend::Dory => dory::wire_registry(cluster_ip), } } + + /// Backend-specific package registry for local provider/config installs. + /// All backends use an in-cluster NodePort registry for Crossplane package + /// pulls (pod network). Host push is always localhost:30500. + pub fn ensure_package_registry(self) -> Result<(), Box> { + crate::commands::local::package_install::ensure_incluster_registry() + } + + /// Pull address used in Crossplane package / ImageConfig refs for this backend. + /// Always the in-cluster Service — Crossplane runs in the pod network. + pub fn registry_pull(self) -> &'static str { + crate::commands::local::package_install::REGISTRY_PULL_INCLUSTER + } + + /// Docker push host:port for package installs on this backend. + pub fn registry_push(self) -> String { + match self { + Backend::Dory => dory::registry_push_addr().unwrap_or_else(|_| { + crate::commands::local::package_install::REGISTRY_PUSH.to_string() + }), + Backend::Colima | Backend::Kind => { + crate::commands::local::package_install::REGISTRY_PUSH.to_string() + } + } + } } impl fmt::Display for Backend { @@ -203,6 +230,11 @@ impl FromStr for Backend { const BACKEND_FILE: &str = "backend"; +/// Persist the Dory desktop context name (`--name`, default hops-dory). +pub fn persist_dory_context_name(name: &str) -> Result<(), Box> { + dory::persist_context_name(name) +} + pub fn platform_default() -> Backend { if cfg!(target_os = "macos") { Backend::Colima @@ -231,17 +263,22 @@ pub fn activate(flag: Option, context: Option<&str>) -> Backend { let backend = resolve(flag); if backend == Backend::Dory { - dory::export_kubeconfig_env(); + // Merge ~/.kube/dory-config → named context and prefer matching docker context. + if let Err(e) = dory::ensure_desktop_integration() { + log::warn!("dory desktop integration incomplete: {e}"); + dory::export_kubeconfig_env(); + } } let explicit_context = context.filter(|ctx| !ctx.is_empty()); + let backend_context = backend.kube_context(); let backend_context_exists = if explicit_context.is_none() { - kube_context_exists(backend.kube_context()) + kube_context_exists(&backend_context) } else { false }; - match kube_context_export(backend, explicit_context, backend_context_exists) { + match kube_context_export(explicit_context, &backend_context, backend_context_exists) { KubeContextExport::Set(ctx) => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx), KubeContextExport::Unset { missing_context } => { std::env::remove_var(HOPS_KUBE_CONTEXT_ENV); @@ -331,7 +368,7 @@ fn registry_cluster_ip() -> Result> { } /// Wire node-level registry pulls for a backend: fetch the registry Service -/// ClusterIP and apply the backend-specific trust/aliasing. +/// ClusterIP and apply the backend-specific trust/aliasing (and Dory host publish). pub fn wire_local_registry(backend: Backend) -> Result<(), Box> { backend.wire_registry(®istry_cluster_ip()?) } @@ -369,19 +406,18 @@ pub fn wire_local_registry_for_target( #[derive(Debug, PartialEq, Eq)] enum KubeContextExport<'a> { Set(&'a str), - Unset { missing_context: &'static str }, + Unset { missing_context: &'a str }, } fn kube_context_export<'a>( - backend: Backend, explicit_context: Option<&'a str>, + backend_context: &'a str, backend_context_exists: bool, ) -> KubeContextExport<'a> { if let Some(ctx) = explicit_context { return KubeContextExport::Set(ctx); } - let backend_context = backend.kube_context(); if backend_context_exists { KubeContextExport::Set(backend_context) } else { @@ -488,7 +524,7 @@ mod tests { #[test] fn explicit_context_is_exported_even_when_backend_context_is_absent() { assert_eq!( - kube_context_export(Backend::Colima, Some("foreign"), false), + kube_context_export(Some("foreign"), "colima", false), KubeContextExport::Set("foreign") ); } @@ -496,11 +532,11 @@ mod tests { #[test] fn backend_context_is_exported_only_when_present() { assert_eq!( - kube_context_export(Backend::Kind, None, true), + kube_context_export(None, "kind-hops", true), KubeContextExport::Set("kind-hops") ); assert_eq!( - kube_context_export(Backend::Kind, None, false), + kube_context_export(None, "kind-hops", false), KubeContextExport::Unset { missing_context: "kind-hops" } diff --git a/src/commands/local/doctor.rs b/src/commands/local/doctor.rs index 248c588..dcbe8d1 100644 --- a/src/commands/local/doctor.rs +++ b/src/commands/local/doctor.rs @@ -72,6 +72,17 @@ pub fn run() -> Result<(), Box> { "registry deployment not Available in crossplane-system".into() }, ); + let push = super::package_install::registry_push(); + let push_ok = package_push_registry_reachable(&push); + d.check( + &format!("package push registry reachable ({push})"), + push_ok, + if push_ok { + String::new() + } else { + format!("expected HTTPS registry at {push} for docker push") + }, + ); d.print(); @@ -289,6 +300,69 @@ fn provider_package(provider: &str) -> String { .unwrap_or_default() } +/// Whether `push` is a host-loopback address (kind/colima NodePort). +/// +/// Dory pushes via the engine docker bridge (`{dory-k8s-ip}:30500`); that IP is +/// not reachable from the Mac, so host curl is the wrong plane and can hang on +/// TCP timeout (~75s) before the engine probe runs. +fn push_host_is_loopback(push: &str) -> bool { + // host:port — IPv6 may be bracketed as [::1]:30500 + let host = if let Some(rest) = push.strip_prefix('[') { + rest.split(']').next().unwrap_or(rest) + } else { + push.split(':').next().unwrap_or(push) + }; + host == "localhost" || host == "127.0.0.1" || host == "::1" +} + +/// Probe the package push registry on the plane docker will use. +/// +/// - Loopback (`localhost:30500`): short host curl (kind/colima). +/// - Otherwise (dory bridge IP): engine-network probe only — never host curl +/// (wrong plane; TCP hang ~75s) and never default docker.sock (can block on +/// password / missing daemon on a personal Mac). +fn package_push_registry_reachable(push: &str) -> bool { + let url = format!("https://{push}/v2/"); + if push_host_is_loopback(push) { + return super::run_cmd_output( + "curl", + &["-skf", "--connect-timeout", "2", "--max-time", "3", &url], + ) + .is_ok(); + } + + // Prefer Dory's engine socket explicitly so we don't touch host Docker. + let dory_sock = std::env::var("HOME") + .ok() + .map(|h| std::path::PathBuf::from(h).join(".dory/dory.sock")) + .filter(|p| p.exists()); + + let mut args: Vec = Vec::new(); + if let Some(sock) = dory_sock { + args.push("-H".into()); + args.push(format!("unix://{}", sock.display())); + } else if std::env::var_os("DOCKER_HOST").is_none() { + // No engine socket and no DOCKER_HOST: skip rather than hang on host sock. + return false; + } + args.extend( + [ + "run", + "--rm", + "alpine:latest", + "wget", + "-qO-", + "--no-check-certificate", + "--timeout=5", + &url, + ] + .into_iter() + .map(String::from), + ); + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + super::run_cmd_output("docker", &arg_refs).is_ok() +} + /// True when `kubectl get ` finds the resource. Uses `--ignore-not-found` /// so a missing resource is `false` rather than an error. fn exists(get_args: &[&str]) -> bool { @@ -385,3 +459,21 @@ impl Doctor { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn push_host_loopback_for_kind_colima() { + assert!(push_host_is_loopback("localhost:30500")); + assert!(push_host_is_loopback("127.0.0.1:30500")); + assert!(push_host_is_loopback("[::1]:30500")); + } + + #[test] + fn push_host_not_loopback_for_dory_bridge() { + assert!(!push_host_is_loopback("192.168.215.2:30500")); + assert!(!push_host_is_loopback("10.43.0.1:30500")); + } +} diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index abb27ac..8cc2751 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -107,6 +107,12 @@ pub struct LocalArgs { /// kind). #[arg(long, global = true, value_enum)] pub backend: Option, + + /// Name for Dory desktop integration (kube context + docker context). + /// Defaults to `hops-dory`. Persisted under `~/.hops/local/dory-name`. + /// Only used with `--backend dory` (or a persisted dory backend). + #[arg(long, global = true, value_name = "NAME")] + pub name: Option, } #[derive(Subcommand, Debug)] @@ -115,7 +121,7 @@ pub enum LocalCommands { Install, /// Reset local Kubernetes state (colima: k8s reset; kind: recreate cluster) Reset, - /// Start local k8s cluster with Crossplane and providers + /// Start local k8s and ensure Crossplane control plane (skips helm when already healthy) Start(start::StartArgs), /// Resize the local cluster VM without destroying cluster state (colima only) Resize(resize::ResizeArgs), @@ -140,12 +146,16 @@ pub enum LocalCommands { } pub fn run(args: &LocalArgs) -> Result<(), Box> { + if let Some(name) = args.name.as_deref().map(str::trim).filter(|s| !s.is_empty()) { + backend::persist_dory_context_name(name)?; + } + let explicit_context = args.context.as_deref().filter(|ctx| !ctx.is_empty()); let install_backend = matches!(&args.command, LocalCommands::Install) .then(|| args.backend.unwrap_or_else(backend::platform_default)); let activation_flag = install_backend.or(args.backend); - let install_context = install_backend.map(backend::Backend::kube_context); - let activation_context = explicit_context.or(install_context); + let install_context = install_backend.map(|b| b.kube_context()); + let activation_context = explicit_context.or(install_context.as_deref()); let backend = backend::activate(activation_flag, activation_context); match &args.command { diff --git a/src/commands/local/package_install.rs b/src/commands/local/package_install.rs index dd25544..1af9b3b 100644 --- a/src/commands/local/package_install.rs +++ b/src/commands/local/package_install.rs @@ -11,13 +11,29 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const REGISTRY_YAML: &str = include_str!("../../../bootstrap/registry/registry.yaml"); -/// Host address for `docker push` (NodePort exposed by the in-cluster registry) +/// Default docker push host for colima/kind (NodePort on localhost). +/// On Dory use [`registry_push`] — engine dockerd is not the Mac. pub const REGISTRY_PUSH: &str = "localhost:30500"; -/// Cluster-internal address used in Crossplane package references -pub const REGISTRY_PULL: &str = "registry.crossplane-system.svc.cluster.local:5000"; +/// Cluster-internal address used in Crossplane package references (all backends). +pub const REGISTRY_PULL_INCLUSTER: &str = + "registry.crossplane-system.svc.cluster.local:5000"; pub const REGISTRY_HOSTNAME: &str = "registry.crossplane-system.svc.cluster.local"; +/// Back-compat alias — prefer [`registry_pull`] when the backend is known. +pub const REGISTRY_PULL: &str = REGISTRY_PULL_INCLUSTER; + +/// Resolve the package pull address for the active local backend. +pub fn registry_pull() -> &'static str { + super::backend::resolve(None).registry_pull() +} + +/// Docker push registry host:port for the active backend. +/// Colima/kind → `localhost:30500`. Dory → `{dory-k8s-ip}:30500` on the engine bridge. +pub fn registry_push() -> String { + super::backend::resolve(None).registry_push() +} + #[derive(Clone, Debug)] pub struct RepoSpec { pub org: String, @@ -169,31 +185,25 @@ pub fn image_config_name(source: &str) -> String { format!("{prefix}{body}-{hash}") } -/// Ensure the in-cluster registry is deployed and available. +/// TLS Secret for the local package registry (server cert + CA). +pub const REGISTRY_TLS_SECRET: &str = "hops-local-registry-tls"; +const REGISTRY_TLS_SECRET_NS: &str = "crossplane-system"; + +/// Ensure the local package registry for the active backend is available. pub fn ensure_registry() -> Result<(), Box> { - let result = run_cmd_output( - "kubectl", - &[ - "get", - "deployment", - "registry", - "-n", - "crossplane-system", - "-o", - "jsonpath={.status.availableReplicas}", - ], - ); + super::backend::resolve(None).ensure_package_registry() +} - if let Ok(replicas) = result { - if replicas.trim() == "1" { - return Ok(()); - } - } +/// Ensure the in-cluster HTTPS NodePort registry is deployed and Crossplane +/// trusts its CA. Crossplane's package manager always uses HTTPS. +pub fn ensure_incluster_registry() -> Result<(), Box> { + ensure_namespace(REGISTRY_TLS_SECRET_NS)?; + ensure_registry_tls_secret()?; - log::info!("Deploying local package registry..."); + log::info!("Deploying local package registry (HTTPS)..."); kubectl_apply_stdin(REGISTRY_YAML)?; - for _ in 0..60 { + for _ in 0..90 { let out = run_cmd_output( "kubectl", &[ @@ -201,13 +211,14 @@ pub fn ensure_registry() -> Result<(), Box> { "deployment", "registry", "-n", - "crossplane-system", + REGISTRY_TLS_SECRET_NS, "-o", "jsonpath={.status.availableReplicas}", ], ); if let Ok(r) = out { if r.trim() == "1" { + ensure_crossplane_trusts_local_registry_ca()?; return Ok(()); } } @@ -217,6 +228,260 @@ pub fn ensure_registry() -> Result<(), Box> { Err("Timed out waiting for registry deployment".into()) } +fn ensure_namespace(ns: &str) -> Result<(), Box> { + let exists = run_cmd_output("kubectl", &["get", "ns", ns, "-o", "name"]).is_ok(); + if exists { + return Ok(()); + } + run_cmd("kubectl", &["create", "namespace", ns]) +} + +/// Create (or reuse) a self-signed CA + server cert for the registry Service DNS. +fn ensure_registry_tls_secret() -> Result<(), Box> { + let exists = run_cmd_output( + "kubectl", + &[ + "get", + "secret", + REGISTRY_TLS_SECRET, + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "name", + ], + ) + .is_ok(); + if exists { + return Ok(()); + } + + log::info!("Generating self-signed TLS for local package registry..."); + let dir = std::env::temp_dir().join(format!( + "hops-registry-tls-{}", + unique_suffix() + )); + fs::create_dir_all(&dir)?; + let ca_key = dir.join("ca.key"); + let ca_crt = dir.join("ca.crt"); + let tls_key = dir.join("tls.key"); + let tls_csr = dir.join("tls.csr"); + let tls_crt = dir.join("tls.crt"); + let san = dir.join("san.cnf"); + + fs::write( + &san, + "[req]\n\ + distinguished_name = dn\n\ + req_extensions = ext\n\ + prompt = no\n\ + [dn]\n\ + CN = registry.crossplane-system.svc.cluster.local\n\ + [ext]\n\ + subjectAltName = @alt\n\ + basicConstraints = CA:FALSE\n\ + keyUsage = digitalSignature, keyEncipherment\n\ + extendedKeyUsage = serverAuth\n\ + [alt]\n\ + DNS.1 = registry\n\ + DNS.2 = registry.crossplane-system\n\ + DNS.3 = registry.crossplane-system.svc\n\ + DNS.4 = registry.crossplane-system.svc.cluster.local\n\ + DNS.5 = localhost\n\ + IP.1 = 127.0.0.1\n", + )?; + + run_cmd( + "openssl", + &[ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + ca_key.to_str().unwrap(), + "-out", + ca_crt.to_str().unwrap(), + "-days", + "3650", + "-subj", + "/CN=hops-local-registry-ca", + ], + )?; + run_cmd( + "openssl", + &[ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + tls_key.to_str().unwrap(), + "-out", + tls_csr.to_str().unwrap(), + "-config", + san.to_str().unwrap(), + ], + )?; + run_cmd( + "openssl", + &[ + "x509", + "-req", + "-in", + tls_csr.to_str().unwrap(), + "-CA", + ca_crt.to_str().unwrap(), + "-CAkey", + ca_key.to_str().unwrap(), + "-CAcreateserial", + "-out", + tls_crt.to_str().unwrap(), + "-days", + "3650", + "-extfile", + san.to_str().unwrap(), + "-extensions", + "ext", + ], + )?; + + run_cmd( + "kubectl", + &[ + "create", + "secret", + "generic", + REGISTRY_TLS_SECRET, + "-n", + REGISTRY_TLS_SECRET_NS, + &format!("--from-file=ca.crt={}", ca_crt.display()), + &format!("--from-file=tls.crt={}", tls_crt.display()), + &format!("--from-file=tls.key={}", tls_key.display()), + ], + )?; + let _ = fs::remove_dir_all(&dir); + Ok(()) +} + +/// Merge the local registry CA into Crossplane's trust store so package pulls +/// over HTTPS succeed without replacing public CAs (xpkg.crossplane.io). +pub fn ensure_crossplane_trusts_local_registry_ca() -> Result<(), Box> { + let xp_ready = run_cmd_output( + "kubectl", + &[ + "get", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "name", + ], + ) + .is_ok(); + if !xp_ready { + log::warn!("crossplane deployment not found yet; skip CA trust patch"); + return Ok(()); + } + + // Already patched? + let has_init = run_cmd_output( + "kubectl", + &[ + "get", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "-o", + "jsonpath={.spec.template.spec.initContainers[*].name}", + ], + ) + .unwrap_or_default(); + if has_init.split_whitespace().any(|n| n == "hops-merge-registry-ca") { + return Ok(()); + } + + log::info!("Patching Crossplane to trust local registry CA..."); + // Strategic merge: initContainer concatenates system CAs + hops CA; SSL_CERT_FILE points at it. + let patch = r#"{ + "spec": { + "template": { + "spec": { + "initContainers": [ + { + "name": "hops-merge-registry-ca", + "image": "alpine:3.20", + "command": ["sh", "-c"], + "args": [ + "set -e; cat /etc/ssl/certs/ca-certificates.crt /hops-ca/ca.crt > /combined/ca-bundle.crt" + ], + "volumeMounts": [ + {"name": "hops-local-registry-ca", "mountPath": "/hops-ca", "readOnly": true}, + {"name": "hops-combined-ca", "mountPath": "/combined"} + ] + } + ], + "containers": [ + { + "name": "crossplane", + "env": [ + {"name": "SSL_CERT_FILE", "value": "/combined/ca-bundle.crt"} + ], + "volumeMounts": [ + {"name": "hops-combined-ca", "mountPath": "/combined", "readOnly": true} + ] + } + ], + "volumes": [ + { + "name": "hops-local-registry-ca", + "secret": { + "secretName": "hops-local-registry-tls", + "items": [{"key": "ca.crt", "path": "ca.crt"}] + } + }, + { + "name": "hops-combined-ca", + "emptyDir": {} + } + ] + } + } + } +}"#; + + run_cmd( + "kubectl", + &[ + "patch", + "deploy", + "crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "--type", + "strategic", + "-p", + patch, + ], + )?; + + // Wait for rollout so package installs see the new trust store. + let _ = run_cmd( + "kubectl", + &[ + "rollout", + "status", + "deploy/crossplane", + "-n", + REGISTRY_TLS_SECRET_NS, + "--timeout=180s", + ], + ); + Ok(()) +} + pub fn interactive_stdio_available() -> bool { io::stdin().is_terminal() && io::stdout().is_terminal() } diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 4f04760..a5d5802 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -13,7 +13,9 @@ const PROVIDER_HELM: &str = include_str!("../../../bootstrap/providers/provider- const PROVIDER_K8S: &str = include_str!("../../../bootstrap/providers/provider-kubernetes.yaml"); const PC_HELM: &str = include_str!("../../../bootstrap/helm/pc.yaml"); const PC_K8S: &str = include_str!("../../../bootstrap/k8s/pc.yaml"); -const REGISTRY: &str = include_str!("../../../bootstrap/registry/registry.yaml"); + +const PROVIDER_K8S_NAME: &str = "crossplane-contrib-provider-kubernetes"; +const PROVIDER_HELM_NAME: &str = "crossplane-contrib-provider-helm"; #[derive(Args, Debug, Clone)] pub struct StartArgs { @@ -23,6 +25,11 @@ pub struct StartArgs { /// Stop and restart a running cluster VM without prompting when requested size differs. #[arg(long)] pub yes: bool, + + /// Force helm upgrade and full bootstrap even when the control plane is + /// already healthy. Default is to skip expensive helm/repo work on resume. + #[arg(long)] + pub bootstrap: bool, } pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box> { @@ -55,6 +62,73 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box bool { + deployment_available("crossplane-system", "crossplane") + && deployment_available("crossplane-system", "registry") + && provider_healthy(PROVIDER_K8S_NAME) + && provider_healthy(PROVIDER_HELM_NAME) +} + +fn deployment_available(namespace: &str, name: &str) -> bool { + run_cmd_output( + "kubectl", + &[ + "get", + "deployment", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Available\")].status}", + ], + ) + .map(|s| s.trim() == "True") + .unwrap_or(false) +} + +fn provider_healthy(provider: &str) -> bool { + run_cmd_output( + "kubectl", + &[ + "get", + "provider.pkg.crossplane.io", + provider, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Healthy\")].status}", + ], + ) + .map(|s| s.trim() == "True") + .unwrap_or(false) +} + +/// Helm + Crossplane + providers + ProviderConfigs (the slow cold-start path). +fn bootstrap_control_plane() -> Result<(), Box> { // 4. Add Crossplane Helm repo log::info!("Adding Crossplane Helm repo..."); run_cmd( @@ -158,27 +232,25 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box> { + // Crossplane package pulls run in the pod network → Service DNS + ClusterIP. + // Docker push: colima/kind → localhost:30500 (host NodePort); dory → + // {dory-k8s-ip}:30500 on the engine docker bridge (daemon is in-engine). wait_for_kubernetes()?; log::info!("Pre-pulling registry:2 (best effort)..."); let _ = run_cmd("docker", &["pull", "registry:2"]); - log::info!("Deploying local package registry..."); - kubectl_apply_stdin(REGISTRY)?; - // Nested virt: image pull + schedule for the registry can exceed the - // default ~5m wait used for lighter resources. + // TLS secret + registry Deployment + Crossplane CA trust (package manager is HTTPS-only). + log::info!("Deploying local package registry (HTTPS)..."); + backend.ensure_package_registry()?; wait_for_deployment_with_diagnostics("crossplane-system", "registry")?; - - // 13. Point the node at the registry Service's ClusterIP so pulls of the - // cluster-internal registry names resolve. backend::wire_local_registry(backend)?; - - log::info!("Local environment is ready"); Ok(()) } @@ -363,3 +435,33 @@ fn wait_for_provider_healthy(provider: &str) -> Result<(), Box> { ) .into()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_args_bootstrap_defaults_false() { + // clap default: bootstrap only when --bootstrap is passed + assert!(!StartArgs { + size: SizeArgs { + cpus: None, + memory: None, + disk: None, + }, + yes: false, + bootstrap: false, + } + .bootstrap); + assert!(StartArgs { + size: SizeArgs { + cpus: None, + memory: None, + disk: None, + }, + yes: false, + bootstrap: true, + } + .bootstrap); + } +} diff --git a/src/commands/provider/install.rs b/src/commands/provider/install.rs index c45a3e6..de0a2da 100644 --- a/src/commands/provider/install.rs +++ b/src/commands/provider/install.rs @@ -2,7 +2,7 @@ use crate::commands::local::backend::{self, Backend}; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout_at, ensure_registry, parse_repo_spec, resolve_repo_install_target, run_watch, sanitize_name_component, RepoInstallTarget, RepoSpec, - REGISTRY_PULL, REGISTRY_PUSH, + registry_pull, registry_push, }; use crate::commands::local::{ kubectl_apply_stdin, run_cmd, run_cmd_output, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, @@ -178,7 +178,7 @@ fn apply_repo_version_spec( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], )?; - let resolved = resolve_provider_target(&provider_name, &providers_json, REGISTRY_PULL)?; + let resolved = resolve_provider_target(&provider_name, &providers_json, registry_pull())?; // Published install pulls directly from ghcr.io — no ImageConfig rewrite, // no local registry push, no runtime-image override. apply_provider_resources( @@ -216,8 +216,8 @@ fn run_local_path( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], )?; - let resolved = resolve_provider_target(&provider_name, &providers_json, REGISTRY_PULL)?; - let upstream_url_prefix = recover_upstream_url_prefix(&resolved, REGISTRY_PULL)?; + let resolved = resolve_provider_target(&provider_name, &providers_json, registry_pull())?; + let upstream_url_prefix = recover_upstream_url_prefix(&resolved, registry_pull())?; let upstream_major = parse_major_version(&resolved.existing_package); ensure_build_submodule(dir)?; @@ -232,7 +232,7 @@ fn run_local_path( let xpkg_path = find_xpkg_for_provider(dir, &provider_name, arch)?; log::info!("Located xpkg: {}", xpkg_path.display()); - let local_image_path_for_tag = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); + let local_image_path_for_tag = format!("{}/hops-ops/{}", registry_pull(), provider_name); let dev_tag = dev_tag_for_file( &xpkg_path, version_prefix, @@ -240,8 +240,8 @@ fn run_local_path( &local_image_path_for_tag, )?; - let push_xpkg_ref = format!("{}/hops-ops/{}:{}", REGISTRY_PUSH, provider_name, dev_tag); - let local_pull_xpkg_path = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); + let push_xpkg_ref = format!("{}/hops-ops/{}:{}", registry_push(), provider_name, dev_tag); + let local_pull_xpkg_path = format!("{}/hops-ops/{}", registry_pull(), provider_name); log::info!("Pushing xpkg to {}...", push_xpkg_ref); crossplane_xpkg_push(&xpkg_path, &push_xpkg_ref)?; @@ -394,7 +394,10 @@ fn local_runtime_image_ref(provider_name: &str, arch: &str, tag: &str) -> String // package manager, so they need the node-pullable local registry address. format!( "{}/hops-ops/{}-{}:{}", - REGISTRY_PUSH, provider_name, arch, tag + registry_push(), + provider_name, + arch, + tag ) } @@ -499,9 +502,10 @@ fn next_local_patch_for_major(image_path: &str, major: u64) -> Result String { } #[test] -fn dory_smoke_workflow_clone_build_install_contract() { +fn dory_smoke_workflow_self_hosted_stock_contract() { let text = workflow_text(); + assert!( - text.contains("patrickleet/dory"), - "must clone patrickleet/dory" + !text.contains("patrickleet/dory"), + "must not clone a hops fork of dory" ); - // Pin a known-good commit of the hops integration work (not a moving branch - // tip — the upstream branch may never merge). Bump intentionally with dory. assert!( - text.contains("f8c61d2fd0fc4d528e5e0da36ffa09b9796b3871"), - "must pin patrickleet/dory to a specific commit SHA" + !text.contains("xcodebuild") && !text.contains("build-dory-ffi"), + "must not build Dory from source" ); assert!( - !text - .lines() - .any(|l| l.trim() == "ref: feat/hops-local-integration"), - "must not track the moving feat/hops-local-integration branch tip" + !text.contains("colima start") + && !text.contains("brew install colima") + && !text.contains("colima-surrogate"), + "must not use Colima as a dory.sock surrogate" ); assert!( - text.contains("xcodebuild") && text.contains("derivedDataPath"), - "must build in-pipeline with deterministic derivedDataPath" + text.contains("dory.sock"), + "must require real ~/.dory/dory.sock" ); assert!( - text.contains("dtolnay/rust-toolchain") && text.contains("brew install protobuf"), - "must install the Rust and protoc prerequisites used by Dory's FFI builder" + text.contains("refusing Colima-backed") || text.contains("Colima-backed dory.sock"), + "must refuse a Colima-symlinked dory.sock" ); - let ffi_build = text - .find("scripts/build-dory-ffi-xcframework.sh --if-needed") - .expect("must materialize DoryFFI from a clean checkout"); - let app_build = text - .find("xcodebuild -project Dory.xcodeproj") - .expect("must build the Dory app"); assert!( - ffi_build < app_build, - "must generate DoryFFI before SwiftPM resolves the Dory app" + text.contains("self-hosted") && text.contains("hops-dory"), + "must target self-hosted runner labeled hops-dory" ); assert!( - text.contains("DoryFFI.xcframework/macos-arm64_x86_64/libdory_ffi.a") - && text.contains("Sources/DoryCore/generated/dory_ffi.swift"), - "must verify both generated Dory FFI artifacts before xcodebuild" + !text.lines().any(|l| { + let t = l.trim(); + t == "runs-on: macos-15" + || t == "runs-on: macos-latest" + || t == "runs-on: macos-15-intel" + }), + "must not use GitHub-hosted macOS" ); assert!( - text.contains("GITHUB_PATH") && text.contains("scripts"), - "must put scripts/dory on PATH" + text.contains("labeled") && text.contains("test-dory"), + "PR smoke must be opt-in via test-dory label" ); +} + +#[test] +fn dory_smoke_workflow_env_only_no_desktop_mutation() { + let text = workflow_text(); + + // Session entirely via env vars. assert!( - text.contains("engine.sock"), - "must ensure ~/.dory/engine.sock before hops start" + text.contains("HOPS_DORY_DESKTOP") && text.contains("\"0\""), + "must set HOPS_DORY_DESKTOP=0 so hops does not rewrite desktop defaults" ); assert!( - !text.contains("brew install --cask") && !text.contains("homebrew/cask"), - "must not use brew cask as primary install" + text.contains("DOCKER_HOST") && text.contains("dory.sock"), + "must drive docker via DOCKER_HOST → dory.sock" ); assert!( - text.contains("workflow_dispatch") && text.contains("pull_request:"), - "must support pull_request (PR-branch runs) and workflow_dispatch" + text.contains("KUBECONFIG") && text.contains("RUNNER_TEMP"), + "must use a job-private KUBECONFIG under RUNNER_TEMP" ); + + // Never switch/restore machine defaults. assert!( - text.contains("labeled") && text.contains("test-dory"), - "PR smoke must start only when opted in with the test-dory label" + !text.contains("kubectl config use-context"), + "must not kubectl config use-context" + ); + assert!( + !text.contains("docker context use"), + "must not docker context use" + ); + assert!( + !text.contains("Restore desktop contexts") && !text.contains("Snapshot desktop"), + "must not snapshot/restore contexts — env-only session needs no restore" + ); + + // Do not destroy product plane. + assert!( + !text.contains("hops-cli local destroy"), + "must not hops local destroy" + ); + assert!( + !text.contains("docker rm -f dory-k8s"), + "must not docker rm dory-k8s" ); assert!( - text.contains("github.event_name == 'workflow_dispatch'") - && text.contains("github.event.pull_request.labels.*.name"), - "manual dispatch must remain available while PR runs are label-gated" + !text.contains("dory engine sleep") && !text.contains("pkill -f"), + "must not sleep engine or kill Dory" ); - // Nested-virt pin for Colima-backed engine.sock on public GHA. assert!( - text.lines() - .any(|l| l.trim() == "runs-on: macos-15-intel"), - "must pin macos-15-intel for nested virt (not bare macos-latest alone)" + !text.contains("hops-cli local stop"), + "must not hops local stop" + ); + + assert!( + text.contains("CARGO_BUILD_JOBS") && text.contains("nice"), + "must limit cargo parallelism and nice the build" ); } #[test] -fn dory_smoke_workflow_kind_parity_when_engine_boots() { +fn dory_smoke_workflow_hops_integration_core() { let text = workflow_text(); for needle in [ "cargo build", "start --backend dory", "local doctor", - "localhost:30500", "registry.crossplane-system.svc.cluster.local:5000", - "--context dory", - "local stop", - "local destroy", + "30500", + "smoke-svc-name", + "smoke-nodeport", ] { assert!( text.contains(needle), - "dory smoke missing kind-parity fragment: {needle}" + "dory smoke missing hops integration fragment: {needle}" ); } - // stop/start resume: start without --backend after stop - assert!( - text.contains("local start\n") - || text - .lines() - .any(|l| l.trim() == "./target/debug/hops-cli local start"), - "must start again without --backend after stop" - ); -} - -#[test] -fn dory_smoke_workflow_carries_colima_lessons() { - let text = workflow_text(); assert!( - text.contains("kube-dns") || text.contains("coredns"), - "must wait for CoreDNS/kube-dns before registry round-trip" + text.contains("NODE_IP") || text.contains("NetworkSettings"), + "must push via engine-plane dory-k8s IP" ); + // Single-platform pull/build avoids multiplatform push warnings. assert!( - text.contains("--timeout=420s"), - "must use a long Ready/Available timeout for nested-virt lag" + text.contains("docker pull --platform") || text.contains("--platform \"$PLATFORM\""), + "registry smoke must pull busybox with an explicit --platform" ); assert!( - text.contains("rollout restart"), - "stop/start resume must rollout-restart stalled deployments" + text.contains("multiplatform") || text.contains("single-platform"), + "must document why single-platform materialization is used" ); +} + +#[test] +fn dory_smoke_workflow_path_based_config_install() { + let text = workflow_text(); assert!( - text.contains("Debug dump on failure") && text.contains("engine.sock"), - "failure dump must capture dory/engine diagnostics" + text.contains("tests/fixtures/config-smoke"), + "must path-install the in-repo config-smoke fixture" ); assert!( - text.contains("describe pod smoke-svc-name") - || text.contains("describe pod smoke-svc-name smoke-localhost"), - "failure dump must describe smoke pods in default ns" + text.contains("config install --path") || text.contains("config install --path "), + "must use hops config install --path (no clone)" ); assert!( - text.to_lowercase().contains("localhost:30500"), - "must exercise localhost:30500 registry path" + !text.contains("config install --repo"), + "must not clone a remote config repo for the smoke" ); -} - -#[test] -fn dory_smoke_workflow_public_gha_engine_fallback() { - let text = workflow_text(); - // Public GHA cannot run dory-hv; workflow must document and implement a - // Colima-backed engine.sock so hops --backend dory remains testable. assert!( - text.contains("colima start") && text.contains("engine.sock"), - "must bootstrap Colima docker as engine.sock when native sock is absent" + text.contains("local/ci-xr.yaml") || text.contains("ci-xr.yaml"), + "must apply the fixture XR" ); assert!( - text.to_lowercase().contains("surrogate") - || text.contains("colima-surrogate") - || text.contains("Colima-backed"), - "must document Colima engine.sock surrogate for public GHA" + text.contains("hops-ci") && text.contains("configmap"), + "must verify ConfigMap in hops-ci namespace" ); assert!( - text.contains("xattr") || text.contains("codesign"), - "must clear quarantine/sign Debug Dory.app before open" + text.contains("command -v up") || text.contains("up CLI"), + "must require Upbound up CLI for path builds" ); } diff --git a/tests/fixtures/config-smoke/.gitignore b/tests/fixtures/config-smoke/.gitignore new file mode 100644 index 0000000..d2e372f --- /dev/null +++ b/tests/fixtures/config-smoke/.gitignore @@ -0,0 +1,4 @@ +_output/ +.up/ +.venv/ +.tmp/ diff --git a/tests/fixtures/config-smoke/README.md b/tests/fixtures/config-smoke/README.md new file mode 100644 index 0000000..0ce3048 --- /dev/null +++ b/tests/fixtures/config-smoke/README.md @@ -0,0 +1,14 @@ +# config-smoke + +Minimal Crossplane configuration fixture for **path-based** hops integration tests. + +```bash +hops local start --backend dory # or kind/colima +hops config install --path tests/fixtures/config-smoke --backend dory +kubectl apply -f tests/fixtures/config-smoke/local/ci-xr.yaml +kubectl -n hops-ci wait --for=condition=Ready configsmoke/hops-ci-smoke --timeout=300s +kubectl -n hops-ci get configmap hops-ci-smoke +``` + +Creates only namespaced resources under `hops-ci` (ConfigMap via provider-kubernetes). +Not a real platform stack — safe on a shared local control plane. diff --git a/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml b/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml new file mode 100644 index 0000000..1c11c03 --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/composition.yaml @@ -0,0 +1,19 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: configsmokes.ci.hops.ops.com.ai +spec: + compositeTypeRef: + apiVersion: ci.hops.ops.com.ai/v1alpha1 + kind: ConfigSmoke + mode: Pipeline + pipeline: + - functionRef: + # Crossplane names the installed Function from the package path + # ghcr.io/hops-ops/config-smoke_render → hops-ops-config-smokerender + # (same convention as gateway-api-stackrender, etc.). + name: hops-ops-config-smokerender + step: render + - functionRef: + name: crossplane-contrib-function-auto-ready + step: crossplane-contrib-function-auto-ready diff --git a/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml b/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml new file mode 100644 index 0000000..4abf95a --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/configuration.yaml @@ -0,0 +1,14 @@ +apiVersion: meta.pkg.crossplane.io/v1alpha1 +kind: Configuration +metadata: + name: config-smoke + annotations: + meta.crossplane.io/maintainer: hops-ops CI + meta.crossplane.io/source: github.com/hops-ops/hops-cli + meta.crossplane.io/description: Minimal CI fixture ConfigMap composition for hops path-based config install. +spec: + dependsOn: + - function: xpkg.crossplane.io/crossplane-contrib/function-auto-ready + version: '>=v0.6.0' + - provider: xpkg.crossplane.io/crossplane-contrib/provider-kubernetes + version: '>=v0.15.0' diff --git a/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml b/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml new file mode 100644 index 0000000..14b9559 --- /dev/null +++ b/tests/fixtures/config-smoke/apis/configsmokes/definition.yaml @@ -0,0 +1,53 @@ +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: configsmokes.ci.hops.ops.com.ai +spec: + group: ci.hops.ops.com.ai + names: + kind: ConfigSmoke + plural: configsmokes + scope: Namespaced + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + description: CI-only composite that materializes a ConfigMap. + type: object + properties: + spec: + type: object + properties: + managementPolicies: + type: array + items: + type: string + default: ["*"] + data: + description: ConfigMap data key/values. + type: object + additionalProperties: + type: string + x-kubernetes-preserve-unknown-fields: true + configMapName: + description: Name of the ConfigMap to create. Defaults to metadata.name. + type: string + providerConfigRef: + type: object + properties: + name: + type: string + kind: + type: string + enum: [ProviderConfig, ClusterProviderConfig] + required: [] + status: + type: object + properties: + ready: + type: boolean + configMapName: + type: string + required: [spec] diff --git a/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml b/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml new file mode 100644 index 0000000..81c7ea3 --- /dev/null +++ b/tests/fixtures/config-smoke/examples/configsmokes/ci.yaml @@ -0,0 +1,13 @@ +apiVersion: ci.hops.ops.com.ai/v1alpha1 +kind: ConfigSmoke +metadata: + name: hops-ci-smoke + namespace: hops-ci +spec: + configMapName: hops-ci-smoke + data: + source: path-based-config-install + smoke: "ok" + providerConfigRef: + name: default + kind: ProviderConfig diff --git a/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl new file mode 100644 index 0000000..9dbb397 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/000-state-init.yaml.gotmpl @@ -0,0 +1,26 @@ +# code: language=yaml +{{- $xr := getCompositeResource . }} +{{- $metadata := $xr.metadata | default dict }} +{{- $spec := $xr.spec | default dict }} + +{{- $name := $metadata.name | default "config-smoke" }} +{{- $namespace := $metadata.namespace | default "default" }} +{{- $configMapName := $spec.configMapName | default $name }} +{{- $managementPolicies := $spec.managementPolicies | default (list "*") }} +{{- $data := $spec.data | default (dict "smoke" "ok") }} +{{- $pc := $spec.providerConfigRef | default dict }} +{{- $providerConfigRef := dict + "name" ($pc.name | default "default") + "kind" ($pc.kind | default "ProviderConfig") +}} + +{{- $state := dict + "name" $name + "namespace" $namespace + "configMapName" $configMapName + "managementPolicies" $managementPolicies + "data" $data + "providerConfigRef" $providerConfigRef + "observed" (dict) + "status" (dict "ready" false "configMapName" $configMapName) +}} diff --git a/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl new file mode 100644 index 0000000..691f1d4 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/010-state-status.yaml.gotmpl @@ -0,0 +1,17 @@ +# code: language=yaml +{{- $raw := $.observed.resources | default dict }} +{{- $entry := get $raw "configmap" | default dict }} +{{- $resource := $entry.resource | default dict }} +{{- $status := $resource.status | default dict }} + +{{- $ready := false }} +{{- range ($status.conditions | default list) }} + {{- if and (eq .type "Ready") (eq .status "True") }} + {{- $ready = true }} + {{- end }} +{{- end }} + +{{- $state = set $state "status" (dict + "ready" $ready + "configMapName" $state.configMapName +) }} diff --git a/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl new file mode 100644 index 0000000..6004660 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/200-configmap.yaml.gotmpl @@ -0,0 +1,28 @@ +# code: language=yaml +{{- $s := $state }} +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: {{ $s.configMapName }} + annotations: + {{ setResourceNameAnnotation "configmap" }} + labels: + hops.ops.com.ai/ci-fixture: "true" + hops.ops.com.ai/config-smoke: {{ $s.name | quote }} +spec: + managementPolicies: {{ $s.managementPolicies | toJson }} + forProvider: + manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ $s.configMapName }} + namespace: {{ $s.namespace }} + labels: + hops.ops.com.ai/ci-fixture: "true" + hops.ops.com.ai/config-smoke: {{ $s.name | quote }} + data: {{ $s.data | toJson }} + providerConfigRef: + name: {{ $s.providerConfigRef.name }} + kind: {{ $s.providerConfigRef.kind }} diff --git a/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl b/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl new file mode 100644 index 0000000..9e6c392 --- /dev/null +++ b/tests/fixtures/config-smoke/functions/render/999-status.yaml.gotmpl @@ -0,0 +1,8 @@ +# code: language=yaml +{{- $xr := getCompositeResource . }} +--- +apiVersion: {{ $xr.apiVersion }} +kind: {{ $xr.kind }} +status: + ready: {{ $state.status.ready }} + configMapName: {{ $state.status.configMapName | quote }} diff --git a/tests/fixtures/config-smoke/local/ci-xr.yaml b/tests/fixtures/config-smoke/local/ci-xr.yaml new file mode 100644 index 0000000..a611f06 --- /dev/null +++ b/tests/fixtures/config-smoke/local/ci-xr.yaml @@ -0,0 +1,36 @@ +# Applied by dory smoke after `hops config install --path tests/fixtures/config-smoke`. +# Lives in the hops-ci namespace only — safe alongside real workloads. +apiVersion: v1 +kind: Namespace +metadata: + name: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +--- +# Namespaced k8s provider config (mirrors bootstrap/k8s/pc.yaml for default/). +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: ProviderConfig +metadata: + name: default + namespace: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +spec: + credentials: + source: InjectedIdentity +--- +apiVersion: ci.hops.ops.com.ai/v1alpha1 +kind: ConfigSmoke +metadata: + name: hops-ci-smoke + namespace: hops-ci + labels: + hops.ops.com.ai/ci-fixture: "true" +spec: + configMapName: hops-ci-smoke + data: + source: path-based-config-install + smoke: "ok" + providerConfigRef: + name: default + kind: ProviderConfig diff --git a/tests/fixtures/config-smoke/upbound.yaml b/tests/fixtures/config-smoke/upbound.yaml new file mode 100644 index 0000000..fb8db98 --- /dev/null +++ b/tests/fixtures/config-smoke/upbound.yaml @@ -0,0 +1,27 @@ +apiVersion: meta.dev.upbound.io/v2alpha1 +kind: Project +metadata: + name: config-smoke +spec: + dependsOn: + - apiVersion: pkg.crossplane.io/v1 + kind: Function + package: xpkg.crossplane.io/crossplane-contrib/function-auto-ready + version: '>=v0.6.0' + - apiVersion: pkg.crossplane.io/v1 + kind: Provider + package: xpkg.crossplane.io/crossplane-contrib/provider-kubernetes + version: '>=v0.15.0' + description: | + Minimal hops CI fixture. Builds from source via `hops config install --path` + and composes a single Kubernetes ConfigMap through provider-kubernetes. + Safe to install on a shared local control plane (namespaced resources only). + license: Apache-2.0 + maintainer: hops-ops CI + readme: | + # config-smoke + + Test-only configuration for hops local integration. Creates a ConfigMap via + the Kubernetes provider. Not a real platform stack. + repository: ghcr.io/hops-ops/config-smoke + source: github.com/hops-ops/hops-cli