From 36d1a15acaeaf77dd184cca06386a8f2f6a6f871 Mon Sep 17 00:00:00 2001 From: Benjamin Ernst Date: Fri, 13 Mar 2026 16:47:43 +0100 Subject: [PATCH 1/4] test dev-deployment to local k3d clusters --- build/make/k8s-component.mk | 16 +++++++++---- build/make/k8s.mk | 48 ++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/build/make/k8s-component.mk b/build/make/k8s-component.mk index 7eaa439..1b1f2bd 100644 --- a/build/make/k8s-component.mk +++ b/build/make/k8s-component.mk @@ -9,6 +9,9 @@ endif ifeq (${RUNTIME_ENV}, local) BINARY_HELM_ADDITIONAL_PUSH_ARGS?=--plain-http endif +ifeq (${RUNTIME_ENV}, k3d) + BINARY_HELM_ADDITIONAL_PUSH_ARGS?=--plain-http +endif BINARY_HELM_ADDITIONAL_PACK_ARGS?= BINARY_HELM_ADDITIONAL_UNINST_ARGS?= BINARY_HELM_ADDITIONAL_UPGR_ARGS?= @@ -18,10 +21,15 @@ HELM_SOURCE_DIR ?= k8s/helm HELM_RELEASE_TGZ=${HELM_TARGET_DIR}/${COMPONENT_ARTIFACT_ID}-${VERSION}.tgz HELM_DEV_RELEASE_TGZ=${HELM_TARGET_DIR}/${COMPONENT_ARTIFACT_ID}-${COMPONENT_DEV_VERSION}.tgz HELM_ARTIFACT_NAMESPACE?=k8s +HELM_PUSH_REGISTRY_HOST?=${CES_REGISTRY_HOST} ifeq (${RUNTIME_ENV}, remote) HELM_ARTIFACT_NAMESPACE=testing/k8s endif +ifeq (${RUNTIME_ENV}, k3d) + HELM_PUSH_REGISTRY_HOST=localhost:5001 +endif $(info HELM_ARTIFACT_NAMESPACE=$(HELM_ARTIFACT_NAMESPACE)) +$(info HELM_PUSH_REGISTRY_HOST=$(HELM_PUSH_REGISTRY_HOST)) K8S_RESOURCE_COMPONENT ?= "${K8S_RESOURCE_TEMP_FOLDER}/component-${COMPONENT_ARTIFACT_ID}-${VERSION}.yaml" K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML ?= $(BUILD_DIR)/make/k8s-component.tpl @@ -98,11 +106,11 @@ helm-reinstall: helm-delete helm-apply ## Uninstalls the current helm chart and .PHONY: helm-chart-import helm-chart-import: ${CHECK_VAR_TARGETS} helm-generate helm-package ${IMAGE_IMPORT_TARGET} ## Imports the currently available chart into the cluster-local registry. @if [[ ${STAGE} == "development" ]]; then \ - echo "Import ${HELM_DEV_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ - ${BINARY_HELM} push ${HELM_DEV_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + echo "Import ${HELM_DEV_RELEASE_TGZ} into K8s cluster ${HELM_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_DEV_RELEASE_TGZ} oci://${HELM_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ else \ - echo "Import ${HELM_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ - ${BINARY_HELM} push ${HELM_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + echo "Import ${HELM_RELEASE_TGZ} into K8s cluster ${HELM_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_RELEASE_TGZ} oci://${HELM_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ fi @echo "Done." diff --git a/build/make/k8s.mk b/build/make/k8s.mk index d8e44a5..64633ea 100644 --- a/build/make/k8s.mk +++ b/build/make/k8s.mk @@ -40,8 +40,12 @@ IMAGE ?= # with development images pointing to CES_REGISTRY_URL_PREFIX. STAGE?=production -# Set the "local" as runtime-environment, to push images to the container-registry of the local cluster and to apply resources to the local cluster. -# Use "remote" as runtime-environment in your .env file to push images to the container-registry at "registry.cloudogu.com/testing" and to apply resources to the configured kubernetes-context in KUBE_CONTEXT_NAME. +# Set "local" as runtime-environment to use the legacy in-cluster registry of the local cluster. +# Set "k3d" as runtime-environment for local k3d development with local registry push/pull: +# - push from host to localhost:5001 +# - pull in-cluster via k3d-registry-proxy.localhost:5000/local-dev +# Use "remote" as runtime-environment in your .env file to push images to the container-registry at +# "registry.cloudogu.com/testing" and to apply resources to the configured kubernetes-context in KUBE_CONTEXT_NAME. RUNTIME_ENV?=local $(info RUNTIME_ENV=$(RUNTIME_ENV)) @@ -49,26 +53,40 @@ $(info RUNTIME_ENV=$(RUNTIME_ENV)) K3S_CLUSTER_FQDN?=k3ces.localdomain K3S_LOCAL_REGISTRY_PORT?=30099 -# The URL of the container-registry to use. Defaults to the registry of the local-cluster. -# If RUNTIME_ENV is "remote" it is "registry.cloudogu.com/testing", if ENVIRONMENT is "ci" it is "registry.cloudogu.com/ci" -# if run on ci (jenkins) the images must be pushed to a separate namespace in order to free space every night after the build. +# The URL or image-prefix host to use for development images. +# If RUNTIME_ENV is "remote" it is "registry.cloudogu.com/testing", if ENVIRONMENT is "ci" it is "registry.cloudogu.com/ci". +# If run on ci (jenkins) the images must be pushed to a separate namespace in order to free space every night after the build. CES_REGISTRY_HOST?=${K3S_CLUSTER_FQDN}:${K3S_LOCAL_REGISTRY_PORT} CES_REGISTRY_NAMESPACE ?= +IMAGE_PUSH_REGISTRY_HOST ?= $(CES_REGISTRY_HOST) +IMAGE_PUSH_REGISTRY_NAMESPACE ?= $(CES_REGISTRY_NAMESPACE) ifeq (${RUNTIME_ENV}, remote) CES_REGISTRY_HOST=registry.cloudogu.com CES_REGISTRY_NAMESPACE=/testing + IMAGE_PUSH_REGISTRY_HOST=$(CES_REGISTRY_HOST) + IMAGE_PUSH_REGISTRY_NAMESPACE=$(CES_REGISTRY_NAMESPACE) ifeq ($(ENVIRONMENT), ci) CES_REGISTRY_NAMESPACE=/ci + IMAGE_PUSH_REGISTRY_NAMESPACE=$(CES_REGISTRY_NAMESPACE) endif endif +ifeq (${RUNTIME_ENV}, k3d) + CES_REGISTRY_HOST=k3d-registry-proxy.localhost:5000 + CES_REGISTRY_NAMESPACE=/local-dev + IMAGE_PUSH_REGISTRY_HOST=localhost:5001 + IMAGE_PUSH_REGISTRY_NAMESPACE=/local-dev +endif $(info CES_REGISTRY_HOST=$(CES_REGISTRY_HOST)) # The name of the kube-context to use for applying resources. # If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is "remote" the currently configured kube-context is used. -# If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is not "remote" the "k3ces.localdomain" is used as kube-context. +# If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is "k3d" the currently configured kube-context is used. +# If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is neither "remote" nor "k3d" the "k3ces.localdomain" is used as kube-context. ifeq (${KUBE_CONTEXT_NAME}, ) ifeq (${RUNTIME_ENV}, remote) KUBE_CONTEXT_NAME = $(shell kubectl config current-context) + else ifeq (${RUNTIME_ENV}, k3d) + KUBE_CONTEXT_NAME = $(shell kubectl config current-context) else KUBE_CONTEXT_NAME = k3ces.localdomain endif @@ -83,6 +101,8 @@ GIT_HASH := $(shell git rev-parse --short HEAD) ## Image URL to use all building/pushing image targets IMAGE_DEV?=$(CES_REGISTRY_HOST)$(CES_REGISTRY_NAMESPACE)/$(ARTIFACT_ID)/$(GIT_BRANCH) IMAGE_DEV_VERSION=$(IMAGE_DEV):$(VERSION) +IMAGE_DEV_PUSH?=$(IMAGE_PUSH_REGISTRY_HOST)$(IMAGE_PUSH_REGISTRY_NAMESPACE)/$(ARTIFACT_ID)/$(GIT_BRANCH) +IMAGE_DEV_PUSH_VERSION=$(IMAGE_DEV_PUSH):$(VERSION) # Variables for the temporary yaml files. These are used as template to generate a development resource containing # the current namespace and the dev image. @@ -174,9 +194,17 @@ ifeq (${IMAGE_DEV},) endif .PHONY: image-import -image-import: check-all-vars check-k8s-artifact-id docker-dev-tag ## Imports the currently available image into the configured ces-registry. - @echo "Import $(IMAGE_DEV_VERSION) into K8s cluster ${KUBE_CONTEXT_NAME}..." - @docker push $(IMAGE_DEV_VERSION) +image-import: check-all-vars check-k8s-artifact-id docker-dev-tag ## Imports the currently available image into the configured runtime target. + @if [[ "${RUNTIME_ENV}" == "k3d" ]]; then \ + echo "Push $(IMAGE_DEV_VERSION) for k3d registry workflow..."; \ + echo "Push target: $(IMAGE_DEV_PUSH_VERSION)"; \ + echo "Pull target: $(IMAGE_DEV_VERSION)"; \ + DOCKER_BUILDKIT=1 docker tag $(IMAGE_DEV_VERSION) $(IMAGE_DEV_PUSH_VERSION); \ + docker push $(IMAGE_DEV_PUSH_VERSION); \ + else \ + echo "Import $(IMAGE_DEV_VERSION) into K8s cluster ${KUBE_CONTEXT_NAME}..."; \ + docker push $(IMAGE_DEV_VERSION); \ + fi @echo "Done." ## Functions @@ -237,5 +265,3 @@ isProduction: else \ echo "Command executed in development stage. Continuing."; \ fi - - From 812b1f59d5d83990c997d3066963687c18530354 Mon Sep 17 00:00:00 2001 From: Benjamin Ernst Date: Wed, 15 Apr 2026 09:13:24 +0200 Subject: [PATCH 2/4] update k3d-handling add overridable defaults for the registries --- README.md | 20 ++++++++++++++++++-- build/make/k8s-component.mk | 2 +- build/make/k8s.mk | 30 +++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 18f93fd..ed5aab9 100644 --- a/README.md +++ b/README.md @@ -377,9 +377,26 @@ tests in `${yourProjectDir}/batsTests` (overrideable with the variable `TESTS_DI The k8s-modules support remote runtimes and container-registries. The environment-variable `RUNTIME_ENV`controls which runtime-environment to use: - * `local`: uses the local k8s-cluster at `k3ces-local` and the container-registry of this local-cluster + * `local`: uses the legacy local k8s-cluster at `k3ces.localdomain` and the in-cluster registry of this local cluster + * `k3d`: uses a local k3d cluster, pushes from the host to a writable local registry and pulls in-cluster from the proxy registry * `remote`: uses the currently configured cluster of the kube-config and the container-registry at `registry.cloudogu.com/testing` +For `k3d`, the defaults are: + +- pull in-cluster via `k3d-registry-proxy.localhost:5000/local-dev` +- push from the host via `localhost:5001/local-dev` +- use the current kube-context unless `KUBE_CONTEXT_NAME` is set explicitly + +You can also set `KUBECONFIG` in the repository-local `.env`. +If `KUBE_CONTEXT_NAME` is not set, the current context is then resolved from this kubeconfig and used for all `kubectl` and `helm` calls. + +The `k3d` registry endpoints can be overridden with: + +- `K3D_PULL_REGISTRY_HOST` +- `K3D_PULL_REGISTRY_NAMESPACE` +- `K3D_PUSH_REGISTRY_HOST` +- `K3D_PUSH_REGISTRY_NAMESPACE` + To manually override the kube-context the environment-variable `KUBE_CONTEXT_NAME` can be used. #### k8s.mk @@ -462,4 +479,3 @@ This module provides a target for scanning dogu images with trivy Usage: `make trivyscan` or `make trivyscan SEVERITY='HIGH,CRITICAL'` - diff --git a/build/make/k8s-component.mk b/build/make/k8s-component.mk index 1b1f2bd..18a196f 100644 --- a/build/make/k8s-component.mk +++ b/build/make/k8s-component.mk @@ -26,7 +26,7 @@ ifeq (${RUNTIME_ENV}, remote) HELM_ARTIFACT_NAMESPACE=testing/k8s endif ifeq (${RUNTIME_ENV}, k3d) - HELM_PUSH_REGISTRY_HOST=localhost:5001 + HELM_PUSH_REGISTRY_HOST=$(IMAGE_PUSH_REGISTRY_HOST) endif $(info HELM_ARTIFACT_NAMESPACE=$(HELM_ARTIFACT_NAMESPACE)) $(info HELM_PUSH_REGISTRY_HOST=$(HELM_PUSH_REGISTRY_HOST)) diff --git a/build/make/k8s.mk b/build/make/k8s.mk index 64633ea..f9f8648 100644 --- a/build/make/k8s.mk +++ b/build/make/k8s.mk @@ -33,6 +33,12 @@ BINARY_CRANE_ARCHIVE_STRIP?=0 SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +ifneq (${KUBECONFIG},) + # Values from the repo-local .env become plain make variables first. Export KUBECONFIG so + # recipe shells and nested kubectl/helm calls use the same kubeconfig file as the make logic. + export KUBECONFIG +endif + # The productive tag of the image IMAGE ?= @@ -42,8 +48,8 @@ STAGE?=production # Set "local" as runtime-environment to use the legacy in-cluster registry of the local cluster. # Set "k3d" as runtime-environment for local k3d development with local registry push/pull: -# - push from host to localhost:5001 -# - pull in-cluster via k3d-registry-proxy.localhost:5000/local-dev +# - push from host to ${K3D_PUSH_REGISTRY_HOST}${K3D_PUSH_REGISTRY_NAMESPACE} +# - pull in-cluster via ${K3D_PULL_REGISTRY_HOST}${K3D_PULL_REGISTRY_NAMESPACE} # Use "remote" as runtime-environment in your .env file to push images to the container-registry at # "registry.cloudogu.com/testing" and to apply resources to the configured kubernetes-context in KUBE_CONTEXT_NAME. RUNTIME_ENV?=local @@ -52,6 +58,10 @@ $(info RUNTIME_ENV=$(RUNTIME_ENV)) # The host and port of the local cluster K3S_CLUSTER_FQDN?=k3ces.localdomain K3S_LOCAL_REGISTRY_PORT?=30099 +K3D_PULL_REGISTRY_HOST?=k3d-registry-proxy.localhost:5000 +K3D_PULL_REGISTRY_NAMESPACE?=/local-dev +K3D_PUSH_REGISTRY_HOST?=localhost:5001 +K3D_PUSH_REGISTRY_NAMESPACE?=$(K3D_PULL_REGISTRY_NAMESPACE) # The URL or image-prefix host to use for development images. # If RUNTIME_ENV is "remote" it is "registry.cloudogu.com/testing", if ENVIRONMENT is "ci" it is "registry.cloudogu.com/ci". @@ -71,19 +81,25 @@ ifeq (${RUNTIME_ENV}, remote) endif endif ifeq (${RUNTIME_ENV}, k3d) - CES_REGISTRY_HOST=k3d-registry-proxy.localhost:5000 - CES_REGISTRY_NAMESPACE=/local-dev - IMAGE_PUSH_REGISTRY_HOST=localhost:5001 - IMAGE_PUSH_REGISTRY_NAMESPACE=/local-dev + CES_REGISTRY_HOST=$(K3D_PULL_REGISTRY_HOST) + CES_REGISTRY_NAMESPACE=$(K3D_PULL_REGISTRY_NAMESPACE) + IMAGE_PUSH_REGISTRY_HOST=$(K3D_PUSH_REGISTRY_HOST) + IMAGE_PUSH_REGISTRY_NAMESPACE=$(K3D_PUSH_REGISTRY_NAMESPACE) endif $(info CES_REGISTRY_HOST=$(CES_REGISTRY_HOST)) # The name of the kube-context to use for applying resources. +# If KUBECONFIG is set and KUBE_CONTEXT_NAME is empty, the current context from this kubeconfig is used. # If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is "remote" the currently configured kube-context is used. # If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is "k3d" the currently configured kube-context is used. +# Set KUBE_CONTEXT_NAME explicitly if the current kube-context does not point to the desired local k3d cluster. # If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is neither "remote" nor "k3d" the "k3ces.localdomain" is used as kube-context. ifeq (${KUBE_CONTEXT_NAME}, ) - ifeq (${RUNTIME_ENV}, remote) + ifneq (${KUBECONFIG}, ) + # Resolve the current context from the explicitly configured kubeconfig instead of the + # user's default ~/.kube/config. This keeps repo-local .env settings self-contained. + KUBE_CONTEXT_NAME = $(shell KUBECONFIG="${KUBECONFIG}" kubectl config current-context) + else ifeq (${RUNTIME_ENV}, remote) KUBE_CONTEXT_NAME = $(shell kubectl config current-context) else ifeq (${RUNTIME_ENV}, k3d) KUBE_CONTEXT_NAME = $(shell kubectl config current-context) From 7206c55640af6993e316b3b49d89bfbbed4e0911 Mon Sep 17 00:00:00 2001 From: Benjamin Ernst Date: Wed, 15 Apr 2026 09:36:15 +0200 Subject: [PATCH 3/4] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ccbd0..69b7bd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Add support for local `k3d` development workflows in the k8s make targets. ## [v10.9.0] - 2026-04-15 - [#259] pass trivy version as parameter for coder target "trivyscanImage" From 56a356d83a0fb4768fd6c7d7d70ad639ec05ede6 Mon Sep 17 00:00:00 2001 From: Benjamin Ernst Date: Wed, 10 Jun 2026 11:28:20 +0200 Subject: [PATCH 4/4] fix push for crd-components --- build/make/k8s-crd.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/make/k8s-crd.mk b/build/make/k8s-crd.mk index aca625d..b9a9c1c 100644 --- a/build/make/k8s-crd.mk +++ b/build/make/k8s-crd.mk @@ -90,11 +90,11 @@ ${HELM_CRD_RELEASE_TGZ}: ${BINARY_HELM} crd-helm-generate ## Generates and packa .PHONY: crd-helm-chart-import crd-helm-chart-import: ${CHECK_VAR_TARGETS} check-k8s-artifact-id crd-helm-generate crd-helm-package ## Imports the currently available Helm CRD chart into the cluster-local registry. @if [[ ${STAGE} == "development" ]]; then \ - echo "Import ${HELM_CRD_DEV_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ - ${BINARY_HELM} push ${HELM_CRD_DEV_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + echo "Import ${HELM_CRD_DEV_RELEASE_TGZ} into K8s cluster ${IMAGE_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_CRD_DEV_RELEASE_TGZ} oci://${IMAGE_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ else \ - echo "Import ${HELM_CRD_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ - ${BINARY_HELM} push ${HELM_CRD_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + echo "Import ${HELM_CRD_RELEASE_TGZ} into K8s cluster ${IMAGE_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_CRD_RELEASE_TGZ} oci://${IMAGE_PUSH_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ fi @echo "Done."