From cb59bfed929e49226f26c45f6503ca079985fa38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20K=C3=A4stner?= Date: Tue, 21 Jul 2026 11:36:50 +0200 Subject: [PATCH] Add Prometheus instance to development setup with Tilt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploy the Prometheus operator and a Prometheus instance into the monitoring namespace. The Tiltfile installs the operator with a cached bundle download and creates the namespace. The Prometheus instance, ServiceAccount and RBAC are in config/develop/prometheus.yaml. Enable the ServiceMonitor for the controller-manager by including config/prometheus in the develop kustomization and patching its namespace. Expose the metrics bind address in the develop manager patch. Port-forward Prometheus to localhost:9090 for local access. Signed-off-by: Felix Kästner --- .gitignore | 3 ++ Makefile | 2 +- Tiltfile | 17 +++++++++ config/develop/kustomization.yaml | 7 ++++ config/develop/manager_patch.yaml | 1 + config/develop/prometheus.yaml | 58 +++++++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 config/develop/prometheus.yaml diff --git a/.gitignore b/.gitignore index e021a935f..1746e313d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ dist/* # Helm dependency chart archives charts/**/*.tgz +# Tilt cache +.tiltcache/ + # editor and IDE paraphernalia .idea .vscode diff --git a/Makefile b/Makefile index 53f42c89d..0c2933bbb 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ setup-test-e2e: kind ## Set up a Kind cluster for e2e tests if it does not exist esac # E2E test dependency versions -E2E_PROMETHEUS_OPERATOR_VERSION ?= v0.82.2 +E2E_PROMETHEUS_OPERATOR_VERSION ?= v0.92.1 E2E_CERTMANAGER_VERSION ?= v1.21.0 .PHONY: test-e2e diff --git a/Tiltfile b/Tiltfile index 9c5f39f86..228c83520 100644 --- a/Tiltfile +++ b/Tiltfile @@ -17,6 +17,23 @@ is_kind = cluster_name.startswith('kind-') load('ext://cert_manager', 'deploy_cert_manager') deploy_cert_manager(version='v1.21.0', load_to_kind=is_kind, kind_cluster_name=cluster_name.removeprefix('kind-')) +def deploy_prometheus_operator(version='v0.92.1'): + local('kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -', quiet=True, echo_off=True) + + print('Installing prometheus-operator') + cache_file = '.tiltcache/prometheus-operator-{}.yaml'.format(version) + if not os.path.exists(cache_file): + local("mkdir -p .tiltcache && curl -sL https://github.com/prometheus-operator/prometheus-operator/releases/download/{}/bundle.yaml | sed 's/namespace: default/namespace: monitoring/g' > {}".format(version, cache_file), quiet=True, echo_off=True) + local('kubectl apply --server-side -n monitoring -f {}'.format(cache_file), quiet=True, echo_off=True) + + print('Waiting for prometheus-operator to start') + local('kubectl wait --for=condition=Available --timeout=300s -n monitoring deployment/prometheus-operator', quiet=True, echo_off=True) + + k8s_yaml('./config/develop/prometheus.yaml') + k8s_resource(new_name = 'prometheus', objects = ['prometheus:prometheus'], port_forwards = '9090', extra_pod_selectors = [{'app.kubernetes.io/name': 'prometheus'}], labels = ['observability']) + +deploy_prometheus_operator() + docker_build('controller:latest', '.', only=[ 'api/', 'cmd/', 'internal/', 'go.mod', 'go.sum' ]) diff --git a/config/develop/kustomization.yaml b/config/develop/kustomization.yaml index 5ccca71e2..495888b9b 100644 --- a/config/develop/kustomization.yaml +++ b/config/develop/kustomization.yaml @@ -1,8 +1,15 @@ resources: - ../default +- ../prometheus patches: - path: manager_patch.yaml target: kind: Deployment name: controller-manager + - target: + kind: ServiceMonitor + patch: | + - op: replace + path: /metadata/namespace + value: network-operator-system diff --git a/config/develop/manager_patch.yaml b/config/develop/manager_patch.yaml index 4fddb21a4..9f18a099a 100644 --- a/config/develop/manager_patch.yaml +++ b/config/develop/manager_patch.yaml @@ -3,6 +3,7 @@ value: - --leader-elect=false - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8443 - --provider=openconfig - --requeue-interval=30s - --max-concurrent-reconciles=5 diff --git a/config/develop/prometheus.yaml b/config/develop/prometheus.yaml new file mode 100644 index 000000000..0c47f7a38 --- /dev/null +++ b/config/develop/prometheus.yaml @@ -0,0 +1,58 @@ +# Prometheus instance for local development. +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: prometheus + namespace: monitoring +spec: + replicas: 1 + serviceAccountName: prometheus + serviceMonitorNamespaceSelector: {} + serviceMonitorSelector: {} + podMonitorNamespaceSelector: {} + podMonitorSelector: {} + resources: + requests: + memory: 400Mi +--- +# RBAC setup based on https://prometheus-operator.dev/docs/platform/rbac/ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/metrics", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: monitoring