From d3a9803bea09f65606799e5238f3ec621895bc9b Mon Sep 17 00:00:00 2001 From: Jiachen Fan Date: Fri, 21 Aug 2026 15:02:58 +0800 Subject: [PATCH 1/2] feat(DIARCHERS-1674): add GKE cluster provisioning REST API Introduces a thin REST API to let external CI consumers (e.g. GitHub Actions) dynamically provision ephemeral GKE clusters via InfraBox, without exposing GCP credentials. The API is a thin orchestration layer on top of the existing GCP operator: POST/GET/DELETE against GKECluster CRs in the worker namespace, plus a GET /kubeconfig endpoint that reads the operator-generated Secret. Endpoints (v0): POST /api/v1/projects//gke-clusters GET /api/v1/projects//gke-clusters/ GET /api/v1/projects//gke-clusters//kubeconfig DELETE /api/v1/projects//gke-clusters/ Changes: - src/api/handlers/projects/gke_clusters.py: new handler (uses requests + the pod's SA token/CA, same pattern as scheduler.py to avoid adding the kubernetes python client dep) - src/api/handlers/projects/__init__.py: register new handler - src/openpolicyagent/policies/projects_gke_clusters.rego: OPA allow rules for the 4 endpoints (default deny -> explicit allow) - deploy/infrabox/templates/api/deployment.yaml: run the API pod under the existing 'infrabox' ServiceAccount (cluster-admin, same as scheduler), and inject INFRABOX_KUBERNETES_MASTER_HOST/PORT so the handler can reach the in-cluster K8s API server --- deploy/infrabox/templates/api/deployment.yaml | 7 + src/api/handlers/projects/__init__.py | 1 + src/api/handlers/projects/gke_clusters.py | 333 ++++++++++++++++++ .../policies/projects_gke_clusters.rego | 46 +++ 4 files changed, 387 insertions(+) create mode 100644 src/api/handlers/projects/gke_clusters.py create mode 100644 src/openpolicyagent/policies/projects_gke_clusters.rego diff --git a/deploy/infrabox/templates/api/deployment.yaml b/deploy/infrabox/templates/api/deployment.yaml index cb229162d..76010259a 100644 --- a/deploy/infrabox/templates/api/deployment.yaml +++ b/deploy/infrabox/templates/api/deployment.yaml @@ -15,6 +15,7 @@ spec: labels: app: infrabox-api spec: + serviceAccountName: infrabox {{ include "imagePullSecret" . | indent 8 }} containers: {{ include "containers_database" . | indent 12 }} @@ -90,6 +91,12 @@ spec: - name: INFRABOX_GERRIT_ENABLED value: {{ .Values.gerrit.enabled | quote }} + - + name: INFRABOX_KUBERNETES_MASTER_HOST + value: "kubernetes.default" + - + name: INFRABOX_KUBERNETES_MASTER_PORT + value: "443" volumes: {{ include "volumes_database" . | indent 16 }} {{ include "volumes_rsa" . | indent 16 }} diff --git a/src/api/handlers/projects/__init__.py b/src/api/handlers/projects/__init__.py index 3d03c21f7..cfe25268f 100644 --- a/src/api/handlers/projects/__init__.py +++ b/src/api/handlers/projects/__init__.py @@ -1,6 +1,7 @@ import api.handlers.projects.build import api.handlers.projects.collaborators import api.handlers.projects.commits +import api.handlers.projects.gke_clusters import api.handlers.projects.jobs import api.handlers.projects.projects import api.handlers.projects.secrets diff --git a/src/api/handlers/projects/gke_clusters.py b/src/api/handlers/projects/gke_clusters.py new file mode 100644 index 000000000..3d024ebae --- /dev/null +++ b/src/api/handlers/projects/gke_clusters.py @@ -0,0 +1,333 @@ +"""GKE cluster provisioning API for external CI consumers (e.g. GitHub Actions). + +Thin orchestration on top of the existing InfraBox GCP operator: creates and +manages `GKECluster` custom resources in the worker namespace. The operator +(already deployed) performs the actual GKE cluster lifecycle. + +v0 endpoints (all under /api/v1/projects//gke-clusters): + POST / Create a cluster (async, returns 202) + GET / Get cluster status + GET //kubeconfig Fetch kubeconfig (only when status=ready) + DELETE / Delete the cluster +""" + +import base64 +import os +import uuid + +import requests +from flask import Response, abort, g, request +from flask_restx import Resource, fields + +from pyinfraboxutils.ibrestplus import api + + +# --------------------------------------------------------------------------- # +# Namespace registration +# --------------------------------------------------------------------------- # + +ns = api.namespace( + 'GKEClusters', + path='/api/v1/projects//gke-clusters', + description='Ephemeral GKE cluster provisioning for external CI consumers', +) + + +# --------------------------------------------------------------------------- # +# Constants +# --------------------------------------------------------------------------- # + +CR_GROUP = 'gcp.service.infrabox.net' +CR_VERSION = 'v1alpha1' +CR_PLURAL = 'gkeclusters' + +# Same shape scheduler.py already uses to talk to the K8s API server. +K8S_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token' +K8S_CA_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' + + +# --------------------------------------------------------------------------- # +# Kubernetes helpers (thin wrapper on top of requests, same pattern as +# scheduler.py so we don't need to add the `kubernetes` python client dep) +# --------------------------------------------------------------------------- # + +def _worker_namespace(): + """Namespace in which GKECluster CRs and their kubeconfig secrets live. + + Same env var that the scheduler reads (see scheduler.py:main). It is + injected into the API pod via the `env_general` Helm template. + """ + return os.environ['INFRABOX_GENERAL_WORKER_NAMESPACE'] + + +def _k8s_api_base(): + """In-cluster K8s API server URL.""" + host = os.environ.get('INFRABOX_KUBERNETES_MASTER_HOST', 'kubernetes.default') + port = os.environ.get('INFRABOX_KUBERNETES_MASTER_PORT', '443') + return 'https://{}:{}'.format(host, port) + + +def _k8s_headers(): + with open(K8S_TOKEN_PATH, 'r') as f: + token = f.read().strip() + return { + 'Authorization': 'Bearer {}'.format(token), + 'Content-Type': 'application/json', + } + + +def _k8s_verify(): + """Path to the CA bundle used to verify the K8s API server certificate.""" + return K8S_CA_PATH if os.path.exists(K8S_CA_PATH) else True + + +def _cr_url(name=None): + base = '{api}/apis/{group}/{version}/namespaces/{ns}/{plural}'.format( + api=_k8s_api_base(), + group=CR_GROUP, + version=CR_VERSION, + ns=_worker_namespace(), + plural=CR_PLURAL, + ) + return '{}/{}'.format(base, name) if name else base + + +def _secret_url(name): + return '{api}/api/v1/namespaces/{ns}/secrets/{name}'.format( + api=_k8s_api_base(), + ns=_worker_namespace(), + name=name, + ) + + +def _handle_k8s_status(resp, on_not_found_msg='not found'): + """Translate a non-2xx K8s response into a Flask abort.""" + if resp.status_code == 404: + abort(404, on_not_found_msg) + if not resp.ok: + abort(502, 'kubernetes api error: {} {}'.format( + resp.status_code, resp.text[:200])) + + +# --------------------------------------------------------------------------- # +# Authorization helpers +# --------------------------------------------------------------------------- # + +def _current_user_id(): + """Return current user id from the JWT. Only user tokens are supported + for v0 (OPA rules require token.type=user).""" + token = getattr(g, 'token', None) or {} + if token.get('type') != 'user': + abort(403, 'user token required') + user = token.get('user') or {} + uid = user.get('id') + if not uid: + abort(403, 'user id missing in token') + return uid + + +def _check_membership(project_id): + """Ensure the current user is a collaborator on project_id.""" + user_id = _current_user_id() + row = g.db.execute_one(''' + SELECT 1 FROM collaborator + WHERE user_id = %s AND project_id = %s + ''', [user_id, project_id]) + if not row: + abort(403, 'not a project collaborator') + + +def _check_ownership(cr, project_id): + """Ensure the CR belongs to the given project (label match). + + Guards against cross-project reads / deletes via crafted URLs. + """ + labels = (cr.get('metadata') or {}).get('labels') or {} + if labels.get('infrabox.net/project-id') != str(project_id): + abort(403, 'cluster does not belong to this project') + + +# --------------------------------------------------------------------------- # +# Request / response models (Swagger docs only, not strict validation) +# --------------------------------------------------------------------------- # + +create_model = api.model('GKEClusterCreate', { + 'zone': fields.String(required=True, description='GCP zone, e.g. us-east1-b'), + 'numNodes': fields.Integer(required=False, description='Node count', default=1), + 'machineType': fields.String(required=False, description='GCP machine type', + default='n1-standard-1'), + 'preemptible': fields.Boolean(required=False, description='Use preemptible VMs', + default=True), + 'diskSize': fields.Integer(required=False, description='Node disk size in GB', + default=100), +}) + +status_model = api.model('GKEClusterStatus', { + 'name': fields.String(description='CR name (also the K8s Secret name)'), + 'status': fields.String(description='pending | creating | ready | error | ...'), + 'message': fields.String(description='Operator-provided status message'), + 'clusterName': fields.String(description='GKE cluster name once created'), +}) + + +# --------------------------------------------------------------------------- # +# Endpoints +# --------------------------------------------------------------------------- # + +@ns.route('') +class GKEClusterList(Resource): + + @api.expect(create_model) + def post(self, project_id): + """Create a new ephemeral GKE cluster. + + Returns 202 with the CR name; poll GET / until status=ready. + """ + _check_membership(project_id) + + body = request.get_json(force=True, silent=True) or {} + zone = body.get('zone') + if not zone: + abort(400, 'zone is required') + + name = 'api-{}'.format(uuid.uuid4().hex[:12]) + + cr = { + 'apiVersion': '{}/{}'.format(CR_GROUP, CR_VERSION), + 'kind': 'GKECluster', + 'metadata': { + 'name': name, + 'namespace': _worker_namespace(), + 'labels': { + # Operator uses this label to name the output Secret. + 'service.infrabox.net/secret-name': name, + # Ownership + provenance labels for GC and audit. + 'infrabox.net/created-by': 'api', + 'infrabox.net/project-id': str(project_id), + 'infrabox.net/created-by-user': _current_user_id(), + }, + }, + 'spec': { + 'zone': zone, + 'numNodes': int(body.get('numNodes', 1)), + 'machineType': body.get('machineType', 'n1-standard-1'), + 'preemptible': bool(body.get('preemptible', True)), + 'diskSize': int(body.get('diskSize', 100)), + }, + } + + resp = requests.post( + _cr_url(), + headers=_k8s_headers(), + verify=_k8s_verify(), + json=cr, + timeout=15, + ) + if resp.status_code == 409: + abort(409, 'cluster already exists') + if not resp.ok: + abort(502, 'kubernetes api error: {} {}'.format( + resp.status_code, resp.text[:200])) + + return {'name': name, 'status': 'pending'}, 202 + + +@ns.route('/') +class GKEClusterItem(Resource): + + def get(self, project_id, name): + """Get the current status of a GKE cluster.""" + _check_membership(project_id) + + resp = requests.get( + _cr_url(name), + headers=_k8s_headers(), + verify=_k8s_verify(), + timeout=15, + ) + _handle_k8s_status(resp, 'cluster not found') + + cr = resp.json() + _check_ownership(cr, project_id) + + st = cr.get('status') or {} + return { + 'name': name, + 'status': st.get('status') or 'pending', + 'message': st.get('message'), + 'clusterName': st.get('clusterName'), + }, 200 + + def delete(self, project_id, name): + """Delete the GKE cluster (async via operator finalizer).""" + _check_membership(project_id) + + # First fetch the CR to enforce ownership, then delete. + resp = requests.get( + _cr_url(name), + headers=_k8s_headers(), + verify=_k8s_verify(), + timeout=15, + ) + _handle_k8s_status(resp, 'cluster not found') + _check_ownership(resp.json(), project_id) + + del_resp = requests.delete( + _cr_url(name), + headers=_k8s_headers(), + verify=_k8s_verify(), + timeout=15, + ) + _handle_k8s_status(del_resp, 'cluster not found') + return {'name': name, 'status': 'deleting'}, 202 + + +@ns.route('//kubeconfig') +class GKEClusterKubeconfig(Resource): + + def get(self, project_id, name): + """Fetch the kubeconfig for a ready cluster. + + Returns 409 if the cluster is not yet ready. + Returns the kubeconfig YAML directly (Content-Type: application/yaml). + """ + _check_membership(project_id) + + # 1) Fetch CR, check ownership + readiness. + cr_resp = requests.get( + _cr_url(name), + headers=_k8s_headers(), + verify=_k8s_verify(), + timeout=15, + ) + _handle_k8s_status(cr_resp, 'cluster not found') + + cr = cr_resp.json() + _check_ownership(cr, project_id) + + cr_status = ((cr.get('status') or {}).get('status')) or 'pending' + if cr_status != 'ready': + abort(409, 'cluster not ready (status={})'.format(cr_status)) + + # 2) Fetch the Secret produced by the operator. By convention the + # Secret is named after the CR (see the `secret-name` label above). + sec_resp = requests.get( + _secret_url(name), + headers=_k8s_headers(), + verify=_k8s_verify(), + timeout=15, + ) + _handle_k8s_status(sec_resp, 'kubeconfig secret not found') + + secret = sec_resp.json() + data = secret.get('data') or {} + kubeconfig_b64 = data.get('kubeconfig') + if not kubeconfig_b64: + abort(500, 'kubeconfig field missing in secret') + + try: + kubeconfig_yaml = base64.b64decode(kubeconfig_b64).decode('utf-8') + except Exception: # pragma: no cover - defensive + abort(500, 'failed to decode kubeconfig from secret') + + return Response(kubeconfig_yaml, mimetype='application/yaml') \ No newline at end of file diff --git a/src/openpolicyagent/policies/projects_gke_clusters.rego b/src/openpolicyagent/policies/projects_gke_clusters.rego new file mode 100644 index 000000000..6f848554d --- /dev/null +++ b/src/openpolicyagent/policies/projects_gke_clusters.rego @@ -0,0 +1,46 @@ +package infrabox + +import input as api +import data.infrabox.collaborators.collaborators + + +# Any collaborator (Owner/Developer/Viewer) of the project may +# manage GKE clusters within it. Fine-grained role gating can be added +# in v1 alongside audit + rate limiting. +gke_project_collaborator([user, project]) { + collaborators[i].project_id = project + collaborators[i].user_id = user +} + + +# POST /api/v1/projects//gke-clusters -> create +allow { + api.method = "POST" + api.path = ["api", "v1", "projects", project, "gke-clusters"] + api.token.type = "user" + gke_project_collaborator([api.token.user.id, project]) +} + +# GET /api/v1/projects//gke-clusters/ -> read status +allow { + api.method = "GET" + api.path = ["api", "v1", "projects", project, "gke-clusters", _] + api.token.type = "user" + gke_project_collaborator([api.token.user.id, project]) +} + +# GET /api/v1/projects//gke-clusters//kubeconfig -> read kubeconfig +allow { + api.method = "GET" + api.path = ["api", "v1", "projects", project, "gke-clusters", _, "kubeconfig"] + api.token.type = "user" + gke_project_collaborator([api.token.user.id, project]) +} + +# DELETE /api/v1/projects//gke-clusters/ -> delete +allow { + api.method = "DELETE" + api.path = ["api", "v1", "projects", project, "gke-clusters", _] + api.token.type = "user" + gke_project_collaborator([api.token.user.id, project]) +} \ No newline at end of file From 9b3bb05249d8d7d1889568e94fccced72b594eda Mon Sep 17 00:00:00 2001 From: Jiachen0715 Date: Mon, 24 Aug 2026 11:23:19 +0800 Subject: [PATCH 2/2] fix(db): make 00046.sql a real no-op so migration job passes --- src/db/migrations/00046.sql | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/db/migrations/00046.sql b/src/db/migrations/00046.sql index db49fbdf0..185f0144b 100644 --- a/src/db/migrations/00046.sql +++ b/src/db/migrations/00046.sql @@ -8,6 +8,13 @@ -- to be skipped on databases already at schema_version 47. -- -- This placeholder fills the gap so the numbering is contiguous again and the --- number/index alignment is restored. It performs no schema change: --- migrate.py strips the file contents and only executes when non-empty, so a --- comment-only file is a safe no-op that still advances schema_version. +-- number/index alignment is restored. It performs no schema change. +-- +-- Note: migrate.py's `apply_migration` does `sql.strip()` and then guards with +-- `if sql:` before calling `cur.execute(sql)`. For a comment-only file, +-- `strip()` does NOT remove SQL comments, so `sql` is a non-empty string +-- containing only `--` lines. psycopg2 then rejects it with +-- `ProgrammingError: can't execute an empty query`, which fails the whole +-- migration job and blocks 00047/00048 from running. A single trivial +-- statement below makes this file a true no-op that psycopg2 accepts. +SELECT 1; \ No newline at end of file