diff --git a/tests/templates/kuttl/openlineage/00-limit-range.yaml b/tests/templates/kuttl/openlineage/00-limit-range.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/openlineage/00-limit-range.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2 b/tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/openlineage/00-rbac.yaml.j2 b/tests/templates/kuttl/openlineage/00-rbac.yaml.j2 new file mode 100644 index 00000000..0b9df7d2 --- /dev/null +++ b/tests/templates/kuttl/openlineage/00-rbac.yaml.j2 @@ -0,0 +1,38 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: + - apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - get + - patch +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/openlineage/10-assert.yaml b/tests/templates/kuttl/openlineage/10-assert.yaml new file mode 100644 index 00000000..e9c60b15 --- /dev/null +++ b/tests/templates/kuttl/openlineage/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-superset-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/openlineage/10-install-postgresql.yaml b/tests/templates/kuttl/openlineage/10-install-postgresql.yaml new file mode 100644 index 00000000..50c5ad67 --- /dev/null +++ b/tests/templates/kuttl/openlineage/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install superset-postgresql + --namespace $NAMESPACE + --version 12.5.6 + -f helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/openlineage/20-assert.yaml.j2 b/tests/templates/kuttl/openlineage/20-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/openlineage/20-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/openlineage/20-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/openlineage/20-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/openlineage/20-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/openlineage/30-assert.yaml b/tests/templates/kuttl/openlineage/30-assert.yaml new file mode 100644 index 00000000..f1a136d8 --- /dev/null +++ b/tests/templates/kuttl/openlineage/30-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: openlineage-receiver +timeout: 240 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openlineage-receiver +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/openlineage/30-install-openlineage-receiver.yaml.j2 b/tests/templates/kuttl/openlineage/30-install-openlineage-receiver.yaml.j2 new file mode 100644 index 00000000..7ae8908f --- /dev/null +++ b/tests/templates/kuttl/openlineage/30-install-openlineage-receiver.yaml.j2 @@ -0,0 +1,96 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openlineage-receiver-script +data: + receiver.py: | + #!/usr/bin/env python + # Minimal OpenLineage receiver used for integration testing. + # The OpenLineage HTTP transport POSTs run events to /api/v1/lineage; this handler + # appends every POST body to a log regardless of path. GET returns everything received + # so far, so the test can assert lineage was emitted. + import http.server + import os + + EVENTS = "/tmp/events.log" + PORT = int(os.environ.get("PORT", "5000")) + + class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) if length else b"" + with open(EVENTS, "ab") as f: + f.write(body + b"\n") + self.send_response(201) + self.end_headers() + + def do_GET(self): + try: + with open(EVENTS, "rb") as f: + data = f.read() + except FileNotFoundError: + data = b"" + self.send_response(200) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args): + pass + + httpd = http.server.HTTPServer(("0.0.0.0", PORT), Handler) + httpd.serve_forever() +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openlineage-receiver +spec: + replicas: 1 + selector: + matchLabels: + app: openlineage-receiver + template: + metadata: + labels: + app: openlineage-receiver + spec: + serviceAccountName: integration-tests-sa + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + containers: + - name: receiver + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + command: ["python", "/scripts/receiver.py"] + env: + - name: PORT + value: "5000" + ports: + - containerPort: 5000 + volumeMounts: + - name: script + mountPath: /scripts + resources: + requests: + memory: "128Mi" + cpu: "256m" + limits: + memory: "128Mi" + cpu: "1" + volumes: + - name: script + configMap: + name: openlineage-receiver-script +--- +apiVersion: v1 +kind: Service +metadata: + name: openlineage-receiver +spec: + selector: + app: openlineage-receiver + ports: + - port: 5000 + targetPort: 5000 diff --git a/tests/templates/kuttl/openlineage/40-assert.yaml b/tests/templates/kuttl/openlineage/40-assert.yaml new file mode 100644 index 00000000..cf2629de --- /dev/null +++ b/tests/templates/kuttl/openlineage/40-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-superset +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true supersetclusters.superset.stackable.tech/superset --timeout 301s diff --git a/tests/templates/kuttl/openlineage/40-install-superset.yaml.j2 b/tests/templates/kuttl/openlineage/40-install-superset.yaml.j2 new file mode 100644 index 00000000..2f151ef7 --- /dev/null +++ b/tests/templates/kuttl/openlineage/40-install-superset.yaml.j2 @@ -0,0 +1,75 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-superset +timeout: 300 +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-admin-credentials +type: Opaque +stringData: + adminUser.username: admin + adminUser.firstname: Superset + adminUser.lastname: Admin + adminUser.email: admin@superset.com + adminUser.password: admin +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-postgresql-credentials +stringData: + username: superset + password: superset +--- +apiVersion: superset.stackable.tech/v1alpha1 +kind: SupersetCluster +metadata: + name: superset +spec: + image: +{% if test_scenario['values']['superset-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['superset-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['superset-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['superset-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + credentialsSecret: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + envOverrides: + # Point the OpenLineage client (installed in the image via the superset_openlineage + # module) at the in-cluster test receiver. The client POSTs run events to + # $OPENLINEAGE_URL/api/v1/lineage. + OPENLINEAGE_URL: http://openlineage-receiver:5000 + SUPERSET_OPENLINEAGE_NAMESPACE: superset + configOverrides: + superset_config.py: + # Wire the (otherwise inert) superset_openlineage module into Superset. FILE_FOOTER is + # appended verbatim to superset_config.py by the operator, so it can hold arbitrary + # Python: the QUERY_LOGGER stashes a pending run per SQL Lab query and the + # OpenLineageEventLogger emits START + COMPLETE events on execute_sql. + FILE_FOOTER: | + from superset_openlineage import build_query_logger, OpenLineageEventLogger, init_app + QUERY_LOGGER = build_query_logger() + EVENT_LOGGER = OpenLineageEventLogger() + def FLASK_APP_MUTATOR(app): + init_app(app) + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/openlineage/50-assert.yaml b/tests/templates/kuttl/openlineage/50-assert.yaml new file mode 100644 index 00000000..58987778 --- /dev/null +++ b/tests/templates/kuttl/openlineage/50-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: python +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/openlineage/50-install-test-container.yaml b/tests/templates/kuttl/openlineage/50-install-test-container.yaml new file mode 100644 index 00000000..f760eb87 --- /dev/null +++ b/tests/templates/kuttl/openlineage/50-install-test-container.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: python + labels: + app: python +spec: + replicas: 1 + selector: + matchLabels: + app: python + template: + metadata: + labels: + app: python + spec: + serviceAccountName: integration-tests-sa + containers: + - name: python + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + stdin: true + tty: true + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" diff --git a/tests/templates/kuttl/openlineage/60-assert.yaml b/tests/templates/kuttl/openlineage/60-assert.yaml new file mode 100644 index 00000000..ac042680 --- /dev/null +++ b/tests/templates/kuttl/openlineage/60-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-openlineage-events +timeout: 480 +commands: + - script: kubectl exec -n $NAMESPACE python-0 -- python /tmp/check_lineage.py --receiver-url http://openlineage-receiver:5000 diff --git a/tests/templates/kuttl/openlineage/60-copy-scripts.yaml b/tests/templates/kuttl/openlineage/60-copy-scripts.yaml new file mode 100644 index 00000000..bd194bb0 --- /dev/null +++ b/tests/templates/kuttl/openlineage/60-copy-scripts.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl cp -n $NAMESPACE ./check_lineage.py python-0:/tmp + timeout: 240 diff --git a/tests/templates/kuttl/openlineage/check_lineage.py b/tests/templates/kuttl/openlineage/check_lineage.py new file mode 100644 index 00000000..b21cdba7 --- /dev/null +++ b/tests/templates/kuttl/openlineage/check_lineage.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +"""Trigger a SQL Lab query and assert that OpenLineage events reached the test receiver. + +Runs inside the `python` testing-tools pod. It: + 1. logs in to Superset and gets a JWT access token, + 2. registers the Postgres metadata database as a SQL Lab data source (idempotent), + 3. runs a synchronous SELECT against `public.ab_user`, + 4. polls the test receiver's `/events` endpoint until it has captured a terminal + `COMPLETE` OpenLineage event for that query (the SQL text is embedded in the event's + SQL facet, so we match on it to prove the event came from *our* query). + +Emission is driven by the `superset_openlineage` module wired into `superset_config.py` +(QUERY_LOGGER + EVENT_LOGGER): a SQL Lab `execute_sql` produces a START and a COMPLETE +RunEvent, POSTed by the OpenLineage HTTP client to `$OPENLINEAGE_URL/api/v1/lineage`. +""" + +import argparse +import sys +import time + +import requests + +SUPERSET_URL = "http://superset-node:8088" +DATABASE_NAME = "postgresql-openlineage" +SQLALCHEMY_URI = "postgresql+psycopg2://superset:superset@superset-postgresql:5432/superset" +# A table that always exists in the Superset metadata database. The SQL text is echoed back +# in the emitted event's SQL facet, so we can match on it at the receiver. +QUERY_SQL = "SELECT username FROM public.ab_user LIMIT 1" +QUERY_MARKER = "ab_user" + + +def new_session(base_url: str) -> requests.Session: + """Log in and return a session ready for state-changing API calls. + + Superset protects its write endpoints (database registration, SQL Lab execute) with + Flask-WTF CSRF, which needs three things sent together: the JWT bearer token, the + `X-CSRFToken` header, and the session cookie the CSRF-token endpoint sets. A shared + `Session` keeps that cookie; `Referer` is required by the CSRF check too. + """ + session = requests.Session() + + resp = session.post( + f"{base_url}/api/v1/security/login", + json={"username": "admin", "password": "admin", "provider": "db", "refresh": "true"}, + timeout=10, + ) + resp.raise_for_status() + token = resp.json()["access_token"] + session.headers.update({"Authorization": f"Bearer {token}", "Referer": base_url}) + + csrf = session.get(f"{base_url}/api/v1/security/csrf_token/", timeout=10) + csrf.raise_for_status() + session.headers.update({"X-CSRFToken": csrf.json()["result"]}) + + return session + + +def get_database_id(session: requests.Session, base_url: str) -> int | None: + """Return the id of our registered database, or None if it is not registered yet.""" + resp = session.get(f"{base_url}/api/v1/database/", timeout=10) + resp.raise_for_status() + for db in resp.json().get("result", []): + if db.get("database_name") == DATABASE_NAME: + return db["id"] + return None + + +def ensure_database(session: requests.Session, base_url: str) -> int: + """Register the metadata database for SQL Lab (idempotent) and return its id.""" + existing = get_database_id(session, base_url) + if existing is not None: + return existing + resp = session.post( + f"{base_url}/api/v1/database/", + json={ + "database_name": DATABASE_NAME, + "sqlalchemy_uri": SQLALCHEMY_URI, + "expose_in_sqllab": True, + }, + timeout=30, + ) + if resp.status_code < 300: + return resp.json()["id"] + # A concurrent/previous run may have created it between our check and our POST. + existing = get_database_id(session, base_url) + if existing is not None: + return existing + raise RuntimeError(f"Could not register database: {resp.status_code} {resp.text}") + + +def run_query(session: requests.Session, base_url: str, database_id: int) -> None: + resp = session.post( + f"{base_url}/api/v1/sqllab/execute/", + json={"database_id": database_id, "runAsync": False, "sql": QUERY_SQL}, + timeout=60, + ) + print(f"Query response: {resp.status_code} {resp.text[:500]}") + resp.raise_for_status() + + +def receiver_has_lineage(receiver_url: str) -> bool: + resp = requests.get(f"{receiver_url}/events", timeout=10) + if resp.status_code != 200: + return False + body = resp.text + return QUERY_MARKER in body and "COMPLETE" in body + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Superset OpenLineage lineage check") + parser.add_argument("--superset-url", default=SUPERSET_URL) + parser.add_argument( + "--receiver-url", + required=True, + help="Base URL of the OpenLineage test receiver, e.g. http://openlineage-receiver:5000", + ) + opts = parser.parse_args() + + deadline = time.time() + 300 + while time.time() < deadline: + try: + session = new_session(opts.superset_url) + database_id = ensure_database(session, opts.superset_url) + run_query(session, opts.superset_url, database_id) + + if receiver_has_lineage(opts.receiver_url): + print("OpenLineage events for the query were received.") + sys.exit(0) + except Exception as e: # noqa: BLE001 + print(f"Retrying after error: {e}") + + time.sleep(10) + + # Timed out - dump what the receiver captured to aid debugging. + try: + dump = requests.get(f"{opts.receiver_url}/events", timeout=10).text + print(f"Receiver events at timeout:\n{dump}") + except Exception as e: # noqa: BLE001 + print(f"Could not read receiver events: {e}") + print("Timed out waiting for OpenLineage events at the receiver.") + sys.exit(1) diff --git a/tests/templates/kuttl/openlineage/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/openlineage/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..2e851682 --- /dev/null +++ b/tests/templates/kuttl/openlineage/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,44 @@ +--- +global: + security: + allowInsecureImages: true # needed starting with Chart version 16.3.0 if modifying images + +image: + repository: bitnamilegacy/postgresql + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + securityContext: + runAsUser: auto + +metrics: + image: + repository: bitnamilegacy/postgres-exporter + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + +shmVolume: + chmod: + enabled: false + +auth: + username: superset + password: superset + database: superset diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 078cd3a0..802dcb3f 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -83,6 +83,10 @@ tests: dimensions: - superset - openshift + - name: openlineage + dimensions: + - superset-latest + - openshift - name: upgrade dimensions: - superset-old