feat: add support for openlineage events

tests/templates/kuttl/openlineage/00-limit-range.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: limit-request-ratio
+spec:
+  limits:
+  - type: "Container"
+    maxLimitRequestRatio:
+      cpu: 5
+      memory: 1

tests/templates/kuttl/openlineage/00-patch-ns.yaml.j2

@@ -0,0 +1,9 @@
+{% if test_scenario['values']['openshift'] == 'true' %}
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
+    timeout: 120
+{% endif %}

tests/templates/kuttl/openlineage/00-rbac.yaml.j2

@@ -0,0 +1,38 @@
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: use-integration-tests-scc
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - secrets
+    verbs:
+      - create
+      - get
+      - patch
+{% if test_scenario['values']['openshift'] == "true" %}
+  - apiGroups: ["security.openshift.io"]
+    resources: ["securitycontextconstraints"]
+    resourceNames: ["privileged"]
+    verbs: ["use"]
+{% endif %}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: integration-tests-sa
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: use-integration-tests-scc
+subjects:
+  - kind: ServiceAccount
+    name: integration-tests-sa
+roleRef:
+  kind: Role
+  name: use-integration-tests-scc
+  apiGroup: rbac.authorization.k8s.io

tests/templates/kuttl/openlineage/10-assert.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: test-superset-postgresql
+timeout: 480
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: superset-postgresql
+status:
+  readyReplicas: 1
+  replicas: 1

tests/templates/kuttl/openlineage/10-install-postgresql.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: >-
+      helm install superset-postgresql
+      --namespace $NAMESPACE
+      --version 12.5.6
+      -f helm-bitnami-postgresql-values.yaml
+      --repo https://charts.bitnami.com/bitnami postgresql
+      --wait
+    timeout: 600

tests/templates/kuttl/openlineage/20-assert.yaml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}

tests/templates/kuttl/openlineage/20-install-vector-aggregator-discovery-configmap.yaml.j2

@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}

tests/templates/kuttl/openlineage/30-assert.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: openlineage-receiver
+timeout: 240
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: openlineage-receiver
+status:
+  readyReplicas: 1
+  replicas: 1

tests/templates/kuttl/openlineage/30-install-openlineage-receiver.yaml.j2

@@ -0,0 +1,96 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: openlineage-receiver-script
+data:
+  receiver.py: |
+    #!/usr/bin/env python
+    # Minimal OpenLineage receiver used for integration testing.
+    # The OpenLineage HTTP transport POSTs run events to /api/v1/lineage; this handler
+    # appends every POST body to a log regardless of path. GET returns everything received
+    # so far, so the test can assert lineage was emitted.
+    import http.server
+    import os
+
+    EVENTS = "/tmp/events.log"
+    PORT = int(os.environ.get("PORT", "5000"))
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def do_POST(self):
+            length = int(self.headers.get("Content-Length", 0))
+            body = self.rfile.read(length) if length else b""
+            with open(EVENTS, "ab") as f:
+                f.write(body + b"\n")
+            self.send_response(201)
+            self.end_headers()
+
+        def do_GET(self):
+            try:
+                with open(EVENTS, "rb") as f:
+                    data = f.read()
+            except FileNotFoundError:
+                data = b""
+            self.send_response(200)
+            self.end_headers()
+            self.wfile.write(data)
+
+        def log_message(self, *args):
+            pass
+
+    httpd = http.server.HTTPServer(("0.0.0.0", PORT), Handler)
+    httpd.serve_forever()
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: openlineage-receiver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: openlineage-receiver
+  template:
+    metadata:
+      labels:
+        app: openlineage-receiver
+    spec:
+      serviceAccountName: integration-tests-sa
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+      containers:
+        - name: receiver
+          image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev
+          command: ["python", "/scripts/receiver.py"]
+          env:
+            - name: PORT
+              value: "5000"
+          ports:
+            - containerPort: 5000
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "256m"
+            limits:
+              memory: "128Mi"
+              cpu: "1"
+      volumes:
+        - name: script
+          configMap:
+            name: openlineage-receiver-script
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openlineage-receiver
+spec:
+  selector:
+    app: openlineage-receiver
+  ports:
+    - port: 5000
+      targetPort: 5000

tests/templates/kuttl/openlineage/40-assert.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: install-superset
+timeout: 300
+commands:
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true supersetclusters.superset.stackable.tech/superset --timeout 301s

tests/templates/kuttl/openlineage/40-install-superset.yaml.j2

@@ -0,0 +1,75 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: superset-admin-credentials
+type: Opaque
+stringData:
+  adminUser.username: admin
+  adminUser.firstname: Superset
+  adminUser.lastname: Admin
+  adminUser.email: admin@superset.com
+  adminUser.password: admin
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: superset-postgresql-credentials
+stringData:
+  username: superset
+  password: superset
+---
+apiVersion: superset.stackable.tech/v1alpha1
+kind: SupersetCluster
+metadata:
+  name: superset
+spec:
+  image:
+{% if test_scenario['values']['superset-latest'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['superset-latest'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['superset-latest'].split(',')[0] }}"
+{% else %}
+    productVersion: "{{ test_scenario['values']['superset-latest'] }}"
+{% endif %}
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    credentialsSecret: superset-admin-credentials
+    metadataDatabase:
+      postgresql:
+        host: superset-postgresql
+        database: superset
+        credentialsSecretName: superset-postgresql-credentials
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  nodes:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    envOverrides:
+      # Point the OpenLineage client (installed in the image via the superset_openlineage
+      # module) at the in-cluster test receiver. The client POSTs run events to
+      # $OPENLINEAGE_URL/api/v1/lineage.
+      OPENLINEAGE_URL: http://openlineage-receiver:5000
+      SUPERSET_OPENLINEAGE_NAMESPACE: superset
+    configOverrides:
+      superset_config.py:
+        # Wire the (otherwise inert) superset_openlineage module into Superset. FILE_FOOTER is
+        # appended verbatim to superset_config.py by the operator, so it can hold arbitrary
+        # Python: the QUERY_LOGGER stashes a pending run per SQL Lab query and the
+        # OpenLineageEventLogger emits START + COMPLETE events on execute_sql.
+        FILE_FOOTER: |
+          from superset_openlineage import build_query_logger, OpenLineageEventLogger, init_app
+          QUERY_LOGGER = build_query_logger()
+          EVENT_LOGGER = OpenLineageEventLogger()
+          def FLASK_APP_MUTATOR(app):
+              init_app(app)
+    roleGroups:
+      default:
+        replicas: 1

tests/templates/kuttl/openlineage/50-assert.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: install-test-container
+timeout: 300
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: python
+status:
+  readyReplicas: 1
+  replicas: 1

tests/templates/kuttl/openlineage/50-install-test-container.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+metadata:
+  name: install-test-container
+timeout: 300
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: python
+  labels:
+    app: python
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: python
+  template:
+    metadata:
+      labels:
+        app: python
+    spec:
+      serviceAccountName: integration-tests-sa
+      containers:
+        - name: python
+          image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev
+          stdin: true
+          tty: true
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "512m"
+            limits:
+              memory: "128Mi"
+              cpu: "1"

tests/templates/kuttl/openlineage/60-assert.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-openlineage-events
+timeout: 480
+commands:
+  - script: kubectl exec -n $NAMESPACE python-0 -- python /tmp/check_lineage.py --receiver-url http://openlineage-receiver:5000

tests/templates/kuttl/openlineage/60-copy-scripts.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl cp -n $NAMESPACE ./check_lineage.py python-0:/tmp
+    timeout: 240