diff --git a/.cruft.json b/.cruft.json index b85834a..0af0317 100644 --- a/.cruft.json +++ b/.cruft.json @@ -4,10 +4,10 @@ "checkout": "main", "context": { "cookiecutter": { - "name": "A high-performance observability data pipeline.", + "name": "Vector", "slug": "vector", "parameter_key": "vector", - "test_cases": "defaults", + "test_cases": "defaults resources", "add_lib": "n", "add_pp": "n", "add_golden": "y", diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 986e75f..ad84919 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,6 +33,7 @@ jobs: matrix: instance: - defaults + - resources defaults: run: working-directory: ${{ env.COMPONENT_NAME }} @@ -48,6 +49,7 @@ jobs: matrix: instance: - defaults + - resources defaults: run: working-directory: ${{ env.COMPONENT_NAME }} diff --git a/Makefile.vars.mk b/Makefile.vars.mk index dfaa27a..4d0b133 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -50,4 +50,4 @@ KUBENT_IMAGE ?= ghcr.io/doitintl/kube-no-trouble:latest KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE) instance ?= defaults -test_instances = tests/defaults.yml +test_instances = tests/defaults.yml tests/resources.yml diff --git a/README.md b/README.md index 0d79dac..53166a5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Commodore Component: A high-performance observability data pipeline. +# Commodore Component: Vector -This is a [Commodore][commodore] Component for A high-performance observability data pipeline.. +This is a [Commodore][commodore] Component for Vector. This repository is part of Project Syn. For documentation on Project Syn and this component, see [syn.tools](https://syn.tools). diff --git a/class/defaults.yml b/class/defaults.yml index 0d13b1a..552564c 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -2,4 +2,17 @@ parameters: vector: =_metadata: multi_tenant: true - namespace: syn-vector + namespace: + name: syn-vector + labels: {} + annotations: {} + + # Cluster identifier exposed for pipelines that label or partition events per cluster. + tenant_id: ${cluster:name} + + charts: + vector: + source: https://helm.vector.dev + version: 0.52.0 + + helm_values: {} diff --git a/class/vector.yml b/class/vector.yml index eee8cec..0d0293f 100644 --- a/class/vector.yml +++ b/class/vector.yml @@ -1,5 +1,11 @@ parameters: kapitan: + dependencies: + - type: helm + source: ${vector:charts:vector:source} + version: ${vector:charts:vector:version} + chart_name: vector + output_path: ${_base_directory}/helmcharts/vector/${vector:charts:vector:version} compile: - input_paths: - ${_base_directory}/component/app.jsonnet @@ -9,3 +15,12 @@ parameters: - ${_base_directory}/component/main.jsonnet input_type: jsonnet output_path: vector/ + # vector Helm chart + - input_paths: + - ${_base_directory}/helmcharts/vector/${vector:charts:vector:version} + input_type: helm + helm_values: ${vector:helm_values} + helm_params: + name: vector + namespace: ${vector:namespace:name} + output_path: vector/01_vector diff --git a/component/app.jsonnet b/component/app.jsonnet index ee199db..ee03466 100644 --- a/component/app.jsonnet +++ b/component/app.jsonnet @@ -3,7 +3,7 @@ local inv = kap.inventory(); local params = inv.parameters.vector; local argocd = import 'lib/argocd.libjsonnet'; -local app = argocd.App('vector', params.namespace); +local app = argocd.App('vector', params.namespace.name); local appPath = local project = std.get(std.get(app, 'spec', {}), 'project', 'syn'); diff --git a/component/main.jsonnet b/component/main.jsonnet index 540e800..57b86e3 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -7,4 +7,10 @@ local params = inv.parameters.vector; // Define outputs below { + '00_namespace': kube.Namespace(params.namespace.name) { + metadata+: { + annotations+: params.namespace.annotations, + labels+: params.namespace.labels, + }, + }, } diff --git a/docs/antora.yml b/docs/antora.yml index ee71662..a4c973f 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -1,5 +1,5 @@ name: vector -title: A high-performance observability data pipeline. +title: Vector version: master start_page: ROOT:index.adoc nav: diff --git a/docs/modules/ROOT/pages/how-tos/talos.adoc b/docs/modules/ROOT/pages/how-tos/talos.adoc new file mode 100644 index 0000000..7cb0b64 --- /dev/null +++ b/docs/modules/ROOT/pages/how-tos/talos.adoc @@ -0,0 +1,52 @@ += Collect Talos host logs + +This guide shows how to forward Talos service and kernel logs into a Vector `socket` source running as a DaemonSet `Agent` on each node. + +== Talos MachineConfig + +Forward Talos service logs to the loopback UDP socket Vector listens on: + +[source,yaml] +---- +machine: + logging: + destinations: + - endpoint: "udp://127.0.0.1:5140/" + format: "json_lines" + extraTags: + cluster: +---- + +Forward kernel ring buffer (`kmsg`) entries to the same socket: + +[source,yaml] +---- +apiVersion: v1alpha1 +kind: KmsgLogConfig +name: remote-kernel-log +url: udp://127.0.0.1:5140/ +---- + +== Vector helm_values + +The Vector Agent must share the host network namespace so the loopback datagrams arrive: + +[source,yaml] +---- +parameters: + vector: + helm_values: + role: Agent + podHostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + customConfig: + sources: + talos_logs: + type: socket + mode: udp + address: 0.0.0.0:5140 + decoding: + codec: json +---- + +See `tests/resources.yml` in this repo for a fuller pipeline example that ships the parsed logs to Loki. diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 321ecf4..83f6b96 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -1,5 +1,6 @@ -= A high-performance observability data pipeline. += Vector -vector is a Commodore component to manage A high-performance observability data pipeline.. +vector is a Commodore component to manage https://vector.dev[Vector]. See the xref:references/parameters.adoc[parameters] reference for further details. +See xref:how-tos/talos.adoc[Collect Talos host logs] for an example. diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 10c3639..aca3388 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -2,18 +2,72 @@ The parent key for all of the following parameters is `vector`. + == `namespace` +[horizontal] +type:: dictionary +default:: https://github.com/projectsyn/component-vector/blob/master/class/defaults.yml[See `class/defaults.yml`] + +Configures the namespace in which to deploy this component. + +=== `namespace.name` + [horizontal] type:: string default:: `syn-vector` -The namespace in which to deploy this component. +The namespace name. + +=== `namespace.labels` + +[horizontal] +type:: dictionary +default:: `{}` + +Additional labels to add to the component's namespace. + +=== `namespace.annotations` + +[horizontal] +type:: dictionary +default:: `{}` + +Additional annotations to add to the component's namespace. + + +== `tenant_id` + +[horizontal] +type:: string +default:: `${cluster:name}` + +Cluster identifier exposed for pipelines that need to label or partition events per cluster. +Reference it from `helm_values` as `${vector:tenant_id}` to inject the same value into env vars, sink tenant fields, or transform expressions. + + +== `charts` + +[horizontal] +type:: dictionary +default:: See https://github.com/projectsyn/component-vector/blob/master/class/defaults.yml[`class/defaults.yml`] + +The Helm chart source and version for the Vector chart. + + +== `helm_values` + +[horizontal] +type:: dictionary +default:: `{}` +Helm values to pass to the Vector Helm chart. +The component does not set any defaults beyond the chart's own. +See the https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/values.yaml[upstream `values.yaml`] for available options. -== Example +Typical fields a consumer will set: -[source,yaml] ----- -namespace: example-namespace ----- +* `role` (`Agent`, `Aggregator`, or `Stateless-Aggregator`). +* `podHostNetwork` (for example required when reading Talos logs from `127.0.0.1:5140`). +* `customConfig` (the full Vector pipeline definition). +* `env` (downward-API entries such as `VECTOR_NODE_NAME`). diff --git a/docs/modules/ROOT/partials/nav.adoc b/docs/modules/ROOT/partials/nav.adoc index 08f9283..6984b36 100644 --- a/docs/modules/ROOT/partials/nav.adoc +++ b/docs/modules/ROOT/partials/nav.adoc @@ -1,2 +1,4 @@ * xref:index.adoc[Home] * xref:references/parameters.adoc[Parameters] +* How-tos +** xref:how-tos/talos.adoc[Collect Talos host logs] diff --git a/tests/golden/defaults/vector/vector/00_namespace.yaml b/tests/golden/defaults/vector/vector/00_namespace.yaml new file mode 100644 index 0000000..24f5433 --- /dev/null +++ b/tests/golden/defaults/vector/vector/00_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: {} + labels: + name: syn-vector + name: syn-vector diff --git a/tests/golden/defaults/vector/vector/01_vector/vector/templates/configmap.yaml b/tests/golden/defaults/vector/vector/01_vector/vector/templates/configmap.yaml new file mode 100644 index 0000000..a54fa64 --- /dev/null +++ b/tests/golden/defaults/vector/vector/01_vector/vector/templates/configmap.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +data: + aggregator.yaml: | + data_dir: /vector-data-dir + api: + enabled: true + address: 0.0.0.0:8686 + sources: + datadog_agent: + address: 0.0.0.0:8282 + type: datadog_agent + fluent: + address: 0.0.0.0:24224 + type: fluent + internal_metrics: + type: internal_metrics + logstash: + address: 0.0.0.0:5044 + type: logstash + splunk_hec: + address: 0.0.0.0:8080 + type: splunk_hec + statsd: + address: 0.0.0.0:8125 + mode: tcp + type: statsd + syslog: + address: 0.0.0.0:9000 + mode: tcp + type: syslog + vector: + address: 0.0.0.0:6000 + type: vector + version: "2" + sinks: + prom_exporter: + type: prometheus_exporter + inputs: [internal_metrics] + address: 0.0.0.0:9090 + stdout: + type: console + inputs: [datadog_agent, fluent, logstash, splunk_hec, statsd, syslog, vector] + encoding: + codec: json +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector diff --git a/tests/golden/defaults/vector/vector/01_vector/vector/templates/service-headless.yaml b/tests/golden/defaults/vector/vector/01_vector/vector/templates/service-headless.yaml new file mode 100644 index 0000000..98f7fa7 --- /dev/null +++ b/tests/golden/defaults/vector/vector/01_vector/vector/templates/service-headless.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: null + labels: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector-headless + namespace: syn-vector +spec: + clusterIP: None + ports: + - name: datadog-agent + port: 8282 + protocol: TCP + - name: fluent + port: 24224 + protocol: TCP + - name: logstash + port: 5044 + protocol: TCP + - name: splunk-hec + port: 8080 + protocol: TCP + - name: statsd + port: 8125 + protocol: TCP + - name: syslog + port: 9000 + protocol: TCP + - name: vector + port: 6000 + protocol: TCP + - name: prom-exporter + port: 9090 + protocol: TCP + selector: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + type: ClusterIP diff --git a/tests/golden/defaults/vector/vector/01_vector/vector/templates/service.yaml b/tests/golden/defaults/vector/vector/01_vector/vector/templates/service.yaml new file mode 100644 index 0000000..0510fb4 --- /dev/null +++ b/tests/golden/defaults/vector/vector/01_vector/vector/templates/service.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: null + labels: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector +spec: + ports: + - name: datadog-agent + port: 8282 + protocol: TCP + - name: fluent + port: 24224 + protocol: TCP + - name: logstash + port: 5044 + protocol: TCP + - name: splunk-hec + port: 8080 + protocol: TCP + - name: statsd + port: 8125 + protocol: TCP + - name: syslog + port: 9000 + protocol: TCP + - name: vector + port: 6000 + protocol: TCP + - name: prom-exporter + port: 9090 + protocol: TCP + selector: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + type: ClusterIP diff --git a/tests/golden/defaults/vector/vector/01_vector/vector/templates/serviceaccount.yaml b/tests/golden/defaults/vector/vector/01_vector/vector/templates/serviceaccount.yaml new file mode 100644 index 0000000..6fbd63a --- /dev/null +++ b/tests/golden/defaults/vector/vector/01_vector/vector/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector diff --git a/tests/golden/defaults/vector/vector/01_vector/vector/templates/statefulset.yaml b/tests/golden/defaults/vector/vector/01_vector/vector/templates/statefulset.yaml new file mode 100644 index 0000000..d3e3f92 --- /dev/null +++ b/tests/golden/defaults/vector/vector/01_vector/vector/templates/statefulset.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: {} + labels: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector +spec: + minReadySeconds: 0 + podManagementPolicy: OrderedReady + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + serviceName: vector-headless + template: + metadata: + annotations: + checksum/config: 9689c15cc2ade8f68f33cf39f534f5877b504362168acc00963d8cac2f9474ee + labels: + app.kubernetes.io/component: Aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + vector.dev/exclude: 'true' + spec: + containers: + - args: + - --config-dir + - /etc/vector/ + env: + - name: VECTOR_LOG + value: info + image: docker.io/timberio/vector:0.55.0-distroless-libc + imagePullPolicy: IfNotPresent + name: vector + ports: + - containerPort: 8282 + name: datadog-agent + protocol: TCP + - containerPort: 24224 + name: fluent + protocol: TCP + - containerPort: 5044 + name: logstash + protocol: TCP + - containerPort: 8080 + name: splunk-hec + protocol: TCP + - containerPort: 8125 + name: statsd + protocol: TCP + - containerPort: 9000 + name: syslog + protocol: TCP + - containerPort: 6000 + name: vector + protocol: TCP + - containerPort: 9090 + name: prom-exporter + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 + volumeMounts: + - mountPath: /vector-data-dir + name: data + - mountPath: /etc/vector/ + name: config + readOnly: true + dnsPolicy: ClusterFirst + serviceAccountName: vector + terminationGracePeriodSeconds: 60 + volumes: + - emptyDir: {} + name: data + - name: config + projected: + sources: + - configMap: + name: vector + volumeClaimTemplates: null diff --git a/tests/golden/resources/vector/apps/vector.yaml b/tests/golden/resources/vector/apps/vector.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/resources/vector/vector/00_namespace.yaml b/tests/golden/resources/vector/vector/00_namespace.yaml new file mode 100644 index 0000000..24f5433 --- /dev/null +++ b/tests/golden/resources/vector/vector/00_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: {} + labels: + name: syn-vector + name: syn-vector diff --git a/tests/golden/resources/vector/vector/01_vector/vector/templates/configmap.yaml b/tests/golden/resources/vector/vector/01_vector/vector/templates/configmap.yaml new file mode 100644 index 0000000..958a496 --- /dev/null +++ b/tests/golden/resources/vector/vector/01_vector/vector/templates/configmap.yaml @@ -0,0 +1,123 @@ +apiVersion: v1 +data: + vector.yaml: | + api: + enabled: false + data_dir: /vector-data-dir + sinks: + loki_pod: + buffer: + max_size: 268435488 + type: disk + when_full: drop_newest + compression: snappy + encoding: + codec: text + except_fields: + - host + - source_type + - kubernetes + - file + - source_ip + - stream + endpoint: http://loki-gateway.test-loki.svc.cluster.local:3100 + healthcheck: + enabled: false + inputs: + - pod_transform + labels: + cluster: '{{ cluster }}' + container: '{{ container }}' + namespace: '{{ namespace }}' + node: '{{ node }}' + pod: '{{ pod }}' + service_name: '{{ container }}' + source: pod + out_of_order_action: rewrite_timestamp + request: + retry_max_duration_secs: 60 + timeout_secs: 90 + tenant_id: c-green-test-1234 + type: loki + loki_talos: + buffer: + max_size: 268435488 + type: disk + when_full: drop_newest + compression: snappy + encoding: + codec: text + except_fields: + - host + - source_type + - port + - talos-time + - talos-level + - facility + - priority + - kubernetes + endpoint: http://loki-gateway.test-loki.svc.cluster.local:3100 + healthcheck: + enabled: false + inputs: + - talos_transform + labels: + cluster: '{{ cluster }}' + node: '{{ node }}' + service_name: '{{ service }}' + source: '{{ source }}' + out_of_order_action: rewrite_timestamp + request: + retry_max_duration_secs: 60 + timeout_secs: 90 + tenant_id: c-green-test-1234 + type: loki + sources: + pod_logs: + ignore_older_secs: 600 + type: kubernetes_logs + talos_logs: + address: 0.0.0.0:5140 + decoding: + codec: json + mode: udp + type: socket + transforms: + pod_transform: + inputs: + - pod_logs + source: | + .cluster = get_env_var!("VECTOR_TENANT_ID") + .node = get_env_var!("VECTOR_NODE_NAME") + .namespace = .kubernetes.pod_namespace + .pod = .kubernetes.pod_name + .container = .kubernetes.container_name + type: remap + talos_transform: + inputs: + - talos_logs + source: | + .cluster = get_env_var!("VECTOR_TENANT_ID") + .node = get_env_var!("VECTOR_NODE_NAME") + if exists(.msg) { .message = del(.msg) } + if exists(.log) { .message = del(.log) } + if exists(."talos-service") { .service = del(."talos-service") } + if exists(.facility) { + .source = "kernel" + del(.priority) + del(.facility) + } else { + .source = "service" + } + type: remap +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector diff --git a/tests/golden/resources/vector/vector/01_vector/vector/templates/daemonset.yaml b/tests/golden/resources/vector/vector/01_vector/vector/templates/daemonset.yaml new file mode 100644 index 0000000..2bda09e --- /dev/null +++ b/tests/golden/resources/vector/vector/01_vector/vector/templates/daemonset.yaml @@ -0,0 +1,110 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector +spec: + minReadySeconds: 0 + selector: + matchLabels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + template: + metadata: + annotations: + checksum/config: 22dede55958655b05f52af4d52dab89d118b9f4d6e3efb7c0c6a5bc698662658 + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + vector.dev/exclude: 'true' + spec: + containers: + - args: + - --config-dir + - /etc/vector/ + env: + - name: VECTOR_LOG + value: info + - name: VECTOR_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_TENANT_ID + value: c-green-test-1234 + - name: VECTOR_SELF_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_SELF_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VECTOR_SELF_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PROCFS_ROOT + value: /host/proc + - name: SYSFS_ROOT + value: /host/sys + image: docker.io/timberio/vector:0.55.0-distroless-libc + imagePullPolicy: IfNotPresent + name: vector + ports: + - containerPort: 5140 + name: talos-logs + protocol: UDP + volumeMounts: + - mountPath: /vector-data-dir + name: data + - mountPath: /etc/vector/ + name: config + readOnly: true + - mountPath: /var/log/ + name: var-log + readOnly: true + - mountPath: /var/lib + name: var-lib + readOnly: true + - mountPath: /host/proc + name: procfs + readOnly: true + - mountPath: /host/sys + name: sysfs + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + serviceAccountName: vector + terminationGracePeriodSeconds: 60 + tolerations: + - operator: Exists + volumes: + - name: config + projected: + sources: + - configMap: + name: vector + - hostPath: + path: /var/lib/vector + name: data + - hostPath: + path: /var/log/ + name: var-log + - hostPath: + path: /var/lib/ + name: var-lib + - hostPath: + path: /proc + name: procfs + - hostPath: + path: /sys + name: sysfs diff --git a/tests/golden/resources/vector/vector/01_vector/vector/templates/rbac.yaml b/tests/golden/resources/vector/vector/01_vector/vector/templates/rbac.yaml new file mode 100644 index 0000000..2381ec6 --- /dev/null +++ b/tests/golden/resources/vector/vector/01_vector/vector/templates/rbac.yaml @@ -0,0 +1,41 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector +rules: + - apiGroups: + - '' + resources: + - namespaces + - nodes + - pods + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vector +subjects: + - kind: ServiceAccount + name: vector + namespace: syn-vector diff --git a/tests/golden/resources/vector/vector/01_vector/vector/templates/service-headless.yaml b/tests/golden/resources/vector/vector/01_vector/vector/templates/service-headless.yaml new file mode 100644 index 0000000..91ac37b --- /dev/null +++ b/tests/golden/resources/vector/vector/01_vector/vector/templates/service-headless.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: null + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector-headless + namespace: syn-vector +spec: + clusterIP: None + ports: + - name: talos-logs + port: 5140 + protocol: UDP + targetPort: 5140 + selector: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + type: ClusterIP diff --git a/tests/golden/resources/vector/vector/01_vector/vector/templates/service.yaml b/tests/golden/resources/vector/vector/01_vector/vector/templates/service.yaml new file mode 100644 index 0000000..ee80649 --- /dev/null +++ b/tests/golden/resources/vector/vector/01_vector/vector/templates/service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: null + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector +spec: + ports: + - name: talos-logs + port: 5140 + protocol: UDP + targetPort: 5140 + selector: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/name: vector + type: ClusterIP diff --git a/tests/golden/resources/vector/vector/01_vector/vector/templates/serviceaccount.yaml b/tests/golden/resources/vector/vector/01_vector/vector/templates/serviceaccount.yaml new file mode 100644 index 0000000..35b0ae1 --- /dev/null +++ b/tests/golden/resources/vector/vector/01_vector/vector/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: Agent + app.kubernetes.io/instance: vector + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: vector + app.kubernetes.io/version: 0.55.0-distroless-libc + helm.sh/chart: vector-0.52.0 + name: vector + namespace: syn-vector diff --git a/tests/resources.yml b/tests/resources.yml new file mode 100644 index 0000000..209a67f --- /dev/null +++ b/tests/resources.yml @@ -0,0 +1,137 @@ +# Test instance: Talos node deployed Agent. +# Tails pod logs (kubernetes_logs) + Talos kernel/service logs (UDP 5140 via hostNetwork) and ships both to Loki with cluster/node labels. +parameters: + vector: + helm_values: + role: Agent + podHostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + + env: + - name: VECTOR_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_TENANT_ID + value: ${vector:tenant_id} + + tolerations: + - operator: Exists + + customConfig: + data_dir: /vector-data-dir + + api: + enabled: false + + sources: + talos_logs: + type: socket + mode: udp + address: 0.0.0.0:5140 + decoding: + codec: json + + pod_logs: + type: kubernetes_logs + ignore_older_secs: 600 + + transforms: + talos_transform: + type: remap + inputs: + - talos_logs + source: | + .cluster = get_env_var!("VECTOR_TENANT_ID") + .node = get_env_var!("VECTOR_NODE_NAME") + if exists(.msg) { .message = del(.msg) } + if exists(.log) { .message = del(.log) } + if exists(."talos-service") { .service = del(."talos-service") } + if exists(.facility) { + .source = "kernel" + del(.priority) + del(.facility) + } else { + .source = "service" + } + + pod_transform: + type: remap + inputs: + - pod_logs + source: | + .cluster = get_env_var!("VECTOR_TENANT_ID") + .node = get_env_var!("VECTOR_NODE_NAME") + .namespace = .kubernetes.pod_namespace + .pod = .kubernetes.pod_name + .container = .kubernetes.container_name + + sinks: + loki_talos: + type: loki + inputs: + - talos_transform + endpoint: http://loki-gateway.test-loki.svc.cluster.local:3100 + tenant_id: ${vector:tenant_id} + out_of_order_action: rewrite_timestamp + labels: + cluster: '{{`{{ cluster }}`}}' + node: '{{`{{ node }}`}}' + source: '{{`{{ source }}`}}' + service_name: '{{`{{ service }}`}}' + encoding: + codec: text + except_fields: + - host + - source_type + - port + - talos-time + - talos-level + - facility + - priority + - kubernetes + compression: snappy + healthcheck: + enabled: false + request: + retry_max_duration_secs: 60 + timeout_secs: 90 + buffer: + type: disk + max_size: 268435488 + when_full: drop_newest + + loki_pod: + type: loki + inputs: + - pod_transform + endpoint: http://loki-gateway.test-loki.svc.cluster.local:3100 + tenant_id: ${vector:tenant_id} + out_of_order_action: rewrite_timestamp + labels: + cluster: '{{`{{ cluster }}`}}' + node: '{{`{{ node }}`}}' + namespace: '{{`{{ namespace }}`}}' + pod: '{{`{{ pod }}`}}' + container: '{{`{{ container }}`}}' + service_name: '{{`{{ container }}`}}' + source: pod + encoding: + codec: text + except_fields: + - host + - source_type + - kubernetes + - file + - source_ip + - stream + compression: snappy + healthcheck: + enabled: false + request: + retry_max_duration_secs: 60 + timeout_secs: 90 + buffer: + type: disk + max_size: 268435488 + when_full: drop_newest