diff --git a/.github/workflows/stackpacks-ci.yml b/.github/workflows/stackpacks-ci.yml new file mode 100644 index 0000000..f07790e --- /dev/null +++ b/.github/workflows/stackpacks-ci.yml @@ -0,0 +1,310 @@ +name: Contrib Stackpacks CI + +run-name: >- + contrib stackpacks - + ${{ github.event_name == 'pull_request' && format('PR #{0} {1}', github.event.number, github.event.pull_request.title) || + github.event_name == 'workflow_dispatch' && 'manual rebuild' || + github.event_name == 'push' && format('{0}: {1}', github.ref_name, github.event.head_commit.message) || + github.ref_name }} + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +concurrency: + group: contrib-stackpacks-${{ github.ref }} + cancel-in-progress: false + +jobs: + + validate: + name: validate (${{ matrix.stackpack }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + stackpack: + - open-telemetry-2 + - otel-k8s-crd + - notification-operator + - suse-observability-2 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Validate ${{ matrix.stackpack }} + env: + STACKPACK: ${{ matrix.stackpack }} + # Pinned snapshot of stackstate-server providing /opt/docker/bin/stack-pack-validator. + # Bumped manually when validating a newer stackstate-server release. + VALIDATOR_IMAGE: quay.io/stackstate/stackstate-server:b7c3604-2.5 + run: | + docker run --rm \ + -v "${PWD}:/workspace" \ + -w /workspace \ + "${VALIDATOR_IMAGE}" \ + /opt/docker/bin/stack-pack-validator -directory "/workspace/stackpacks/${STACKPACK}" + + package: + name: package (${{ matrix.stackpack }}) + needs: [validate] + if: | + always() && + needs.validate.result == 'success' + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + stackpack: + - open-telemetry-2 + - otel-k8s-crd + - notification-operator + - suse-observability-2 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install yq + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends yq + + - name: Install StackState CLI + run: | + version=$(curl --fail --silent --show-error https://dl.stackstate.com/stackstate-cli/LATEST_VERSION) + version="${version#v}" + arch=$(uname -m) + tmp=$(mktemp -d) + curl --fail --silent --show-error --location \ + "https://dl.stackstate.com/stackstate-cli/v${version}/stackstate-cli-${version}.linux-${arch}.tar.gz" \ + | tar xz -C "${tmp}" + sudo install -m 0755 "${tmp}/sts" /usr/local/bin/sts + sts version + + - name: Package and publish ${{ matrix.stackpack }} + env: + STACKPACK: ${{ matrix.stackpack }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + RUN_NUMBER: ${{ github.run_number }} + SHA: ${{ github.sha }} + EVENT_NAME: ${{ github.event_name }} + REF: ${{ github.ref }} + run: | + cd "stackpacks/${STACKPACK}" + base_version=$(yq -r '.version' stackpack.yaml) + echo "Version from stackpack.yaml: ${base_version}" + + if [[ "${EVENT_NAME}" == "push" && "${REF}" == "refs/heads/main" ]]; then + stackpack_version=${base_version} + else + major=$(echo "${base_version}" | cut -d. -f1) + minor=$(echo "${base_version}" | cut -d. -f2) + patch=$(echo "${base_version}" | cut -d. -f3) + new_patch=$((patch + 1)) + branch_sanitized=$(echo "${BRANCH_NAME}" | sed 's/[^a-zA-Z0-9]/-/g') + short_sha="${SHA:0:7}" + stackpack_version="${major}.${minor}.${new_patch}-${branch_sanitized}-${RUN_NUMBER}-${short_sha}-SNAPSHOT" + sed -i "s/^version: .*/version: \"${stackpack_version}\"/" stackpack.yaml + fi + + export STS_EXPERIMENTAL_STACKPACKS=1 + sts stackpack package + mkdir -p target + mv -- *.sts target/ + ls -lh target/*.sts + + - name: Upload ${{ matrix.stackpack }} sts artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sts-${{ matrix.stackpack }} + path: stackpacks/${{ matrix.stackpack }}/target/*.sts + if-no-files-found: error + + check-version-bump: + name: Check version bump + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + stackpack: + - open-telemetry-2 + - otel-k8s-crd + - notification-operator + - suse-observability-2 + env: + TARGET_BRANCH: ${{ github.base_ref || 'main' }} + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true + fetch-depth: 0 + + - name: Install yq + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends yq + + - name: Compare versions + run: ./scripts/ci/verify_versions_bumped.sh ${{ matrix.stackpack }} + + build-and-push-docker-image: + name: Docker image build, scan, and push + needs: [package] + if: | + always() && + github.reposity == 'StackVista/contrib-stackpacks' && + (needs.package.result == 'success' || needs.package.result == 'skipped') + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download stackpack artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sts-* + path: artifacts + merge-multiple: true + + - name: Stage stackpacks + run: | + dest=docker/rootfs/stackpacks + rm -rf "${dest}" + mkdir -p "${dest}" + cp artifacts/*.sts "${dest}/" + + - name: Compute image tag + id: tag + run: | + git_branch="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" + git_commit_sha_short=$(git rev-parse --short=7 HEAD) + git_last_commit_timestamp=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:'%Y%m%d%H%M%S') + echo "value=${git_last_commit_timestamp}-${git_branch}-${git_commit_sha_short}" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + with: + image: tonistiigi/binfmt:qemu-v10.0.4 + cache-image: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Build local image for scan + id: local-image + env: + TAG: ${{ steps.tag.outputs.value }} + run: | + local_ref="local/contrib-stackpacks:${TAG}" + docker buildx build \ + --platform linux/amd64 \ + --load \ + --provenance=false \ + --sbom=false \ + --tag "${local_ref}" \ + docker + echo "ref=${local_ref}" >> "$GITHUB_OUTPUT" + + - name: Smoke test image + env: + LOCAL_IMAGE_REF: ${{ steps.local-image.outputs.ref }} + run: | + mkdir -p smoke-output + docker run --rm -v "${PWD}/smoke-output:/output" "${LOCAL_IMAGE_REF}" /output + copied=$(find smoke-output -type f -name '*.sts' | wc -l | tr -d ' ') + if [[ "${copied}" -eq 0 ]]; then + echo "Smoke test produced no .sts files" >&2 + exit 1 + fi + + - name: Scan image + uses: StackVista/image-pipeline/.github/actions/scan-image@f7e766e074b516bb754891a240e6ab2df1e60e4c + with: + image: ${{ steps.local-image.outputs.ref }} + mode: gate + severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL + with-grype: true + upload-sarif: false + + - name: Apply OCI labels + id: oci-labels + uses: StackVista/image-pipeline/.github/actions/apply-oci-labels@f7e766e074b516bb754891a240e6ab2df1e60e4c + with: + image-name: contrib-stackpacks + tag: ${{ steps.tag.outputs.value }} + title: SUSE Observability Community Stackpacks + description: Community stackpacks distribution bundle + component: contrib-stackpacks + dockerfile: docker/Dockerfile + base-name: registry.suse.com/bci/bci-minimal:15.7 + + - name: Push single-arch image (amd64) + uses: StackVista/image-pipeline/.github/actions/push-single-arch@f7e766e074b516bb754891a240e6ab2df1e60e4c + with: + image: quay.io/stackstate/contrib-stackpacks + tag: ${{ steps.tag.outputs.value }} + arch: amd64 + docker-context: docker + dockerfile: docker/Dockerfile + labels: ${{ steps.oci-labels.outputs.labels }} + target-registry: quay.io + target-registry-user: ${{ secrets.QUAY_USER }} + target-registry-password: ${{ secrets.QUAY_PASSWORD }} + + - name: Push single-arch image (arm64) + uses: StackVista/image-pipeline/.github/actions/push-single-arch@f7e766e074b516bb754891a240e6ab2df1e60e4c + with: + image: quay.io/stackstate/contrib-stackpacks + tag: ${{ steps.tag.outputs.value }} + arch: arm64 + docker-context: docker + dockerfile: docker/Dockerfile + labels: ${{ steps.oci-labels.outputs.labels }} + target-registry: quay.io + target-registry-user: ${{ secrets.QUAY_USER }} + target-registry-password: ${{ secrets.QUAY_PASSWORD }} + + - name: Merge multi-arch manifest + uses: StackVista/image-pipeline/.github/actions/merge-multiarch@f7e766e074b516bb754891a240e6ab2df1e60e4c + with: + image: quay.io/stackstate/contrib-stackpacks + tag: ${{ steps.tag.outputs.value }} + arches: amd64,arm64 + target-registry: quay.io + target-registry-user: ${{ secrets.QUAY_USER }} + target-registry-password: ${{ secrets.QUAY_PASSWORD }} + + update-stackpacks-version-in-helm: + name: Update stackpacks version in helm-charts-internal + needs: + - build-and-push-docker-image + if: | + always() && + github.event_name == 'push' && github.ref == 'refs/heads/main' && + needs.build-and-push-docker-image.result == 'success' + uses: ./.github/workflows/update-stackpacks-version-in-helm.yml + secrets: + HELM_CHARTS_INTERNAL_GH_APP_CLIENT_ID: ${{ secrets.HELM_CHARTS_INTERNAL_GH_APP_CLIENT_ID }} + HELM_CHARTS_INTERNAL_GH_APP_PRIVATE_KEY: ${{ secrets.HELM_CHARTS_INTERNAL_GH_APP_PRIVATE_KEY }} + diff --git a/.github/workflows/update-stackpacks-version-in-helm.yml b/.github/workflows/update-stackpacks-version-in-helm.yml new file mode 100644 index 0000000..9c9a022 --- /dev/null +++ b/.github/workflows/update-stackpacks-version-in-helm.yml @@ -0,0 +1,54 @@ +name: Update Stackpacks Version in Helm + +# Triggers the helm-charts-internal updatecli workflow after stackpacks have +# been built and uploaded. Invoked from the CI workflow on master. +# The standalone `workflow_dispatch` entry remains as a manual fallback. +on: + workflow_call: + secrets: + HELM_CHARTS_INTERNAL_GH_APP_CLIENT_ID: + required: true + HELM_CHARTS_INTERNAL_GH_APP_PRIVATE_KEY: + required: true + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +jobs: + update-stackpacks-version-in-helm: + runs-on: small + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.HELM_CHARTS_INTERNAL_GH_APP_CLIENT_ID }} + private-key: ${{ secrets.HELM_CHARTS_INTERNAL_GH_APP_PRIVATE_KEY }} + repositories: helm-charts-internal + permission-actions: write + + - name: Trigger helm-charts-internal updatecli workflow + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + curl --fail-with-body --request POST \ + --header "Accept: application/vnd.github+json" \ + --header "Authorization: Bearer ${GH_TOKEN}" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --data '{ + "ref": "master", + "inputs": { + "pipeline": "stackpacks", + "target_branch": "master", + "stackpacks_source_branch": "master", + "stackpacks_contrib_source_branch": "main", + "stackpacks_suse_source_branch": "main", + "dry_run": "false" + } + }' \ + "https://api.github.com/repos/StackVista/helm-charts-internal/actions/workflows/updatecli.yml/dispatches" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c8ff264 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: check-added-large-files + stages: [pre-commit] + exclude: "^stackstate-generated-api/.*$" + - id: check-case-conflict + stages: [pre-commit] + - id: check-merge-conflict + stages: [pre-commit] + - id: detect-private-key + stages: [pre-commit] + - id: detect-aws-credentials + stages: [pre-commit] + args: + - --allow-missing-credentials + - repo: https://github.com/zizmorcore/zizmor-pre-commit + # Zizmor version. + rev: v1.25.0 + hooks: + # Run the linter. + - id: zizmor diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..6d987a8 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,7 @@ +FROM registry.suse.com/bci/bci-minimal:15.7 + +COPY rootfs / + +USER 1001 + +ENTRYPOINT [ "/copy-stackpacks.sh" ] diff --git a/docker/rootfs/copy-stackpacks.sh b/docker/rootfs/copy-stackpacks.sh new file mode 100755 index 0000000..d89b426 --- /dev/null +++ b/docker/rootfs/copy-stackpacks.sh @@ -0,0 +1,20 @@ +#! /bin/sh + +set -e + +target_dir=$1 + +if [ -z "${target_dir}" ]; then + echo "Usage: $0 [--clear]" + exit 1 +fi + +if [[ $# -ge 2 && $2 == "--clear" ]]; then + echo "Cleaning up current Stackpacks in ${target_dir}/" + rm -rf "${target_dir:?}/"* || true +fi + +echo "Copying Stackpacks from /stackpacks to ${target_dir}/" + +# Copy .sts files +cp /stackpacks/*.sts "${target_dir}/" 2>/dev/null || true diff --git a/scripts/ci/verify_versions_bumped.sh b/scripts/ci/verify_versions_bumped.sh new file mode 100755 index 0000000..ae7801a --- /dev/null +++ b/scripts/ci/verify_versions_bumped.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Verifies if a version of a stackpack has been "bumped" if not then it returns an error. +# The script is executed when there are some changes in a stackpack directory. +# +# Required env: TARGET_BRANCH (the PR base branch). In GitHub Actions, set this +# from ${{ github.base_ref }} in the workflow step. + +stackpack="$1" +: "${TARGET_BRANCH:?TARGET_BRANCH must be set (e.g. from github.base_ref)}" + +git fetch origin ${TARGET_BRANCH} + +# Gate: skip if THIS BRANCH didn't modify stackpacks//. We use the merge-base +# (not ${TARGET_BRANCH}'s tip) because if master moves forward after this +# branch was cut — e.g. someone else legitimately bumps a stackpack on master — the +# tip comparison would report our branch as "having changes" for stackpacks we +# never touched, then the version-bump check below would falsely demand a bump. +# Comparing to the merge-base reflects only what this branch introduced. +# +# Note: the VERSION comparison below intentionally still uses the tip — if we +# *did* modify the stackpack, we want to fail loudly if master has bumped past us +# in the meantime (concurrent bumps would otherwise silently downgrade on +# merge or force an awkward stackpack.yaml conflict). +base=$(git merge-base "origin/${TARGET_BRANCH}" HEAD 2>/dev/null || echo "") +if [ -z "$base" ]; then + echo "Cannot determine merge-base with origin/${TARGET_BRANCH}; running version check against tip" +elif git diff --quiet "$base" HEAD -- "stackpacks/${stackpack}/"; then + echo "stackpacks/${stackpack}/ has no changes on this branch (since merge-base ${base}); skipping version-bump check" + exit 0 +fi + +check_if_stackpack_exists=$(git ls-tree -r "origin/${TARGET_BRANCH}" -- "stackpacks/${stackpack}/stackpack.yaml") +if [ -z "$check_if_stackpack_exists" ]; then + echo "stackpack.yaml doesnt exist in the target branch, probably it is the first commit for the stackpack. Skipping version check!!!" + exit 0 +fi + +remote_version=$(git show "origin/${TARGET_BRANCH}:stackpacks/${stackpack}/stackpack.yaml" | yq ".version") +new_version=$(yq ".version" "stackpacks/${stackpack}/stackpack.yaml") + +if [ "$(printf '%s\n' "$remote_version" "$new_version" | sort -V | head -n1)" = "$new_version" ]; then + echo "Version of the stackpack should be updated" + exit 1 +fi diff --git a/stackpacks/notification-operator/README.md b/stackpacks/notification-operator/README.md new file mode 100644 index 0000000..9f6f904 --- /dev/null +++ b/stackpacks/notification-operator/README.md @@ -0,0 +1,109 @@ +# Notification Operator StackPack + +This StackPack provides monitoring and metrics for the Notification Operator, which manages notifications in SUSE Observability through Kubernetes Custom Resources (CRs). + +## What's Included + +This StackPack provides: +- **Monitors**: Health checks for controller reconciliation errors, notification sync status, and disabled notifications +- **Metric Bindings**: Charts for reconciliation rates and notification status tracking +- **Remediation Hints**: Guided troubleshooting for common notification operator issues + +## Project Structure + +``` +notification-operator/ +├── README.md +├── provisioning/ +│ ├── metricbindings.sty # Notification operator metric charts +│ ├── monitors.sty # Health monitors for notification operator +│ ├── remediation-hints/ # Troubleshooting guides +│ │ ├── controller-reconcile-errors.md.hbs +│ │ ├── notification-disabled.md.hbs +│ │ └── notification-not-synced.md.hbs +│ └── stackpack.sty # Main provisioning template +├── resources/ # UI resources and documentation +│ ├── deprovisioning.md +│ ├── error.md +│ ├── installed.md +│ ├── logo.png +│ ├── notinstalled.md +│ ├── overview.md +│ ├── provisioning.md +│ └── waitingfordata.md +└── stackpack.conf # StackPack configuration +``` + + +## Prerequisites + +The notification operator must be deployed in your Kubernetes cluster with the following requirements: +- The notification operator pod must be labeled with `app.kubernetes.io/name=notification-operator` +- The operator must expose Prometheus metrics on the standard `/metrics` endpoint +- SUSE Observability must have access to scrape metrics from the notification operator pod + +## Monitors Included + +This StackPack provides the following monitors: + +### Controller Runtime Reconcile Errors +- **Purpose**: Detects errors in the controller runtime reconcile process +- **Metric**: `increase(notification_operator_controller_runtime_reconcile{result="error"}[10m])` +- **Threshold**: Any increase in errors triggers a CRITICAL alert +- **Remediation**: Includes troubleshooting hints for common reconciliation issues + +### Enabled Notification Resources Not Synced +- **Purpose**: Identifies enabled notifications that are not properly synced +- **Metric**: `sum(notification_operator_suse_observability_notification_enabled == 1 and notification_operator_suse_observability_notification_synced == 0)` +- **Threshold**: Any unsynced enabled notifications trigger a CRITICAL alert +- **Remediation**: Provides guidance for resolving synchronization issues + +### Notification Disabled +- **Purpose**: Tracks disabled SUSE Observability notifications +- **Metric**: `sum(notification_operator_suse_observability_notification_enabled == 0)` +- **Threshold**: Any disabled notifications trigger a DEVIATING alert +- **Remediation**: Explains why notifications might be disabled and how to re-enable them + +## Metric Bindings Included + +### Total Reconciliations per Controller/Cluster +- **Chart Type**: Line chart +- **Metric**: `sum(increase(notification_operator_controller_runtime_reconcile[1m])) by (cluster_name, result)` +- **Purpose**: Shows reconciliation activity and success/failure rates over time + +### Notification Sync Status +- **Chart Type**: Line chart +- **Metric**: `sum(notification_operator_suse_observability_notification_synced) by (cluster_name, name)` +- **Purpose**: Shows whether notifications are properly synced with SUSE Observability + +### Notification Enabled Status +- **Chart Type**: Line chart +- **Metric**: `sum(notification_operator_suse_observability_notification_enabled) by (cluster_name, name)` +- **Purpose**: Shows whether notifications are enabled or disabled + +## Installation + +1. **Deploy the Notification Operator** in your Kubernetes cluster following the operator's installation guide +2. **Ensure proper labeling** of the notification operator pod with `app.kubernetes.io/name=notification-operator` +3. **Install this StackPack** through the SUSE Observability UI or CLI +4. **Verify metrics collection** by checking that the notification operator metrics are being scraped + +## Troubleshooting + +If monitors are not triggering or metrics are not appearing: + +1. **Check pod labeling**: Verify the notification operator pod has the correct label +2. **Verify metrics endpoint**: Ensure the operator exposes metrics on `/metrics` +3. **Check network access**: Confirm SUSE Observability can reach the operator pod +4. **Review logs**: Check both notification operator and SUSE Observability logs for errors + +## Development and Customization + +To modify this StackPack: + +1. **Edit monitors**: Modify `provisioning/monitors.sty` to adjust thresholds or add new monitors +2. **Update metric bindings**: Edit `provisioning/metricbindings.sty` to create custom charts +3. **Add remediation hints**: Create new `.md.hbs` files in `provisioning/remediation-hints/` +4. **Package and test**: Use `sbt notification-operator/package` to build the StackPack + +For more details on StackPack development, see the main repository documentation. diff --git a/stackpacks/notification-operator/includes/remediation-hints/controller-reconcile-errors.md.hbs b/stackpacks/notification-operator/includes/remediation-hints/controller-reconcile-errors.md.hbs new file mode 100644 index 0000000..ea9afd5 --- /dev/null +++ b/stackpacks/notification-operator/includes/remediation-hints/controller-reconcile-errors.md.hbs @@ -0,0 +1,5 @@ +The possible causes for this issue are: +- Incorrect configuration of the Notifications. +- Issues with SUSE Observability instance. + +1. Inspect the [Logs](/#/components/{{ componentUrnForUrl }}#logs) for the error messages. diff --git a/stackpacks/notification-operator/includes/remediation-hints/notification-disabled.md.hbs b/stackpacks/notification-operator/includes/remediation-hints/notification-disabled.md.hbs new file mode 100644 index 0000000..e6452a4 --- /dev/null +++ b/stackpacks/notification-operator/includes/remediation-hints/notification-disabled.md.hbs @@ -0,0 +1,14 @@ +Notification Details: + - cluster: {{ labels.cluster_name }} + - notification: {{ labels.notification_name }} + - notification CR name: {{ labels.name }} + - notification CR namespace: {{ labels.namespace }} + +The SUSE Observability Notification can be disabled for the following reasons: +- By Deployment pipeline to avoid false positive alerts during redeployments. + +The possible causes for this issue are: +- Deployment pipeline using this Notification has not been completed yet. +- Deployment pipeline using this Notification failed to complete or delete the notification. + +Check the Deployment pipeline to find out more. diff --git a/stackpacks/notification-operator/includes/remediation-hints/notification-not-synced.md.hbs b/stackpacks/notification-operator/includes/remediation-hints/notification-not-synced.md.hbs new file mode 100644 index 0000000..4f2f74d --- /dev/null +++ b/stackpacks/notification-operator/includes/remediation-hints/notification-not-synced.md.hbs @@ -0,0 +1,11 @@ +Notification Details: + - cluster: {{ labels.cluster_name }} + - notification: {{ labels.notification_name }} + - notification CR name: {{ labels.name }} + - notification CR namespace: {{ labels.namespace }} + +The possible causes for this issue are: +- Incorrect configuration of the Notifications. +- Issues with SUSE Observability instance. + +1. Inspect the [Logs](/#/components/{{ componentUrnForUrl }}#logs) for the error messages. diff --git a/stackpacks/notification-operator/resources/deprovisioning.md b/stackpacks/notification-operator/resources/deprovisioning.md new file mode 100644 index 0000000..1de3476 --- /dev/null +++ b/stackpacks/notification-operator/resources/deprovisioning.md @@ -0,0 +1,2 @@ +# notification-operator Stackpack +Information the user is going to see during the uninstallation process. diff --git a/stackpacks/notification-operator/resources/error.md b/stackpacks/notification-operator/resources/error.md new file mode 100644 index 0000000..26e43c1 --- /dev/null +++ b/stackpacks/notification-operator/resources/error.md @@ -0,0 +1,2 @@ +# notification-operator Stackpack +Error message to show when an error occurred during provisioning of the StackPack. diff --git a/stackpacks/notification-operator/resources/installed.md b/stackpacks/notification-operator/resources/installed.md new file mode 100644 index 0000000..385222f --- /dev/null +++ b/stackpacks/notification-operator/resources/installed.md @@ -0,0 +1,2 @@ +# notification-operator Stackpack +The Stackpack has been successfully installed and is ready to use. The monitors and metrics are added to the Notification Operator's pod. diff --git a/stackpacks/notification-operator/resources/logo.png b/stackpacks/notification-operator/resources/logo.png new file mode 100644 index 0000000..bdab9e4 Binary files /dev/null and b/stackpacks/notification-operator/resources/logo.png differ diff --git a/stackpacks/notification-operator/resources/notinstalled.md b/stackpacks/notification-operator/resources/notinstalled.md new file mode 100644 index 0000000..83546a2 --- /dev/null +++ b/stackpacks/notification-operator/resources/notinstalled.md @@ -0,0 +1,2 @@ +# notification-operator Stackpack +Information the user is going to see when the StackPack is not installed. diff --git a/stackpacks/notification-operator/resources/overview.md b/stackpacks/notification-operator/resources/overview.md new file mode 100644 index 0000000..2ccf546 --- /dev/null +++ b/stackpacks/notification-operator/resources/overview.md @@ -0,0 +1,4 @@ +# notification-operator Stackpack +The Stackpack for the Notification Operator provides metrics and monitors for the Notification Operator, which is used to manage notifications in SUSE Observability with Kubernetes Custom resources (CRs). + +The notification operator's pod must be labeled with `app.kubernetes.io/name=notification-operator` to ensure that the metrics and monitors are applied correctly. diff --git a/stackpacks/notification-operator/resources/provisioning.md b/stackpacks/notification-operator/resources/provisioning.md new file mode 100644 index 0000000..3bb1cb4 --- /dev/null +++ b/stackpacks/notification-operator/resources/provisioning.md @@ -0,0 +1,2 @@ +# notification-operator Stackpack +Information the user is going to see during the installation process. diff --git a/stackpacks/notification-operator/resources/waitingfordata.md b/stackpacks/notification-operator/resources/waitingfordata.md new file mode 100644 index 0000000..533e262 --- /dev/null +++ b/stackpacks/notification-operator/resources/waitingfordata.md @@ -0,0 +1,2 @@ +# notification-operator Stackpack +Information about user actions required for the StackPack instance to start receiving data. diff --git a/stackpacks/notification-operator/settings/monitors.sty b/stackpacks/notification-operator/settings/monitors.sty new file mode 100644 index 0000000..2fa5655 --- /dev/null +++ b/stackpacks/notification-operator/settings/monitors.sty @@ -0,0 +1,58 @@ +nodes: +- _type: Monitor + function: urn:stackpack:common:monitor-function:threshold + arguments: + threshold: 0.0 + comparator: GT + failureState: CRITICAL + metric: + aliasTemplate: "cluster: ${cluster_name} status: ${result}" + query: increase(notification_operator_controller_runtime_reconcile{result="error"}[10m]) + unit: short + urnTemplate: urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name} + description: "This monitor checks for errors in the controller runtime reconcile process." + identifier: urn:stackpack:notification-operator:monitor:notification-operator:controller-reconcile-errors + intervalSeconds: 30 + name: Controller Runtime Reconcile Errors + remediationHint: !include remediation-hints/controller-reconcile-errors.md.hbs + status: ENABLED + tags: + - notification-operator +- _type: Monitor + function: urn:stackpack:common:monitor-function:threshold + arguments: + threshold: 0.0 + comparator: GT + failureState: CRITICAL + metric: + aliasTemplate: "cluster: ${cluster_name} namespace: ${namespace} name: ${name}" + query: count(notification_operator_suse_observability_notification_enabled == 1 and notification_operator_suse_observability_notification_synced == 0) by (cluster_name, name, namespace, pod_name) + unit: short + urnTemplate: urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name} + description: "This monitor checks if there are any enabled Notification resources that are not synced." + identifier: urn:stackpack:notification-operator:monitor:notification-operator:notification-enabled-not-synced + intervalSeconds: 30 + name: Enabled Notification Resources Not Synced + remediationHint: !include remediation-hints/notification-not-synced.md.hbs + status: ENABLED + tags: + - notification-operator +- _type: Monitor + function: urn:stackpack:common:monitor-function:threshold + arguments: + threshold: 0.0 + comparator: GT + failureState: DEVIATING + metric: + aliasTemplate: "cluster: ${cluster_name} namespace: ${namespace} name: ${name}" + query: count(notification_operator_suse_observability_notification_enabled == 0) by (cluster_name, name, namespace, pod_name) + unit: short + urnTemplate: urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name} + description: This monitor checks if there are any disabled SUSE Observability Notifications. + identifier: urn:stackpack:notification-operator:monitor:notification-operator:notification-disabled + intervalSeconds: 30 + name: Notification Disabled + remediationHint: !include remediation-hints/notification-disabled.md.hbs + status: ENABLED + tags: + - notification-operator diff --git a/stackpacks/notification-operator/settings/presentation.sty b/stackpacks/notification-operator/settings/presentation.sty new file mode 100644 index 0000000..f58d098 --- /dev/null +++ b/stackpacks/notification-operator/settings/presentation.sty @@ -0,0 +1,45 @@ +nodes: + - _type: ComponentPresentation + name: "Notification operator metrics" + description: "Metrics for notification operator" + identifier: urn:stackpack:notification-operator:component-presentation:metrics + binding: + _type: ComponentPresentationQueryBinding + query: label = "stackpack:kubernetes" and type = "pod" + rank: + specificity: 2 + presentation: + metricPerspective: + tabs: + - + tabId: notification-controller + title: Notification controller + order: 80 + sections: + - + sectionId: notification-controller + title: Notification Controller + order: 3 + metrics: + - metricId: controller_runtime_reconcile + - metricId: notification_synced + - metricId: notification_enabled + metrics: + - metricId: controller_runtime_reconcile + name: Total number of reconciliations per controller/cluster + queries: + - alias: "cluster: ${cluster_name} status: ${result}" + expression: sum(increase(notification_operator_controller_runtime_reconcile[1m])) by (cluster_name, result) + unit: short + - metricId: notification_synced + name: Notification Sync Status + queries: + - alias: "cluster: ${cluster_name} name: ${name}" + expression: sum(notification_operator_suse_observability_notification_synced) by (cluster_name, name) + unit: short + - metricId: urn:stackpack:notification-operator:metric-binding:notification_enabled + name: Notification Enabled Status + queries: + - alias: "cluster: ${cluster_name} name: ${name}" + expression: sum(notification_operator_suse_observability_notification_enabled) by (cluster_name, name) + unit: short diff --git a/stackpacks/notification-operator/stackpack.yaml b/stackpacks/notification-operator/stackpack.yaml new file mode 100644 index 0000000..6479010 --- /dev/null +++ b/stackpacks/notification-operator/stackpack.yaml @@ -0,0 +1,25 @@ +# schemaVersion -- Stackpack specification version. +schemaVersion: "2.0" +# name -- Name of the StackPack. This is what is used to uniquely identify the StackPack. +name: "notification-operator" +# displayName -- Name that's displayed on both the StackPack listing page and on the title of the StackPack page. +displayName: "Notification Operator" +# version -- Semantic version of the StackPack. StackPacks with the same major version are considered compatible. +version: "0.0.3" +# isNew -- This specifies whether the StackPack is new, as in the StackPack version is the first publicly available version. The values can be yes/no/true/false. +# logoUrl -- Specifies the logo used as a badge for the StackPack. +logoUrl: !resource "logo.png" +# categories -- These are keywords using which the StackPacks can be filtered. Any list of relevant labels can be passed here. It's recommended to keep labels in capitalized letters. +categories: [ "Internal" ] +# Markdown resource with general information about the StackPack. +overviewUrl: !resource "overview.md" +# configurationUrls -- Contains the Markdown resources relevant for various states of StackPack provisioning. +configurationUrls: + INSTALLED: !resource "installed.md" + NOT_INSTALLED: !resource "notinstalled.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + DEPROVISIONING: !resource "deprovisioning.md" + ERROR: !resource "error.md" +provision: + companionStackPacks: [ ] diff --git a/stackpacks/open-telemetry-2/README.md b/stackpacks/open-telemetry-2/README.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/open-telemetry-2/icons/databases.svg b/stackpacks/open-telemetry-2/icons/databases.svg new file mode 100644 index 0000000..49e2227 --- /dev/null +++ b/stackpacks/open-telemetry-2/icons/databases.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry-2/icons/open-telemetry.svg b/stackpacks/open-telemetry-2/icons/open-telemetry.svg new file mode 100644 index 0000000..923736b --- /dev/null +++ b/stackpacks/open-telemetry-2/icons/open-telemetry.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/stackpacks/open-telemetry-2/icons/queues.svg b/stackpacks/open-telemetry-2/icons/queues.svg new file mode 100644 index 0000000..8c580ba --- /dev/null +++ b/stackpacks/open-telemetry-2/icons/queues.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry-2/icons/service-instances.svg b/stackpacks/open-telemetry-2/icons/service-instances.svg new file mode 100644 index 0000000..b4ff7f8 --- /dev/null +++ b/stackpacks/open-telemetry-2/icons/service-instances.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry-2/icons/service-namespaces.svg b/stackpacks/open-telemetry-2/icons/service-namespaces.svg new file mode 100644 index 0000000..4db9548 --- /dev/null +++ b/stackpacks/open-telemetry-2/icons/service-namespaces.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry-2/icons/services.svg b/stackpacks/open-telemetry-2/icons/services.svg new file mode 100644 index 0000000..d6f1259 --- /dev/null +++ b/stackpacks/open-telemetry-2/icons/services.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/stackpacks/open-telemetry-2/includes/remediation-hints/database-span-duration.md.hbs b/stackpacks/open-telemetry-2/includes/remediation-hints/database-span-duration.md.hbs new file mode 100644 index 0000000..b5be314 --- /dev/null +++ b/stackpacks/open-telemetry-2/includes/remediation-hints/database-span-duration.md.hbs @@ -0,0 +1,12 @@ +SUSE Observability has detected that this service has a 95th percentile span duration above {{threshold}}s for spans. +This signals that a significant number of users are experiencing a slow service. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Load on the database has increased + +### Load on the database has increased + +Check if the number of requests to the database has increased by looking at the "Span rate" chart on the [database highlight page](/#/components/{{ componentUrnForUrl }}). If so, a quick fix can be to give the database more resources like CPU and memory. diff --git a/stackpacks/open-telemetry-2/includes/remediation-hints/service-instance-span-duration.md.hbs b/stackpacks/open-telemetry-2/includes/remediation-hints/service-instance-span-duration.md.hbs new file mode 100644 index 0000000..38456ac --- /dev/null +++ b/stackpacks/open-telemetry-2/includes/remediation-hints/service-instance-span-duration.md.hbs @@ -0,0 +1,48 @@ +SUSE Observability has detected that this service instance has a 95th percentile span duration above {{threshold}}ms for spans. +This signals that a significant number of users are experiencing a slow service. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent start of the instance +- Load on the service instance has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors + +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces +perspective](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or instances that this instance depends on (focus on the lowest dependency). If you find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent start of the service instance + +Check if the instance was recently started: + +- If the trace metrics charts on the [highlight page](/#/components/{{ componentUrnForUrl }}) have no data at all, or only for very recent time period, the instance has usually just started. +- If the instance has just started, it might be that it is still warming-up. Compare the span duration metrics +for the current instance with the previous instances using the span duration metrics on the service highlights page for this service. Quickly navigate to the service via the link in the "About" section. +- Check if the instance is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. + +### Load on the service instance has increased + +Check if the amount of requests to the instance has increased by looking at the "Span rate" chart on the [service instance highlight page](/#/components/{{ +componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it might be an issue with the instance itself or the server the is running on. Check if other service instances running in the same location show similar issues. If that's the case consider replacing the entire server. \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/includes/remediation-hints/service-instance-span-error-ratio.md.hbs b/stackpacks/open-telemetry-2/includes/remediation-hints/service-instance-span-error-ratio.md.hbs new file mode 100644 index 0000000..bef737c --- /dev/null +++ b/stackpacks/open-telemetry-2/includes/remediation-hints/service-instance-span-error-ratio.md.hbs @@ -0,0 +1,54 @@ +SUSE Observability has detected that more than {{threshold}}% spans for your service instance have an Error +status, this may lead to noticeable disruptions or degraded performance for end-users. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent update of the application +- Load on the application has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces perspective](/#/components/{{ +componentUrnForUrl}}/traces?filterBy=status--Error__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or instances that this instance depends on (focus on the lowest dependency). If you +find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent update of the service + +Check if the instance was recently started: + +- If the trace metrics charts on the [highlight page](/#/components/{{ componentUrnForUrl }}) have no data at all, or +only for very recent time period, the instance has usually just started. +- If the instance has just started, it might be that it is still warming-up. Compare the span duration metrics +for the current instance with the previous instances using the span duration metrics on the service highlights page for +this service. Quickly navigate to the service via the link in the "About" section. +- Check if the instance is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. + +### Load on the service has increased + +Check if the amount of requests to the instance has increased by looking at the "Span rate" chart on the [service +instance highlight page](/#/components/{{ +componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more +resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it +might be an issue with the instance itself or the server the is running on. Check if other service instances running in +the same location show similar issues. If that's the case consider replacing the entire server. \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/includes/remediation-hints/service-span-duration.md.hbs b/stackpacks/open-telemetry-2/includes/remediation-hints/service-span-duration.md.hbs new file mode 100644 index 0000000..1227b4e --- /dev/null +++ b/stackpacks/open-telemetry-2/includes/remediation-hints/service-span-duration.md.hbs @@ -0,0 +1,55 @@ +SUSE Observability has detected that this service has a 95th percentile span duration above {{threshold}}ms for spans. +This signals that a significant number of users are experiencing a slow service. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent update of the service +- Load on the service has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors + +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces +perspective](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 +}}__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or service instances that this service depends on (focus on the lowest dependency). If you find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent update of the service + +Check if new instances for service were recently deployed: + +- Check if the instances are recently updated / redeployed. The "Service instances" metric chart on +the [service highlight page](/#/components/{{ componentUrnForUrl }}) will show a change in instance count, if an instance was restarted or replaced it shows a short spike in the chart. +- If application has just started, it might be that the service has not warmed up yet. Compare the response time metrics +for the current deployment with the previous deployment by checking the response time metric chart with a time interval +including both. +- Check if application is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. +- Repeat these steps for related services if needed + +### Load on the service has increased + +Check if the amount of requests to the service has increased by looking at the "Span rate" chart on the [service +highlight page](/#/components/{{ componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more +resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it +might be an issue with the instance itself or the server the is running on. Check if other service instances running in +the same location show similar issues. If that's the case consider replacing the entire server. diff --git a/stackpacks/open-telemetry-2/includes/remediation-hints/service-span-error-ratio.md.hbs b/stackpacks/open-telemetry-2/includes/remediation-hints/service-span-error-ratio.md.hbs new file mode 100644 index 0000000..46ef041 --- /dev/null +++ b/stackpacks/open-telemetry-2/includes/remediation-hints/service-span-error-ratio.md.hbs @@ -0,0 +1,53 @@ +SUSE Observability has detected that more than {{threshold}}% spans for your service have an Error +status, this may lead to noticeable disruptions or degraded performance for end-users. + +Take the following steps to diagnose the problem: + +## Possible causes + +- Slow dependency or dependency serving errors +- Recent update of the application +- Load on the application has increased +- Environment issues (e.g. certain nodes, database or services that the service depends on) + +### Slow dependency or dependency serving errors +When troubleshooting slow dependencies or dependencies that are serving errors, their are two ways to find the root +cause: + +1. Analyze traces: begin by examining the traces with an error status in the [traces perspective](/#/components/{{ +componentUrnForUrl}}/traces?filterBy=status--Error__parentType--External%2CRoot). Expanding a span will show +the full trace for that span. Spans with an error status are marked and slow services will be displayed as long spans in +the trace. If you find an erroneous or slow span you can follow the link to the component to continue the investigation. + +2. Review Related Health Violations: Check, in the related health violations of this monitor if there are any health +violations on one of the services or service instances that this service depends on (focus on the lowest dependency). If you find a +violation to click on the component to see the related health violations. You can than continue your troubleshooting by +following those health violations. The related health violations can be found in the expanded version if you read this +in the pinned minimised version. + +### Recent update of the service + +Check if the service was recently updated: + +- Go to the "Events" section in the middle of the [service highlight page](/#/components/{{ componentUrnForUrl }}) +- Check if there was a recent deployment by clicking on the Deployment section. +- Check if any of the pods are recently updated by clicking on "Pods of this service" in "Related resource" section of +the [service highlight page](/#/components/{{ componentUrnForUrl }}) and look if their Age is recent. +- If application has just started, it might be that the service has not warmed up yet. Compare the response time metrics +for the current deployment with the previous deployment by checking the response time metric chart with a time interval +including both. +- Check if application is using more resources than before, consider scaling it up or giving it more resources. +- If increased latency is crucial, consider rolling back the service to the previous version: + - if that helps, then the issue is likely with new deployment + - if that does not help, then the issue may be in the environment (e.g. network issues or issues with the underlying + infrastructure, e. g. database). [Traces](/#/components/{{componentUrnForUrl}}/traces?filterBy=duration--{{ multiply + threshold 1000000 }}__parentType--External%2CRoot) can be used to look deeper. +- Repeat these steps for related services if needed + +Check if the amount of requests to the service has increased by looking at the "Span rate" chart on the [service highlight page](/#/components/{{ componentUrnForUrl }}). If so, a quick fix can be to start more instances for the service or giving the instances more resources like CPU and memory. + +### Environment issues + +Check the span duration of the different instances of the service. If only certain instances are having problems, it +might be an issue with the instance itself or the server the is running on. Check if other service instances running in +the same location show similar issues. If that's the case consider replacing the entire server. \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/resources/RELEASE.md b/stackpacks/open-telemetry-2/resources/RELEASE.md new file mode 100644 index 0000000..595de92 --- /dev/null +++ b/stackpacks/open-telemetry-2/resources/RELEASE.md @@ -0,0 +1,2 @@ +# Open Telemetry + diff --git a/stackpacks/open-telemetry-2/resources/configuration.md b/stackpacks/open-telemetry-2/resources/configuration.md new file mode 100644 index 0000000..37e534f --- /dev/null +++ b/stackpacks/open-telemetry-2/resources/configuration.md @@ -0,0 +1,3 @@ +## Installation + +You can click on `Install` to install the Open Telemetry StackPack. Then follow the instructions here and in [the documentation](https://l.stackstate.com/open-telemetry-setup). to finish the Open Telemetry setup. diff --git a/stackpacks/open-telemetry-2/resources/deprovisioning.md b/stackpacks/open-telemetry-2/resources/deprovisioning.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/open-telemetry-2/resources/detailed-overview.md b/stackpacks/open-telemetry-2/resources/detailed-overview.md new file mode 100644 index 0000000..dd44b19 --- /dev/null +++ b/stackpacks/open-telemetry-2/resources/detailed-overview.md @@ -0,0 +1,3 @@ +## Prerequisites + +The Open Telemetry StackPack expects traces and metrics to be provided via the Open Telemetry protocol. The easiest way to do this is by instrumenting your applications with one of the Open Telemetry SDKs and/or installing the Open Telemetry collector to send observability data to SUSE Observability. See the [SUSE Observability Open Telemetry documentation](https://l.stackstate.com/open-telemetry-setup). diff --git a/stackpacks/open-telemetry-2/resources/enabled.md b/stackpacks/open-telemetry-2/resources/enabled.md new file mode 100644 index 0000000..8317e75 --- /dev/null +++ b/stackpacks/open-telemetry-2/resources/enabled.md @@ -0,0 +1,9 @@ +## The Open Telemetry StackPack is installed + +### What's next + +Instrument one or more applications with Open Telemetry SDKs to generate traces and metrics and install and configure the Open Telemetry collector to send data to SUSE Observability. See the [SUSE Observability Open Telemetry documentation](https://l.stackstate.com/open-telemetry-setup). + +To send data to SUSE Observability a service token is needed. + + \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/resources/error.md b/stackpacks/open-telemetry-2/resources/error.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/open-telemetry-2/resources/logo.png b/stackpacks/open-telemetry-2/resources/logo.png new file mode 100644 index 0000000..ac6f4bd Binary files /dev/null and b/stackpacks/open-telemetry-2/resources/logo.png differ diff --git a/stackpacks/open-telemetry-2/resources/overview.md b/stackpacks/open-telemetry-2/resources/overview.md new file mode 100644 index 0000000..d276aca --- /dev/null +++ b/stackpacks/open-telemetry-2/resources/overview.md @@ -0,0 +1,9 @@ +## Open Telemetry StackPack + +The Open Telemetry StackPack provides out-of-the-box support for traces and metrics ingested via the Open Telemetry protocol. Based on the traces a topology is created from services and service instances and their relations. Components are also created for other resources, like hosts, databases, and Kubernetes resources, that can be derived from the Open Telemetry data. + +Next to this also metric bindings and monitors are provided for traces and an increasing number of the languages supported by Open Telemetry like: +* Java +* .NET + +You can add custom metric bindings and monitors to support your application-specific use cases via Open Telemetry as well. diff --git a/stackpacks/open-telemetry-2/resources/provisioning.md b/stackpacks/open-telemetry-2/resources/provisioning.md new file mode 100644 index 0000000..a9522b1 --- /dev/null +++ b/stackpacks/open-telemetry-2/resources/provisioning.md @@ -0,0 +1,3 @@ +## Provisioning... + +Provisioning Open Telemetry monitoring. Please wait a moment. You can already start configuring your application for [Open Telemetry observability](https://l.stackstate.com/open-telemetry-setup). diff --git a/stackpacks/open-telemetry-2/resources/waitingfordata.md b/stackpacks/open-telemetry-2/resources/waitingfordata.md new file mode 100644 index 0000000..8317e75 --- /dev/null +++ b/stackpacks/open-telemetry-2/resources/waitingfordata.md @@ -0,0 +1,9 @@ +## The Open Telemetry StackPack is installed + +### What's next + +Instrument one or more applications with Open Telemetry SDKs to generate traces and metrics and install and configure the Open Telemetry collector to send data to SUSE Observability. See the [SUSE Observability Open Telemetry documentation](https://l.stackstate.com/open-telemetry-setup). + +To send data to SUSE Observability a service token is needed. + + \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/databases.sty b/stackpacks/open-telemetry-2/settings/component-mappings/databases.sty new file mode 100644 index 0000000..4e3d1d9 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/databases.sty @@ -0,0 +1,44 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Database" + description: "Represents a database service" + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:database" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'database' + vars: + - name: "clientServiceNamespace" + value: "datapoint.attributes['client_service.namespace']" + - name: "client" + value: "datapoint.attributes['client']" + - name: "server" + value: "datapoint.attributes['server']" + output: + identifier: >- + 'urn:opentelemetry:namespace/' + vars.clientServiceNamespace + ':service/' + vars.client + ':database/' + vars.server + name: "vars.server" + typeName: "'database'" + required: + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.clientServiceNamespace" + target: "service.namespace" + - source: "vars.client" + target: "service.name" + expireAfter: 900000 + rank: + specificity: 200 diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/ecs-tasks.sty b/stackpacks/open-telemetry-2/settings/component-mappings/ecs-tasks.sty new file mode 100644 index 0000000..14ba1b3 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/ecs-tasks.sty @@ -0,0 +1,34 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Task Component" + description: "Represents an ECS task" + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:task" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'aws.ecs.task.id' in resource.attributes" + vars: + - name: "awsEcsTaskId" + value: "resource.attributes['aws.ecs.task.id']" + output: + identifier: "'urn:opentelemetry:task/' + vars.awsEcsTaskId" + name: "vars.awsEcsTaskId" + typeName: "'task'" + required: + tags: + - source: "'open-telemetry'" + target: "stackpack" + optional: + tags: + - source: "resource.attributes" + pattern: "^aws.ecs\\.(.*)" + target: "ecs.${1}" + - source: "resource.attributes" + pattern: "^cloud\\.(.*)" + target: "cloud.${1}" + expireAfter: 900000 + rank: + specificity: 110 diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/functions.sty b/stackpacks/open-telemetry-2/settings/component-mappings/functions.sty new file mode 100644 index 0000000..39c14dd --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/functions.sty @@ -0,0 +1,32 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Function Component" + description: "Represents a serverless function (faas)." + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:function" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'faas.id' in resource.attributes" + vars: + - name: "faasId" + value: "resource.attributes['faas.id']" + output: + identifier: "'urn:opentelemetry:function/' + vars.faasId" + name: "vars.faasId" + typeName: "'function'" + required: + version: "resource.attributes['faas.version']" + tags: + - source: "'open-telemetry'" + target: "stackpack" + optional: + tags: + - source: "resource.attributes" + pattern: "^(faas|cloud)\\.(.*)" + target: "${1}.${2}" + expireAfter: 900000 + rank: + specificity: 200 diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/hosts.sty b/stackpacks/open-telemetry-2/settings/component-mappings/hosts.sty new file mode 100644 index 0000000..a003cb1 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/hosts.sty @@ -0,0 +1,31 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Host Component" + description: "Represents a machine host" + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:host" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'host.id' in resource.attributes" + vars: + - name: "hostId" + value: "resource.attributes['host.id']" + output: + identifier: "'urn:opentelemetry:host/' + vars.hostId" + name: "vars.hostId" + typeName: "'host'" + required: + tags: + - source: "'open-telemetry'" + target: "stackpack" + optional: + tags: + - source: "resource.attributes" + pattern: "^(os|host|cloud|azure|gcp)\\.(.*)" + target: "${1}.${2}" + expireAfter: 900000 + rank: + specificity: 50 diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/k8s-pods.sty b/stackpacks/open-telemetry-2/settings/component-mappings/k8s-pods.sty new file mode 100644 index 0000000..db8ef8b --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/k8s-pods.sty @@ -0,0 +1,46 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Pod" + description: "Represents a Kubernetes Pod" + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:pod" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: >- + 'k8s.cluster.name' in resource.attributes && + 'k8s.namespace.name' in resource.attributes && + 'k8s.pod.name' in resource.attributes + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "namespaceName" + value: "resource.attributes['k8s.namespace.name']" + - name: "podName" + value: "resource.attributes['k8s.pod.name']" + output: + identifier: >- + 'urn:opentelemetry:kubernetes:/' + vars.clusterName + ':' + vars.namespaceName + ':pod/' + vars.podName + name: "vars.podName" + typeName: "'pod'" + required: + additionalIdentifiers: + - "'urn:kubernetes:/' + vars.clusterName + ':' + vars.namespaceName + ':pod/' + vars.podName" + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.namespaceName" + target: "namespace" + - source: "vars.clusterName + '/' + vars.namespaceName" + target: "k8s-scope" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + expireAfter: 900000 + rank: + specificity: 120 diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/namespaces.sty b/stackpacks/open-telemetry-2/settings/component-mappings/namespaces.sty new file mode 100644 index 0000000..2ea55aa --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/namespaces.sty @@ -0,0 +1,40 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Namespace" + description: "Represents a logical service namespace in Kubernetes" + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:namespace" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.namespace' in resource.attributes" + vars: + - name: "namespace" + value: "resource.attributes['service.namespace']" + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "k8sNamespaceName" + value: "resource.attributes['k8s.namespace.name']" + output: + identifier: "'urn:opentelemetry:namespace/' + vars.namespace" + name: "vars.namespace" + typeName: "'service namespace (otel)'" + required: + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.k8sNamespaceName" + target: "namespace" + - source: "vars.clusterName + '/' + vars.k8sNamespaceName" + target: "k8s-scope" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + expireAfter: 900000 + rank: + specificity: 100 diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/service-instances.sty b/stackpacks/open-telemetry-2/settings/component-mappings/service-instances.sty new file mode 100644 index 0000000..bd1b32f --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/service-instances.sty @@ -0,0 +1,69 @@ +nodes: + - _type: "OtelComponentMapping" + name: "OTel Service Instance" + description: "Represents an instance of an OTel service" + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:service-instance" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.namespace' in resource.attributes" + vars: + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "version" + value: >- + 'service.version' in resource.attributes ? + resource.attributes['service.version'] : + 'undefined' + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + - name: "instanceName" + value: >- + 'service.instance.id' in resource.attributes ? + resource.attributes['service.name'] + " - " + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + " - instance" + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "k8sNamespaceName" + value: "resource.attributes['k8s.namespace.name']" + output: + identifier: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + name: "vars.instanceName" + typeName: "'otel service instance'" + required: + version: "vars.version" + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.service" + target: "service.name" + - source: "vars.namespace" + target: "service.namespace" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.k8sNamespaceName" + target: "namespace" + - source: "vars.clusterName + '/' + vars.k8sNamespaceName" + target: "k8s-scope" + - source: "resource.attributes" + pattern: "(.*)" + target: "${1}" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + expireAfter: 900000 + rank: + specificity: 210 diff --git a/stackpacks/open-telemetry-2/settings/component-mappings/services.sty b/stackpacks/open-telemetry-2/settings/component-mappings/services.sty new file mode 100644 index 0000000..78f4f8e --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/component-mappings/services.sty @@ -0,0 +1,61 @@ +nodes: + - _type: "OtelComponentMapping" + name: "OTel Service Component" + description: "Represents an OTel service based on service.name and service.namespace" + identifier: "urn:stackpack:open-telemetry-2:otel-component-mapping:service" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.namespace' in resource.attributes" + vars: + - name: "namespace" + value: >- + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "version" + value: >- + 'service.version' in resource.attributes ? + resource.attributes['service.version'] : + 'undefined' + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "k8sNamespaceName" + value: "resource.attributes['k8s.namespace.name']" + output: + identifier: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + name: "vars.service" + typeName: "'otel service'" + required: + version: "vars.version" + tags: + - source: "'open-telemetry'" + target: "stackpack" + - source: "vars.service" + target: "service.name" + - source: "vars.namespace" + target: "service.namespace" + - source: "vars.version" + target: "service.version" + - source: "vars.clusterName" + target: "cluster-name" + - source: "vars.k8sNamespaceName" + target: "namespace" + - source: "vars.clusterName + '/' + vars.k8sNamespaceName" + target: "k8s-scope" + - source: "resource.attributes" + pattern: "telemetry.sdk\\.(.*)" + target: "telemetry.sdk.${1}" + optional: + tags: + - source: "resource.attributes['deployment.environment']" + target: "deployment-environment" + expireAfter: 900000 + rank: + specificity: 200 diff --git a/stackpacks/open-telemetry-2/settings/main-menu.sty b/stackpacks/open-telemetry-2/settings/main-menu.sty new file mode 100644 index 0000000..c7edfc9 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/main-menu.sty @@ -0,0 +1,7 @@ +nodes: + - _type: MainMenuGroup + name: Open Telemetry + iconbase64: !icon open-telemetry.svg + defaultOpen: true + items: [] + identifier: "urn:stackpack:open-telemetry-2:main-menu-group:open-telemetry" \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/monitors/database-span-duration/monitor.sty b/stackpacks/open-telemetry-2/settings/monitors/database-span-duration/monitor.sty new file mode 100644 index 0000000..9ee728b --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/monitors/database-span-duration/monitor.sty @@ -0,0 +1,21 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + query: 'histogram_quantile(0.95, sum by(client_service_namespace, client, server, le)(rate(traces_service_graph_request_client_seconds_bucket{connection_type="database"}[5m])))' + aliasTemplate: Span duration 95th percentile + unit: s + threshold: 5 + titleTemplate: Span duration + urnTemplate: "urn:opentelemetry:namespace/${client_service_namespace}:service/${client}:database/${server}" + description: | + Monitors the duration of database spans. When the 95th percentile of the duration is greater than the threshold (default 5s) the monitor has a Deviating state. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry-2:monitor:database-span-duration + intervalSeconds: 30 + name: Span duration for Open Telemetry database + status: ENABLED + tags: ["database", "traces"] + remediationHint: !include remediation-hints/database-span-duration.md.hbs \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/monitors/service-instance-span-duration/monitor.sty b/stackpacks/open-telemetry-2/settings/monitors/service-instance-span-duration/monitor.sty new file mode 100644 index 0000000..03c274d --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/monitors/service-instance-span-duration/monitor.sty @@ -0,0 +1,21 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + query: 'histogram_quantile(0.95, sum by(service_namespace, service_name, service_instance_id, le)(rate(otel_span_duration_milliseconds_bucket{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])))' + aliasTemplate: Span duration 95th percentile + unit: ms + threshold: 5000 + titleTemplate: Span duration + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + description: | + Monitors the duration of the server and consumer spans. When the 95th percentile of the duration is greater than the threshold (default 5000ms) the monitor has a Deviating state. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry-2:monitor:otel-service-instance-span-duration + intervalSeconds: 30 + name: Span duration for Open Telemetry service instances + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-instance-span-duration.md.hbs \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/monitors/service-instance-span-error-ratio/monitor.sty b/stackpacks/open-telemetry-2/settings/monitors/service-instance-span-error-ratio/monitor.sty new file mode 100644 index 0000000..ac4bd2d --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/monitors/service-instance-span-error-ratio/monitor.sty @@ -0,0 +1,21 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + query: 'sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) or (present_over_time(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) -1)) / sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])) * 100' + aliasTemplate: Error ratio (errors/total) + unit: percent + threshold: 5 + titleTemplate: Span error ratio + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + description: | + Percentage of server and consumer spans that are in an error state for the service instance. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry-2:monitor:otel-service-instance-span-error-ratio + intervalSeconds: 30 + name: Span error ratio for Open Telemetry service instances + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-instance-span-error-ratio.md.hbs \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/monitors/service-span-duration/monitor.sty b/stackpacks/open-telemetry-2/settings/monitors/service-span-duration/monitor.sty new file mode 100644 index 0000000..0d5a615 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/monitors/service-span-duration/monitor.sty @@ -0,0 +1,21 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + query: 'histogram_quantile(0.95, sum by(service_namespace, service_name, le)(rate(otel_span_duration_milliseconds_bucket{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])))' + aliasTemplate: Span duration 95th percentile + unit: ms + threshold: 5000 + titleTemplate: Span duration + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + description: | + Monitors the duration of the server and consumer spans. When the 95th percentile of the duration is greater than the threshold (default 5000ms) the monitor has a Deviating state. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry-2:monitor:otel-service-span-duration + intervalSeconds: 30 + name: Span duration for Open Telemetry services + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-span-duration.md.hbs \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/monitors/service-span-error-ratio/monitor.sty b/stackpacks/open-telemetry-2/settings/monitors/service-span-error-ratio/monitor.sty new file mode 100644 index 0000000..ee47569 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/monitors/service-span-error-ratio/monitor.sty @@ -0,0 +1,21 @@ +nodes: + - _type: Monitor + arguments: + comparator: GT + failureState: DEVIATING + metric: + query: 'sum by(service_namespace, service_name)(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) or (present_over_time(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m]) -1)) / sum by(service_namespace, service_name)(rate(otel_span_calls_total{span_kind=~"SPAN_KIND_SERVER|SPAN_KIND_CONSUMER"}[5m])) * 100' + aliasTemplate: Error ratio (errors/total) + unit: percent + threshold: 5 + titleTemplate: Span error ratio + urnTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + description: | + Percentage of server and consumer spans that are in an error state for the service. + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:open-telemetry-2:monitor:otel-service-span-error-ratio + intervalSeconds: 30 + name: Span error ratio for Open Telemetry services + status: ENABLED + tags: ["service", "traces"] + remediationHint: !include remediation-hints/service-span-error-ratio.md.hbs diff --git a/stackpacks/open-telemetry-2/settings/presentations/common_filters.sty b/stackpacks/open-telemetry-2/settings/presentations/common_filters.sty new file mode 100644 index 0000000..95a67f5 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/presentations/common_filters.sty @@ -0,0 +1,44 @@ +nodes: + - _type: ComponentPresentation + name: "Otel common columns" + description: "ComponentPresentation common for Otel components" + identifier: urn:stackpack:open-telemetry-2:component-presentation:common_filters + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("stackpack:open-telemetry") + rank: + specificity: 2 + presentation: + filters: + - filterId: "otel_namespace" + displayName: + singular: "OTEL namespace" + plural: "OTEL namespaces" + filter: + _type: "TagFilter" + tagKey: "service.namespace" + moreTab: Otel + - filterId: "otel_service" + displayName: + singular: "OTEL service" + plural: "OTEL services" + filter: + _type: "TagFilter" + tagKey: "service.name" + moreTab: Otel + - filterId: "k8s_namespace" + displayName: + singular: "K8s namespace" + plural: "K8s namespaces" + filter: + _type: "TagFilter" + tagKey: "namespace" + moreTab: K8s + - filterId: "k8s_cluster" + displayName: + singular: "K8s cluster" + plural: "K8s clusters" + filter: + _type: "TagFilter" + tagKey: "cluster-name" + moreTab: K8s \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/presentations/component.sty b/stackpacks/open-telemetry-2/settings/presentations/component.sty new file mode 100644 index 0000000..ceff146 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/presentations/component.sty @@ -0,0 +1,61 @@ +nodes: + - _type: ComponentPresentation + name: "Otel component" + description: "ComponentPresentation for Otel components" + identifier: urn:stackpack:open-telemetry-2:component-presentation:component + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("stackpack:open-telemetry") + rank: + specificity: 2 + presentation: + overview: + name: + singular: "otel component" + plural: "otel components" + title: "Otel component" + columns: + - columnId: "health" + title: "Health" + projection: + _type: "HealthProjection" + value: "healthState" + - columnId: "otel_name" + title: "Otel component" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: "otel_namespace" + title: "Namespace" + projection: + _type: "ComponentLinkProjection" + name: "tags.singleValue('service.namespace')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace')" + - columnId: otel_sdk_language + title: "Language" + projection: + _type: "TextProjection" + value: "tags.singleValue('telemetry.sdk.language')" + highlight: + title: "Otel Component" + fields: + - fieldId: "type" + title: "Type" + order: 100.0 + projection: + _type: "TextProjection" + value: "typeName" + - fieldId: "health" + title: "Health" + order: 90.0 + projection: + _type: "HealthProjection" + value: "healthState" + - fieldId: "labels" + title: "Labels" + order: 80.0 + projection: + _type: "TagProjection" + events: + showEvents: true diff --git a/stackpacks/open-telemetry-2/settings/presentations/databases.sty b/stackpacks/open-telemetry-2/settings/presentations/databases.sty new file mode 100644 index 0000000..dc0b06c --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/presentations/databases.sty @@ -0,0 +1,192 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Databases presentation" + description: "ComponentPresentation for Otel Services" + identifier: urn:stackpack:open-telemetry-2:component-presentation:database + binding: + _type: ComponentPresentationQueryBinding + query: (label IN ("stackpack:open-telemetry") AND type IN ("database")) + rank: + specificity: 10 + presentation: + icon: !icon databases.svg + filters: + - filterId: "otel_namespace" + - filterId: "otel_service" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "database" + plural: "databases" + title: "Databases" + mainMenu: + group: "Open Telemetry" + order: 98 + columns: + - columnId: health + - columnId: "otel_name" + title: "Database" + - columnId: otel_namespace + - columnId: otel_database_span_error_rate + title: Span error % + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: percentunit + query: |- + sum by(client_service_namespace, client, server) (rate(traces_service_graph_request_failed_total{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[5m]) or 0) / sum by (client_service_namespace, client, server)(rate(traces_service_graph_request_total{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[5m])) + metricId: database-spans-error-rate + - columnId: otel_database_span_duration + title: Span duration + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: s + query: |- + histogram_quantile(0.95, sum(rate(traces_service_graph_request_client_seconds_bucket{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[${__rate_interval}])) by (client_service_namespace, client, server, le)) + metricId: database-spans-duration + - columnId: otel_database_span_rate + title: Span rate + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: cps + query: |- + sum by (client_service_namespace, client, server)(rate(traces_service_graph_request_total{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[5m])) + metricId: database-spans-rate + sort: + - columnId: health + - columnId: otel_name + highlight: + title: "Otel database" + fields: [] + provisioning: + externalComponentSelector: "external.mapping.identifier == 'urn:stackpack:open-telemetry-2:otel-component-mapping:database'" + showConfiguration: false + showStatus: false + relatedResources: + - resourceId: "otel_database_service_instances" + title: "Related service instances" + order: 2.0 + topologyQuery: '(withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance"' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service-instance" + - resourceId: "otel_datbase_services" + title: "Related services" + order: 1.0 + topologyQuery: '(withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance") , levels = "1") and type = "otel service")' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service" + metrics: + - sectionId: traces + title: "Traces" + description: "Rate, Errors and Duration metrics for the trace spans" + order: 100 + metrics: + - metricId: database-spans-rate + order: 100 + - metricId: database-spans-error-rate + order: 90 + - metricId: database-spans-duration + order: 80 + summary: + metrics: + - + summaryMetricId: database-spans-rate + name: Span rate + order: 100 + decimalPlaces: 2 + unit: cps + query: |- + sum by (client_service_namespace, client, server)(rate(traces_service_graph_request_total{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[5m])) + metricId: database-spans-rate + - + summaryMetricId: database-spans-error-rate + name: Span error % + order: 90 + decimalPlaces: 2 + unit: percentunit + query: |- + sum by(client_service_namespace, client, server) (rate(traces_service_graph_request_failed_total{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[5m]) or 0) / sum by (client_service_namespace, client, server)(rate(traces_service_graph_request_total{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[5m])) + metricId: database-spans-error-rate + - + summaryMetricId: database-spans-duration + name: Span duration + order: 80 + decimalPlaces: 2 + unit: s + query: |- + histogram_quantile(0.95, sum(rate(traces_service_graph_request_client_seconds_bucket{ + client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}" + }[${__rate_interval}])) by (client_service_namespace, client, server, le)) + metricId: database-spans-duration + metricPerspective: + tabs: + - + tabId: traces-performance + title: Traces Performance + order: 10 + sections: + - + sectionId: spans + title: Spans + order: 10 + metrics: + - metricId: database-spans-error-rate + - metricId: database-spans-duration + - metricId: database-spans-rate + metrics: + - + metricId: database-spans-error-rate + description: The percentage of errors per second + name: Span error % + metricQueries: + - expression: |- + sum(rate(traces_service_graph_request_failed_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}"}[5m]) or 0) + / sum(rate(traces_service_graph_request_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}"}[5m])) + alias: Error rate + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: database-spans-duration + description: + name: Span duration + metricQueries: + - expression: histogram_quantile(0.95, sum(rate(traces_service_graph_request_client_seconds_bucket{client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}"}[${__rate_interval}])) by (le)) + alias: '95th percentile' + chart: + _type: TimeSeriesChart + unit: s + - + metricId: database-spans-rate + description: The number of spans for the database per second + name: Span rate + metricQueries: + - expression: sum(rate(traces_service_graph_request_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${tags.singleValue('service.name')}", server="${name}"}[${__rate_interval}])) + alias: Query rate + chart: + _type: TimeSeriesChart + unit: cps diff --git a/stackpacks/open-telemetry-2/settings/presentations/service-instances.sty b/stackpacks/open-telemetry-2/settings/presentations/service-instances.sty new file mode 100644 index 0000000..53dec68 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/presentations/service-instances.sty @@ -0,0 +1,384 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Service Instances presentation" + description: "ComponentPresentation for Otel Service Instances" + identifier: urn:stackpack:open-telemetry-2:component-presentation:service-instance + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:open-telemetry" and type = "otel service instance") + rank: + specificity: 10 + presentation: + icon: !icon service-instances.svg + filters: + - filterId: "otel_namespace" + - filterId: "otel_service" + - filterId: "k8s_cluster" + - filterId: "k8s_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "service instance" + plural: "service instances" + title: "Service instances" + mainMenu: + group: "Open Telemetry" + order: 99 + columns: + - columnId: health + - columnId: otel_name + title: Service Instance + - columnId: otel_namespace + - columnId: otel_service + title: "Service" + projection: + _type: "ComponentLinkProjection" + name: "tags.singleValue('service.name')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace') + ':service/' + tags.singleValue('service.name')" + - columnId: otel_service_instance_span_error_percent_summary + title: Span error % + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: percentunit + query: |- + (sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{ + status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[5m])) or 0) / + sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[5m])) + metricId: service-instance-spans-error-rate + - columnId: otel_service_instance_span_duration_summary + title: Span duration + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: ms + query: |- + histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (service_namespace, service_name, service_instance_id, le)) + metricId: service-instance-spans-duration + - columnId: otel_service_instance_span_rate_summary + title: Span rate + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: cps + query: |- + sum by(service_namespace, service_name, service_instance_id)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) + metricId: service-instance-spans-rate + - columnId: otel_sdk_language + sort: + - columnId: health + - columnId: otel_name + highlight: + title: "Otel service" + fields: + - fieldId: "namespace" + title: "Namespace" + order: 85.0 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('service.namespace')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace')" + - fieldId: "service" + title: "Service" + order: 82.5 + projection: + _type: ComponentLinkProjection + name: "tags.singleValue('service.name')" + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace') + ':service/' + tags.singleValue('service.name')" + provisioning: + externalComponentSelector: "external.mapping.identifier == 'urn:stackpack:open-telemetry-2:otel-component-mapping:service-instance'" + showConfiguration: false + showStatus: false + relatedResources: + - resourceId: "otel_service_instance_service_instances" + title: "Related service instances" + order: 2.0 + topologyQuery: '(withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance"' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service-instance" + - resourceId: "otel_service_instance_services" + title: "Related services" + order: 1.0 + topologyQuery: '(withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance") , levels = "1") and type = "otel service")' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service" + events: + showEvents: true + relatedResourcesQuery: | + ((withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance") + OR + (withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance") , levels = "1") and type = "otel service") + OR + (identifier = "${identifiers[0]}") + metrics: + - sectionId: traces + title: Traces + description: "Rate, Errors and Duration metrics for the trace spans" + order: 100 + metrics: + - metricId: service-instance-spans-rate + order: 100 + - metricId: service-instance-spans-error-rate + order: 90 + - metricId: service-instance-spans-duration + order: 80 + + summary: + metrics: + - summaryMetricId: service-instance-spans-rate + name: Span rate + description: The number of spans for the service instance per second + order: 100 + unit: cps + query: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}])) + metricId: service-instance-spans-rate + - + summaryMetricId: service-instance-spans-error-rate + name: Span error % + description: The percentage of errors per second + order: 90 + unit: percentunit + query: | + (sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) or 0) / + sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) + metricId: service-instance-spans-error-rate + - + summaryMetricId: service-instance-spans-duration + name: Span duration + description: + order: 80 + unit: ms + query: |- + histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (le)) + metricId: service-instance-spans-duration + + metricPerspective: + tabs: + - + tabId: dotnet + title: .NET + order: 80 + sections: + - + sectionId: gc + title: GC + order: 30 + metrics: + - metricId: dotnet-gc-collections + - metricId: dotnet-gc-duration + - metricId: dotnet-gc-allocated + - metricId: dotnet-gc-objects-and-committed-memory + - metricId: dotnet-gc-heap-size + - metricId: dotnet-gc-heap-fragmentation + - + tabId: jvm + title: JVM + order: 80 + sections: + - + sectionId: gc + title: GC + order: 20 + metrics: + - metricId: jvm_gc_durations_95th_quantile + - metricId: jvm_gc_per_second + - metricId: jvm_mem_after_last_gc + - + sectionId: mem + title: Memory + order: 30 + metrics: + - metricId: jvm_heap_memory + - metricId: jvm_non_heap_memory + - + tabId: traces + title: Traces Performance + order: 10 + sections: + - + sectionId: spans + title: Spans + order: 10 + metrics: + - metricId: service-instance-spans-rate + - metricId: service-instance-spans-error-rate + - metricId: service-instance-spans-duration + metrics: + - + metricId: dotnet-gc-collections + description: 'GC Collections per Second' + name: .NET GC Collections + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_collections_count_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: Generation - ${generation} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: dotnet-gc-duration + description: 'GC Duration per Collection' + name: .NET GC Duration + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_duration_nanoseconds_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: duration(ns) + chart: + _type: TimeSeriesChart + unit: ns + - + metricId: dotnet-gc-allocated + description: 'Bytes allocated to GC Heap' + name: .NET GC Allocated + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_allocations_size_bytes_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: allocated + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-objects-and-committed-memory + description: 'Size of Uncollected Objects and Committed Vmem size for GC Heap' + name: .NET GC Objects and Committed Memory + metricQueries: + - expression: |- + process_runtime_dotnet_gc_objects_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: uncollected objects + - expression: |- + process_runtime_dotnet_gc_committed_memory_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: committed vmem heap size + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-size + description: 'GC Heap size (including fragmentation)' + name: .NET GC Heap Size Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: Generation - ${generation} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-fragmentation + description: 'Heap Fragmentation size in Bytes' + name: .NET GC Heap Fragmentation Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_fragmentation_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: Generation - ${generation} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_gc_durations_95th_quantile + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(jvm_gc_duration_seconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: jvm_gc_per_second + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(jvm_gc_duration_seconds_count{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: jvm_mem_after_last_gc + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + jvm_memory_used_after_last_gc_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"} + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_heap_memory + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}",jvm_memory_type="heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_non_heap_memory + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}",jvm_memory_type="non_heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: service-instance-spans-rate + description: The number of spans for the service instance per second + name: Span rate + metricQueries: + - expression: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}])) by (span_kind) + alias: 'All ${span_kind}' + - expression: sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[${__rate_interval}])) by (span_kind) + alias: 'Errors ${span_kind}' + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: service-instance-spans-error-rate + description: The percentage of errors per second + name: Span error rate + metricQueries: + - expression: |- + ((sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) by (span_kind)) or 0) + / sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}"}[5m])) by (span_kind) + alias: '${span_kind}' + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: service-instance-spans-duration + description: + name: Span duration + metricQueries: + - expression: |- + histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}", service_instance_id="${tags.singleValue('service.instance.id')}" + }[${__rate_interval}])) by (k8s_cluster_name, k8s_namespace_name, k8s_pod_name, span_kind, le)) + alias: '95th percentile ${span_kind}' + chart: + _type: TimeSeriesChart + unit: ms \ No newline at end of file diff --git a/stackpacks/open-telemetry-2/settings/presentations/service-namespaces.sty b/stackpacks/open-telemetry-2/settings/presentations/service-namespaces.sty new file mode 100644 index 0000000..fb59e9b --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/presentations/service-namespaces.sty @@ -0,0 +1,62 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Service Namespaces presentation" + description: "ComponentPresentation for Otel Service Namespaces" + identifier: urn:stackpack:open-telemetry-2:component-presentation:service-namespace + binding: + _type: ComponentPresentationQueryBinding + query: (label IN ("stackpack:open-telemetry") AND type IN ("service namespace (otel)")) + rank: + specificity: 10 + presentation: + icon: !icon service-namespaces.svg + filters: + - filterId: "k8s_cluster" + - filterId: "k8s_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "service namespace" + plural: "service namespaces" + title: "Service Namespaces" + mainMenu: + group: "Open Telemetry" + order: 97 + columns: + - columnId: health + - columnId: "otel_name" + title: "Namespace" + provisioning: + externalComponentSelector: "external.mapping.identifier == 'urn:stackpack:open-telemetry-2:otel-component-mapping:namespace'" + showConfiguration: false + showStatus: false + sort: + - columnId: health + - columnId: otel_name + highlight: + title: "Otel namespace" + fields: [] + relatedResources: + - resourceId: "otel_namespace_service_instances" + title: "Service instances" + order: 1.0 + topologyQuery: 'type = "otel service instance" AND label in ("service.namespace:${name}")' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service-instance" + - resourceId: "otel_namespace_services" + title: "Services" + order: 2.0 + topologyQuery: 'type = "otel service" AND label in ("service.namespace:${name}")' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service" + events: + showEvents: true + relatedResourcesQuery: | + type IN ("otel service", "otel service instance") AND label IN ("service.namespace:${name}") + OR + identifier = "${identifiers[0]}" diff --git a/stackpacks/open-telemetry-2/settings/presentations/services.sty b/stackpacks/open-telemetry-2/settings/presentations/services.sty new file mode 100644 index 0000000..8b740a7 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/presentations/services.sty @@ -0,0 +1,543 @@ +nodes: + - _type: ComponentPresentation + name: "Otel Services presentation" + description: "ComponentPresentation for Otel Services" + identifier: urn:stackpack:open-telemetry-2:component-presentation:service + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:open-telemetry" and type = "otel service") + rank: + specificity: 10 + presentation: + icon: !icon services.svg + filters: + - filterId: "otel_namespace" + - filterId: "k8s_cluster" + - filterId: "k8s_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "service" + plural: "services" + title: Services + mainMenu: + group: "Open Telemetry" + order: 100 + columns: + - columnId: health + - columnId: otel_name + title: Service + - columnId: otel_namespace + - columnId: otel_service_span_error_percent_summary + title: Span error % + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: percentunit + query: |- + (sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) or 0) / + sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) + metricId: service-spans-error-rate + - columnId: otel_service_span_duration_summary + title: Span duration + projection: + _type: MetricProjection + sparkline: true + decimalPlaces: 2 + unit: ms + query: |- + histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) by (service_namespace, service_name, le)) + metricId: service-spans-duration + - columnId: otel_service_span_rate_summary + title: "Span rate" + projection: + _type: "MetricProjection" + sparkline: true + decimalPlaces: 2 + unit: "cps" + query: |- + sum by (service_namespace, service_name)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) + metricId: service-spans-rate + - columnId: otel_sdk_language + sort: + - columnId: health + - columnId: otel_name + highlight: + title: "Otel service" + fields: + - fieldId: "namespace" + title: "Otel Namespace" + order: 85.0 + projection: + _type: ComponentLinkProjection + name: tags.singleValue('service.namespace') + identifier: "'urn:opentelemetry:namespace/' + tags.singleValue('service.namespace')" + provisioning: + externalComponentSelector: "external.mapping.identifier == 'urn:stackpack:open-telemetry-2:otel-component-mapping:service'" + showConfiguration: false + showStatus: false + relatedResources: + - resourceId: "otel_service_service_instances" + title: "Instances of this service" + order: 2.0 + topologyQuery: '(withNeighborsOf(direction = "down", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance"' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service-instance" + - resourceId: "otel_service_services" + title: "Related services" + order: 1.0 + topologyQuery: '(withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (withNeighborsOf(direction = "down", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance"), levels = "1") and type = "otel service instance") , levels = "1") and type = "otel service")' + presentationIdentifier: "urn:stackpack:open-telemetry-2:component-presentation:service" + events: + showEvents: true + relatedResourcesQuery: | + (withNeighborsOf(direction = "up", components = (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and type = "otel service instance"), levels = "1") and type = "otel service") + OR + ((withNeighborsOf(direction = "down", components = (identifier = "${identifiers[0]}"), levels = "1")) and type = "otel service instance") + OR + identifier = "${identifiers[0]}" + metrics: + - sectionId: traces + title: Traces + description: "Rate, Errors and Duration metrics for the trace spans" + order: 100 + metrics: + - metricId: service-spans-rate + order: 100 + - metricId: service-spans-error-rate + order: 90 + - metricId: service-spans-duration + order: 80 + - sectionId: service + title: Service + description: "Open Telemetry service metrics" + order: 90 + metrics: + - metricId: service-instance-count + order: 100 + summary: + metrics: + - summaryMetricId: otel_service_span_error_percent_summary + name: Span error % + order: 100 + decimalPlaces: 2 + unit: percentunit + query: |- + (sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) or 0) / + sum by (service_namespace, service_name) (rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[5m])) + metricId: service-spans-error-rate + - summaryMetricId: otel_service_span_duration_summary + name: Span duration + order: 90 + decimalPlaces: 2 + unit: ms + query: |- + histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) by (service_namespace, service_name, le)) + metricId: service-spans-duration + - summaryMetricId: otel_service_span_rate_summary + name: Span rate + order: 80 + decimalPlaces: 2 + unit: cps + query: |- + sum by (service_namespace, service_name)(rate(otel_span_calls_total{ + service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}" + }[${__rate_interval}])) + metricId: service-spans-rate + metricPerspective: + tabs: + - + tabId: dotnet + title: .NET + order: 80 + sections: + - + sectionId: gc + title: GC + order: 3 + metrics: + - metricId: dotnet-gc-collections + - metricId: dotnet-gc-duration + - metricId: dotnet-gc-allocated + - metricId: dotnet-gc-objects-and-committed-memory + - metricId: dotnet-gc-heap-size + - metricId: dotnet-gc-heap-fragmentation + - + tabId: jvm + title: JVM + order: 80 + sections: + - + sectionId: gc + title: GC + order: 3 + metrics: + - metricId: jvm_gc_durations_95th_quantile + - metricId: jvm_gc_per_second + - metricId: jvm_mem_after_last_gc + - + sectionId: mem + title: Memory + order: 3 + metrics: + - metricId: jvm_heap_memory + - metricId: jvm_non_heap_memory + - + tabId: service + title: Service + order: 20 + sections: + - + sectionId: instances + title: Instances + order: 10 + metrics: + - metricId: service-instance-count + - + tabId: traces-performance + title: Traces Performance + order: 10 + sections: + - + sectionId: spans + title: Spans + order: 10 + metrics: + - metricId: service-spans-rate + - metricId: service-spans-error-rate + - metricId: service-spans-duration + - + sectionId: spans-per-instance + title: Spans per instance + order: 20 + metrics: + - metricId: service-spans-rate-per-instance + - metricId: service-spans-error-rate-per-instance + - metricId: service-spans-duration-per-instance + - + sectionId: server-spans + title: Server Spans + order: 30 + metrics: + - metricId: service-server-spans-failed-percentage + - metricId: service-server-spans-rate + - metricId: service-server-spans-duration + - + sectionId: client-spans + title: Client Spans + order: 40 + metrics: + - metricId: service-client-spans-failed-percentage + - metricId: service-client-spans-rate + - metricId: service-client-spans-duration + metrics: + - + metricId: dotnet-gc-collections + description: 'GC Collections per Second' + name: .NET GC Collections + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_collections_count_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: Generation - ${generation} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: dotnet-gc-duration + description: 'GC Duration per Collection' + name: .NET GC Duration + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_duration_nanoseconds_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: duration(ns) + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: ns + - + metricId: dotnet-gc-allocated + description: 'Bytes allocated to GC Heap' + name: .NET GC Allocated + metricQueries: + - expression: |- + rate(process_runtime_dotnet_gc_allocations_size_bytes_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: allocated + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-objects-and-committed-memory + description: 'Size of Uncollected Objects and Committed Vmem size for GC Heap' + name: .NET GC Objects and Committed Memory + metricQueries: + - expression: |- + process_runtime_dotnet_gc_objects_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: uncollected objects + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + - expression: |- + process_runtime_dotnet_gc_committed_memory_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: committed vmvem heap size + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-size + description: 'GC Heap size (including fragmentation)' + name: .NET GC Heap Size Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: Generation - ${generation} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: dotnet-gc-heap-fragmentation + description: 'Heap Fragmentation size in Bytes' + name: .NET GC Heap Fragmentation Bytes + metricQueries: + - expression: |- + process_runtime_dotnet_gc_heap_fragmentation_size_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: Generation - ${generation} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_gc_durations_95th_quantile + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(jvm_gc_duration_seconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: jvm_gc_per_second + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(jvm_gc_duration_seconds_count{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"}[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: jvm_mem_after_last_gc + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + jvm_memory_used_after_last_gc_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}"} + alias: Memory Pool - ${jvm_memory_pool_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_heap_memory + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}",jvm_memory_type="heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: jvm_non_heap_memory + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + jvm_memory_used_bytes{service_namespace="${tags.singleValue('service.namespace')}", service_name="${tags.singleValue('service.name')}",jvm_memory_type="non_heap"} + alias: Memory Pool - ${jvm_memory_pool_name} + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}" + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: service-spans-rate + description: The number of spans for the otel service per second + name: Span rate + metricQueries: + - expression: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind) + alias: 'All ${span_kind}' + - expression: sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind) + alias: 'Errors ${span_kind}' + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: service-spans-error-rate + description: The percentage of errors per second + name: Span error rate + metricQueries: + - expression: |- + ((sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (span_kind)) or 0) + / sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (span_kind) + alias: '${span_kind}' + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: service-spans-duration + description: + name: Span duration + metricQueries: + - expression: histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind, le)) + alias: '95th percentile ${span_kind}' + - expression: histogram_quantile(0.99, sum(rate(otel_span_duration_milliseconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (span_kind, le)) + alias: '99th percentile ${span_kind}' + chart: + _type: TimeSeriesChart + unit: ms + - + metricId: service-spans-rate-per-instance + description: The number of spans for the otel service per second + name: Span rate + metricQueries: + - expression: sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (service_instance_id, service_name, service_namespace) + alias: '${service_instance_id}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: service-spans-error-rate-per-instance + description: The percentage of errors per second + name: Span error rate + metricQueries: + - expression: |- + ((sum(rate(otel_span_calls_total{status_code="STATUS_CODE_ERROR", service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (service_instance_id, service_name, service_namespace)) or 0) + / sum(rate(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[5m])) by (service_instance_id, service_name, service_namespace) + alias: '${service_instance_id}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: service-spans-duration-per-instance + description: + name: Span duration + metricQueries: + - expression: histogram_quantile(0.95, sum(rate(otel_span_duration_milliseconds_bucket{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}])) by (service_instance_id, service_name, service_namespace, le)) + alias: '95th percentile ${service_instance_id}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${service_namespace}:service/${service_name}:serviceInstance/${service_instance_id}" + chart: + _type: TimeSeriesChart + unit: ms + - + metricId: service-instance-count + description: + name: Service instance count + metricQueries: + - expression: count(count(otel_span_calls_total{service_namespace="${tags.singleValue('service.namespace')}", service_name="${name}"}[${__rate_interval}]) by (service_instance_id)) + alias: Service instance count + chart: + _type: TimeSeriesChart + unit: short + - + metricId: service-server-spans-failed-percentage + name: Span error rate + metricQueries: + - expression: | + ( + ((sum by (client_service_namespace, client) (rate(traces_service_graph_request_failed_total{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}]))) or 0) + / + sum by (client_service_namespace, client) (rate(traces_service_graph_request_total{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}])) + ) * 100 + componentIdentifierTemplate: "urn:opentelemetry:namespace/${client_service_namespace}:service/${client}" + alias: "Failed call rate ${client}" + chart: + _type: TimeSeriesChart + unit: percent + - + metricId: service-server-spans-rate + name: Span rate + metricQueries: + - expression: sum by (client_service_namespace, client) (rate(traces_service_graph_request_total{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}])) + componentIdentifierTemplate: "urn:opentelemetry:namespace/${client_service_namespace}:service/${client}" + alias: "Client ${client}" + chart: + _type: TimeSeriesChart + unit: reqps + - + metricId: service-server-spans-duration + description: + name: Span duration + metricQueries: + - expression: histogram_quantile(0.95, sum(rate(traces_service_graph_request_server_seconds_bucket{server_service_namespace="${tags.singleValue('service.namespace')}", server="${name}"}[${__rate_interval}])) by (client, client_service_namespace, le)) + alias: '95th percentile ${client}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${client_service_namespace}:service/${client}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: service-client-spans-failed-percentage + name: Span error rate + metricQueries: + - expression: | + ( + ((sum by (server_service_namespace, server) (rate(traces_service_graph_request_failed_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}]))) or 0) + / + sum by (server_service_namespace, server) (rate(traces_service_graph_request_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}])) + ) * 100 + componentIdentifierTemplate: "urn:opentelemetry:namespace/${server_service_namespace}:service/${server}" + alias: "${server}" + chart: + _type: TimeSeriesChart + unit: percent + - + metricId: service-client-spans-rate + name: Span rate + metricQueries: + - expression: sum by (server_service_namespace, server) (rate(traces_service_graph_request_total{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}])) + componentIdentifierTemplate: "urn:opentelemetry:namespace/${server_service_namespace}:service/${server}" + alias: "Server ${server}" + chart: + _type: TimeSeriesChart + unit: reqps + - + metricId: service-client-spans-duration + description: + name: Span duration + metricQueries: + - expression: histogram_quantile(0.95, sum(rate(traces_service_graph_request_client_seconds_bucket{client_service_namespace="${tags.singleValue('service.namespace')}", client="${name}"}[${__rate_interval}])) by (server, server_service_namespace, le)) + alias: '95th percentile ${server}' + componentIdentifierTemplate: "urn:opentelemetry:namespace/${server_service_namespace}:service/${server}" + chart: + _type: TimeSeriesChart + unit: s diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-consumer-peer.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-consumer-peer.sty new file mode 100644 index 0000000..210e8b7 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-consumer-peer.sty @@ -0,0 +1,38 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Async Relation (Consumer -> Peer)" + description: "Creates a relation from the consumer service instance to the message broker (peer service)" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:asynchronous-consumer-peer" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'server_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'messaging_system' && + 'client_peer.service' in datapoint.attributes + vars: + - name: "sourceId" + value: >- + 'server_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server'] + - name: "targetId" + value: >- + 'urn:opentelemetry:namespace/' + datapoint.attributes['client_service.namespace'] + ':service/' + datapoint.attributes['client_peer.service'] + output: + sourceId: "vars.sourceId" + targetId: "vars.targetId" + typeName: "'asynchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-producer-consumer.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-producer-consumer.sty new file mode 100644 index 0000000..d2f3963 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-producer-consumer.sty @@ -0,0 +1,40 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Async Relation (Producer -> Consumer)" + description: "Creates a direct asynchronous relation from producer to consumer" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:asynchronous-producer-consumer" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'server_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'messaging_system' && + !('client_peer.service' in datapoint.attributes) + vars: + - name: "sourceId" + value: >- + 'client_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client'] + - name: "targetId" + value: >- + 'server_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['server_service.namespace'] + ":service/" + datapoint.attributes['server'] + ":serviceInstance/" + datapoint.attributes['server'] + output: + sourceId: "vars.sourceId" + targetId: "vars.targetId" + typeName: "'asynchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-producer-peer.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-producer-peer.sty new file mode 100644 index 0000000..03f8121 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/asynchronous-producer-peer.sty @@ -0,0 +1,37 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Async Relation (Producer -> Peer)" + description: "Creates a relation from the producer to the message broker (peer service)" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:asynchronous-producer-peer" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'messaging_system' && + 'client_peer.service' in datapoint.attributes + vars: + - name: "sourceId" + value: >- + 'client_service.instance.id' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client_service.instance.id'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":serviceInstance/" + datapoint.attributes['client'] + - name: "targetId" + value: >- + 'urn:opentelemetry:namespace/' + datapoint.attributes['client_service.namespace'] + ':service/' + datapoint.attributes['client_peer.service'] + output: + sourceId: "vars.sourceId" + targetId: "vars.targetId" + typeName: "'asynchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/database.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/database.sty new file mode 100644 index 0000000..4038089 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/database.sty @@ -0,0 +1,43 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Database Relation" + description: "Adds a database connection to a service" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:database" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == 'database' + vars: + - name: "clientServiceNamespace" + value: "datapoint.attributes['client_service.namespace']" + - name: "client" + value: "datapoint.attributes['client']" + - name: "clientServiceInstanceId" + value: >- + 'client_service.instance.id' in datapoint.attributes ? + datapoint.attributes['client_service.instance.id'] : + datapoint.attributes['client'] + - name: "targetId" + value: >- + 'client_peer.service' in datapoint.attributes ? + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client_peer.service'] : + "urn:opentelemetry:namespace/" + datapoint.attributes['client_service.namespace'] + ":service/" + datapoint.attributes['client'] + ":database/" + datapoint.attributes['server'] + output: + sourceId: >- + 'urn:opentelemetry:namespace/' + vars.clientServiceNamespace + ':service/' + vars.client + ':serviceInstance/' + vars.clientServiceInstanceId + targetId: "vars.targetId" + typeName: "'database'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/executes-ecs-task.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/executes-ecs-task.sty new file mode 100644 index 0000000..933f988 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/executes-ecs-task.sty @@ -0,0 +1,36 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Executes Relation (Task)" + description: "Tasks executes a service instance" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:executes-task" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'aws.ecs.task.id' in resource.attributes" + vars: + - name: "awsEcsTaskId" + value: "resource.attributes['aws.ecs.task.id']" + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes && + resource.attributes['service.instance.id'] != '' ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: "'urn:opentelemetry:task/' + vars.awsEcsTaskId" + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'executes'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/executes-function.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/executes-function.sty new file mode 100644 index 0000000..ee187a0 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/executes-function.sty @@ -0,0 +1,36 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Executes Relation (Function)" + description: "Functions executes a service instance" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:executes-function" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'faas.id' in resource.attributes" + vars: + - name: "faasId" + value: "resource.attributes['faas.id']" + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes && + resource.attributes['service.instance.id'] != '' ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: "'urn:opentelemetry:function/' + vars.faasId" + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'executes'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/executes-host.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/executes-host.sty new file mode 100644 index 0000000..fb55997 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/executes-host.sty @@ -0,0 +1,36 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Executes Relation (Host)" + description: "Host executes a service instance" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:executes-host" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'host.id' in resource.attributes" + vars: + - name: "hostId" + value: "resource.attributes['host.id']" + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes && + resource.attributes['service.instance.id'] != '' ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: "'urn:opentelemetry:host/' + vars.hostId" + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'executes'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/k8s-pod-to-otel.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/k8s-pod-to-otel.sty new file mode 100644 index 0000000..d86fc01 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/k8s-pod-to-otel.sty @@ -0,0 +1,46 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Kubernetes to OTEL Relation" + description: "Relates a Kubernetes pod to a service instance" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:kubernetes-to-otel" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: >- + 'k8s.cluster.name' in resource.attributes && + 'k8s.namespace.name' in resource.attributes && + 'k8s.pod.name' in resource.attributes + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "namespaceName" + value: "resource.attributes['k8s.namespace.name']" + - name: "podName" + value: "resource.attributes['k8s.pod.name']" + - name: "serviceNamespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: >- + 'service.name' in resource.attributes ? + resource.attributes['service.name'] : + '' + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: >- + 'urn:opentelemetry:kubernetes:/' + vars.clusterName + ':' + vars.namespaceName + ':pod/' + vars.podName + targetId: >- + 'urn:opentelemetry:namespace/' + vars.serviceNamespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'kubernetes-to-otel'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/provided-by.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/provided-by.sty new file mode 100644 index 0000000..da78878 --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/provided-by.sty @@ -0,0 +1,34 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel service instance provided-by service relation" + description: "Links a service to its instance" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:provided-by" + input: + signal: + - "TRACES" + - "METRICS" + resource: + action: "'CREATE'" + condition: "'service.name' in resource.attributes" + vars: + - name: "namespace" + value: >- + 'service.namespace' in resource.attributes && + resource.attributes['service.namespace'] != '' ? + resource.attributes['service.namespace'] : + 'default' + - name: "service" + value: "resource.attributes['service.name']" + - name: "instanceId" + value: >- + 'service.instance.id' in resource.attributes ? + resource.attributes['service.instance.id'] : + resource.attributes['service.name'] + output: + sourceId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + targetId: >- + 'urn:opentelemetry:namespace/' + vars.namespace + ':service/' + vars.service + ':serviceInstance/' + vars.instanceId + typeName: "'provided-by'" + dependencyType: "'HIERARCHICAL'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/settings/relation-mappings/synchronous.sty b/stackpacks/open-telemetry-2/settings/relation-mappings/synchronous.sty new file mode 100644 index 0000000..5f7f9ca --- /dev/null +++ b/stackpacks/open-telemetry-2/settings/relation-mappings/synchronous.sty @@ -0,0 +1,49 @@ +nodes: + - _type: "OtelRelationMapping" + name: "OTel Synchronous Relation" + description: "Adds a synchronous connection to a service" + identifier: "urn:stackpack:open-telemetry-2:otel-relation-mapping:synchronous" + input: + signal: + - "METRICS" + resource: + scope: + condition: "scope.name == 'traces_service_graph'" + metric: + condition: "metric.name == 'traces_service_graph_request_total'" + datapoint: + action: "'CREATE'" + condition: >- + 'client' in datapoint.attributes && + 'server' in datapoint.attributes && + 'client_service.namespace' in datapoint.attributes && + 'server_service.namespace' in datapoint.attributes && + 'connection_type' in datapoint.attributes && + datapoint.attributes['connection_type'] == '' + vars: + - name: "clientServiceNamespace" + value: "datapoint.attributes['client_service.namespace']" + - name: "serverServiceNamespace" + value: "datapoint.attributes['server_service.namespace']" + - name: "client" + value: "datapoint.attributes['client']" + - name: "server" + value: "datapoint.attributes['server']" + - name: "clientServiceInstanceId" + value: >- + 'client_service.instance.id' in datapoint.attributes ? + datapoint.attributes['client_service.instance.id'] : + datapoint.attributes['client'] + - name: "serverServiceInstanceId" + value: >- + 'server_service.instance.id' in datapoint.attributes ? + datapoint.attributes['server_service.instance.id'] : + datapoint.attributes['server'] + output: + sourceId: >- + 'urn:opentelemetry:namespace/' + vars.clientServiceNamespace + ':service/' + vars.client + ':serviceInstance/' + vars.clientServiceInstanceId + targetId: >- + 'urn:opentelemetry:namespace/' + vars.serverServiceNamespace + ':service/' + vars.server + ':serviceInstance/' + vars.serverServiceInstanceId + typeName: "'synchronous'" + dependencyType: "'CONNECTION'" + expireAfter: 900000 diff --git a/stackpacks/open-telemetry-2/stackpack.yaml b/stackpacks/open-telemetry-2/stackpack.yaml new file mode 100644 index 0000000..2c63899 --- /dev/null +++ b/stackpacks/open-telemetry-2/stackpack.yaml @@ -0,0 +1,20 @@ +name: "open-telemetry-2" +version: "0.0.16" +schemaVersion: "2.0" +displayName: "Open Telemetry 2.0" +categories: [ "Open Telemetry" ] +logoUrl: !resource "logo.png" +# information that will go in the overview section +overviewUrl: !resource "overview.md" +# information that will go to the detailed-overview section +detailedOverviewUrl: !resource "detailed-overview.md" +# information that will go in the configuration section +configurationUrls: + NOT_INSTALLED: !resource "configuration.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + INSTALLED: !resource "enabled.md" + DEPROVISIONING: !resource "configuration.md" + ERROR: !resource "configuration.md" +provision: + companionStackPacks: [] diff --git a/stackpacks/otel-k8s-crd/README.md b/stackpacks/otel-k8s-crd/README.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/otel-k8s-crd/icons/cr.svg b/stackpacks/otel-k8s-crd/icons/cr.svg new file mode 100644 index 0000000..2fa473b --- /dev/null +++ b/stackpacks/otel-k8s-crd/icons/cr.svg @@ -0,0 +1,83 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + diff --git a/stackpacks/otel-k8s-crd/icons/crd.svg b/stackpacks/otel-k8s-crd/icons/crd.svg new file mode 100644 index 0000000..c59c325 --- /dev/null +++ b/stackpacks/otel-k8s-crd/icons/crd.svg @@ -0,0 +1,83 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + diff --git a/stackpacks/otel-k8s-crd/resources/configuration.md b/stackpacks/otel-k8s-crd/resources/configuration.md new file mode 100644 index 0000000..726ae2e --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/configuration.md @@ -0,0 +1,110 @@ +## Prerequisites + +- SUSE Observability Agent installed with the k8s resource collector enabled +- Network connectivity from the agent to the platform's OTLP ingest endpoint + +## Configuration + +Enable StackPacks 2.0 support in your agent Helm values. This activates the OTel components, including the k8s resource collector. The collector is enabled by default when OTel is active, but can still be configured under `otel.k8sResourceCollector`: + +```yaml +global: + features: + experimentalStackpacks: true +otel: + k8sResourceCollector: + crdDiscovery: + discoveryMode: api_groups # or "all" to watch every API group + snapshotInterval: 5m # periodic full-state emission from the informer cache (min: 1m) +``` + +If you are not enabling StackPacks 2.0 globally, set `otel.enabled: true` instead. + +### API Group Filtering + +`apiGroupFilters.include` and `apiGroupFilters.exclude` are maps of pattern → bool. Wildcards are supported (e.g. `*.example.com`). To disable a default pattern from an overlay values file, set its value to `false`: + +```yaml +otel: + k8sResourceCollector: + crdDiscovery: + discoveryMode: api_groups + apiGroupFilters: + include: + # Disable the chart default `"*": true` so only the patterns below match. + "*": false + "*.example.com": true + exclude: + "internal.example.com": true +``` + +The chart default is `include: { "*": true }`. There must be at least one truthy entry when `discoveryMode` is `api_groups`. + +### Restricting RBAC + +By default the collector is granted wildcard read permissions for every custom resource in the cluster. To restrict it to specific API groups, set `useWildcard: false` and list the groups under `crdApiGroups` (map of group → bool, same disable semantics): + +```yaml +otel: + k8sResourceCollector: + rbac: + useWildcard: false + crdApiGroups: + "policies.kubewarden.io": true + "longhorn.io": true +``` + +### Watching Additional Kubernetes Resources + +`objects` lets you watch arbitrary Kubernetes resources (built-ins or third-party) alongside the CRD-discovered custom resources. Each entry is keyed by the resource plural (required) and maps to a spec: + +```yaml +otel: + k8sResourceCollector: + objects: + pods: + group: "" # core group + namespaces: ["kube-system"] + deployments: + group: apps + labelSelector: "app=foo" + fieldSelector: "status.phase=Running" +``` + +Spec fields: + +- `group` — API group; empty string (`""`) for the core group +- `version` — optional; defaults to the API server's preferred version +- `namespaces` — optional; cluster-wide when empty +- `labelSelector` / `fieldSelector` — optional Kubernetes selectors + +Entries that overlap a CRD covered by `crdDiscovery.apiGroupFilters` are rejected at startup. When `rbac.useWildcard: false`, the chart auto-derives `get`/`list`/`watch` RBAC for each `objects` entry (deduped per group) — no need to add the group to `crdApiGroups`. + +To disable a default `objects` entry from an overlay file, set its value to `null` or `false`. + +`deniedObjects` extends the receiver's built-in denylist (core `Secrets`, `ConfigMaps`) with resources that must not appear under `objects`. The key is the resource plural; the spec only needs `group`: + +```yaml +otel: + k8sResourceCollector: + deniedObjects: + certificates: + group: cert-manager.io +``` + +### OTLP Endpoint Override + +By default, telemetry is sent to `/otel` over HTTP (proxied via the platform's Envoy router; the default path is HTTP-only). If your platform exposes a dedicated OTLP ingress, override the OTel platform endpoint. HTTP endpoints take precedence over gRPC endpoints when both are configured. + +```yaml +otel: + # gRPC: bare host:port, no scheme. A port is required. + platformGrpcOtlpEndpoint: otlp-my-instance.stackstate.io:443 +``` + +For a dedicated HTTP ingress, provide a full URL with scheme: + +```yaml +otel: + platformHttpOtlpEndpoint: https://otlp-http-my-instance.stackstate.io:4318 +``` diff --git a/stackpacks/otel-k8s-crd/resources/detailed-overview.md b/stackpacks/otel-k8s-crd/resources/detailed-overview.md new file mode 100644 index 0000000..9de123d --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/detailed-overview.md @@ -0,0 +1,18 @@ +## How it works + +The StackPack defines SUSE Observability settings (component and relation mappings) that the platform's OTel collector applies to telemetry produced by the `k8scrdreceiver` OpenTelemetry receiver, which runs in the SUSE Observability Agent's k8s CRD collector. The receiver discovers CRDs and their CR instances using watch mode (real-time informer events) plus periodic snapshots (full-state emission from the informer cache, controlled by `snapshotInterval`). + +### Components created + +- **Custom Resource Definition** - Each CRD installed in the cluster (e.g., `helmcharts.helm.cattle.io`) +- **Custom Resource** - Each CR instance (e.g., a specific HelmChart resource) + +### Relations created + +- **instance of** - Links each Custom Resource to its defining Custom Resource Definition + +### Data sources + +- CRD and CR discovery via the Kubernetes API server +- Watch mode for real-time change detection +- Periodic snapshots from the informer cache for state reconciliation diff --git a/stackpacks/otel-k8s-crd/resources/enabled.md b/stackpacks/otel-k8s-crd/resources/enabled.md new file mode 100644 index 0000000..553cf52 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/enabled.md @@ -0,0 +1,5 @@ +## The Open Telemetry Kubernetes CRD StackPack is installed + +### What's next + +Explore the discovered Custom Resources and their relationships in SUSE Observability. diff --git a/stackpacks/otel-k8s-crd/resources/error.md b/stackpacks/otel-k8s-crd/resources/error.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/otel-k8s-crd/resources/logo.svg b/stackpacks/otel-k8s-crd/resources/logo.svg new file mode 100644 index 0000000..7f5d1f3 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/logo.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/stackpacks/otel-k8s-crd/resources/overview.md b/stackpacks/otel-k8s-crd/resources/overview.md new file mode 100644 index 0000000..2cbd870 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/overview.md @@ -0,0 +1,5 @@ +## Open Telemetry Kubernetes CRD StackPack + +The **Kubernetes Custom Resources** StackPack discovers Custom Resource Definitions (CRDs) and their Custom Resource (CR) instances in your Kubernetes clusters, creating topology components and relations for each. + +This provides visibility into the custom extensions installed in your clusters, including operators, controllers, and platform add-ons. diff --git a/stackpacks/otel-k8s-crd/resources/provisioning.md b/stackpacks/otel-k8s-crd/resources/provisioning.md new file mode 100644 index 0000000..0935880 --- /dev/null +++ b/stackpacks/otel-k8s-crd/resources/provisioning.md @@ -0,0 +1,3 @@ +## Provisioning... + +Provisioning Kubernetes Custom Resource discovery. Please wait a moment. diff --git a/stackpacks/otel-k8s-crd/resources/waitingfordata.md b/stackpacks/otel-k8s-crd/resources/waitingfordata.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resource-definitions.sty b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resource-definitions.sty new file mode 100644 index 0000000..610c4db --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resource-definitions.sty @@ -0,0 +1,45 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Custom Resource Definition" + description: "Represents a CRD installed in a Kubernetes cluster" + identifier: "urn:stackpack:otel-k8s-crd:otel-component-mapping:custom-resource-definition" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: log.eventName == 'KubernetesCustomResourceDefinitionEvent' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "group" + value: "log.attributes['k8s.resource.group']" + - name: "kind" + value: "log.attributes['k8s.resource.kind']" + output: + identifier: "'urn:k8s:crd:/cluster/' + vars.clusterName + ':group/' + vars.group + ':kind/' + vars.kind" + name: "vars.name" + typeName: "'custom resource definition'" + required: + configuration: "omit(log.body['object'], ['status', 'metadata'])" + status: "log.body['object']['status']" + tags: + - source: "'otel-k8s-crd'" + target: "stackpack" + - source: "vars.group" + target: "k8s.crd.group" + - source: "vars.kind" + target: "k8s.crd.kind" + - source: "vars.clusterName" + target: "cluster-name" + optional: + tags: + - source: "log.body['object']['metadata']['creationTimestamp']" + target: "k8s.creationTimestamp" + expireAfter: 900000 + rank: + specificity: 100 diff --git a/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resources.sty b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resources.sty new file mode 100644 index 0000000..077b0e0 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/component-mappings/custom-resources.sty @@ -0,0 +1,51 @@ +nodes: + - _type: "OtelComponentMapping" + name: "Kubernetes Custom Resource" + description: "Represents a CR instance of a CRD in a Kubernetes cluster" + identifier: "urn:stackpack:otel-k8s-crd:otel-component-mapping:custom-resource" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: log.eventName == 'KubernetesCustomResourceEvent' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "kind" + value: "log.attributes['k8s.resource.kind']" + - name: "group" + value: "log.attributes['k8s.resource.group']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "namespace" + value: "'k8s.namespace.name' in log.attributes ? log.attributes['k8s.namespace.name'] : ''" + output: + identifier: "'urn:k8s:cr:/cluster/' + vars.clusterName + ':namespace/' + vars.namespace + ':group/' + vars.group + ':kind/' + vars.kind + ':name/' + vars.name" + name: "vars.name" + typeName: "'custom resource'" + required: + configuration: "omit(log.body['object'], ['status', 'metadata'])" + tags: + - source: "'otel-k8s-crd'" + target: "stackpack" + - source: "vars.kind" + target: "k8s.resource.kind" + - source: "vars.group" + target: "k8s.resource.group" + - source: "vars.name" + target: "k8s.resource.name" + - source: "vars.clusterName" + target: "cluster-name" + optional: + status: "log.body['object']['status']" + tags: + - source: "vars.namespace" + target: "namespace" + - source: "log.body['object']['metadata']['creationTimestamp']" + target: "k8s.creationTimestamp" + expireAfter: 900000 + rank: + specificity: 100 diff --git a/stackpacks/otel-k8s-crd/settings/main-menu.sty b/stackpacks/otel-k8s-crd/settings/main-menu.sty new file mode 100644 index 0000000..5311723 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/main-menu.sty @@ -0,0 +1,7 @@ +nodes: + - _type: MainMenuGroup + name: Custom Resources + iconbase64: !icon crd.svg + defaultOpen: true + items: [] + identifier: "urn:stackpack:otel-k8s-crd:main-menu-group:custom-resources" diff --git a/stackpacks/otel-k8s-crd/settings/presentations/common-filters.sty b/stackpacks/otel-k8s-crd/settings/presentations/common-filters.sty new file mode 100644 index 0000000..57cfef2 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/presentations/common-filters.sty @@ -0,0 +1,60 @@ +nodes: + - _type: ComponentPresentation + name: "K8s CRD Common Filters" + description: "Common filters for Kubernetes Custom Resource components" + identifier: "urn:stackpack:otel-k8s-crd:component-presentation:common-filters" + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("stackpack:otel-k8s-crd") + rank: + specificity: 1 + presentation: + filters: + - filterId: "crd_cluster" + displayName: + singular: "Cluster" + plural: "Clusters" + filter: + _type: "TagFilter" + tagKey: "cluster-name" + moreTab: "Kubernetes" + - filterId: "crd_api_group" + displayName: + singular: "API Group" + plural: "API Groups" + filter: + _type: "TagFilter" + tagKey: "k8s.crd.group" + moreTab: "Custom Resources" + - filterId: "crd_kind" + displayName: + singular: "Kind" + plural: "Kinds" + filter: + _type: "TagFilter" + tagKey: "k8s.crd.kind" + moreTab: "Custom Resources" + - filterId: "cr_api_group" + displayName: + singular: "API Group" + plural: "API Groups" + filter: + _type: "TagFilter" + tagKey: "k8s.resource.group" + moreTab: "Custom Resources" + - filterId: "cr_kind" + displayName: + singular: "Kind" + plural: "Kinds" + filter: + _type: "TagFilter" + tagKey: "k8s.resource.kind" + moreTab: "Custom Resources" + - filterId: "cr_namespace" + displayName: + singular: "Namespace" + plural: "Namespaces" + filter: + _type: "TagFilter" + tagKey: "namespace" + moreTab: "Kubernetes" diff --git a/stackpacks/otel-k8s-crd/settings/presentations/custom-resource-definitions.sty b/stackpacks/otel-k8s-crd/settings/presentations/custom-resource-definitions.sty new file mode 100644 index 0000000..b696650 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/presentations/custom-resource-definitions.sty @@ -0,0 +1,113 @@ +nodes: + - _type: ComponentPresentation + name: "Custom Resource Definition" + description: "Presentation for Kubernetes Custom Resource Definitions" + identifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource-definition" + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:otel-k8s-crd" and label = "source-type:custom resource definition") + rank: + specificity: 10 + presentation: + icon: !icon crd.svg + filters: + - filterId: "crd_cluster" + - filterId: "crd_api_group" + - filterId: "crd_kind" + overview: + name: + singular: "custom resource definition" + plural: "custom resource definitions" + title: "Custom Resource Definitions" + mainMenu: + group: "Custom Resources" + order: 2 + columns: + - columnId: health + title: "Health" + projection: + _type: "HealthProjection" + value: "healthState" + - columnId: crd_name + title: "Name" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: crd_api_group + title: "API Group" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.group')" + - columnId: crd_kind + title: "Kind" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.kind')" + - columnId: crd_cluster + title: "Cluster" + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - columnId: crd_age + title: "Age" + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + highlight: + title: "Custom Resource Definition" + fields: + - fieldId: "type" + title: "Type" + order: 100.0 + projection: + _type: "TextProjection" + value: "typeName" + - fieldId: "api_group" + title: "API Group" + order: 90.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.group')" + - fieldId: "kind" + title: "Kind" + order: 80.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.crd.kind')" + - fieldId: "cluster" + title: "Cluster" + order: 70.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - fieldId: "age" + title: "Age" + order: 60.0 + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + - fieldId: "labels" + title: "Labels" + order: 10.0 + projection: + _type: "TagProjection" + provisioning: + externalComponentSelector: "external.mapping.identifier == 'urn:stackpack:otel-k8s-crd:otel-component-mapping:custom-resource-definition'" + showConfiguration: true + showStatus: true + relatedResources: + - resourceId: "crd_instances" + title: "Custom Resource instances" + order: 1.0 + topologyQuery: >- + (withNeighborsOf(direction = "both", + components = (identifier = "${identifiers[0]}"), + levels = "1")) and label = "source-type:custom resource" + presentationIdentifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource" + events: + showEvents: true + relatedResourcesQuery: | + identifier = "${identifiers[0]}" + OR + (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and label = "source-type:custom resource") diff --git a/stackpacks/otel-k8s-crd/settings/presentations/custom-resources.sty b/stackpacks/otel-k8s-crd/settings/presentations/custom-resources.sty new file mode 100644 index 0000000..66e0654 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/presentations/custom-resources.sty @@ -0,0 +1,134 @@ +nodes: + - _type: ComponentPresentation + name: "Custom Resource" + description: "Presentation for Kubernetes Custom Resource instances" + identifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource" + binding: + _type: ComponentPresentationQueryBinding + query: (label = "stackpack:otel-k8s-crd" and label = "source-type:custom resource") + rank: + specificity: 10 + presentation: + icon: !icon cr.svg + filters: + - filterId: "crd_cluster" + - filterId: "cr_api_group" + - filterId: "cr_kind" + - filterId: "cr_namespace" + topology: + groupingEnabled: true + showIndirectRelations: false + minimumGroupSize: 8 + groupedByLayers: false + groupedByDomains: true + autoGrouping: true + connectedComponents: true + overview: + name: + singular: "custom resource" + plural: "custom resources" + title: "Custom Resources" + mainMenu: + group: "Custom Resources" + order: 1 + columns: + - columnId: health + title: "Health" + projection: + _type: "HealthProjection" + value: "healthState" + - columnId: cr_name + title: "Name" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: cr_kind + title: "Kind" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.resource.kind')" + - columnId: cr_api_group + title: "API Group" + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.resource.group')" + - columnId: cr_namespace + title: "Namespace" + projection: + _type: "TextProjection" + value: "tags.singleValue('namespace')" + - columnId: cr_cluster + title: "Cluster" + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - columnId: cr_age + title: "Age" + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + highlight: + title: "Custom Resource" + fields: + - fieldId: "type" + title: "Type" + order: 100.0 + projection: + _type: "TextProjection" + value: "typeName" + - fieldId: "kind" + title: "Kind" + order: 90.0 + projection: + _type: "ComponentLinkProjection" + name: "tags.singleValue('k8s.resource.kind')" + identifier: "'urn:k8s:crd:/cluster/' + tags.singleValue('cluster-name') + ':group/' + tags.singleValue('k8s.resource.group') + ':kind/' + tags.singleValue('k8s.resource.kind')" + - fieldId: "api_group" + title: "API Group" + order: 80.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('k8s.resource.group')" + - fieldId: "namespace" + title: "Namespace" + order: 60.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('namespace')" + - fieldId: "cluster" + title: "Cluster" + order: 50.0 + projection: + _type: "TextProjection" + value: "tags.singleValue('cluster-name')" + - fieldId: "age" + title: "Age" + order: 40.0 + projection: + _type: "DurationProjection" + startTime: "tags.singleValue('k8s.creationTimestamp')" + - fieldId: "labels" + title: "Labels" + order: 10.0 + projection: + _type: "TagProjection" + provisioning: + externalComponentSelector: "external.mapping.identifier == 'urn:stackpack:otel-k8s-crd:otel-component-mapping:custom-resource'" + showConfiguration: true + showStatus: true + relatedResources: + - resourceId: "cr_definition" + title: "Custom Resource Definition" + order: 1.0 + topologyQuery: >- + (withNeighborsOf(direction = "both", + components = (identifier = "${identifiers[0]}"), + levels = "1")) and label = "source-type:custom resource definition" + presentationIdentifier: "urn:stackpack:otel-k8s-crd:component-presentation:custom-resource-definition" + events: + showEvents: true + relatedResourcesQuery: | + identifier = "${identifiers[0]}" + OR + (withNeighborsOf(direction = "both", components = (identifier = "${identifiers[0]}"), levels = "1") and label = "source-type:custom resource definition") diff --git a/stackpacks/otel-k8s-crd/settings/relation-mappings/cr-instance-of-crd.sty b/stackpacks/otel-k8s-crd/settings/relation-mappings/cr-instance-of-crd.sty new file mode 100644 index 0000000..b33c813 --- /dev/null +++ b/stackpacks/otel-k8s-crd/settings/relation-mappings/cr-instance-of-crd.sty @@ -0,0 +1,30 @@ +nodes: + - _type: "OtelRelationMapping" + name: "Custom Resource instance of CRD" + description: "Links a Custom Resource to the Custom Resource Definition it is an instance of" + identifier: "urn:stackpack:otel-k8s-crd:otel-relation-mapping:cr-instance-of-crd" + input: + signal: + - "LOGS" + resource: + scope: + log: + action: "log.body['type'] in ['ADDED', 'MODIFIED'] ? 'CREATE' : log.body['type'] == 'DELETED' ? 'DELETE' : 'CONTINUE'" + condition: log.eventName == 'KubernetesCustomResourceEvent' + vars: + - name: "clusterName" + value: "resource.attributes['k8s.cluster.name']" + - name: "kind" + value: "log.attributes['k8s.resource.kind']" + - name: "group" + value: "log.attributes['k8s.resource.group']" + - name: "name" + value: "log.attributes['k8s.object.name']" + - name: "namespace" + value: "'k8s.namespace.name' in log.attributes ? log.attributes['k8s.namespace.name'] : ''" + output: + sourceId: "'urn:k8s:cr:/cluster/' + vars.clusterName + ':namespace/' + vars.namespace + ':group/' + vars.group + ':kind/' + vars.kind + ':name/' + vars.name" + targetId: "'urn:k8s:crd:/cluster/' + vars.clusterName + ':group/' + vars.group + ':kind/' + vars.kind" + typeName: "'instance of'" + dependencyType: "'UNCLASSIFIED'" + expireAfter: 900000 diff --git a/stackpacks/otel-k8s-crd/stackpack.yaml b/stackpacks/otel-k8s-crd/stackpack.yaml new file mode 100644 index 0000000..f022a97 --- /dev/null +++ b/stackpacks/otel-k8s-crd/stackpack.yaml @@ -0,0 +1,17 @@ +name: "otel-k8s-crd" +version: "0.0.6" +schemaVersion: "2.0" +displayName: "Kubernetes Custom Resources" +categories: [ "Kubernetes" ] +logoUrl: !resource "logo.svg" +overviewUrl: !resource "overview.md" +detailedOverviewUrl: !resource "detailed-overview.md" +configurationUrls: + NOT_INSTALLED: !resource "configuration.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + INSTALLED: !resource "enabled.md" + DEPROVISIONING: !resource "configuration.md" + ERROR: !resource "error.md" +provision: + companionStackPacks: [] diff --git a/stackpacks/suse-observability-2/README.md b/stackpacks/suse-observability-2/README.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/suse-observability-2/icons/suse-observability.svg b/stackpacks/suse-observability-2/icons/suse-observability.svg new file mode 100644 index 0000000..0c19ca3 --- /dev/null +++ b/stackpacks/suse-observability-2/icons/suse-observability.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/generic/sts-apps-overrun-cpu-request.md.hbs b/stackpacks/suse-observability-2/includes/generic/sts-apps-overrun-cpu-request.md.hbs new file mode 100644 index 0000000..46e2919 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/generic/sts-apps-overrun-cpu-request.md.hbs @@ -0,0 +1,26 @@ +The Average CPU usage for container (`{{ tags.pod_name }}/{{ tags.container }}`) exceeds CPU requests by more than 100%. +This might not be an issue if there are enough resources in the node where the container is running, but is that's not the case performance degradation can be expected in the service. + +### Mitigation +1. Verify with the metrics chart if the average CPU usage has increased for a long time or if the monitor was triggered just due to a spike. +2. In the latter case you can increase the CPU request of the service related to the container `{{ tags.pod_name }}/{{ tags.container }}` +3. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU. +4. Based on the pod locate the relevant section of the `values.yaml` and update the CPU requests to give more resources, for example if the pod with the issue if kafka then we would need to update + +``` +kafka: + resources: + requests: + cpu: 1000m +``` + +5. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously + +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/generic/sts-java-long-gc.md.hbs b/stackpacks/suse-observability-2/includes/generic/sts-java-long-gc.md.hbs new file mode 100644 index 0000000..b4edc6c --- /dev/null +++ b/stackpacks/suse-observability-2/includes/generic/sts-java-long-gc.md.hbs @@ -0,0 +1,22 @@ +GC collection times have been excessive for `{{ tags.pod_name }}`. This means that the CPU has not been able to do any kind of work but GC during those pauses. + +### Mitigation +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested Memory. +2. Based on the pod locate the relevant section of the `values.yaml` and increase the Memory requests to give more resources, for example if the pod with the issue if kafka then we would need to update + +``` +kafka: + resources: + requests: + memory: 3Gi +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/checks/monitors-failure-percentage-pod.md.hbs b/stackpacks/suse-observability-2/includes/services/checks/monitors-failure-percentage-pod.md.hbs new file mode 100644 index 0000000..2be6e53 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/checks/monitors-failure-percentage-pod.md.hbs @@ -0,0 +1,10 @@ +The monitor failures monitor triggers when a monitor running in SUSE Observability is failing more than expected. This will impact the +monitor results by being late, incorrect or incomplete. + +## Possible solutions + +- Look into the logs of the pod running the monitor to understand what is causing the monitor to fail. +- It could be SUSE observability is under provisioned. Investigate the CPU usage and throttling of the server or checks pod + running the monitor. If the pod is being throttled, increase CPU requests and limits. It could also be an upstream like VictoriaMetrics is throttled + slowing down execution of a monitor. +- If this does not solve the issue, please contact SUSE Observability [support](https://www.stackstate.com/company/contact-us/). diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs new file mode 100644 index 0000000..593e2a5 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs @@ -0,0 +1,5 @@ +ClickHouse increase `ReplicatedPartFailedFetches` in `system.events` table. +It mean server was failed to download data part from replica of a ReplicatedMergeTree table. +Please check following things: +- connections between ClickHouse pod and his replicas (see remote_server section in /etc/clickhouse-server/) +- [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs new file mode 100644 index 0000000..b98f3df --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs @@ -0,0 +1,14 @@ +ClickHouse increase DataAfterMergeDiffersFromReplica in `system.events` table. +It mean Data after merge is not byte-identical to data on another replicas. +There could be several reasons: +- Using newer version of compression library after server update. +- Using another compression method. +- Non-deterministic compression algorithm (highly unlikely). +- Non-deterministic merge algorithm due to logical error in code. +- Data corruption in memory due to bug in code. +- Data corruption in memory due to hardware issue. +- Manual modification of source data after server startup. +- Manual modification of checksums stored in ZooKeeper. +- Part format related settings like 'enable_mixed_granularity_parts' are different on different replicas. + +Server will download merged part from replica to force byte-identical result. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs new file mode 100644 index 0000000..a7bb60d --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs @@ -0,0 +1,2 @@ +ClickHouse has INSERT statements that are throttled due to high number of active data parts for partition in a MergeTree, +please decrease INSERT frequency https://clickhouse.com/docs/en/development/architecture/#merge-tree \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs new file mode 100644 index 0000000..c3dcc03 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs @@ -0,0 +1,16 @@ +ClickHouse has too much files which not insert to `*MergeTree` tables via `Distributed` table engine +Check not synced .bin files via + +```kubectl exec -n {{labels.namespace}} pod/{{labels.pod_name}} -- bash -c "ls -la /var/lib/clickhouse/data/*/*/*/*.bin"``` + +Also, check documentation: +https://clickhouse.com/docs/en/engines/table-engines/special/distributed/ +When you insert data to `Distributed` table. +Data is written to target `*MergreTree` tables asynchronously. +When inserted in the table, the data block is just written to the local file system. +The data is sent to the remote servers in the background as soon as possible. +The period for sending data is managed by the `distributed_directory_monitor_sleep_time_ms` and `distributed_directory_monitor_max_sleep_time_ms` settings. +The Distributed engine sends each file with inserted data separately, but you can enable batch sending of files with the `distributed_directory_monitor_batch_inserts` setting + +Also, you can manage distributed tables: +https://clickhouse.com/docs/en/sql-reference/statements/system/#managing-distributed-tables \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs new file mode 100644 index 0000000..e81007c --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs @@ -0,0 +1,4 @@ +ClickHouse increase DistributedSyncInsertionTimeoutExceeded in `system.events` table. +It mean Synchronous distributed insert timeout exceeded after successfully distributed connection. +Please check documentation https://clickhouse.com/docs/en/operations/settings/settings/#insert_distributed_sync +And check connection between the Pod and all nodes in shards from remote_servers config section diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs new file mode 100644 index 0000000..41fc116 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs @@ -0,0 +1,4 @@ +ClickHouse increase ReadBufferFromFileDescriptorReadFailed or ReadBufferFromFileDescriptorReadFailed in `system.events` table. +It mean the read (read/pread) or writes (write/pwrite) to a file descriptor. Does not include sockets. +System can't read or write to some files. +Please check [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs new file mode 100644 index 0000000..f09357f --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs @@ -0,0 +1,5 @@ +ClickHouse has too many parts in one partition. +ClickHouse MergeTree table engine split each INSERT statement to partitions (PARTITION BY expression) +and add one or more PARTS per INSERT inside each partition, after that background merge process run, +and when you have too much unmerged parts inside partition, +SELECT statements performance can significant degrade, so clickhouse tries to delay or reject INSERT \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-readonly-replica.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-readonly-replica.md.hbs new file mode 100644 index 0000000..a361aaf --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-readonly-replica.md.hbs @@ -0,0 +1,10 @@ +ClickHouse has ReplicatedMergeTree tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured. +Please check following things: +- kubernetes nodes have free enough RAM and Disk via `kubectl top node` +- status of ClickHouse Pod +- connection between ClickHouse pods and ZooKeeper ```kubectl exec -n {{labels.namespace}} pod/{{labels.pod_name}} -- bash -c "clickhouse-client -u admin --password CHANGE_ME -q \"SELECT * FROM system.zookeeper WHERE path='/' FORMAT Vertical\""``` +- connection between ClickHouse pods via kubernetes services ```kubectl exec -n {{labels.namespace}} pod/{{labels.pod_name}} -- bash -c "clickhouse-client -u admin --password CHANGE_ME -q \"SELECT host_name, errors_count FROM system.clusters WHERE errors_count > 0 FORMAT PrettyCompactMonoBlock\""``` +- status of PersistentVolumeClaims for the Pod + +Also read documentation: +https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-failures diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-rejected-insert.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-rejected-insert.md.hbs new file mode 100644 index 0000000..48345fd --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-rejected-insert.md.hbs @@ -0,0 +1,5 @@ +ClickHouse has INSERT statements that are rejected due to high number of active data parts for partition in a MergeTree, +please decrease INSERT frequency: +- [MergeTreeArchitecture](https://clickhouse.com/docs/en/development/architecture/#merge-tree) +- [system.part_log](https://clickhouse.com/docs/en/operations/system-tables/part_log) +- [system.merge_tree_settings](https://clickhouse.com/docs/en/operations/system-tables/merge_tree_settings) \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-replicated-data-loss.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-replicated-data-loss.md.hbs new file mode 100644 index 0000000..158fcfb --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-replicated-data-loss.md.hbs @@ -0,0 +1,6 @@ +ClickHouse increase ReplicatedDataLoss in `system.events` table. +It mean data part that server wanted doesn't exist on any replica (even on replicas that are offline right now). +That data parts are definitely lost. This is normal due to asynchronous replication (if quorum inserts were not enabled), +when the replica on which the data part was written was failed and when it became online after fail it doesn't contain that data part. + +Please check [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs new file mode 100644 index 0000000..80ecc2b --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs @@ -0,0 +1 @@ +ClickHouse increase ReplicatedPartCheckFailed in `system.events` table. Please check [logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-slow-read.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-slow-read.md.hbs new file mode 100644 index 0000000..1a39f83 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-slow-read.md.hbs @@ -0,0 +1,4 @@ +ClickHouse increase SlowRead in `system.events` table. +It mean reads from a files that were slow. This indicate system overload. Thresholds are controlled by `SELECT * FROM system.settings WHERE name LIKE 'read_backoff_%'`. +System will reduce the number of threads which used for processing queries. +Check you disks utilization and hardware failures. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-too-many-connections.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-too-many-connections.md.hbs new file mode 100644 index 0000000..7399cda --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-too-many-connections.md.hbs @@ -0,0 +1,5 @@ +ClickHouse has too many open connections. +The ClickHouse is adapted to run not a very large number of parallel SQL requests, not every HTTP/TCP(Native)/MySQL protocol connection means a running SQL request, but a large number of open connections can cause a spike in sudden SQL requests, resulting in performance degradation. + +Also read documentation: +- https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#max-concurrent-queries diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-too-many-running-queries.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-too-many-running-queries.md.hbs new file mode 100644 index 0000000..6218428 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-too-many-running-queries.md.hbs @@ -0,0 +1,9 @@ +Please analyze your workload. +Each concurrent SELECT statements use memory in JOINs use CPU for running aggregation function and can read lot of data from disk when scan parts in partitions and utilize disk I/O. +Each concurrent INSERT statements, allocate around 1MB per each column in an inserted table and utilize disk I/O. + +Look at following documentation parts: +- https://clickhouse.com/docs/en/operations/settings/query-complexity/ +- https://clickhouse.com/docs/en/operations/quotas/ +- https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#max-concurrent-queries +- https://clickhouse.com/docs/en/operations/system-tables/query_log/ \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs new file mode 100644 index 0000000..08dc982 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs @@ -0,0 +1,2 @@ +ClickHouse has unexpected Network errors and similar with communication with Zookeeper. +ClickHouse should reinitialize ZooKeeper session in case of these errors. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-zookeeper-session.md.hbs b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-zookeeper-session.md.hbs new file mode 100644 index 0000000..3d7c883 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/clickhouse/clickhouse-zookeeper-session.md.hbs @@ -0,0 +1,3 @@ +Number of sessions (connections) from ClickHouse to ZooKeeper shall be no more than one, +because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) +that ZooKeeper consistency model allows. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/correlate/latency.md.hbs b/stackpacks/suse-observability-2/includes/services/correlate/latency.md.hbs new file mode 100644 index 0000000..18c2215 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/correlate/latency.md.hbs @@ -0,0 +1,6 @@ +The Correlator pod consumes observations made from our agent and transforms them into topology elements and telemetry. +When the latency of the transformation process is high we might have components and/or relations missing in the topology, and metrics +not present or arriving late. + +### Investigate +There are a few potential causes of the high latency. To identify the root cause, please contact SUSE Observability [support](https://www.stackstate.com/company/contact-us/). \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/hbase/slow-gets.md.hbs b/stackpacks/suse-observability-2/includes/services/hbase/slow-gets.md.hbs new file mode 100644 index 0000000..0373818 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/hbase/slow-gets.md.hbs @@ -0,0 +1,6 @@ +HBase is slow to response to get requests. To proceed take the following steps: + +### Remediation +1. Check the resource usage (CPU/Memory) of the pod slow to repsond to get requests. A common issue is memory pressure +2. Look in the logs for signs of memory pressure +3. If this does not resolve it, contact SUSE Observability support. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/health-sync/latency.md.hbs b/stackpacks/suse-observability-2/includes/services/health-sync/latency.md.hbs new file mode 100644 index 0000000..df11f62 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/health-sync/latency.md.hbs @@ -0,0 +1,51 @@ +The Health Synchronization has high latency. + +The Health Synchronization consumes all results emmited by the monitors, persists them and links them to the corresponding topology elements. +When the latency of the process is high we might have components missing health states or with health states not shown timely in the components. + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +1. Add more resources to the Health Synchronization. + +### Add more resources to the Health Synchronization. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + healthSync: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/kafka-to-es/latency.md.hbs b/stackpacks/suse-observability-2/includes/services/kafka-to-es/latency.md.hbs new file mode 100644 index 0000000..87609d0 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/kafka-to-es/latency.md.hbs @@ -0,0 +1,51 @@ +Data ingestion for `{{ labels.data_type }}` has high latency. + +The Kafka-to-es pod consumes `{{ labels.data_type }}`(s) sent by an agent and persists them, so we can visualize them on topology elements. +When the latency of the process is high, components might miss events (they will show up later). + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +1. Add more resources to the Kafka-to-es pod. + +### Add more resources to the Kafka-to-es pod. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + e2es: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` diff --git a/stackpacks/suse-observability-2/includes/services/metrics-store-clients/metric-store-unreachable.md.hbs b/stackpacks/suse-observability-2/includes/services/metrics-store-clients/metric-store-unreachable.md.hbs new file mode 100644 index 0000000..e5530b2 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/metrics-store-clients/metric-store-unreachable.md.hbs @@ -0,0 +1 @@ +An unreachable metric store can be caused by either networking issues but more likely the metric store, Victoria Metrics, is down or restarting. If this happens outside of an upgrade continue troubleshooting on the Victoria Metrics pod(s), look for monitors in critical or deviating state or for errors in the log. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-export-failure.md.hbs b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-export-failure.md.hbs new file mode 100644 index 0000000..e464b31 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-export-failure.md.hbs @@ -0,0 +1,17 @@ +`stackstate_otelcol_exporter_send_failed_metric_points` metric in your OpenTelemetry collector indicates that metric points +are failing to be sent to the receiving endpoint (e.g., Prometheus server, Victoria Metrics). + +Potential Reasons: +- Network Issues: + - **Connectivity**: Verify the collector's ability to connect to the receiving endpoint. Use tools like ping or network diagnostics to confirm connectivity. + - **Firewall Rules**: Ensure firewall rules don't block communication between the collector and the endpoint. Adjust rules as necessary. + - **DNS Resolution**: Check if the collector can resolve the hostname of the endpoint correctly. +- Receiving Endpoint Overload: + - **Monitor Resources**: Track resource usage (CPU, memory) of the receiving endpoint (open Metrics of the receiver component(s)). If overloaded, consider scaling it up. + - **Endpoint Configuration**: Ensure the endpoint is configured to accept metrics from the collector correctly. Refer to the endpoint's documentation for specific configuration details. +- Collector Configuration: + - **Exporter Configuration**: Double-check your exporter configuration for typos, incorrect endpoint URLs, or invalid authentication credentials. + - **Batch Size and Interval**: Adjust batch size and sending interval settings in your exporter configuration to manage data flow better. Smaller batches and shorter intervals might help with overloaded endpoints. + - **Queue Management**: Monitor metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is often full, consider increasing its capacity or adjusting batch sizes/intervals. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-processor-drops.md.hbs b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-processor-drops.md.hbs new file mode 100644 index 0000000..12e7a34 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-processor-drops.md.hbs @@ -0,0 +1,18 @@ +`stackstate_otelcol_processor_dropped_metric_points` metric in your OpenTelemetry collector indicates that metric points +are being dropped by processors within your OpenTelemetry collector before they can be exported. + +Potential Reasons: +- Processor Configuration: + - **Incorrect configuration**: Double-check your processor configuration for errors like typos, invalid parameters, or incorrect filtering rules. + - **Disabled processor**: Ensure the relevant processor is enabled and not accidentally commented out. + - **Memory limits**: Some processors have memory limits for storing processed data. If exceeded, metric points might be dropped. +- Resource Limitations: + - **Processor overload**: If the processor is overloaded with processing tasks, it might drop metric points to avoid resource exhaustion. Monitor CPU and memory usage of the collector by checking `Metrics` tab. + - **Queue overflows**: Check related metrics like `stackstate_otelcol_processor_queue_size` and `stackstate_otelcol_processor_queue_capacity`. If the queue is constantly full, metric points might be dropped due to lack of space. +- Metric Point Data Issues: + - **Invalid format**: Ensure the format of your metric points (e.g., Prometheus format) adheres to the processor's requirements. + - **Data type mismatches**: Check if the data types of your metric points (e.g., gauge, counter) match what the processor expects. +- Other Potential Causes: + - **Bugs in the processor**: While less likely, consider checking for known issues or bugs in the specific processor you're using. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-refused-receiver.md.hbs b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-refused-receiver.md.hbs new file mode 100644 index 0000000..bc00015 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-metric-refused-receiver.md.hbs @@ -0,0 +1,19 @@ +`stackstate_otelcol_receiver_refused_metric_points` metric in your OpenTelemetry collector indicates that metric points are being rejected by the +collector. This could be happening for several reasons: + +Common Reasons: +- Receiver configuration: + - **Disabled receiver**: Ensure the receiver you're trying to send metrics to (e.g., Prometheus exporter) is actually enabled and configured correctly in your collector config. + - **Invalid configuration**: Double-check your receiver configuration for typos, incorrect values, or any other errors that might prevent it from accepting metrics. + - **Metric name conflicts**: If you're using the Prometheus exporter, verify that metric names don't clash with existing metrics collected by Prometheus itself. +- Metric data format: + - **Invalid or unsupported format**: Prometheus expects metrics in a specific format. Ensure your metric points adhere to this format (e.g., timestamp, name, value, labels). + - **Data type mismatches**: Check if the data types of your metric values (e.g., gauge, counter) match what the receiver expects. +- Resource limitations: + - **Processor or memory overload**: If the collector is overloaded, it might refuse metrics to avoid further resource exhaustion. Monitor your collector's resource usage by checking `Metrics` tab and consider scaling if necessary + - **Queue overflows**: Check related metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is consistently full, you might need to adjust batch sizes, sending intervals, or increase queue capacity. +- Network issues: + - **Connectivity problems**: Ensure the collector can establish and maintain a connection to the receiving endpoint (e.g., backend). Connectivity issues might prevent metrics delivery. + - **Firewall restrictions**: Check for any firewall rules that might block communication between the collector and the receiving endpoint. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-export-failure.md.hbs b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-export-failure.md.hbs new file mode 100644 index 0000000..884f881 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-export-failure.md.hbs @@ -0,0 +1,17 @@ +`stackstate_otelcol_exporter_send_failed_spans` metric in your OpenTelemetry collector indicates that spans are failing +to be sent to the receiving backend. + +Potential Reasons: +- Network Issues: + - **Connectivity**: Verify that the collector can connect to the receiving backend/database. You can use tools like ping or network diagnostics to check connectivity. + - **Firewall Rules**: Ensure firewall rules do not block communication between the collector and the backend. Adjust rules as needed. + - **DNS Resolution**: Check if the collector can resolve the hostname of the backend correctly. +- Backend Overload: + - **Monitor Backend Resources**: Check the resource usage (CPU, memory) of the receiving backend (open Metrics of the receiver component(s)). If overloaded, consider scaling it up. + - **Backend Configuration**: Ensure the backend is configured to accept spans from the collector correctly. Refer to the backend's documentation for specific configuration details. +- Collector Configuration: + - **Exporter Configuration**: Double-check your exporter configuration for typos, incorrect endpoint URLs, or invalid authentication credentials. + - **Batch Size and Interval**: Adjust batch size and sending interval settings in your exporter configuration to manage data flow better. Smaller batches and shorter intervals might help with overloaded backends. + - **Queue Management**: Monitor metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is often full, consider increasing its capacity or adjusting batch sizes/intervals. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-processor-drops.md.hbs b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-processor-drops.md.hbs new file mode 100644 index 0000000..86cac12 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-processor-drops.md.hbs @@ -0,0 +1,18 @@ +`stackstate_otelcol_processor_dropped_spans` metric in your OpenTelemetry collector indicates that spans are being dropped +by processors within your OpenTelemetry collector before they can be exported. + +Potential Reasons: +- Processor Configuration: + - **Incorrect configuration**: Double-check your processor configuration for errors like typos, invalid parameters, or incorrect sampling rules. + - **Disabled processor**: Ensure the relevant processor is enabled and not accidentally commented out. + - **Memory limits**: Some processors have memory limits for storing processed data. If exceeded, spans might be dropped. +- Resource Limitations: + - **Processor overload**: If the processor is overloaded with processing tasks, it might drop spans to avoid resource exhaustion. Monitor CPU and memory usage of the collector by checking `Metrics` tab. + - **Queue overflows**: Check related metrics like `stackstate_otelcol_processor_queue_size` and `stackstate_otelcol_processor_queue_capacity`. If the queue is constantly full, spans might be dropped due to lack of space. +- Span Data Issues: + - **Invalid format**: Ensure the format of your spans (e.g., Protobuf) adheres to the processor's requirements. + - **Size limits**: Some processors have size limits for individual spans or batches. If your spans exceed those limits, they might be dropped. +- Other Potential Causes: + - **Bugs in the processor**: While less likely, consider checking for known issues or bugs in the specific processor you're using. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-refused-receiver.md.hbs b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-refused-receiver.md.hbs new file mode 100644 index 0000000..b4e3ba9 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/otel-collector/otel-collector-span-refused-receiver.md.hbs @@ -0,0 +1,18 @@ +`stackstate_otelcol_receiver_refused_spans` metric in your OpenTelemetry collector indicates that spans are being rejected by the +collector. This could be happening for several reasons: + +Common Reasons: +- Receiver configuration: + - **Disabled receiver**: Ensure the receiver you're trying to send spans to (e.g., OTLP) is actually enabled and configured correctly in your collector config. + - **Invalid configuration**: Double-check your receiver configuration for typos, incorrect values, or any other errors that might prevent it from accepting spans. +- Span data format: + - **Invalid or unsupported format**: Verify that the format of your spans (e.g., Protobuf) is valid and supported by the chosen receiver. + - **Size limits**: Some receivers might have size limits for individual spans or batches. Ensure your spans are within those limits. +- Resource limitations: + - **Processor or memory overload**: If the collector is overloaded, it might refuse spans to avoid further resource exhaustion. Monitor your collector's resource usage by checking `Metrics` tab and consider scaling if necessary. + - **Queue overflows**: Check related metrics like `stackstate_otelcol_exporter_queue_size` and `stackstate_otelcol_exporter_queue_capacity`. If the queue is consistently full, you might need to adjust batch sizes, sending intervals, or increase queue capacity. +- Network issues: + - **Connectivity problems**: Ensure the collector can establish and maintain a connection to the receiving endpoint (e.g., backend). Connectivity issues might prevent span delivery. + - **Firewall restrictions**: Check for any firewall rules that might block communication between the collector and the receiving endpoint. + +You may find more details by checking [logs](/#/components/{{ componentUrnForUrl }}#logs). \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/receiver/budget-saturation.md.hbs b/stackpacks/suse-observability-2/includes/services/receiver/budget-saturation.md.hbs new file mode 100644 index 0000000..d3ba362 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/receiver/budget-saturation.md.hbs @@ -0,0 +1,35 @@ +Element budget **nearly saturated for `{{ labels.element_type }}`**, has high unique element budget saturation level. When the budget is saturated data starts being dropped which translates in missing topology or metrics. + +### Increase the budget of elements allowed +1. We can control the budget by setting extra environment variables for the receiver pod. +Here are the default settings: +``` + receiver: + extraEnv: + open: + CONFIG_FORCE_stackstate_receiver_processAgent_maxAgents: "2000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxComponents: "5000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxConnections: "150000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxRelations: "15000" +``` + +2. Identify which budget got saturated and increment the right environment variable to allow more elements to be passed: + +| element type | environment variable | +|--------------|--------------------------------------------------------------| +| agent | CONFIG_FORCE_stackstate_receiver_processAgent_maxAgents | +| component | CONFIG_FORCE_stackstate_receiver_processAgent_maxComponents | +| connection | CONFIG_FORCE_stackstate_receiver_processAgent_maxConnections | +| relation | CONFIG_FORCE_stackstate_receiver_processAgent_maxRelations | + +Keep in mind that allowing more elements to be processed will have an impact on downstream services such as the Correlator and the Topology Sync. + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/receiver/hourly-budget-saturation.md.hbs b/stackpacks/suse-observability-2/includes/services/receiver/hourly-budget-saturation.md.hbs new file mode 100644 index 0000000..d38f1a4 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/receiver/hourly-budget-saturation.md.hbs @@ -0,0 +1,31 @@ +Element budget **nearly saturated for `{{ labels.element_type }}`**, has high element hourly budget saturation level. When the budget is saturated data starts being dropped which translates in missing topology or metrics. + +### Increase the budget of elements allowed +1. We can control the budget by setting extra environment variables for the receiver pod. +Here are the default settings: +``` + receiver: + extraEnv: + open: + CONFIG_FORCE_stackstate_receiver_processAgent_maxNewComponentsHourly: "2000" + CONFIG_FORCE_stackstate_receiver_processAgent_maxNewRelationsHourly: "5000" +``` + +2. Identify which budget got saturated and increment the right environment variable to allow more elements to be passed: + +| element type | environment variable | +|--------------|--------------------------------------------------------------| +| component | CONFIG_FORCE_stackstate_receiver_processAgent_maxNewComponentsHourly | +| relation | CONFIG_FORCE_stackstate_receiver_processAgent_maxNewRelationsHourly | + +Keep in mind that allowing more elements to be processed will have an impact on downstream services such as the Correlator and the Topology Sync. + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/receiver/http-growing-requests.md.hbs b/stackpacks/suse-observability-2/includes/services/receiver/http-growing-requests.md.hbs new file mode 100644 index 0000000..20ce4f5 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/receiver/http-growing-requests.md.hbs @@ -0,0 +1,11 @@ +The # of active requests for SUSE Observability receiver has only been growing for the last 5 minutes. +This either means that there is a huge and sudden increase in load but much more likely it means that the receiver is “stuck” and not able to write data to Kafka, eventually the requests will all timeout and will be retried later. The end result will be that data doesn't receive any more data and metrics, health states etc. go stale, components disappear. + +### Troubleshoot steps +1. Go to the [metrics](/#/components/{{ componentUrnForUrl }}/metrics +2. Verify the `Active HTTP Requests` chart, normal behavior is that it fluctuates but is pretty much at a constant level even quite low at about 0 to 5 active requests. If this alert triggers it is expected to be a continuously increasing value. +3. After confirming the increasing value restart the receiver pod using +``` +kubectl delete pod {{ labels.pod_name }} -n {{ labels.namespace }} +``` +4. After restart check the same charts again for the next 10 minutes to make sure issue is fixed. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/receiver/memory-limiter.md.hbs b/stackpacks/suse-observability-2/includes/services/receiver/memory-limiter.md.hbs new file mode 100644 index 0000000..e3bf7e1 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/receiver/memory-limiter.md.hbs @@ -0,0 +1,37 @@ +SUSE Observability requests are rejected is the memory budget is exhausted and that can cause data loss. + +### Increase the memory +1. Check if this monitor triggers for a long time, in that case add more memory to the pod +2. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of memory. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` +2. Update your helm `values.yaml` to add more memory. +``` + stackstate: + components: + receiver: + resources: + limits: + memory: "4500Mi" + requests: + memory: "4500Mi" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/receiver/rejected-logs.md.hbs b/stackpacks/suse-observability-2/includes/services/receiver/rejected-logs.md.hbs new file mode 100644 index 0000000..801e7ab --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/receiver/rejected-logs.md.hbs @@ -0,0 +1,50 @@ +SUSE Observability logs are rejected when one or more log sources (for example pods) are generating more logs than their budget based on the log retention and disk space available. Solutions: + +### Troubleshoot steps +- The cause can be an increase in log volume for 1 or more sources, for example because debug logging was enabled for an application. The solution in that case is to reduce the log volume for these pods. +- If the log volume is legitimate and not expected to change either the available disk space can be increased or the retention can be decreased. + +### Increase Disk Space for ElasticSearch +1. Verify the current ElasticSearch disk space and inspect its configuration to check the current size. + * Verify that the used storage class can be resized. For that please follow our [resizing documentation] https://docs.stackstate.com/self-hosted-setup/data-management/data_retention#resizing-storage +2. Update the values.yaml adding more +``` +elasticsearch: + volumeClaimTemplate: + resources: + requests: + storage: 250Gi +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + +For more detailed information visit our [data-retention](https://docs.stackstate.com/self-hosted-setup/data-management/data_retention) documentation. + +### Decrease retention for logs +1. By default, SUSE Observability sets a 7-day retention for logs +2. Update the values.yaml with a lower retention +``` +stackstate: + components: + receiver: + # Number of days to keep the logs data on Es + retention: 7 +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/topo-sync/blocked.md.hbs b/stackpacks/suse-observability-2/includes/services/topo-sync/blocked.md.hbs new file mode 100644 index 0000000..8675478 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/topo-sync/blocked.md.hbs @@ -0,0 +1,10 @@ +The Topology Synchronization pod uses several workers and one of them if blocked and can't resume work. + +### Remediation +1. Restart the pod so the synchronization of topology data resumes. +``` +kubectl delete pod {{ labels.pod_name }} -n {{ labels.namespace }} +``` +`Restarting the pod can help by either unblocking the issue or making it more evident in the logs` + +2. Inspect the [Logs](/#/components/{{ componentUrnForUrl }}#logs) of the pod in order to find the root cause of the blockage. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/topo-sync/internal-latency.md.hbs b/stackpacks/suse-observability-2/includes/services/topo-sync/internal-latency.md.hbs new file mode 100644 index 0000000..bdbaff8 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/topo-sync/internal-latency.md.hbs @@ -0,0 +1,56 @@ +The Topology Synchronization for `{{ labels.integration_type }}` has high latency. + +The Topology Synchronization pod consumes data sent by an agent and transforms them into topology elements. +When the latency of the transformation process is high we might have components and/or relations missing in the topology. + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +* Add more resources to the Topology Synchronization. +* Reducing the frequency of topology updates for {{ labels.integration_type }} + +### Add more resources to the Topology Synchronization. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. More CPU's translate into more +parallel work achieved by the pod. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + sync: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + +### Reduce the frequency of the topology updates. +Update the settings of the agent collection of data for integration: `{{ labels.integration_type }}`, +Reducing the frequency of updates will remove pressure on the Topology Synchronization but as well means that the topology data will be less up-to-date as we don't refresh it that often. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/topo-sync/latency.md.hbs b/stackpacks/suse-observability-2/includes/services/topo-sync/latency.md.hbs new file mode 100644 index 0000000..bdbaff8 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/topo-sync/latency.md.hbs @@ -0,0 +1,56 @@ +The Topology Synchronization for `{{ labels.integration_type }}` has high latency. + +The Topology Synchronization pod consumes data sent by an agent and transforms them into topology elements. +When the latency of the transformation process is high we might have components and/or relations missing in the topology. + +### Investigate +Evaluate with the monitor metric chart if the deviation was caused by a burst of incoming data or if the load has consistently increased. If the high latency +is present regularly you can consider: +* Add more resources to the Topology Synchronization. +* Reducing the frequency of topology updates for {{ labels.integration_type }} + +### Add more resources to the Topology Synchronization. +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of requested CPU and memory. More CPU's translate into more +parallel work achieved by the pod. +``` +resources: + limits: + cpu: 1000m + ephemeral-storage: 1Mi + memory: 3Gi + requests: + cpu: 750m + ephemeral-storage: 1Mi + memory: 2Gi +``` + +* Verify with the memory, JVM heap, JVM Gc metric charts if the pod needs more memory. + + +2. Update your helm `values.yaml` to add more cpu cores. +``` +stackstate: + components: + sync: + resources: + limits: + memory: "4500Mi" + cpu: "3000m" + requests: + memory: "4500Mi" + cpu: "2000m" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + +### Reduce the frequency of the topology updates. +Update the settings of the agent collection of data for integration: `{{ labels.integration_type }}`, +Reducing the frequency of updates will remove pressure on the Topology Synchronization but as well means that the topology data will be less up-to-date as we don't refresh it that often. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/topo-sync/multiple-hosts.md.hbs b/stackpacks/suse-observability-2/includes/services/topo-sync/multiple-hosts.md.hbs new file mode 100644 index 0000000..96f264d --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/topo-sync/multiple-hosts.md.hbs @@ -0,0 +1,6 @@ +The Topology Synchronization gets data from the agent. Snapshots of data should come from a single agent. If multiple agents +send snapshot data to the same synchronization, this is regarded a violation, because there are multiple sources of truth. + +### Remediation +1. Investigate the error in details through the stackstate cli (`sts-cli topology-sync`) +2. Remediate by disabling the agent sending duplicate data. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/topo-sync/snapshot-already-open.md.hbs b/stackpacks/suse-observability-2/includes/services/topo-sync/snapshot-already-open.md.hbs new file mode 100644 index 0000000..99bb290 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/topo-sync/snapshot-already-open.md.hbs @@ -0,0 +1,6 @@ +The Topology Synchronization snapshot model requires that snapshots are properly closed in order to perform the cleanup of obsolete topology items. +Observing subsequent starts of a snapshot is regarded as an error as it will lead to inconsistent topology. + +### Remediation +1. Investigate the error in details through the stackstate cli (`sts-cli topology-sync`) +2. Remediate by inspecting the agent or integration and finding the cause of the truncated snapshots. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/victoria-metrics/errors.md.hbs b/stackpacks/suse-observability-2/includes/services/victoria-metrics/errors.md.hbs new file mode 100644 index 0000000..404ae1f --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/victoria-metrics/errors.md.hbs @@ -0,0 +1 @@ +Requests to path victoria metrics are receiving errors. Please verify if clients are sending correct requests by inspecting the [logs](/#/components/{{ componentUrnForUrl }}#logs). diff --git a/stackpacks/suse-observability-2/includes/services/victoria-metrics/labels-limit.md.hbs b/stackpacks/suse-observability-2/includes/services/victoria-metrics/labels-limit.md.hbs new file mode 100644 index 0000000..ba0936a --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/victoria-metrics/labels-limit.md.hbs @@ -0,0 +1,38 @@ +VictoriaMetrics limits the number of labels per each metric with '-maxLabelsPerTimeseries' command-line flag.\n +This prevents from ingesting metrics with too many labels. Please verify that '-maxLabelsPerTimeseries' is configured +correctly or that clients which send these metrics aren't misbehaving. + +### Troubleshooting steps +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the `-maxLabelsPerTimeseries` argument. +``` +spec: + automountServiceAccountToken: true + containers: + - args: + - --dedup.minScrapeInterval=1ms + - --envflag.enable=true + - --envflag.prefix=VM_ + - --loggerFormat=json + - --maxLabelsPerTimeseries=35 + - --retentionPeriod=15d + - --storageDataPath=/storage +``` + +Please verify that '-maxLabelsPerTimeseries' is configured correctly or that clients which send these metrics aren't misbehaving. + +2. Update your helm `values.yaml` to allow more labels just in case you really need them and can't remove any. +``` + victoria-metrics-0: + extraArgs: + maxLabelsPerTimeseries: 35 +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/victoria-metrics/rejected.md.hbs b/stackpacks/suse-observability-2/includes/services/victoria-metrics/rejected.md.hbs new file mode 100644 index 0000000..b3c8ef1 --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/victoria-metrics/rejected.md.hbs @@ -0,0 +1 @@ +VM is rejecting to ingest rows due to the following reason: {{ labels.reason }} diff --git a/stackpacks/suse-observability-2/includes/services/victoria-metrics/slow-insert-rate.md.hbs b/stackpacks/suse-observability-2/includes/services/victoria-metrics/slow-insert-rate.md.hbs new file mode 100644 index 0000000..95d037d --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/victoria-metrics/slow-insert-rate.md.hbs @@ -0,0 +1,34 @@ +High rate of slow inserts may be a sign of resource exhaustion for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series. + +### Troubleshooting steps +1. Inspect the [Configuration](/#/components/{{ componentUrnForUrl }}#configuration) of this pod and look for the amount of memory. +``` +resources: + limits: + cpu: 1000m + memory: 3Gi + requests: + cpu: 750m + memory: 2Gi +``` +Please verify that `-maxLabelsPerTimeseries` is configured correctly or that clients which send these metrics aren't misbehaving. + +2. Update your helm `values.yaml` to add more memory +``` + victoria-metrics-0: + resources: + limits: + memory: "4500Mi" + requests: + memory: "4500Mi" +``` + +3. Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` \ No newline at end of file diff --git a/stackpacks/suse-observability-2/includes/services/vmagent/persistentqueue-dropping-data.md.hbs b/stackpacks/suse-observability-2/includes/services/vmagent/persistentqueue-dropping-data.md.hbs new file mode 100644 index 0000000..eb6356a --- /dev/null +++ b/stackpacks/suse-observability-2/includes/services/vmagent/persistentqueue-dropping-data.md.hbs @@ -0,0 +1,29 @@ +Vmagent dropped data from persistent queue. Vmagent receives data and tries to push it to victoria metrics, the persistent queue helps as a buffer while writting to victoria metrics. + +### Troubleshooting steps +1. Inspect the [Logs](/#/components/{{ componentUrnForUrl }}#logs) to verify if there's in issue while writing to victoria metrics. + 1.1 Continue the troubleshooting process if you find errors in the logs by inspecting victoria metrics telemetry. +2. If there are no issues regarding writing the problem could be related to a producer faster than what we can write to victoria metrics. + 2a. Consider giving Victoria metrics more resources in the values.yaml. You can check the current settings on the victoria metrics pod's configuration. + ``` + victoria-metrics-0: + resources: + requests: + cpu: 300m + memory: 3584Mi + limits: + memory: 4Gi + cpu: 1 + ``` + + Redeploy via a `helm upgrade` including any other values files or extra settings you applied previously +``` +helm upgrade \ +--install \ +--namespace suse-observability \ +--values values.yaml \ +suse-observability \ +suse-observability/suse-observability +``` + + 2b. Consider to limit the ingestion rate. \ No newline at end of file diff --git a/stackpacks/suse-observability-2/resources/RELEASE.md b/stackpacks/suse-observability-2/resources/RELEASE.md new file mode 100644 index 0000000..73bd0eb --- /dev/null +++ b/stackpacks/suse-observability-2/resources/RELEASE.md @@ -0,0 +1,6 @@ +# SUSE Observability v2 + +## 0.0.1 + +**Features** +First pre-release version \ No newline at end of file diff --git a/stackpacks/suse-observability-2/resources/configuration.md b/stackpacks/suse-observability-2/resources/configuration.md new file mode 100644 index 0000000..0bf48e1 --- /dev/null +++ b/stackpacks/suse-observability-2/resources/configuration.md @@ -0,0 +1,3 @@ +## Installation + +You can click on `Install` for installing the SUSE Observability StackPack. The StackPack only supports the Kubernetes installation of SUSE Observability for adding metrics and monitors. Metrics and monitors will be automatically added to the SUSE Observability related Kubernetes components. diff --git a/stackpacks/suse-observability-2/resources/deprovisioning.md b/stackpacks/suse-observability-2/resources/deprovisioning.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/suse-observability-2/resources/detailed-overview.md b/stackpacks/suse-observability-2/resources/detailed-overview.md new file mode 100644 index 0000000..52c405a --- /dev/null +++ b/stackpacks/suse-observability-2/resources/detailed-overview.md @@ -0,0 +1,4 @@ +## Prerequisites + +* The [Kubernetes StackPack](#/stackpacks/kubernetes-v2) is installed for the cluster where SUSE Observability is installed +* The suse-observability-agent must be installed on the Observability platform cluster. Check the [matching instructions](#/stackpacks/kubernetes-v2) for your cluster. diff --git a/stackpacks/suse-observability-2/resources/enabled.md b/stackpacks/suse-observability-2/resources/enabled.md new file mode 100644 index 0000000..6279a33 --- /dev/null +++ b/stackpacks/suse-observability-2/resources/enabled.md @@ -0,0 +1,11 @@ +## The SUSE Observability StackPack is installed + +Congratulations! The SUSE Observability StackPack is configured correctly. + +### What's next + +The topology that is created by the Kubernetes StackPack will be enriched with specific SUSE Observability monitors and metric bindings. See the SUSE Observability view that has been created to discover SUSE Observability's own topology. + +### No data + +Make sure the suse-observability-agent is installed on the Observability platform cluster. Check the [matching instructions](#/stackpacks/kubernetes-v2) for your cluster in the Kubernetes stackpack. diff --git a/stackpacks/suse-observability-2/resources/error.md b/stackpacks/suse-observability-2/resources/error.md new file mode 100644 index 0000000..e69de29 diff --git a/stackpacks/suse-observability-2/resources/logo.png b/stackpacks/suse-observability-2/resources/logo.png new file mode 100644 index 0000000..5fa03e2 Binary files /dev/null and b/stackpacks/suse-observability-2/resources/logo.png differ diff --git a/stackpacks/suse-observability-2/resources/overview.md b/stackpacks/suse-observability-2/resources/overview.md new file mode 100644 index 0000000..c46ae3a --- /dev/null +++ b/stackpacks/suse-observability-2/resources/overview.md @@ -0,0 +1,3 @@ +## SUSE Observability StackPack + +This StackPack adds SUSE Observability self-monitoring and other SUSE Observability instances to the standard Kubernetes StackPack via monitors and metric bindings. diff --git a/stackpacks/suse-observability-2/resources/provisioning.md b/stackpacks/suse-observability-2/resources/provisioning.md new file mode 100644 index 0000000..55f8623 --- /dev/null +++ b/stackpacks/suse-observability-2/resources/provisioning.md @@ -0,0 +1,3 @@ +## Provisioning... + +Provisioning SUSE Observability monitoring. Please wait a moment. diff --git a/stackpacks/suse-observability-2/resources/waitingfordata.md b/stackpacks/suse-observability-2/resources/waitingfordata.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/stackpacks/suse-observability-2/resources/waitingfordata.md @@ -0,0 +1 @@ + diff --git a/stackpacks/suse-observability-2/settings/dashboards/data-ingestion.sty b/stackpacks/suse-observability-2/settings/dashboards/data-ingestion.sty new file mode 100644 index 0000000..d2e9d47 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/dashboards/data-ingestion.sty @@ -0,0 +1,323 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability-2:shared:dashboard:data-ingestion + name: SUSE Observability - Data ingestion + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + YHvrlowP7KLrAuLWFrWBd: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: binBps + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (local_pod_name)(rate(pod_received_bytes_count{local_pod_name=~".*-(receiver|collector)-.*", + cluster_name='${cluster}', direction="incoming", + intra_pod!='true', + local_pod_ns='${namespace}'}[${__rate_interval}])) + alias: ${local_pod_name} + display: + name: Ingested data / second + description: Volume of ingested data per pod. The 2 main entry points for data + in the platform are the receiver pod(s) and the OTel collector + pod(s) that are part of the platform. + KP9lrGvp3jTeJbIDkdzP8: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: binBps + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(rate(pod_received_bytes_count{local_pod_name=~".*-(receiver|collector)-.*", + cluster_name='${cluster}', direction="incoming", + intra_pod!='true', + local_pod_ns='${namespace}'}[${__rate_interval}])) + alias: Total Data Volume + display: + name: Total data volume + description: The total amount of data received on the SUSE Observability API, + this includes topology, metrics, logs, traces, events, and health. + qpExljGT-9X2iu50hKfx_: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type='agent'}[${__interval}])) + alias: node + display: + name: Incoming data + description: The number of standard nodes (4 vCPU and 16GiB of memory) from + which data is received. The real nodes can be a different size, so + the real node count can be significantly lower. + P_IBNZlmdKtK-uXWJWE8z: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base"}) / sum by + (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) / sum by + (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the data ingestion pipeline. + High throttling can lead to delays in data processing, a profile + upgrade or CPU limit increase can resolve the problem. + IFl4huEAwoMRTyV0l5IeP: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + label: something + range: + min: + max: + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base"}[${__interval}])) + / 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="opentelemetry-collector"}[${__interval}])) / + 1000000000 + alias: ${kube_app_name} + display: + name: CPU Usage Processing + description: "For the receiver and opentelemetry collector: the 2 APIs that + accept telemetry data." + L9E1jtO844cup6_P4Po1g: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*(kafka-|hbase-|victoria-metrics|clickhouse|vmagent|zookeeper|sync).*"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*(kafka-|hbase-|victoria-metrics|clickhouse|vmagent|zookeeper|sync).*"}[5m]), + "storage","$1", "persistentvolumeclaim", + ".*-(kafka-.*|hbase-.*|victoria-metrics-.*|clickhouse-.*|vmagent-.*|zookeeper-.*|sync-tmp)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by data ingestion components." + layouts: + - kind: Grid + spec: + items: + - x: 6 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/YHvrlowP7KLrAuLWFrWBd" + - x: 3 + y: 0 + width: 3 + height: 2 + content: + $ref: "#/spec/panels/KP9lrGvp3jTeJbIDkdzP8" + - x: 0 + y: 0 + width: 3 + height: 2 + content: + $ref: "#/spec/panels/qpExljGT-9X2iu50hKfx_" + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/P_IBNZlmdKtK-uXWJWE8z" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/IFl4huEAwoMRTyV0l5IeP" + - x: 0 + y: 4 + width: 24 + height: 2 + content: + $ref: "#/spec/panels/L9E1jtO844cup6_P4Po1g" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability-2/settings/dashboards/logs-processing.sty b/stackpacks/suse-observability-2/settings/dashboards/logs-processing.sty new file mode 100644 index 0000000..d93ae09 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/dashboards/logs-processing.sty @@ -0,0 +1,318 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability-2:shared:dashboard:logs-processing + name: SUSE Observability - Logs processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + HxCR4e7TCojuRrKLl9T-y: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + format: + unit: bytes/sec(IEC) + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: rate(stackstate_stackstate_receiver_events_accepted_size{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs", + element_type="k8sLogs"}[${__rate_interval}]) + alias: Accepted + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: rate(stackstate_stackstate_receiver_events_rejected_size{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs", + element_type="k8sLogs"}[${__rate_interval}]) + alias: Rejected + display: + name: SUSE Observability Receiver - Log volume + description: "Rate of log data accepted and rejected by the receiver." + HILv2hOoS5lHs6NjEv72p: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: bytes/sec(IEC) + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(stackstate_stackstate_receiver_logs_budget_per_pod{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}) + alias: Logs budget per pod per time window + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(10, + rate(stackstate_stackstate_receiver_logs_rejected_size{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}[${__rate_interval}] + @ end())) + alias: Logs from ${limited_pod_name} rejected + display: + name: SUSE Observability Receiver - Rejected Log volume per pod + description: "Log volume rejected per pod due to budget limits. The budget is determined based on available storage and the number of active log streams." + qX3IWlWiFY_lnvFHOqc8W: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}) / sum by + (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_name=~"elasticsearch"}) / + sum by (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_name=~"elasticsearch"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the logs ingestion pipeline. + mvjWBYx0ldBjryYE8MEwg: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-logs"}[${__interval}])) + / 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="elasticsearch"}[${__interval}])) / 1000000000 + alias: ${kube_app_name} + display: + name: CPU Usage Processing + description: "CPU usage of the receiver and Elasticsearch pods." + hTa7uB5XaIDsbVQluJF4d: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)"}[5m]), + "storage","$1", "persistentvolumeclaim", + "data-.*-(elasticsearch-.*)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by Elasticsearch." + mt7i8AHw8Ke4R4EJ269Rs: + spec: + plugin: + kind: Markdown + spec: + text: >- + # More details + + These are the pods and workloads involved in the logs pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-logs"%2C%20"app.kubernetes.io%2Fname%3Aelasticsearch"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Aelasticsearch"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + + * + [Deployments](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:deployments/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C"app.kubernetes.io%2Fcomponent%3Areceiver-logs"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"deployment"%29) + links: [] + queries: [] + display: + name: "-" + description: "Links to related Pods, StatefulSets, and Deployments." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/HxCR4e7TCojuRrKLl9T-y" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/HILv2hOoS5lHs6NjEv72p" + - x: 0 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/qX3IWlWiFY_lnvFHOqc8W" + - x: 8 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/mvjWBYx0ldBjryYE8MEwg" + - x: 0 + y: 0 + width: 6 + height: 2 + content: + $ref: "#/spec/panels/hTa7uB5XaIDsbVQluJF4d" + - x: 6 + y: 0 + width: 4 + height: 2 + content: + $ref: "#/spec/panels/mt7i8AHw8Ke4R4EJ269Rs" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability-2/settings/dashboards/metrics-processing.sty b/stackpacks/suse-observability-2/settings/dashboards/metrics-processing.sty new file mode 100644 index 0000000..eb7dd9c --- /dev/null +++ b/stackpacks/suse-observability-2/settings/dashboards/metrics-processing.sty @@ -0,0 +1,462 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability-2:shared:dashboard:metrics-processing + name: SUSE Observability - Metrics processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + eIFw6RjqVknqJoWc309aa: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: + - color: "#E31A1C" + value: 0.02 + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max(rate(stackstate_vm_slow_row_inserts_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}]) + / + rate(stackstate_vm_rows_added_to_storage_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}])) + alias: Percentage of slow inserts + display: + name: VictoriaMetrics - Slow inserts + description: "Percentage of inserts into VictoriaMetrics that are considered slow." + 6RpPvPOqS5SdnjYcbMZtB: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(rate(stackstate_vm_slow_queries_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}])) + alias: Slow queries rate + display: + name: VictoriaMetrics - Slow queries rate + description: "Rate of slow queries executed against VictoriaMetrics." + b2OMg07XzmxsM0C6TiFoP: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(increase(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name="suse-observability-victoria-metrics-0-0"}[24h])) + alias: Time series created over 24 hrs + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(rate(stackstate_vm_new_timeseries_created_total{cluster_name="${cluster}", + namespace="${namespace}", + pod_name="suse-observability-victoria-metrics-0-0"}[${__rate_interval}])) + alias: Time series created / second + display: + name: VictoriaMetrics - Churn rate + description: "Rate of new time series creation in VictoriaMetrics." + tSVthJSBDQ55Ag0zWATTh: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 10 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~".*-vmagent-.*", + cluster_name="${cluster}", namespace="${namespace}"}[1h])) + alias: vmagent + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (statefulset)(increase(stackstate_vm_rows_ignored_total{pod_name=~".*-victoria-metrics-.*", + cluster_name="${cluster}", namespace="${namespace}"}[1h])) + alias: VictoriaMetrics + display: + name: Dropped timeseries in the past hour + description: >- + VictoriaMetrics limits the number of labels per timeseries with + `-maxLabelsPerTimeseries` command-line flag. + + This prevents from ingesting metrics with too many labels. The value + of `maxLabelsPerTimeseries` must be adjusted for your workload. + + When limit is exceeded (graph is > 0) - the timeseries with too many + labels are dropped, which results in missing metric data. + H5gtvG8QRQgcH2_9kvYLJ: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|vmagent"}) / sum + by (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|vmagent"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector|victoria-metrics-\\d"}) + / sum by (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector|victoria-metrics-\\d"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the data ingestion pipeline. + High throttling can lead to delays in data processing, a profile + upgrade or CPU limit increase can resolve the problem. + 1LYJbBBKSUMFsD1dbNDvi: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|vmagent"}[${__interval}])) + / 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="opentelemetry-collector"}[${__interval}])) / + 1000000000 + alias: ${kube_app_name} + display: + name: CPU Usage Processing + description: "CPU usage of receiver, vmagent, and OpenTelemetry collector pods." + jGL_6sXlItqnyvKC2NU_D: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + pod_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + pod_name=~"suse-observability-(victoria-metrics).*"}[${__interval}])) + / 1000000000 + alias: ${pod_name} + display: + name: CPU Usage VictoriaMetrics + description: "CPU usage of VictoriaMetrics pods." + LENPxpypcWV-l3jYWsgPA: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m]), + "storage","$1", "persistentvolumeclaim", + "server-volume-.*-(victoria-metrics-.+)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by VictoriaMetrics." + Ao0rRQFkI3j5Va7erPJP3: + spec: + plugin: + kind: Markdown + spec: + text: >- + # More details + + These are the pods and workloads involved in the metrics pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fname%3Avictoria-metrics-0"%2C%20"app.kubernetes.io%2Fname%3Avictoria-metrics-1"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Avictoria-metrics-0"%2C%20"app.kubernetes.io%2Fname%3Avictoria-metrics-1"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + + * + [Deployments](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:deployments/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C"app.kubernetes.io%2Fcomponent%3Areceiver-base"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"deployment"%29) + links: [] + queries: [] + display: + name: "-" + description: "Links to related Pods, StatefulSets, and Deployments." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/eIFw6RjqVknqJoWc309aa" + - x: 16 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/6RpPvPOqS5SdnjYcbMZtB" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/b2OMg07XzmxsM0C6TiFoP" + - x: 0 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/tSVthJSBDQ55Ag0zWATTh" + - x: 0 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/H5gtvG8QRQgcH2_9kvYLJ" + - x: 8 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/1LYJbBBKSUMFsD1dbNDvi" + - x: 16 + y: 4 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/jGL_6sXlItqnyvKC2NU_D" + - x: 8 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/LENPxpypcWV-l3jYWsgPA" + - x: 16 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/Ao0rRQFkI3j5Va7erPJP3" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability-2/settings/dashboards/topology-processing.sty b/stackpacks/suse-observability-2/settings/dashboards/topology-processing.sty new file mode 100644 index 0000000..a199194 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/dashboards/topology-processing.sty @@ -0,0 +1,642 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability-2:shared:dashboard:topology-processing + name: SUSE Observability - Topology processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + dBMublgJcGkzpzDGSKUvq: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (integration_type) + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="enterReceiver", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${integration_type} + display: + name: Topology Sync Internal Latency + description: "Internal latency within the topology synchronization process." + zv3RIlz8UQBwAoCBrrpRC: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: s + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by (pod_name, integration_type) + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="source", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${integration_type} + display: + name: Topology Sync Latency + description: "Total latency of the topology synchronization process from source to commit." + 3Py1h-ak88rlmOZpULtQB: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: cps + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: topk(5, sum by (pod_name, topic) + (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{cluster_name="${cluster}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"})) + alias: ${topic} + display: + name: Top 5 Topology Sync Topic lag + description: "Lag in processing topology synchronization topics for the top 5 consumers." + iiJ3ngNvJpch-G_1kggvS: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (element_type)(stackstate_stackstate_sync_topo_total_counts{cluster_name="${cluster}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${element_type} + display: + name: Topology Sync Elements count + description: "Total count of topology elements synchronized." + bLeT4qQjqsZBvePCyyctd: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percentunit + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) / sum by + (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) + alias: ${element_type} + display: + name: Profile budget saturation + description: "Ratio of processed elements to the maximum allowed budget." + zf9-XhnHykeb7b_PZPALv: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) / sum by + (element_type) + (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{cluster_name="${cluster}", + namespace="${namespace}"}[${__interval}])) + alias: ${element_type} + display: + name: Profile budget saturation + description: "Ratio of processed elements to the maximum allowed budget." + hExuFO5EwYoceUpJ9gHv4: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 50 + - color: "#FF7F00" + value: 30 + mode: absolute + max: 60 + format: + unit: s + decimalPlaces: 1 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="enterReceiver", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: Latency internal + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max + (stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="source", + end_stage="commit", quantile="1.0", + cluster_name="${cluster}", namespace="${namespace}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: Latency total + display: + name: Topology Sync Latency + description: "Current internal and total latency for topology synchronization." + bIYe-LnD5LgOMa47Cou8W: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: table + values: + - max + - min + - mean + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type='agent'}[${__interval}])) + alias: node + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type!='agent'}[${__interval}])) + alias: ${element_type} + display: + name: Process agent traffic + description: "Traffic volume of unique elements passed by the process agent vs other agents." + GpHAhwMv6nirtbgom2vA5: + spec: + plugin: + kind: Markdown + spec: + text: >- + # Overview + + Topology input at a quick glance. How close to the profile limits + are you and how up-to-date is the topology? + + + These are the pods and workloads involved in the topology + pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-process-agent"%2C%20"app.kubernetes.io%2Fcomponent%3Async"%2C%20"app.kubernetes.io%2Fcomponent%3Aserver"%2C%20"app.kubernetes.io%2Fname%3Ahbase"%2C%20"app.kubernetes.io%2Fname%3Akafka"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-process-agent"%2C%20"app.kubernetes.io%2Fcomponent%3Async"%2C%20"app.kubernetes.io%2Fcomponent%3Aserver"%2C%20"app.kubernetes.io%2Fname%3Ahbase"%2C%20"app.kubernetes.io%2Fname%3Akafka"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + + * + [Deployments](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:deployments/overview?query=label%20IN%20%28"app.kubernetes.io%2Fcomponent%3Areceiver"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-base"%2C%20"app.kubernetes.io%2Fcomponent%3Areceiver-process-agent"%2C%20"app.kubernetes.io%2Fcomponent%3Async"%2C%20"app.kubernetes.io%2Fcomponent%3Aserver"%2C%20"app.kubernetes.io%2Fname%3Ahbase"%2C%20"app.kubernetes.io%2Fname%3Akafka"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"deployment"%29) + links: [] + queries: [] + display: + name: "-" + description: "Overview of topology input and links to related workloads." + 4GEucAw6zR3hmI1J-lO4H: + spec: + plugin: + kind: Markdown + spec: + text: >- + # Detailed status + + A more in-depth overview over time of the topology input in volume + and latency. + links: [] + queries: [] + display: + name: "-" + description: "Detailed status of topology input volume and latency over time." + q5ZUHAngpZY0H0SwYKSdt: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by + (element_type)(stackstate_stackstate_sync_topo_total_counts{cluster_name="${cluster}", + kube_app_component=~"sync|server", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}) + alias: ${element_type} + display: + name: Topology Sync Elements count + description: "Total count of topology elements processed." + Q_uoMywjEMcIh5Tp3bAd8: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: short + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type='agent'}[${__interval}])) + alias: node + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{cluster_name="${cluster}", + namespace="${namespace}", + element_type!='agent'}[${__interval}])) + alias: ${element_type} + display: + name: Incoming data + description: "Maximum count of unique elements received from agents." + d7kCYa0DT6HuQXfjirCoe: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: right + mode: table + values: + - mean + - last + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|kafka|sync"}) / + sum by (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"receiver|receiver-base|kafka|sync"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) / sum by + (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the services that are part of the topology + processing pipeline. + K_Av99mNhjRFhJOD-Gfud: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)"}[5m]) + / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)"}[5m]), + "storage","$1", "persistentvolumeclaim", + ".*-(kafka-.*|hbase-.*|sync-tmp)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by Kafka, HBase, and Sync." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 9 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/dBMublgJcGkzpzDGSKUvq" + - x: 8 + y: 9 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/zv3RIlz8UQBwAoCBrrpRC" + - x: 16 + y: 9 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/3Py1h-ak88rlmOZpULtQB" + - x: 8 + y: 7 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/iiJ3ngNvJpch-G_1kggvS" + - x: 16 + y: 7 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/bLeT4qQjqsZBvePCyyctd" + - x: 8 + y: 2 + width: 6 + height: 2 + content: + $ref: "#/spec/panels/zf9-XhnHykeb7b_PZPALv" + - x: 19 + y: 2 + width: 5 + height: 2 + content: + $ref: "#/spec/panels/hExuFO5EwYoceUpJ9gHv4" + - x: 0 + y: 7 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/bIYe-LnD5LgOMa47Cou8W" + - x: 0 + y: 0 + width: 24 + height: 2 + content: + $ref: "#/spec/panels/GpHAhwMv6nirtbgom2vA5" + - x: 0 + y: 6 + width: 24 + height: 1 + content: + $ref: "#/spec/panels/4GEucAw6zR3hmI1J-lO4H" + - x: 14 + y: 2 + width: 5 + height: 2 + content: + $ref: "#/spec/panels/q5ZUHAngpZY0H0SwYKSdt" + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/Q_uoMywjEMcIh5Tp3bAd8" + - x: 0 + y: 11 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/d7kCYa0DT6HuQXfjirCoe" + - x: 0 + y: 4 + width: 24 + height: 2 + content: + $ref: "#/spec/panels/K_Av99mNhjRFhJOD-Gfud" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability-2/settings/dashboards/traces-processing.sty b/stackpacks/suse-observability-2/settings/dashboards/traces-processing.sty new file mode 100644 index 0000000..619a53a --- /dev/null +++ b/stackpacks/suse-observability-2/settings/dashboards/traces-processing.sty @@ -0,0 +1,272 @@ +nodes: + - _type: Dashboard + identifier: urn:stackpack:suse-observability-2:shared:dashboard:traces-processing + name: SUSE Observability - Traces processing + description: "" + dashboard: + metadata: + project: '{"saveVariables":false}' + spec: + panels: + ExAp1uCSS2aYmub4u_Hpt: + spec: + plugin: + kind: StatChart + spec: + calculation: last + format: + unit: "" + decimalPlaces: 0 + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 5 + - color: "#FF7F00" + value: 1 + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(increase(stackstate_ClickHouseProfileEvents_DelayedInserts{cluster_name="${cluster}", + namespace="${namespace}"}[1h])) by (cluster_name, namespace) + alias: Delayed inserts (last hour) + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum(increase(stackstate_ClickHouseProfileEvents_RejectedInserts{cluster_name="${cluster}", + namespace="${namespace}"}[1h])) by (cluster_name, namespace) + alias: Rejected inserts (last hour) + display: + name: ClickHouse + description: Traces are stored in ClickHouse, if data is delayed or rejected it + means some traces are later or not at all available. + Rv6ehkvi0i3ws1Z7KA9wf: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: percent + decimalPlaces: 0 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_component) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_component=~"clickhouse"}) + / sum by (kube_app_component) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", kube_app_component=~"clickhouse"}) + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: 100 * sum by (kube_app_name) + (container_cpu_throttled_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) / sum by + (kube_app_name) + (container_cpu_elapsed_periods{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name=~"opentelemetry-collector"}) + alias: ${kube_app_name} + display: + name: CPU Throttling + description: CPU throttling for the main pods in the data ingestion pipeline. + High throttling can lead to delays in data processing, a profile + upgrade or CPU limit increase can resolve the problem. + wYSjcYqXSXLRvrKEeAgZQ: + spec: + plugin: + kind: TimeSeriesChart + spec: + yAxis: + show: true + format: + unit: short + decimalPlaces: 2 + legend: + show: true + position: bottom + mode: list + values: [] + thresholds: + mode: absolute + steps: [] + visual: + connectNulls: false + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, pod_name, + kube_app_component)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_component=~"clickhouse"}[${__interval}])) / + 1000000000 + alias: ${kube_app_component} + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: sum by (cluster_name, namespace, + kube_app_name)(max_over_time(container_cpu_usage{cluster_name="${cluster}", + namespace="${namespace}", + kube_app_name="opentelemetry-collector"}[${__interval}])) / + 1000000000 + alias: ${kube_app_name} + display: + name: CPU Usage Processing + description: "CPU usage of ClickHouse and OpenTelemetry collector pods." + 7nAaWHcI1m_yktoqpY5gH: + spec: + plugin: + kind: GaugeChart + spec: + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + max: 1 + format: + unit: percentunit + decimalPlaces: 0 + calculation: last + links: [] + queries: + - kind: TimeSeriesQuery + spec: + plugin: + kind: PrometheusTimeSeriesQuery + spec: + query: max by + (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)"}[5m]) / + max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{cluster_name="${cluster}", + namespace="${namespace}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)"}[5m]), + "storage","$1", "persistentvolumeclaim", + "data-.*-(clickhouse-.*)")) + alias: ${storage} + display: + name: Volume usage + description: "Percentage of persistent volume capacity used by ClickHouse." + HgyHHI7knLfl4MHuvyvbF: + spec: + plugin: + kind: Markdown + spec: + text: >- + # More details + + These are the pods and workloads involved in the metrics pipeline: + + + * + [Pods](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:pods/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Aclickhouse"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20%3D%20"pod") + + * + [Statefulsets](/#/views/urn:stackpack:kubernetes-v2:shared:query-view:statefulsets/overview?query=label%20IN%20%28"app.kubernetes.io%2Fname%3Aclickhouse"%2C%20"app.kubernetes.io%2Fname%3Aopentelemetry-collector"%29%20AND%20label%20%3D%20"cluster-name%3A${cluster}"%20AND%20label%20%3D%20"namespace%3A${namespace}"%20AND%20type%20IN%28"statefulset"%29) + links: [] + queries: [] + display: + name: "-" + description: "Links to related Pods and StatefulSets." + layouts: + - kind: Grid + spec: + items: + - x: 0 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/ExAp1uCSS2aYmub4u_Hpt" + - x: 0 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/Rv6ehkvi0i3ws1Z7KA9wf" + - x: 8 + y: 2 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/wYSjcYqXSXLRvrKEeAgZQ" + - x: 8 + y: 0 + width: 8 + height: 2 + content: + $ref: "#/spec/panels/7nAaWHcI1m_yktoqpY5gH" + - x: 16 + y: 0 + width: 5 + height: 2 + content: + $ref: "#/spec/panels/HgyHHI7knLfl4MHuvyvbF" + variables: + - kind: ListVariable + spec: + plugin: + kind: MetricLabelValues + spec: + labelName: cluster_name + matchers: + - stackstate_stackstate_receiver_element_create_passed_count + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: cluster + display: + name: Cluster + description: "" + - kind: ListVariable + spec: + plugin: + kind: MetricPromQL + spec: + labelName: namespace + expr: stackstate_stackstate_receiver_element_create_passed_count{cluster_name="${cluster}"} + sort: alphabetical-asc + allowMultiple: false + allowAllValue: false + name: namespace + display: + name: Namespace + description: "" diff --git a/stackpacks/suse-observability-2/settings/generic/monitors/sts-apps-overrun-cpu-request.sty b/stackpacks/suse-observability-2/settings/generic/monitors/sts-apps-overrun-cpu-request.sty new file mode 100644 index 0000000..febe74c --- /dev/null +++ b/stackpacks/suse-observability-2/settings/generic/monitors/sts-apps-overrun-cpu-request.sty @@ -0,0 +1,24 @@ +nodes: + - _type: "Monitor" + name: "Overrunning requested CPU" + tags: + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-apps-overrun-cpu-request + status: "ENABLED" + description: "Critical state when average CPU usage is consistently more than 100% higher than requested CPU." + arguments: + titleTemplate: "Overrunning requested CPU ${container}" + comparator: "GT" + failureState: "DEVIATING" + threshold: 2 + metric: + query: |- + (sum(avg_over_time(container_cpu_usage{kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) / 1000000000) / sum(avg_over_time(kubernetes_cpu_requests{kube_app_name=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) + or + (sum(avg_over_time(container_cpu_usage{kube_app_part_of=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) / 1000000000) / sum(avg_over_time(kubernetes_cpu_requests{kube_app_part_of=~"stackstate|stackstate-k8s|suse-observability"}[30m])) by (pod_name, namespace, cluster_name, container) + unit: percentunit + aliasTemplate: "CPU usage as percentage of requested" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include generic/sts-apps-overrun-cpu-request.md.hbs diff --git a/stackpacks/suse-observability-2/settings/generic/monitors/sts-java-long-gc.sty b/stackpacks/suse-observability-2/settings/generic/monitors/sts-java-long-gc.sty new file mode 100644 index 0000000..ac6e9e4 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/generic/monitors/sts-java-long-gc.sty @@ -0,0 +1,80 @@ +nodes: + - _type: "Monitor" + name: "Long GCs" + tags: + - jvm + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-java-long-gc + status: "ENABLED" + description: "Warn when the time of old-gen garbage collects gets above a certain threshold." + arguments: + comparator: "GT" + titleTemplate: "Java long GCs" + failureState: "DEVIATING" + threshold: 4 + metric: + query: |- + sum by (pod_name, namespace, cluster_name) ( + (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m]) + / + rate(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m])) + or (present_over_time(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m]) - 1)) + or + sum by (pod_name, namespace, cluster_name) ( + (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m]) + / + rate(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m]) + ) or (present_over_time(stackstate_jvm_gc_collection_seconds_count{gc="G1 Old Generation"}[5m]) - 1) + ) + aliasTemplate: "${gc}" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include generic/sts-java-long-gc.md.hbs + - _type: "Monitor" + name: "Total time spend in Old Gen GC" + tags: + - jvm + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-java-gc-time-spent + status: "ENABLED" + description: "Deviating when the amount of time spent on old generation garbagecollect exceeds a threshold." + arguments: + comparator: "GT" + titleTemplate: "Java GC time spent" + failureState: "DEVIATING" + threshold: 0.2 + metric: + query: |- + sum by (pod_name, namespace, cluster_name) (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m])) + or + sum by (pod_name, namespace, cluster_name) (rate(stackstate_jvm_gc_collection_seconds_sum{gc="G1 Old Generation"}[5m])) + aliasTemplate: "${gc}" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include generic/sts-java-long-gc.md.hbs + - _type: "Monitor" + name: "Java threads deadlocked" + tags: + - jvm + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-java-threads-deadlocked + status: "ENABLED" + description: "Critical state when there are deadlocked threads in the JVM." + arguments: + comparator: "GT" + titleTemplate: "Java threads deadlocked" + failureState: "CRITICAL" + threshold: 1.5 + metric: + query: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_deadlocked{}[5m])) + aliasTemplate: "# threads deadlocked" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: "Deadlocked threads indicate a problem within the application that needs to be resolved via a code change. A short-term, temporary, mitigation is to restart the application but the deadlock may return. Depending on the likelihood of the deadlock it may return immediately or only after a long period of time." diff --git a/stackpacks/suse-observability-2/settings/generic/presentations/deployments.sty b/stackpacks/suse-observability-2/settings/generic/presentations/deployments.sty new file mode 100644 index 0000000..cbf15cd --- /dev/null +++ b/stackpacks/suse-observability-2/settings/generic/presentations/deployments.sty @@ -0,0 +1,342 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability deployments" + description: "ComponentPresentation for SUSE Observability Deployments" + identifier: urn:stackpack:suse-observability-2:component-presentation:deployments + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/part-of:stackstate", "app.kubernetes.io/part-of:stackstate-k8s", "app.kubernetes.io/name:suse-observability", "app.kubernetes.io/part-of:suse-observability") and type in ("deployment", "statefulset") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: jvm + title: JVM + order: 100 + sections: + - + sectionId: gc + title: GC + order: 100 + metrics: + - metricId: sts-java-gc-count-deployments + - metricId: java-gc-duration-deployments + - metricId: java-gc-total-deployments + - metricId: agent-otel-jvm-gc-durations-95th-quantile-deployments + - metricId: agent-otel-jvm-gc-per-second-deployments + - metricId: agent-otel-jvm-mem-after-last-gc-deployments + - + sectionId: memory + title: Memory + order: 90 + metrics: + - metricId: sts-java-heap-memory-deployments + - metricId: sts-java-nonheap-memory-deployments + - metricId: agent-otel-jvm-heap-memory-deployments + - metricId: agent-otel-jvm-non-heap-memory-deployments + - + sectionId: threads + title: Threads + order: 80 + metrics: + - metricId: sts-java-threads-deployments + metrics: + - + metricId: java-gc-duration-deployments + description: 'Java GC duration' + name: Java GC Duration + metricQueries: + - expression: |- + max by (gc, pod_name, cluster_name, namespace) (rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]) / rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: ${gc} - ${pod_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: java-gc-total-deployments + description: 'Java total GC' + name: Java GC Total + metricQueries: + - expression: |- + max by (gc, pod_name, cluster_name, namespace) (rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: ${gc} - ${pod_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-java-gc-count-deployments + description: 'Java GCs/s' + name: Java GC count + metricQueries: + - expression: |- + max by (gc, pod_name, cluster_name, namespace) (rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: ${gc} - ${pod_name} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-java-heap-memory-deployments + description: 'Java Heap Memory' + name: Java Heap Memory + metricQueries: + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_used_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_used{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Used - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_max_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_max{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Max - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_committed_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_committed{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Committed - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-nonheap-memory-deployments + description: 'Java Non-Heap Memory' + name: Java Non-Heap Memory + metricQueries: + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_used_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_used{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Used - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_max_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_max{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Max - ${pod_name} + - expression: |- + max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_committed_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) or max by (pod_name, cluster_name, namespace) (max_over_time(stackstate_jvm_memory_bytes_committed{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Committed - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-threads-deployments + description: 'Java Threads' + name: Java Threads + metricQueries: + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Current - ${pod_name} + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Peak - ${pod_name} + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_deadlocked{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Deadlocked - ${pod_name} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: agent-otel-jvm-gc-durations-95th-quantile-deployments + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(stackstate_jvm_gc_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: agent-otel-jvm-gc-per-second-deployments + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_duration_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: agent-otel-jvm-mem-after-last-gc-deployments + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + stackstate_jvm_memory_used_after_last_gc_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + } + alias: Memory Pool - ${jvm_memory_pool_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-heap-memory-deployments + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + jvm_memory_type="heap" + } + alias: Memory Pool - ${jvm_memory_pool_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-non-heap-memory-deployments + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + jvm_memory_type="non_heap" + } + alias: Memory Pool - ${jvm_memory_pool_name} - pod - ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) diff --git a/stackpacks/suse-observability-2/settings/generic/presentations/overview.sty b/stackpacks/suse-observability-2/settings/generic/presentations/overview.sty new file mode 100644 index 0000000..1667673 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/generic/presentations/overview.sty @@ -0,0 +1,63 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Stackgraph overview" + description: "ComponentPresentation for SUSE Observability overview" + identifier: urn:stackpack:suse-observability-2:component-presentation:overview + binding: + _type: ComponentPresentationQueryBinding + query: label in ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/part-of:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/part-of:stackstate-k8s", "app.kubernetes.io/name:suse-observability", "app.kubernetes.io/part-of:suse-observability") AND type in ("pod", "service", "deployment", "statefulset") + rank: + specificity: 10 + presentation: + icon: !icon suse-observability.svg + overview: + name: + singular: "SUSE Observability component" + plural: "SUSE Observability components" + title: "SUSE Observability" + mainMenu: + group: "SUSE Observability" + order: 1 + columns: + - columnId: "health" + title: "Health" + projection: + _type: "HealthProjection" + value: "healthState" + - columnId: "name" + title: "Name" + projection: + _type: "ComponentLinkProjection" + name: "name" + identifier: "identifiers[0]" + - columnId: "type" + title: "Type" + projection: + _type: "TextProjection" + value: "typeName" + topology: + showIndirectRelations: false + connectedComponents: true + groupedByDomains: true + groupedByLayers: true + groupedByRelations: true + groupingEnabled: true + autoGrouping: true + minimumGroupSize: 2 + filters: + - filterId: "namespace" + displayName: + singular: "namespace" + plural: "namespaces" + filter: + _type: "TagFilter" + tagKey: "namespace" + moreTab: K8s + - filterId: "cluster" + displayName: + singular: "cluster" + plural: "clusters" + filter: + _type: "TagFilter" + tagKey: "cluster-name" + moreTab: K8s diff --git a/stackpacks/suse-observability-2/settings/generic/presentations/pods.sty b/stackpacks/suse-observability-2/settings/generic/presentations/pods.sty new file mode 100644 index 0000000..46806bb --- /dev/null +++ b/stackpacks/suse-observability-2/settings/generic/presentations/pods.sty @@ -0,0 +1,302 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Otel pods" + description: "ComponentPresentation for SUSE Observability Pods" + identifier: urn:stackpack:suse-observability-2:component-presentation:pods + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ( + "app.kubernetes.io/name:stackstate", + "app.kubernetes.io/name:stackstate-k8s", + "app.kubernetes.io/part-of:stackstate", + "app.kubernetes.io/part-of:stackstate-k8s", + "app.kubernetes.io/name:suse-observability", + "app.kubernetes.io/part-of:suse-observability" + ) and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: jvm + title: JVM + order: 100 + sections: + - + sectionId: gc + title: GC + order: 100 + metrics: + - metricId: sts-java-gc-count-pod + - metricId: java-gc-duration-pod + - metricId: java-gc-total-pod + - metricId: agent-otel-jvm-gc-durations-95th-quantile-pod + - metricId: agent-otel-jvm-mem-after-last-gc-pod + - metricId: agent-otel-jvm-gc-per-second-pod + - + sectionId: memory + title: Memory + order: 90 + metrics: + - metricId: sts-java-heap-memory-pod + - metricId: sts-java-nonheap-memory-pod + - metricId: agent-otel-jvm-heap-memory-pod + - metricId: agent-otel-jvm-non-heap-memory-pod + - + sectionId: threads + title: Threads + order: 80 + metrics: + - metricId: sts-java-threads-pod + metrics: + - + metricId: java-gc-duration-pod + description: 'Java GC duration' + name: Java GC Duration + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) / rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: "${gc}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: java-gc-total-pod + description: 'Java total GC' + name: Java GC Total + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_collection_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: "${gc}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-java-gc-count-pod + description: 'Java GCs/s' + name: Java GC count + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_collection_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: "${gc}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-java-heap-memory-pod + description: 'Java Heap Memory' + name: Java Heap Memory + metricQueries: + - expression: |- + max_over_time(stackstate_jvm_memory_used_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_used{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Used + - expression: |- + max_over_time(stackstate_jvm_memory_max_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_max{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Max + - expression: |- + max_over_time(stackstate_jvm_memory_committed_bytes{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_committed{ + area="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Committed + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-nonheap-memory-pod + description: 'Java Non-Heap Memory' + name: Java Non-Heap Memory + metricQueries: + - expression: |- + max_over_time(stackstate_jvm_memory_used_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_used{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Used + - expression: |- + max_over_time(stackstate_jvm_memory_max_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_max{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Max + - expression: |- + max_over_time(stackstate_jvm_memory_committed_bytes{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) or max_over_time(stackstate_jvm_memory_bytes_committed{ + area="nonheap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Committed + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-java-threads-pod + description: 'Java Threads' + name: Java Threads + metricQueries: + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Current + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_current{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Peak + - expression: |- + sum by (pod_name, namespace, cluster_name) (max_over_time(stackstate_jvm_threads_deadlocked{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Deadlocked + chart: + _type: TimeSeriesChart + unit: short + - + metricId: agent-otel-jvm-gc-durations-95th-quantile-pod + description: 'GC operations that are slower than 95% of total gc operations.' + name: JVM GC Durations 95th Quantile + metricQueries: + - expression: |- + histogram_quantile(0.95, (rate(stackstate_jvm_gc_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]))) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: agent-otel-jvm-gc-per-second-pod + description: 'Garbage collections per second.' + name: JVM GC Per Second + metricQueries: + - expression: |- + rate(stackstate_jvm_gc_duration_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: GC Name - ${jvm_gc_name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: agent-otel-jvm-mem-after-last-gc-pod + description: 'JVM memory size after last garbage collection.' + name: JVM Mem After Last GC + metricQueries: + - expression: |- + stackstate_jvm_memory_used_after_last_gc_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-heap-memory-pod + description: 'JVM heap memory.' + name: JVM Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + jvm_memory_type="heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) + - + metricId: agent-otel-jvm-non-heap-memory-pod + description: 'JVM Non-Heap memory.' + name: JVM Non-Heap Memory + metricQueries: + - expression: |- + stackstate_jvm_memory_used_bytes{ + jvm_memory_type="non_heap", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } + alias: Memory Pool - ${jvm_memory_pool_name} + chart: + _type: TimeSeriesChart + unit: bytes(IEC) diff --git a/stackpacks/suse-observability-2/settings/generic/presentations/stackgraph-pods.sty b/stackpacks/suse-observability-2/settings/generic/presentations/stackgraph-pods.sty new file mode 100644 index 0000000..1264dfc --- /dev/null +++ b/stackpacks/suse-observability-2/settings/generic/presentations/stackgraph-pods.sty @@ -0,0 +1,95 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Stackgraph pods" + title: "SUSE Observability Stackgraph pods" + description: "ComponentPresentation for SUSE Observability Stackgraph pods" + identifier: urn:stackpack:suse-observability-2:component-presentation:stackgraph-pods + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("stackstate.com/connects-to-stackgraph:true") and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: stackgraph + title: StackGraph + order: 100 + metrics: + - metricId: sts-stackgraph-pod-read-cache-utilization + - metricId: sts-stackgraph-pod-read-cache-survival-time + - metricId: sts-stackgraph-pod-transaction-duration + metrics: + - + metricId: sts-stackgraph-pod-read-cache-utilization + description: 'ReadCache space utilization' + name: ReadCache space utilization + metricQueries: + - expression: |- + sum(stackstate_stackgraph_readcache_size_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: Used + - expression: |- + sum(stackstate_stackgraph_readcache_max_size_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: Max + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-stackgraph-pod-read-cache-survival-time + description: 'ReadCache entry survival time' + name: ReadCache entry survival time + metricQueries: + - expression: |- + histogram_quantile(0.50, (sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])) or sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])))) + alias: 0.5 quantile + - expression: |- + histogram_quantile(0.95, (sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])) or sum by (le) (rate(stackstate_stackgraph_readcache_retention_time_ms_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[20m])))) + alias: 0.95 quantile + chart: + _type: TimeSeriesChart + unit: ms + - + metricId: sts-stackgraph-pod-transaction-duration + description: 'StackGraph transaction duration' + name: StackGraph transaction duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackgraph_transaction_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + })) + alias: Used - ${name} + chart: + _type: TimeSeriesChart + unit: s \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/generic/presentations/stackgraph-services.sty b/stackpacks/suse-observability-2/settings/generic/presentations/stackgraph-services.sty new file mode 100644 index 0000000..92817b7 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/generic/presentations/stackgraph-services.sty @@ -0,0 +1,85 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Stackgraph services" + title: "SUSE Observability Stackgraph services" + description: "ComponentPresentation for SUSE Observability Stackgraph services" + identifier: urn:stackpack:suse-observability-2:component-presentation:stackgraph-services + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("stackstate.com/connects-to-stackgraph:true") and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: stackgraph + title: StackGraph + order: 100 + metrics: + - metricId: sts-stackgraph-deployment-read-cache-utilization + - metricId: sts-stackgraph-deployment-transaction-duration + - metricId: sts-stackgraph-deployment-read-cache-survival-time + metrics: + - + metricId: sts-stackgraph-deployment-read-cache-utilization + description: 'ReadCache space utilization' + name: ReadCache space utilization + metricQueries: + - expression: |- + sum by (pod_name) (stackstate_stackgraph_readcache_size_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }) + alias: Used ${pod_name} + chart: + _type: TimeSeriesChart + unit: bytes + - + metricId: sts-stackgraph-deployment-transaction-duration + description: 'StackGraph transaction duration' + name: StackGraph transaction duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackgraph_transaction_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + })) + alias: Used ${pod_name} - ${name} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-stackgraph-deployment-read-cache-survival-time + description: 'ReadCache entry survival time' + name: ReadCache entry survival time + metricQueries: + - expression: |- + histogram_quantile(0.50, (sum by (pod_name, le) (rate(stackstate_stackgraph_readcache_retention_time_ms{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[20m])) or sum by (pod_name, le) (rate(stackstate_stackgraph_readcache_retention_time_ms_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[20m])))) + alias: .5 quantile - ${pod_name} + chart: + _type: TimeSeriesChart + unit: ms \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/main-menu/main-menu.sty b/stackpacks/suse-observability-2/settings/main-menu/main-menu.sty new file mode 100644 index 0000000..363626a --- /dev/null +++ b/stackpacks/suse-observability-2/settings/main-menu/main-menu.sty @@ -0,0 +1,7 @@ +nodes: + - _type: MainMenuGroup + name: SUSE Observability + iconbase64: !icon suse-observability.svg + defaultOpen: true + items: [] + identifier: "urn:stackpack:suse-observability-2:main-menu-group:overview" \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/aad/presentation.sty b/stackpacks/suse-observability-2/settings/services/aad/presentation.sty new file mode 100644 index 0000000..0da31e4 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/aad/presentation.sty @@ -0,0 +1,66 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability AAD" + description: "ComponentPresentation for SUSE Observability AAD" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-aad + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:spotlight") and type = "pod" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: aad + title: AAD + order: 100 + sections: + - + sectionId: streams + title: Streams + order: 100 + metrics: + - metricId: aad-checked + - metricId: aad-duration + - metricId: aad-throughput + metrics: + - + metricId: aad-checked + description: 'Number of streams checked' + name: AAD - Streams checked + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name)(stackstate_spotlight_streams_checked{ + cluster_name="${tags.singleValue('cluster-name')}", namespace="${tags.singleValue('namespace')}", pod_name="${name}" + }[${__rate_interval}]) + alias: Streams checked + chart: + _type: TimeSeriesChart + unit: short + - + metricId: aad-duration + description: 'Time taken by AAD steps' + name: AAD - 95% Step duration + metricQueries: + - expression: |- + histogram_quantile(0.95, sum by (cluster_name, namespace, step, le)(rate(stackstate_spotlight_step_timings_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", namespace="${tags.singleValue('namespace')}", pod_name="${name}" + }[5m]))) + alias: Step ${step} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: aad-throughput + description: 'Number of AAD steps' + name: AAD - Throughput + metricQueries: + - expression: |- + sum by (cluster_name, namespace, step)(rate(stackstate_spotlight_step_timings_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", namespace="${tags.singleValue('namespace')}", pod_name="${name}" + }[5m])) + alias: Step ${step} + chart: + _type: TimeSeriesChart + unit: ops \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/api/presentation.sty b/stackpacks/suse-observability-2/settings/services/api/presentation.sty new file mode 100644 index 0000000..48a2692 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/api/presentation.sty @@ -0,0 +1,92 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability API" + description: "ComponentPresentation for SUSE Observability API" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-api + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:api", "app.kubernetes.io/component:server", "app.kubernetes.io/component:view-health") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: topology-api + title: Topology API + order: 100 + metrics: + - metricId: sts-topology-query-duration + - metricId: sts-topology-query-latency + - metricId: sts-topology-query-execution-rate + - metricId: sts-topology-query-timeout-rate + metrics: + - + metricId: sts-topology-query-duration + description: 'Topology Api - Duration of a query execution on the topology query service' + name: Topology Api Query Execution duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_topology_query_execute_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topology-query-latency + description: 'Topology Api - Latency of a query execution on the topology query service' + name: Topology Api Query Execution latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_topology_query_execute_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topology-query-execution-rate + description: 'Topology Api - Number of times we execute the query on the topology query service' + name: Topology Api Query execution rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_topology_query_execute{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-topology-query-timeout-rate + description: 'Topology Api - Number of queries that got rejected due to the queue being full on the topology query service' + name: Topology Api Query timeout rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_topology_query_timeout{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - ${query} + chart: + _type: TimeSeriesChart + unit: short \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/checks/deployment-presentation.sty b/stackpacks/suse-observability-2/settings/services/checks/deployment-presentation.sty new file mode 100644 index 0000000..dee6546 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/checks/deployment-presentation.sty @@ -0,0 +1,64 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Checks" + description: "ComponentPresentation for SUSE Observability Checks" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-checks-service + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:checks") + and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: monitors + title: Monitors + order: 100 + metrics: + - metricId: sts-checks-deployment-active-monitors-count + - metricId: sts-checks-deployment-failing-monitors + metrics: + - + metricId: sts-checks-deployment-active-monitors-count + description: 'Active monitors' + name: Active monitors count + metricQueries: + - expression: | + count(count without (monitorResult) (stackstate_stackstate_monitor_last_run_timestamp { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + })) + alias: Active monitors + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-checks-deployment-failing-monitors + description: 'Failing monitors' + name: Failing monitors + metricQueries: + - expression: | + sum by (monitorName) (count_over_time(stackstate_stackstate_monitor_last_run_timestamp { + monitorResult = "FAILURE", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}]) > 0) + alias: "${monitorName} fail count" + chart: + _type: TimeSeriesChart + unit: short \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/checks/monitors.sty b/stackpacks/suse-observability-2/settings/services/checks/monitors.sty new file mode 100644 index 0000000..fe1800c --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/checks/monitors.sty @@ -0,0 +1,26 @@ +nodes: + - _type: "Monitor" + name: "Failure percentage" + tags: + - monitors + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:monitors-failure-percentage-pod + arguments: + metric: + query: |- + ( + (sum by(cluster_name, namespace, pod_name, monitorName) (changes(stackstate_stackstate_monitor_last_run_timestamp{monitorResult="FAILURE"}[10m])) + / (sum by(cluster_name, namespace, pod_name, monitorName) (changes(stackstate_stackstate_monitor_last_run_timestamp{}[10m])))) + and (sum by(cluster_name, namespace, pod_name, monitorName) (changes(stackstate_stackstate_monitor_last_run_timestamp{}[10m]) > 5)) + ) + unit: "percentunit" + aliasTemplate: "Failed run %" + comparator: GT + threshold: 0.3 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Monitor ${monitorName} execution" + intervalSeconds: 30 + remediationHint: !include services/checks/monitors-failure-percentage-pod.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability-2/settings/services/checks/pod-presentation.sty b/stackpacks/suse-observability-2/settings/services/checks/pod-presentation.sty new file mode 100644 index 0000000..611342c --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/checks/pod-presentation.sty @@ -0,0 +1,61 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Checks" + title: "SUSE Observability Checks" + description: "ComponentPresentation for SUSE Observability Checks" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-checks-pods + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:checks") + and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: monitors + title: Monitors + order: 100 + metrics: + - metricId: sts-checks-pod-active-monitors-count + - metricId: sts-checks-pod-failing-monitors + metrics: + - + metricId: sts-checks-pod-active-monitors-count + description: 'Active monitors' + name: Active monitors count + metricQueries: + - expression: | + count(count without (monitorResult) (stackstate_stackstate_monitor_last_run_timestamp { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + })) + alias: Active monitors + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-checks-pod-failing-monitors + description: 'Failing monitors' + name: Failing monitors + metricQueries: + - expression: | + sum by (monitorName) (count_over_time(stackstate_stackstate_monitor_last_run_timestamp { + monitorResult = "FAILURE", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) > 0) + alias: "${monitorName} fail count" + chart: + _type: TimeSeriesChart + unit: short \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/clickhouse/monitors.sty b/stackpacks/suse-observability-2/settings/services/clickhouse/monitors.sty new file mode 100644 index 0000000..b380f54 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/clickhouse/monitors.sty @@ -0,0 +1,358 @@ +nodes: + - _type: "Monitor" + name: "ClickHouse - backups performed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-backups-not-executed + arguments: + metric: + query: |- + sum(increase(stackstate_clickhouse_backup_successful_uploads{kube_app_name="clickhouse"}[12h])) by (cluster_name, namespace, statefulset) or sum(stackstate_clickhouse_backup_number_backups_remote_expected{kube_app_name="clickhouse"}) by (cluster_name, namespace, statefulset) * 0 + unit: "short" + aliasTemplate: "Number of backups execution in 12h window" + comparator: LTE + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:statefulset/${statefulset}" + titleTemplate: "Backup performed in the last 12 hours" + intervalSeconds: 3600 + remediationHint: "Potential reasons why backups haven't been executed:\n- wrong configuration of MinIO (like secrets, bucket name and so on)\n- interval between backups is longer than 12h (you have to change the configuration of the Monitor)\n\nTo investigate the problem you should check:\n- logs of `backup` container\n- logs/status of `MinIO` pods" + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many Distributed Files to insert" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-distributed-files-to-insert-high + arguments: + metric: + query: sum(stackstate_ClickHouseMetrics_DistributedFilesToInsert) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of pending files to process for asynchronous insertion into Distributed tables" + comparator: GT + threshold: 50 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - has too many Distributed Files to insert" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-distributed-files-to-insert-high.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - rejected insert" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-rejected-insert + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_RejectedInserts[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the INSERT of a block to a MergeTree table was rejected" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - rejected INSERT statements occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-rejected-insert.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - delayed insert" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-delayed-insert-throttling + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_DelayedInserts[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - delayed INSERT statements occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-delayed-insert-throttling.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many parts per partition" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-max-part-count-for-partition + arguments: + metric: + query: sum (stackstate_ClickHouseAsyncMetrics_MaxPartCountForPartition) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Maximum number of parts per partition across all partitions of all tables of MergeTree family" + comparator: GT + threshold: 300 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many parts per a partition" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-max-part-count-for-partition.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - readonly replica" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-readonly-replica + arguments: + metric: + query: sum (stackstate_ClickHouseMetrics_ReadonlyReplica) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of Replicated tables that are currently in readonly state" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - readOnly replica occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-readonly-replica.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many connections" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-too-many-connections + arguments: + metric: + query: sum (stackstate_ClickHouseMetrics_HTTPConnection + stackstate_ClickHouseMetrics_TCPConnection + stackstate_ClickHouseMetrics_MySQLConnection) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of active connections" + comparator: GT + threshold: 100 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many connections to ClickHouse" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-too-many-connections.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many queries" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-too-many-running-queries + arguments: + metric: + query: sum ((stackstate_ClickHouseMetrics_Query - stackstate_ClickHouseMetrics_PendingAsyncInsert) or stackstate_ClickHouseMetrics_Query) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of executing queries" + comparator: GT + threshold: 80 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many running queries" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-too-many-running-queries.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - ZooKeeper hardware exceptions" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-zookeeper-hardware-exceptions + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of exceptions while working with ZooKeeper" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many exceptions of ZooKeeper" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-zookeeper-hardware-exceptions.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - too many ZooKeeper session" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-zookeeper-session + arguments: + metric: + query: max(stackstate_ClickHouseMetrics_ZooKeeperSession) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of sessions (connections) to ZooKeeper" + comparator: GT + threshold: 1 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - too many ZooKeeper sessions" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-zookeeper-session.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - replicated part checks failed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-replicated-part-checks-failed + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReplicatedPartChecksFailed[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the advanced search for a data part on replicas did not give result or when unexpected part has been found and moved away" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - advanced search for a data part on replicas did not give result or when unexpected part has been found and moved away occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-replicated-part-checks-failed.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - replicated part fetches failed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-replicated-part-failed-fetches + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReplicatedPartFailedFetches[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times a data part was failed to download from replica of a ReplicatedMergeTree table" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - data part was failed to download from replica of a ReplicatedMergeTree table" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-clickhouse-replicated-part-failed-fetches.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - replicated data loss" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-replicated-data-loss + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReplicatedDataLoss[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times a data part that we wanted doesn't exist on any replica" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - data part that we wanted doesn't exist on any replica" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-replicated-data-loss.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - data after merge differs from replica" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-data-after-merge-differs-from-replica + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_DataAfterMergeDiffersFromReplica[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times data after merge is not byte-identical to the data on another replicas" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - data after merge is not byte-identical to the data on another replicas" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-data-after-merge-differs-from-replica.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - distributed sync insertion timeout" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-distributed-sync-insertion-timeout-exceeded + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times a timeout has exceeded while waiting for shards during synchronous insertion into a Distributed table" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - a timeout has exceeded while waiting for shards during synchronous insertion into a Distributed table" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-distributed-sync-insertion-timeout-exceeded.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - file descriptor buffer read or write failed" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-file-descriptor-buffer-read-or-write-failed + arguments: + metric: + query: sum(increase(stackstate_ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed[1m]) + increase(stackstate_ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteFailed[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of times the write or read to a file descriptor have failed" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - write or read operation to a file descriptor have failed occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-file-descriptor-buffer-read-or-write-failed.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "ClickHouse - slow reads" + tags: + - clickhouse + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:clickhouse-slow-read + arguments: + metric: + query: sum (increase(stackstate_ClickHouseProfileEvents_SlowRead[1m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Number of reads from a file that were slow" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "ClickHouse - slow reads occurred" + intervalSeconds: 60 + remediationHint: !include services/clickhouse/clickhouse-slow-read.md.hbs + status: "ENABLED" \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/clickhouse/pod-presentation.sty b/stackpacks/suse-observability-2/settings/services/clickhouse/pod-presentation.sty new file mode 100644 index 0000000..9ecd7db --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/clickhouse/pod-presentation.sty @@ -0,0 +1,83 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability ClickHouse" + description: "ComponentPresentation for SUSE Observability ClickHouse" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-clickhouse-pod + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("statefulset.kubernetes.io/pod-name:stackstate-clickhouse-shard0-0", "statefulset.kubernetes.io/pod-name:suse-observability-clickhouse-shard0-0") + AND type = "pod" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Backup + order: 100 + metrics: + - metricId: clickhouse-backup-duration-pod + - + sectionId: storage + title: Storage + order: 100 + metrics: + - metricId: clickhouse-storage + metrics: + - + metricId: clickhouse-backup-duration-pod + name: ClickHouse - Backup duration + description: Duration of ClickHouse backups + metricQueries: + - expression: |- + (stackstate_clickhouse_backup_last_create_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + } + stackstate_clickhouse_backup_last_upload_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }) / 1000000000 + alias: "${kube_app_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: clickhouse-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by ClickHouse." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]), "storage"," $1", "persistentvolumeclaim", "data-.*-(clickhouse-.*)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + mode: absolute + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/clickhouse/statefulset-presentation.sty b/stackpacks/suse-observability-2/settings/services/clickhouse/statefulset-presentation.sty new file mode 100644 index 0000000..faa235a --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/clickhouse/statefulset-presentation.sty @@ -0,0 +1,84 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability ClickHouse" + title: "SUSE Observability ClickHouse" + description: "ComponentPresentation for SUSE Observability ClickHouse" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-clickhouse-statefulset + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("statefulset:stackstate-clickhouse-shard0", "statefulset:suse-observability-clickhouse-shard0") + AND type = "statefulset" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Backup + order: 100 + metrics: + - metricId: clickhouse-backup-duration-statefulset + - + sectionId: storage + title: Storage + order: 100 + metrics: + - metricId: clickhouse-storage + metrics: + - + metricId: clickhouse-backup-duration-statefulset + name: ClickHouse - Backup duration + description: Duration of ClickHouse backups + metricQueries: + - expression: |- + (stackstate_clickhouse_backup_last_create_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + statefulset="${tags.singleValue('statefulset')}" + } + stackstate_clickhouse_backup_last_upload_duration{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + statefulset="${tags.singleValue('statefulset')}" + }) / 1000000000 + alias: "${kube_app_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: clickhouse-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by ClickHouse." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(clickhouse-.*)" + }[5m]), "storage"," $1", "persistentvolumeclaim", "data-.*-(clickhouse-.*)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + mode: absolute + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 diff --git a/stackpacks/suse-observability-2/settings/services/correlate/deployment-presentation.sty b/stackpacks/suse-observability-2/settings/services/correlate/deployment-presentation.sty new file mode 100644 index 0000000..b1b2316 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/correlate/deployment-presentation.sty @@ -0,0 +1,277 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Correlate" + description: "ComponentPresentation for SUSE Observability Correlate" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-correlate-deployment + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:correlate") and type in ("service", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Correlate Performance + order: 100 + metrics: + - metricId: sts-correlate-deployment-total-observations + - metricId: sts-correlate-deployment-internal-latency + - + sectionId: backup + title: Correlate Produced + order: 90 + metrics: + - metricId: sts-correlate-deployment-approximate-count-of-elements-produced + - metricId: sts-correlate-deployment-approximate-count-of-elements-produced-hour + - + sectionId: backup + title: Correlate Details + order: 80 + metrics: + - metricId: sts-correlate-deployment-buffers-occupancy + - metricId: sts-correlate-deployment-http-trace-observations + - metricId: sts-correlate-deployment-connection-observations + - metricId: sts-correlate-deployment-http-trace-observations-cluster + - metricId: sts-correlate-deployment-connection-observations-cluster + metrics: + - + metricId: sts-correlate-deployment-total-observations + name: SUSE Observability Correlate - total observations + description: Total observations at the correlator + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Http + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Connections + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-internal-latency + description: 'Correlate Internal Latency per data type measured from reception time to correlation finished' + name: Stackstate Correlate - Internal Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, sum by (data_type, le) (rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }))) + alias: Latency to ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-correlate-deployment-approximate-count-of-elements-produced + name: SUSE Observability Correlate - Approximate count of elements produced + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_correlate_unique_active_element_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-deployment-approximate-count-of-elements-produced-hour + name: SUSE Observability Correlate - Approximate count of created elements in the last hour + description: Approximate count of new elements create in the past hour + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_correlate_element_create_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-deployment-buffers-occupancy + name: SUSE Observability Correlate - buffers occupancy + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + sum by (correlation_type) (avg_over_time(stackstate_stackstate_correlate_buffer_occupancy{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${correlation_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-correlate-deployment-http-trace-observations + name: SUSE Observability HTTP Correlate - bservation correlation + description: HTTP Trace observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_subsumed{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Subsumed + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Dropped + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-connection-observations + name: SUSE Observability Connection Correlate - observation correlation + description: Layer 4 connection observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Dropped + - expression: |- + sum(rate(stackstate_stackstate_correlate_ambiguous_endpoints{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Ambiguous + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-http-trace-observations-cluster + name: SUSE Observability HTTP Correlations - per cluster + description: HTTP Trace observations received and processing results + metricQueries: + - expression: |- + sum by (agent_cluster) (rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: "${agent_cluster}" + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-deployment-connection-observations-cluster + name: SUSE Observability Connection Correlate - per cluster + description: Layer 4 connection observations received and processing results + metricQueries: + - expression: |- + sum by (agent_cluster) (rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: "${agent_cluster}" + chart: + _type: TimeSeriesChart + unit: ops \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/correlate/monitors.sty b/stackpacks/suse-observability-2/settings/services/correlate/monitors.sty new file mode 100644 index 0000000..8e5e32c --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/correlate/monitors.sty @@ -0,0 +1,43 @@ +nodes: + - _type: "Monitor" + name: "Correlate - Processing latency" + tags: + - correlate + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-pod-correlation-latency + status: "ENABLED" + description: "Critical state when the internal correlation latency is too high. The latency is measured from the time the data is received until a metric is produced." + arguments: + comparator: GT + failureState: "DEVIATING" + threshold: 300.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, le)(rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{data_type="prometheus_earliest"}[5m]))) + aliasTemplate: "Latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/correlate/latency.md.hbs + - _type: "Monitor" + name: "Correlate - Processing latency" + tags: + - correlate + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-correlation-deployment-latency + status: "ENABLED" + description: "Critical state when the internal correlation latency is too high. The latency is measured from the time the data is received until a metric is produced." + arguments: + comparator: GT + failureState: "DEVIATING" + threshold: 300.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, deployment, le)(rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{data_type="prometheus_earliest"}[5m]))) + aliasTemplate: "Latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:deployment/${deployment}" + intervalSeconds: 30 + remediationHint: !include services/correlate/latency.md.hbs diff --git a/stackpacks/suse-observability-2/settings/services/correlate/pod-presentation.sty b/stackpacks/suse-observability-2/settings/services/correlate/pod-presentation.sty new file mode 100644 index 0000000..07325d7 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/correlate/pod-presentation.sty @@ -0,0 +1,209 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Correlate" + description: "ComponentPresentation for SUSE Observability Correlate" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-correlate-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:correlate") and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Correlate Performance + order: 100 + metrics: + - metricId: sts-correlate-pod-total-observations + - metricId: sts-correlate-pod-internal-latency + - + sectionId: backup + title: Correlate Produced + order: 90 + metrics: + - metricId: sts-correlate-pod-approximate-count-of-elements-produced + - metricId: sts-correlate-pod-approximate-count-of-elements-produced-hour + - + sectionId: backup + title: Correlate Details + order: 80 + metrics: + - metricId: sts-correlate-pod-buffers-occupancy + - metricId: sts-correlate-pod-http-trace-observations + - metricId: sts-correlate-pod-connection-observations + metrics: + - + metricId: sts-correlate-pod-total-observations + name: SUSE Observability Correlate - total observations + description: Total observations at the correlator + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Http + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Connections + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-pod-internal-latency + description: 'Correlate Internal Latency per data type measured from reception time to correlation finished' + name: Stackstate Correlate - Internal Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_correlate_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + })) + alias: Latency to ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-correlate-pod-approximate-count-of-elements-produced + name: SUSE Observability Correlate - Approximate count of elements produced + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + avg_over_time(stackstate_stackstate_correlate_unique_active_element_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-pod-approximate-count-of-elements-produced-hour + name: SUSE Observability Correlate - Approximate count of created elements in the last hour + description: Approximate count of new elements create in the past hour + metricQueries: + - expression: |- + avg_over_time(stackstate_stackstate_correlate_element_create_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-correlate-pod-buffers-occupancy + name: SUSE Observability Correlate - buffers occupancy + description: Amount of elements produced in the correlator + metricQueries: + - expression: |- + avg_over_time(stackstate_stackstate_correlate_buffer_occupancy{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: "${correlation_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-correlate-pod-http-trace-observations + name: SUSE Observability HTTP Correlate - observation correlation + description: HTTP Trace observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_subsumed{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Subsumed + - expression: |- + sum(rate(stackstate_stackstate_correlate_http_observations_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Dropped + chart: + _type: TimeSeriesChart + unit: ops + - + metricId: sts-correlate-pod-connection-observations + name: SUSE Observability Connection Correlate - observation correlation + description: Layer 4 connection observations received and processing results + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_added_to_state{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Total + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_correlated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Correlated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_uncorrelated{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Uncorrelated + - expression: |- + sum(rate(stackstate_stackstate_correlate_endpoints_data_dropped{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Dropped + - expression: |- + sum(rate(stackstate_stackstate_correlate_ambiguous_endpoints{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) + alias: Ambiguous + chart: + _type: TimeSeriesChart + unit: ops \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/es/presentation.sty b/stackpacks/suse-observability-2/settings/services/es/presentation.sty new file mode 100644 index 0000000..64b9940 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/es/presentation.sty @@ -0,0 +1,71 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Elasticsearch" + description: "ComponentPresentation for SUSE Observability Elasticsearch" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-elasticsearch-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:elasticsearch") AND label IN("app.kubernetes.io/part-of:stackstate-k8s", "app.kubernetes.io/part-of:suse-observability") and type in ("service", "pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: elasticsearch + title: Elasticsearch + order: 100 + metrics: + - metricId: sts-es-current-index-size + - metricId: sts-es-storage + metrics: + - + metricId: sts-es-current-index-size + name: SUSE Observability ElasticSearch - Current index disk space + description: Amount of data stored on ElasticSearch + metricQueries: + - expression: |- + sum(label_replace(stackstate_elasticsearch_indices_store_size_bytes_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}" + }, "sts_index","$1","index", ".*(sts_k8s_logs|sts_multi_metrics|sts_internal_events|sts_topology_events|sts_generic_events|sts_trace_events|sts_annotations).*")) by (sts_index) / 1000000000 + alias: "${sts_index}" + chart: + _type: TimeSeriesChart + unit: gbytes + - + metricId: sts-es-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by Elasticsearch." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"data-.*-(elasticsearch-master-\\d)" + }[5m]), "storage", "$1", "persistentvolumeclaim", "data-.*-(elasticsearch-.*)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/hbase/monitors.sty b/stackpacks/suse-observability-2/settings/services/hbase/monitors.sty new file mode 100644 index 0000000..14abf9f --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/hbase/monitors.sty @@ -0,0 +1,23 @@ +nodes: + - _type: "Monitor" + name: "HBase slow gets" + tags: + - suse-observability + - hbase + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-hbase-slow-get-count + status: "ENABLED" + description: "Monitors the amount of slow get counts." + arguments: + comparator: GT + titleTemplate: "HBase - slow get count" + failureState: "DEVIATING" + threshold: 0 + metric: + query: |- + sum by (cluster_name, namespace, pod_name)(rate(stackstate_hbase_slowgetcount{}[5m])) + aliasTemplate: "slow get count" + unit: ops/sec (ops) + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/hbase/slow-gets.md.hbs \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/health-sync/monitors.sty b/stackpacks/suse-observability-2/settings/services/health-sync/monitors.sty new file mode 100644 index 0000000..3814379 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/health-sync/monitors.sty @@ -0,0 +1,22 @@ +nodes: + - _type: "Monitor" + name: "Health sync data processing latency" + tags: + - health-sync + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-health-sync-data-processing-latency + status: "ENABLED" + description: "Critical state when the processing latency is too high. The latency is measured from the time the data is received until it is stored as a health state." + arguments: + comparator: GT + failureState: "DEVIATING" + threshold: 30.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, le)(rate(stackstate_stackstate_health_sync_latency_seconds_bucket{}[5m]))) + aliasTemplate: "Latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/health-sync/latency.md.hbs diff --git a/stackpacks/suse-observability-2/settings/services/health-sync/presentation.sty b/stackpacks/suse-observability-2/settings/services/health-sync/presentation.sty new file mode 100644 index 0000000..10530b9 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/health-sync/presentation.sty @@ -0,0 +1,92 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Elasticsearch" + description: "ComponentPresentation for SUSE Observability Elasticsearch" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-health-sync-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:health-sync", "app.kubernetes.io/component:server") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: health-sync + title: Health Sync + order: 100 + metrics: + - metricId: sts-health-sync-latency + - metricId: sts-health-sync-messages-rate + - metricId: sts-health-sync-operations-rate + - metricId: sts-health-sync-kafka-lag + metrics: + - + description: 'Health Sync latency' + name: Health Sync Latency + metricId: sts-health-sync-latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_health_sync_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${partition} + chart: + _type: TimeSeriesChart + unit: s + - + description: 'Rate of all received check state messages that were received for a stream and substreams' + name: Health Sync Received messages rate + metricId: sts-health-sync-messages-rate + metricQueries: + - expression: |- + stackstate_stackstate_health_sync_all_check_state_message_rate{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + } + alias: ${pod_name} - ${stream} + chart: + _type: TimeSeriesChart + unit: cps + - + description: 'Rate of all check state operations that were performed for the stream and substreams' + name: Health Sync Received operations rate + metricId: sts-health-sync-operations-rate + metricQueries: + - expression: |- + stackstate_health_sync_all_check_state_operation_rate{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + } > 0 + alias: ${pod_name} - ${stream} - ${operation} + chart: + _type: TimeSeriesChart + unit: cps + - + description: 'Kafka consumer lag per topic' + name: Health Sync Topic lag + metricId: sts-health-sync-kafka-lag + metricQueries: + - expression: |- + sum by (pod_name, topic) (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${topic} + chart: + _type: TimeSeriesChart + unit: cps \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/kafka-to-es/monitors.sty b/stackpacks/suse-observability-2/settings/services/kafka-to-es/monitors.sty new file mode 100644 index 0000000..54c3bb2 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/kafka-to-es/monitors.sty @@ -0,0 +1,45 @@ +nodes: + - _type: "Monitor" + name: "Elasticsearch data ingestion latency" + tags: + - kafka-to-es + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-es-data-ingestion-latency + status: "ENABLED" + description: "Critical state when the ingestion latency is too high. The latency includes the time taken from producing the data to it being received on the SUSE Observability api." + arguments: + comparator: GT + failureState: "DEVIATING" + titleTemplate: "Elasticsearch ingestion latency for ${data_type}" + threshold: 120.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, data_type, le)(rate(stackstate_stackstate_kafka2es_data_latency_seconds_bucket{}[5m]))) + aliasTemplate: "${data_type} latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/kafka-to-es/latency.md.hbs + - _type: "Monitor" + name: "Elasticsearch data processing latency" + tags: + - kafka-to-es + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:java-es-data-processing-latency + status: "ENABLED" + description: "Critical state when the processing latency is too high. The latency is measured from the time the data is received until it is stored in Elasticsearch." + arguments: + comparator: GT + titleTemplate: "Elasticsearch processing latency for ${data_type}" + failureState: "DEVIATING" + threshold: 30.0 + metric: + query: |- + histogram_quantile(0.98, sum by (cluster_name, namespace, pod_name, data_type, le)(rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_bucket{}[5m]))) + aliasTemplate: "${data_type} latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/kafka-to-es/latency.md.hbs \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/kafka-to-es/presentation.sty b/stackpacks/suse-observability-2/settings/services/kafka-to-es/presentation.sty new file mode 100644 index 0000000..f3c7acf --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/kafka-to-es/presentation.sty @@ -0,0 +1,80 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Kafka2ES" + description: "ComponentPresentation for SUSE Observability Kafka2ES" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-kafka-to-es-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:trace2es", "app.kubernetes.io/component:mm2es", "app.kubernetes.io/component:e2es") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: kafka2es + title: Kafka2ES + order: 100 + metrics: + - metricId: sts-topo-k2es-latency + - metricId: sts-topo-k2es-internal-latency + - metricId: sts-topo-k2es-catch-up + metrics: + - + metricId: sts-topo-k2es-latency + description: 'Kafka2Es Latency per data type measured from collection time to database' + name: Kafka2Es Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_kafka2es_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-k2es-internal-latency + description: 'Kafka2Es Internal Latency per data type measured from reception time to database' + name: Kafka2Es Internal Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${data_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-k2es-catch-up + description: 'Kafka2Es time to catch up' + name: Kafka2Es Time to catch up + metricQueries: + - expression: |- + histogram_quantile(0.98, sum by(data_type, pod_name, le)(rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }))) / sum by(data_type, pod_name)(rate(stackstate_stackstate_kafka2es_received_data_latency_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) > 0.05) + alias: ${pod_name} - ${data_type} + chart: + _type: TimeSeriesChart + unit: s \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/metrics-store-clients/monitors.sty b/stackpacks/suse-observability-2/settings/services/metrics-store-clients/monitors.sty new file mode 100644 index 0000000..f851088 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/metrics-store-clients/monitors.sty @@ -0,0 +1,23 @@ +nodes: + - _type: "Monitor" + name: "Metric store unreachable" + tags: + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-metric-store-unreachable + status: "ENABLED" + description: | + Critical state when the service cannot connect to the metric store. + A circuit breaker can have 3 states, closed (0), open (1) or half-open (2). When closed, metrics queries are working normally, when open there were problems running recent queries. While open queries will be failed immediately without querying the metric store. To test connectivity the circuit breaker will switch regularly to a half-open state where 1 query request is send to the store as a test. If that passes the circuit breaker will close again. + arguments: + comparator: GT + failureState: "CRITICAL" + threshold: 0 + metric: + query: |- + max by (cluster_name, namespace, pod_name)(min_over_time(stackstate_stackstate_metrics_circuit_breaker_status[2m])) + unit: short + aliasTemplate: "Connection status (0 = connected, > 0 = unreachable)" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/metrics-store-clients/metric-store-unreachable.md.hbs diff --git a/stackpacks/suse-observability-2/settings/services/metrics-store-clients/presentations.sty b/stackpacks/suse-observability-2/settings/services/metrics-store-clients/presentations.sty new file mode 100644 index 0000000..8cc90b2 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/metrics-store-clients/presentations.sty @@ -0,0 +1,130 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Metrics Store Clients - Circuit Breaker" + description: "ComponentPresentation for SUSE Observability Metrics Store Clients - Circuit Breaker" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-metrics-store-clients-circuit-breaker + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:api", "app.kubernetes.io/component:checks", "app.kubernetes.io/component:sync", "app.kubernetes.io/component:health-sync") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: metrics + title: Metrics + order: 100 + metrics: + - metricId: sts-metric-store-circuit-breaker + metrics: + - + metricId: sts-metric-store-circuit-breaker + description: 'States: closed (0), open (1) or half-open (2)' + name: Metrics client - Circuit Breaker + metricQueries: + - expression: |- + sum by (pod_name)(max_over_time(stackstate_stackstate_metrics_circuit_breaker_status{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}])) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: short + + - _type: ComponentPresentation + name: "SUSE Observability Metrics Store Clients - PromQL Hit" + title: "SUSE Observability Metrics Store Clients - PromQL Hit" + description: "ComponentPresentation for SUSE Observability Metrics Store Clients - PromQL Hit" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-metrics-store-clients-promql-hit-deployment + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:api", "app.kubernetes.io/component:checks") and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: metrics + title: Metrics + order: 100 + metrics: + - metricId: sts-checks-deployment-promql-hit-pct + metrics: + - + metricId: sts-checks-deployment-promql-hit-pct + description: 'PromQL cache hit %' + name: PromQL cache hit % + metricQueries: + - expression: | + sum by (cluster_name, namespace, pod_name) (rate(stackstate_caffeine_cache_hit{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + } )) / sum by (cluster_name, namespace, pod_name) (rate(stackstate_caffeine_cache_requests{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + } )) + alias: hit % - ${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: TimeSeriesChart + unit: percentunit + + - _type: ComponentPresentation + name: "SUSE Observability Metrics Store Clients - PromQL Hit" + title: "SUSE Observability Metrics Store Clients - PromQL Hit" + description: "ComponentPresentation for SUSE Observability Metrics Store Clients - PromQL Hit" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-metrics-store-clients-promql-hit-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server", "app.kubernetes.io/component:api", "app.kubernetes.io/component:checks") and type in ("pod") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: metrics + title: Metrics + order: 100 + metrics: + - metricId: sts-checks-pod-promql-hit-pct + metrics: + - + metricId: sts-checks-pod-promql-hit-pct + description: 'PromQL cache hit %' + name: PromQL cache hit % + metricQueries: + - expression: | + sum(rate(stackstate_caffeine_cache_hit{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } )) / sum(rate(stackstate_caffeine_cache_requests{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + } )) + alias: hit % + chart: + _type: TimeSeriesChart + unit: percentunit diff --git a/stackpacks/suse-observability-2/settings/services/otel-collector/monitors.sty b/stackpacks/suse-observability-2/settings/services/otel-collector/monitors.sty new file mode 100644 index 0000000..e0eb052 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/otel-collector/monitors.sty @@ -0,0 +1,126 @@ +nodes: + - _type: "Monitor" + name: "OTel Collector - spans refused by a receiver" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:otel-collector-span-refused-receiver + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_receiver_refused_spans[1m])) + unit: "short" + aliasTemplate: "Number of spans that could not be pushed into the pipeline" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - spans refused by a receiver" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-span-refused-receiver.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - metric points refused by a receiver" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:otel-collector-metric-refused-receiver + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_receiver_refused_metric_points[1m])) + unit: "short" + aliasTemplate: "Number of metrics that could not be pushed into the pipeline" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - metric points refused by a receiver" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-metric-refused-receiver.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - spans export failures" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:otel-collector-span-export-failure + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_exporter_send_failed_spans[1m])) + unit: "short" + aliasTemplate: "Number of spans in failed attempts to send to destination" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - spans export failures" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-span-export-failure.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - metric points export failures" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:otel-collector-metric-export-failure + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_exporter_send_failed_metric_points[1m])) + unit: "short" + aliasTemplate: "Number of metric points in failed attempts to send to destination" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - metric points export failures" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-metric-export-failure.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - a processor drops spans" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:otel-collector-span-processor-drops + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_processor_dropped_spans[1m])) + unit: "short" + aliasTemplate: "Number of spans that were dropped" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - a processor drops spans" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-span-processor-drops.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "OTel Collector - a processor drops metric points" + tags: + - otel-collector + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:otel-collector-metric-processor-drops + arguments: + metric: + query: sum by (cluster_name, namespace, pod_name) (rate(stackstate_otelcol_processor_dropped_metric_points[1m])) + unit: "short" + aliasTemplate: "Number of metric points that were dropped" + comparator: GT + threshold: 1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "OTel Collector - a processor drops metric points" + intervalSeconds: 60 + remediationHint: !include services/otel-collector/otel-collector-metric-processor-drops.md.hbs + status: "ENABLED" \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/otel-collector/presentation.sty b/stackpacks/suse-observability-2/settings/services/otel-collector/presentation.sty new file mode 100644 index 0000000..bdd0976 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/otel-collector/presentation.sty @@ -0,0 +1,240 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability OTel Collector" + description: "ComponentPresentation for SUSE Observability OTel Collector" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-otel-collector-pod + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:opentelemetry-collector") and type = "pod" + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: otel-collector + title: OTel Collector + order: 100 + metrics: + - metricId: otel-collector-accepted-metrics + - metricId: otel-collector-accepted-spans + - + sectionId: topology-connector + title: Topology connector + order: 90 + metrics: + - metricId: otel-collector-connector-topology-topology-produced + - metricId: otel-collector-connector-topology-mappings-removed + - metricId: otel-collector-connector-topology-mapping-errors + - metricId: otel-collector-connector-topology-mapping-duration + - metricId: otel-collector-connector-topology-signal-processing-duration + - + sectionId: settings-provider-extension + title: Settings provider extension + order: 80 + metrics: + - metricId: otel-collector-extension-incomplete-snapshots + - metricId: otel-collector-extension-successful-snapshot-ends + - metricId: otel-collector-extension-settings-count + - metricId: otel-collector-extension-settings-size + metrics: + - + metricId: otel-collector-accepted-metrics + description: 'Number of metric points successfully pushed into the pipeline' + name: OTel Collector - accepted metric points + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, le)(increase(stackstate_otelcol_receiver_accepted_metric_points{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}])) + alias: Metric points + chart: + _type: TimeSeriesChart + unit: short + - + metricId: otel-collector-accepted-spans + description: 'Number of spans successfully pushed into the pipeline' + name: OTel Collector - accepted spans + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, le)(increase(stackstate_otelcol_receiver_accepted_spans{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}])) + alias: Spans + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-topology-produced + description: 'Total number of topology produced by the topology mapping connector' + name: OTel topology connector - components produced + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, type, signal)( + rate(stackstate_otelcol_connector_topology_topology_produced_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Components produced + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-mappings-removed + description: 'Total number of mappings removed by the topology mapping connector' + name: OTel topology connector - mappings removed + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, type)( + rate(stackstate_otelcol_connector_topology_mappings_removed_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Components produced + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-mapping-errors + description: 'Number of mapping errors observed by the topology connector' + name: OTel topology connector - mapping errors + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name, type, signal)( + rate(stackstate_otelcol_connector_topology_mapping_errors_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Mapping errors + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-mapping-duration + description: '95th percentile of mapping input to topology duration in seconds' + name: OTel topology connector - input mapping duration (p95) + metricQueries: + - expression: |- + histogram_quantile(0.95, + sum by (le, cluster_name, namespace, pod_name, signal)( + rate(stackstate_otelcol_connector_topology_mapping_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + ) + alias: P95 Mapping duration + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-connector-topology-signal-processing-duration + description: '95th percentile of processing a single input signal request in seconds' + name: OTel topology connector - single input signal processing duration (p95) + metricQueries: + - expression: |- + histogram_quantile(0.95, + sum by (le, cluster_name, namespace, pod_name, signal)( + rate(stackstate_otelcol_connector_topology_mapping_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + ) + alias: P95 Mapping duration + chart: + _type: TimeSeriesChart + unit: s + - + metricId: otel-collector-extension-incomplete-snapshots + description: 'Total number of incomplete setting snapshots (replaced by new snapshot starts messages)' + name: OTel settings provider extension - incomplete snapshots + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name)( + rate(stackstate_otelcol_extension_sts_settings_provider_settings_snapshot_incomplete{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Incomplete snapshots + chart: + _type: TimeSeriesChart + unit: short + - + metricId: otel-collector-extension-successful-snapshot-ends + description: 'Total number of setting snapshots processed completely' + name: OTel settings provider extension - successful snapshots + metricQueries: + - expression: |- + sum by (cluster_name, namespace, pod_name)( + rate(stackstate_otelcol_extension_sts_settings_provider_settings_snapshot_complete{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Successful snapshots + chart: + _type: TimeSeriesChart + unit: short + - + metricId: otel-collector-extension-settings-count + description: 'Settings count per snapshot by type' + name: OTel settings provider extension - settings count + metricQueries: + - expression: | + sum by (cluster_name, namespace, pod_name, setting_type)( + rate(stackstate_otelcol_extension_sts_settings_provider_settings_count_by_type_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) + ) + alias: Settings count by type + chart: + _type: TimeSeriesChart + unit: settings + - + metricId: otel-collector-extension-settings-size + description: 'Maximum average size of completed settings snapshots (in bytes)' + name: OTel settings provider extension - settings size + metricQueries: + - expression: | + sum by (cluster_name, namespace, setting_type)( + max_over_time(stackstate_otelcol_extension_sts_settings_provider_settings_size_by_type_bytes_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__interval}]) + ) + / + sum by (cluster_name, namespace, setting_type)( + max_over_time(stackstate_otelcol_extension_sts_settings_provider_settings_size_by_type_bytes_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__interval}]) + ) + alias: Max average snapshot size (bytes) + chart: + _type: TimeSeriesChart + unit: By \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/receiver/deployment-presentation.sty b/stackpacks/suse-observability-2/settings/services/receiver/deployment-presentation.sty new file mode 100644 index 0000000..280fd91 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/receiver/deployment-presentation.sty @@ -0,0 +1,295 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Receiver" + description: "ComponentPresentation for SUSE Observability Receiver" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-receiver-deployment + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:receiver-base", "app.kubernetes.io/component:receiver-process-agent", "app.kubernetes.io/component:receiver") and type in ("deployment", "service") + rank: + specificity: 10 + presentation: + highlight: + title: "SUSE Observability Receiver" + fields: [] + links: + - linkId: data-ingestion-dashboard + title: "Data ingestion dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:data-ingestion" + tooltip: "Open the bundled SUSE Observability data ingestion dashboard" + order: 100.0 + - linkId: topology-processing-dashboard + title: "Topology processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:topology-processing" + tooltip: "Open the bundled SUSE Observability topology processing dashboard" + order: 90.0 + - linkId: traces-processing-dashboard + title: "Traces processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:traces-processing" + tooltip: "Open the bundled SUSE Observability traces processing dashboard" + order: 80.0 + - linkId: logs-processing-dashboard + title: "Logs processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:logs-processing" + tooltip: "Open the bundled SUSE Observability logs processing dashboard" + order: 70.0 + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: receiver + title: Receiver + order: 100 + metrics: + - metricId: sts-receiver-incoming-data + - metricId: sts-receiver-deployment-current-incoming-element-count + - metricId: sts-receiver-deployment-current-incoming-element-created + - metricId: sts-receiver-deployment-budget-saturation + - metricId: sts-receiver-deployment-created-budget-saturation + - metricId: sts-receiver-deployment-agent-count + - metricId: sts-receiver-deployment-created-elements + - metricId: sts-receiver-deployment-active-http-requests + - metricId: sts-receiver-deployment-log-volume + - metricId: sts-receiver-deployment-rejected-log-volume + metrics: + - + metricId: sts-receiver-deployment-current-incoming-element-count + name: SUSE Observability Receiver - Current incoming element (approximate) count + description: Amount of data being offered to the receiver at this moment + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-current-incoming-element-created + name: SUSE Observability Receiver - Created elements in the last hour (approximate) count + description: Amount of elements expected to result in new element creates in SUSE Observability + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type}" + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: Budget ${element_type} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-budget-saturation + name: SUSE Observability Receiver - Element budget saturation + description: Percentage of agent element processing budget filled up + metricQueries: + - expression: |- + sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) / sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type} - ${pod_name}" + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + - + metricId: sts-receiver-deployment-created-budget-saturation + name: SUSE Observability Receiver - Created elements hourly budget saturation + description: Percentage of newly created elements budget filled up + metricQueries: + - expression: |- + sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) / sum by (element_type, cluster_name, namespace, pod_name) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}])) + alias: "${element_type} - ${pod_name}" + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-receiver-deployment-agent-count + name: SUSE Observability Receiver - Connected agent count + description: Number of currently connected agents + metricQueries: + - expression: |- + max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + element_type="agent", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__interval}]) + alias: Agent count - ${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-created-elements + name: SUSE Observability Receiver - Created elements hourly total + description: Total elements created per hour + metricQueries: + - expression: |- + sum by (element_type) (stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-active-http-requests + name: SUSE Observability Receiver - Active HTTP Requests + description: Currently active http requests, this should not be an ever increasing line (see monitor) + metricQueries: + - expression: |- + sum(delta(stackstate_akka_http_requests_active{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Active requests + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-deployment-log-volume + name: SUSE Observability Receiver - Log volume + description: The volume of logs received by Stackstate, accepted logs are stored, rejected logs are dropped because they are over the allocated disk-space + metricQueries: + - expression: |- + sum(rate(stackstate_stackstate_receiver_events_accepted_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Accepted + - expression: |- + sum(rate(stackstate_stackstate_receiver_events_rejected_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) + alias: Rejected + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-deployment-rejected-log-volume + name: SUSE Observability Receiver - Rejected Log volume per pod + description: The volume of logs rejected by Stackstate per pod + metricQueries: + - expression: |- + topk(10, rate(stackstate_stackstate_receiver_logs_rejected_size{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }[${__rate_interval}])) @ end() + alias: ${limited_pod_name} Rejected by - ${cluster_name}/${namespace}/${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}/pod/${limited_pod_uid}" + - expression: |- + max(stackstate_stackstate_receiver_logs_budget_per_pod { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_name="${tags.singleValue('app.kubernetes.io/name')}", + kube_app_instance="${tags.singleValue('app.kubernetes.io/instance')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}" + }) + alias: Logs budget per pod per time window + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-incoming-data + name: Incoming data + description: "Maximum count of unique elements received from agents." + metricQueries: + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster')}", + namespace="${tags.singleValue('namespace')}", + element_type='agent' + }[${__interval}])) + alias: node + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + element_type!='agent' + }[${__interval}])) + alias: ${element_type} + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] diff --git a/stackpacks/suse-observability-2/settings/services/receiver/monitors.sty b/stackpacks/suse-observability-2/settings/services/receiver/monitors.sty new file mode 100644 index 0000000..44fe259 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/receiver/monitors.sty @@ -0,0 +1,181 @@ +nodes: + - _type: "Monitor" + name: "Receiver - unique element budget saturation" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-budget-saturation + status: "ENABLED" + description: "The SUSE Observability receiver has a budget for the number of unique elements (agents, connections, processes, etc...) it will accept data for within a certain time. If this budget is exceeded, the receiver will stop accepting new elements. This monitor will alert when the budget is exceeded." + arguments: + comparator: GT + failureState: "DEVIATING" + titleTemplate: "Receiver - unique element budget saturation - ${element_type}" + threshold: 0.95 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{kube_app_component=~"receiver.*"}[5m]) / stackstate_stackstate_receiver_unique_element_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "${element_type} saturation" + unit: percentunit + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - hourly element budget saturation" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-hourly-budget-saturation + status: "ENABLED" + description: "The SUSE Observability receiver has an budget for the number of elements (agents, connections, processes, etc..) it will accept data for within an hour. If this budget is exceeded, the receiver will stop accepting new elements. This monitor will alert when the budget is exceeded." + arguments: + comparator: GT + titleTemplate: "Receiver - hourly element budget saturation - ${element_type}" + failureState: "DEVIATING" + threshold: 0.95 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{kube_app_component=~"receiver.*"}[5m]) / stackstate_stackstate_receiver_element_create_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "${element_type} hourly saturation" + unit: percentunit + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/hourly-budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - Maximum number of elements exceeded" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-max-elements-exceeded + status: "ENABLED" + description: "The SUSE Observability receiver has a limit on the maximum number of unqiue elements (agents, connections, processes, etc..) it will accept data for. If this limit is exceeded, the receiver will stop accepting new elements. This monitor will alert when the limit is exceeded." + arguments: + comparator: GTE + titleTemplate: "Receiver - Maximum number of elements exceeded - ${element_type}" + failureState: "DEVIATING" + threshold: 1.0 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{kube_app_component=~"receiver.*"}[5m])) + / sum by (namespace, cluster_name, pod_name, element_type) (stackstate_stackstate_receiver_unique_element_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "# of ${element_type} elements" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - Maximum number of hourly elements exceeded" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-hourly-max-elements-exceeded + status: "ENABLED" + description: "The SUSE Observability receiver has a limit on the maximum number of elements (agents, connections, processes, etc..) it will accept data for within an hour. If this hourly limit is exceeded, the receiver will stop accepting new elements for the rest of the hour. This monitor will alert when the limit is exceeded." + arguments: + comparator: GTE + titleTemplate: "Receiver - Maximum number of hourly elements exceeded - ${element_type}" + failureState: "DEVIATING" + threshold: 1.0 + metric: + query: |- + sum by (namespace, cluster_name, pod_name, element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{kube_app_component=~"receiver.*"}[5m])) + / sum by (namespace, cluster_name, pod_name, element_type) (stackstate_stackstate_receiver_element_create_passed_max{kube_app_component=~"receiver.*"}) + aliasTemplate: "# of ${element_type} elements" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/hourly-budget-saturation.md.hbs + - _type: "Monitor" + name: "Receiver - Active number of http requests growing" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-active-http-requests-growing + status: "ENABLED" + description: "The number of active http requests should not keep growing. This can happen for short periods of time (for example with a sudden spike of traffic), but if it keeps growing, it is a sign of a problem." + arguments: + comparator: GT + failureState: "CRITICAL" + threshold: 0.0 + metric: + query: |- + sum by(namespace, cluster_name, pod_name) (min_over_time(delta(stackstate_receiver_akka_http_requests_active{kube_app_component=~"receiver.*"}[1m])[5m:1m])) + aliasTemplate: "# of active requests" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/http-growing-requests.md.hbs + - _type: "Monitor" + name: "Receiver - No data received" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-no-data + status: "ENABLED" + description: "When SUSE Observability is not receiving data this monitor will trigger." + arguments: + failureState: "DEVIATING" + threshold: 0.00000001 + comparator: LTE + metric: + query: |- + sum by (cluster_name, namespace, pod_name) (rate(stackstate_receiver_agent_request{valid_api_key="true"}[5m])) + aliasTemplate: "# of requests, valid_api_key=${valid_api_key}" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: |- + No more data flowing into SUSE Observability will result in most of the topology slowly disappearing based on timeouts. + Common causes are: + - An invalid api key is used by the agent, if this is the case there will be 2 lines in the chart, one for valid_api_key="true" and one for valid_api_key="false" + - SUSE Observability agents in the cluster are not (yet) running + - Network connectivity issues between the agents in the cluster and SUSE Observability, these can also be caused by, for example firewalls or proxy servers. + + In case of invalid api keys try [this query in the metrics explorer](/#/metrics?promql=rate(stackstate_receiver_agent_request{valid_api_key%3D"false"}[%24{__rate_interval}])) to see the network addresses for the agents with the invalid api key and redeploy them with a valid api key. + - _type: "Monitor" + name: "Receiver - Logs rejected" + tags: + - receiver + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-logs-rejected + status: "ENABLED" + description: "SUSE Observability reserves disk space for logs such that the retention period (default 7 days) can be met for all logs. Based on retention and available disk space each log source is allocated a fixed amount of log space per hour, any excessive logs will be rejected (reported as truncated in the log) however short-lived spikes are accepted. When logs are rejected this monitor triggers." + arguments: + failureState: "DEVIATING" + threshold: 0 + comparator: GT + metric: + query: |- + sum by (cluster_name, namespace, pod_name) (rate(stackstate_stackstate_receiver_events_rejected_size{element_type="k8sLogs"}[5m])) + aliasTemplate: "Rejected logs (bytes) per second" + unit: bytes/sec(IEC) + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/receiver/rejected-logs.md.hbs + - _type: "Monitor" + name: "Receiver - Memory Limiter saturation" + tags: + - receiver + - memory + function: urn:stackpack:common:monitor-function:threshold + arguments: + metric: + query: "stackstate_stackstate_receiver_memory_limiter_claimed_size / stackstate_stackstate_receiver_memory_limiter_limit" + unit: "percentunit" + aliasTemplate: "Memory Limiter saturation % = ${namespace}" + threshold: 0.8 + comparator: GTE + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-receiver-memory-limiter-saturation + status: "ENABLED" + description: "he SUSE Observability receiver rejects requests if the allocated memory is (almost) exhausted. This protects the receiver from sudden spikes in the incoming data volume. However if this happens permanently or for long periods data will be dropped and the receiver needs more memory." + intervalSeconds: 60 + remediationHint: !include services/receiver/memory-limiter.md.hbs diff --git a/stackpacks/suse-observability-2/settings/services/receiver/pod-presentation.sty b/stackpacks/suse-observability-2/settings/services/receiver/pod-presentation.sty new file mode 100644 index 0000000..6ee30ef --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/receiver/pod-presentation.sty @@ -0,0 +1,257 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Receiver" + description: "ComponentPresentation for SUSE Observability Receiver" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-receiver-pod + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:receiver-base", "app.kubernetes.io/component:receiver-process-agent", "app.kubernetes.io/component:receiver") + and type in ("pod") + rank: + specificity: 10 + presentation: + highlight: + title: "SUSE Observability Receiver" + fields: [] + links: + - linkId: data-ingestion-dashboard + title: "Data ingestion dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:data-ingestion" + tooltip: "Open the bundled SUSE Observability data ingestion dashboard" + order: 100.0 + - linkId: topology-processing-dashboard + title: "Topology processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:topology-processing" + tooltip: "Open the bundled SUSE Observability topology processing dashboard" + order: 90.0 + - linkId: traces-processing-dashboard + title: "Traces processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:traces-processing" + tooltip: "Open the bundled SUSE Observability traces processing dashboard" + order: 80.0 + - linkId: logs-processing-dashboard + title: "Logs processing dashboard" + target: "#/dashboards/urn:stackpack:suse-observability-2:shared:dashboard:logs-processing" + tooltip: "Open the bundled SUSE Observability logs processing dashboard" + order: 70.0 + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: receiver + title: Receiver + order: 100 + metrics: + - metricId: sts-receiver-incoming-data + - metricId: sts-receiver-pod-current-incoming-element-count + - metricId: sts-receiver-pod-current-incoming-element-created + - metricId: sts-receiver-pod-budget-saturation + - metricId: sts-receiver-pod-created-budget-saturation + - metricId: sts-receiver-pod-agent-count + - metricId: sts-receiver-pod-created-elements + - metricId: sts-receiver-pod-active-http-requests + - metricId: sts-receiver-pod-log-volume + - metricId: sts-receiver-pod-rejected-log-volume + metrics: + - + metricId: sts-receiver-pod-current-incoming-element-count + name: SUSE Observability Receiver - Current incoming element (approximate) count + description: Amount of data being offered to the receiver at this moment + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-current-incoming-element-created + name: SUSE Observability Receiver - Created elements in the last hour (approximate) count + description: Amount of elements expected to result in new element creates in SUSE Observability + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_request_approx_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: Budget ${element_type} + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-budget-saturation + name: SUSE Observability Receiver - Element budget saturation + description: Percentage of agent element processing budget filled up + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) / sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_unique_element_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-receiver-pod-created-budget-saturation + name: SUSE Observability Receiver - Created elements hourly budget saturation + description: Percentage of newly created elements budget filled up + metricQueries: + - expression: |- + sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) / sum by (element_type) (avg_over_time(stackstate_stackstate_receiver_element_create_passed_max{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}])) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: sts-receiver-pod-agent-count + name: SUSE Observability Receiver - Connected agent count + description: Number of currently connected agents + metricQueries: + - expression: |- + max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + element_type="agent", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__interval}]) + alias: Agent Count + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-created-elements + name: SUSE Observability Receiver - Created elements hourly total + description: Total elements created per hour + metricQueries: + - expression: |- + sum by (element_type) (stackstate_stackstate_receiver_element_create_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: "${element_type}" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-active-http-requests + name: SUSE Observability Receiver - Active HTTP Requests + description: Currently active http requests, this should not be an ever increasing line (see monitor) + metricQueries: + - expression: |- + delta(stackstate_akka_http_requests_active{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Active Requests + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-receiver-pod-log-volume + name: SUSE Observability Receiver - Log volume + description: The volume of logs received by Stackstate, accepted logs are stored, rejected logs are dropped because they are over the allocated disk-space + metricQueries: + - expression: |- + rate(stackstate_stackstate_receiver_events_accepted_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Accepted + - expression: |- + rate(stackstate_stackstate_receiver_events_rejected_size{ + element_type="k8sLogs", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}]) + alias: Rejected + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-pod-rejected-log-volume + name: SUSE Observability Receiver - Rejected Log volume per pod + description: The volume of logs rejected by Stackstate per pod + metricQueries: + - expression: |- + topk(10, rate(stackstate_stackstate_receiver_logs_rejected_size{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }[${__rate_interval}])) @ end() + alias: ${limited_pod_name} Rejected by - ${cluster_name}/${namespace}/${pod_name} + componentIdentifierTemplate: "urn:kubernetes:/${cluster_name}/pod/${limited_pod_uid}" + - expression: |- + max(stackstate_stackstate_receiver_logs_budget_per_pod { + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${name}" + }) + alias: Logs budget per pod per time window + chart: + _type: TimeSeriesChart + unit: bytes/sec(IEC) + - + metricId: sts-receiver-incoming-data + name: Incoming data + description: "Maximum count of unique elements received from agents." + metricQueries: + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster')}", + namespace="${tags.singleValue('namespace')}", + element_type='agent' + }[${__interval}])) + alias: node + - expression: |- + max by (element_type)(max_over_time(stackstate_stackstate_receiver_unique_element_passed_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + element_type!='agent' + }[${__interval}])) + alias: ${element_type} + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: [] diff --git a/stackpacks/suse-observability-2/settings/services/server/presentation.sty b/stackpacks/suse-observability-2/settings/services/server/presentation.sty new file mode 100644 index 0000000..e78f7c7 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/server/presentation.sty @@ -0,0 +1,41 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Server" + description: "ComponentPresentation for SUSE Observability Server" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-server + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:server") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: consumer-lag + title: Consumer lag + order: 100 + metrics: + - metricId: sts-server-kafka-lag + metrics: + - + metricId: sts-server-kafka-lag + description: 'Kafka consumer lag per topic' + name: Kafka Topic lag + metricQueries: + - expression: |- + sum by (pod_name, topic) (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${topic} + chart: + _type: TimeSeriesChart + unit: cps \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/state/presentation.sty b/stackpacks/suse-observability-2/settings/services/state/presentation.sty new file mode 100644 index 0000000..0487dba --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/state/presentation.sty @@ -0,0 +1,134 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability State Propagation" + description: "ComponentPresentation for SUSE Observability State Propagation" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-state + binding: + _type: ComponentPresentationQueryBinding + query: label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") and label IN ("app.kubernetes.io/component:state", "app.kubernetes.io/component:server") and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: state-propagation + title: State Propagation + order: 100 + metrics: + - metricId: sts-topo-state-calc-duration + - metricId: sts-topo-state-calc-batches-rate + - metricId: sts-topo-state-calc-steps-duration + - metricId: sts-topo-state-persistence-duration + - metricId: sts-topo-state-queued-rate + - metricId: sts-topo-state-latency + metrics: + - + metricId: sts-topo-state-calc-duration + description: 'State Propagation - Calculation duration' + name: State Propagation - Calculation duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_propagation_calculation_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-state-calc-batches-rate + description: 'State Propagation - Calculation batches rate' + name: State Propagation - Calculation batches rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_state_propagation_calculation_duration_seconds_count{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-topo-state-calc-steps-duration + description: 'State Propagation - Calculation steps duration' + name: State Propagation - Calculation steps duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_propagation_calculation_steps_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: ${pod_name} - ${step} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-state-persistence-duration + description: 'State Propagation - Persistence duration' + name: State Propagation - Persistence duration + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_propagation_persistence_duration_seconds_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-state-queued-rate + description: 'State Propagation - Dequeued/Enqueued rate' + name: State Propagation - Dequeued commands rate + metricQueries: + - expression: |- + rate(stackstate_stackstate_state_propagation_dequeued_commands{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - Dequeued + - expression: |- + rate(stackstate_stackstate_state_propagation_enqueued_commands{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[${__interval}]) + alias: ${pod_name} - Enqueued + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-topo-state-latency + description: 'State State Propagation - Latency' + name: State Propagation - Latency + metricQueries: + - expression: |- + histogram_quantile(0.98, rate(stackstate_stackstate_state_calculation_latency_bucket{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + })) + alias: "${pod_name}" + chart: + _type: TimeSeriesChart + unit: s \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/topo-sync/monitors.sty b/stackpacks/suse-observability-2/settings/services/topo-sync/monitors.sty new file mode 100644 index 0000000..042a201 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/topo-sync/monitors.sty @@ -0,0 +1,115 @@ +nodes: + - _type: "Monitor" + name: "Topology Sync - latency too high" + tags: + - jvm + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-topo-sync-latency-too-high + status: "ENABLED" + description: "Monitors the latency of the topology sync and goes to a DEVIATING state when it is too high. Latency is measured from time of collection to commit of the data" + arguments: + comparator: GT + titleTemplate: "Topology Sync - latency too high - ${integration_type}" + failureState: DEVIATING + threshold: 45.000000 + metric: + query: |- + max by (cluster_name, namespace, pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="source", end_stage="commit", quantile="1.0"}) + aliasTemplate: "${integration_type} latency" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/latency.md.hbs + + - _type: "Monitor" + name: "Topology Sync - high processing time" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-topo-sync-high-processing-time-per-message + status: "ENABLED" + description: "Monitors the average processing time and goes to a DEVIATING state when it is considered too high. Processing time is measured from receiver to commit of the data." + arguments: + comparator: GT + threshold: 30.0 + failureState: DEVIATING + metric: + query: max by (cluster_name, namespace, pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{start_stage="enterReceiver", end_stage="commit", quantile="1.0"}) + aliasTemplate: "${integration_type} processing time" + unit: s + titleTemplate: "Topology Sync - high processing time - ${integration_type}" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/internal-latency.md.hbs + + - _type: "Monitor" + name: "Topology Sync - Sync Workers blocked" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-topo-sync-workers-blocked + status: "ENABLED" + description: "Monitors the total time for the sync workers have not been processing any messages due to an error." + arguments: + comparator: GT + titleTemplate: "Topology Sync - Workers blocked - ${worker_name}" + failureState: "CRITICAL" + threshold: 180 + metric: + query: |- + max by (worker_name, pod_name, namespace, cluster_name)(last_over_time(stackstate_stackstate_sync_blocked_duration_seconds{}[5m])) + aliasTemplate: "${worker_name} blockage" + unit: s + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/blocked.md.hbs + + - _type: "Monitor" + name: "Topology Sync - SnapshotDataFromMultipleHosts error" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-topo-sync-snapshot-data-multiple-hosts + status: "ENABLED" + description: "Monitors whether snapshot data for a single synchronization is coming from multiple hosts." + arguments: + comparator: GT + titleTemplate: "Topology Sync - SnapshotDataFromMultipleHosts - ${name}" + failureState: DEVIATING + threshold: 0 + metric: + query: |- + max by (name, pod_name, namespace, cluster_name)(rate(stackstate_stackstate_sync_extTopo_batch_errors{category="SnapshotDataFromMultipleHosts"}[5m])) + aliasTemplate: "${name} SnapshotDataFromMultipleHosts errors" + unit: ops/sec (ops) + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/multiple-hosts.md.hbs + + - _type: "Monitor" + name: "Topology Sync - SnapshotAlreadyOpen error" + tags: + - suse-observability + - topo-sync + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:sts-topo-sync-snapshot-already-open + status: "ENABLED" + description: "Monitors whether snapshots are properly closed." + arguments: + comparator: GT + titleTemplate: "Topology Sync - SnapshotAlreadyOpen - ${name}" + failureState: DEVIATING + threshold: 0 + metric: + query: |- + max by (name, pod_name, namespace, cluster_name)(stackstate_stackstate_sync_extTopo_batch_errors{category="SnapshotAlreadyOpen"}[5m]) > 0 + aliasTemplate: "${name} SnapshotAlreadyOpen errors" + unit: ops + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 30 + remediationHint: !include services/topo-sync/snapshot-already-open.md.hbs \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/topo-sync/presentation.sty b/stackpacks/suse-observability-2/settings/services/topo-sync/presentation.sty new file mode 100644 index 0000000..48292a0 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/topo-sync/presentation.sty @@ -0,0 +1,160 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Topology Sync" + description: "ComponentPresentation for SUSE Observability Topology Sync" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-topology-sync + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:sync", "app.kubernetes.io/component:server") + and type in ("pod", "deployment") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: topology-sync + title: Topology Sync + order: 100 + metrics: + - metricId: sts-topo-sync-elements-total + - metricId: sts-topo-sync-latency + - metricId: sts-topo-sync-internal-latency + - metricId: sts-topo-sync-changes-rate + - metricId: sts-sync-kafka-lag + - + sectionId: storage + title: Storage + order: 100 + metrics: + - metricId: sts-topo-sync-storage + metrics: + - + metricId: sts-topo-sync-elements-total + description: 'Topology Sync total number of elements' + name: Topology Sync Elements count + metricQueries: + - expression: |- + stackstate_stackstate_sync_topo_total_counts{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + } + alias: ${pod_name} - ${element_type}s + chart: + _type: TimeSeriesChart + unit: short + - + metricId: sts-topo-sync-latency + description: 'Topology Sync latency per integration type measured from collection time to database' + name: Topology Sync Latency + metricQueries: + - expression: |- + max by (pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{ + start_stage="source", + end_stage="commit", + quantile="1.0", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${integration_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-sync-internal-latency + description: 'Topology Sync internal latency per integration type measured from reception time to database' + name: Topology Sync Internal Latency + metricQueries: + - expression: |- + max by (pod_name, integration_type)(stackstate_stackstate_sync_stage_latency_seconds_quantile{ + start_stage="enterReceiver", + end_stage="commit", + quantile="1.0", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${integration_type} + chart: + _type: TimeSeriesChart + unit: s + - + metricId: sts-topo-sync-changes-rate + description: 'Topology Sync changes rate per synchronization' + name: Topology Sync Changes rate + metricQueries: + - expression: |- + (sum by (node_id, operation_type, pod_name)(rate(stackstate_stackstate_sync_extTopo_changes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[1m])) or sum by (node_id, operation_type, pod_name)(rate(stackstate_stackstate_sync_extTopo_changes_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }[1m]))) > 0 + alias: ${pod_name} - ${node_id} - ${operation_type} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-sync-kafka-lag + description: 'Kafka consumer lag per topic' + name: Topology Sync Topic lag + metricQueries: + - expression: |- + sum by (pod_name, topic) (stackstate_kafka_consumer_consumer_fetch_manager_metrics_records_lag{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + kube_app_component="${tags.singleValue('app.kubernetes.io/component')}", + kube_app_name=~"stackstate|stackstate-k8s|suse-observability" + }) + alias: ${pod_name} - ${topic} + chart: + _type: TimeSeriesChart + unit: cps + - + metricId: sts-topo-sync-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by Kafka, HBase, and Sync." + metricQueries: + - expression: |- + max by (storage)(label_replace( + max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)" + }[5m]) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~".*-(kafka-\\d|hbase-.*|sync-tmp)" + }[5m]), "storage", "$1", "persistentvolumeclaim", ".*-(kafka-.*|hbase-.*|sync-tmp)")) + alias: ${storage} + chart: + _type: GaugeChart + unit: percentunit + decimals: 0 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + diff --git a/stackpacks/suse-observability-2/settings/services/victoria-metrics/monitors.sty b/stackpacks/suse-observability-2/settings/services/victoria-metrics/monitors.sty new file mode 100644 index 0000000..fac2c54 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/victoria-metrics/monitors.sty @@ -0,0 +1,234 @@ +nodes: + - _type: "Monitor" + name: "VictoriaMetrics - backups performed" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-0-backups-not-executed + arguments: + metric: + query: |- + sum(increase(stackstate_supercronic_successful_executions{kube_app_name=~"victoria-metrics-.*"}[12h])) by (cluster_name, namespace, pod_name) or sum(kubernetes_state_container_running{pod_name=~"stackstate-victoria-metrics-.*", container="vmbackup"}) by (cluster_name, namespace, pod_name) * 0 + unit: "short" + aliasTemplate: "Number of backups execution in 12h window" + comparator: LTE + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Backup performed in the last 12 hours" + intervalSeconds: 3600 + remediationHint: "Potential reasons why backups haven't been executed:\n- wrong configuration of MinIO (like secrets, bucket name and so on)\n- interval between backups is longer than 12h (you have to change the configuration of the Monitor)\n\nTo investigate the problem you should check:\n- logs of `vmbackup` container\n- logs/status of `MinIO` pods" + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Disk runs out of space in 3 days" + description: "Monitor that triggers when free disk space will be enough only for less than 3 days" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-disk-space-3days + arguments: + metric: + query: |- + sum ( + stackstate_vm_free_disk_space_bytes / ignoring(path) + ( + ( + rate(stackstate_vm_rows_added_to_storage_total[1d]) - + ignoring(type) rate(stackstate_vm_deduplicated_samples_total{type="merge"}[1d]) + ) + * scalar( + sum(stackstate_vm_data_size_bytes{type!~"indexdb.*"}) / + sum(stackstate_vm_rows{type!~"indexdb.*"}) + ) + ) + ) by (cluster_name, namespace, pod_name) + unit: "s" + aliasTemplate: "Time to fill the disk space" + comparator: LTE + threshold: 259200 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Time to fill the disk space" + intervalSeconds: 90 + remediationHint: "Taking into account current ingestion rate, free disk space will be enough only + for less than 3 days.\n + Consider to limit the ingestion rate, decrease retention or scale the disk space if possible." + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Disk runs out of space" + description: "Monitor that triggers when disk utilisation on the instance is more than 80%." + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-disk-space + arguments: + metric: + query: |- + sum(stackstate_vm_data_size_bytes) by (cluster_name, namespace, pod_name) / + ( + sum(stackstate_vm_free_disk_space_bytes) by (cluster_name, namespace, pod_name) + + sum(stackstate_vm_data_size_bytes) by (cluster_name, namespace, pod_name) + ) + unit: "percentunit" + aliasTemplate: "Disk space usage" + comparator: GT + threshold: 0.8 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Disk space usage" + intervalSeconds: 90 + remediationHint: "Disk utilisation on the instance is more than 80%.\n + Having less than 20% of free disk space could cripple merges processes and overall performance. + Consider to limit the ingestion rate, decrease retention or scale the disk space if possible." + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Requests errors to api" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-errors + arguments: + metric: + query: |- + sum(increase(stackstate_vm_http_request_errors_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Errors" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Request errors to api" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/errors.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Rows rejected on ingestion" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-rows-rejected + arguments: + metric: + query: |- + sum(rate(stackstate_vm_rows_ignored_total[5m])) by (cluster_name, namespace, pod_name, instance, reason) + unit: "short" + aliasTemplate: "Rows rejected" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Rows rejected due to: ${reason}" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/rejected.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Too high churn rate" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-churn-rate + arguments: + metric: + query: |- + sum(rate(stackstate_vm_new_timeseries_created_total[5m])) by (cluster_name, namespace, pod_name) + / + sum(rate(stackstate_vm_rows_inserted_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Time series created" + comparator: GT + threshold: 0.1 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "High churn rate" + intervalSeconds: 90 + description: "VM constantly creates new time series.\n + This effect is known as Churn Rate.\n + High Churn Rate tightly connected with database performance and may + result in unexpected OOM's or slow queries." + remediationHint: "The [cardinality explorer](https://docs.victoriametrics.com/#cardinality-explorer) of Victoria Metrics can help determine which metrics have the highest cardinality. These are likely sources of churn. Only known solution is to investigate the source of the metric and remove labels that change frequently." + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Too high churn rate 24hrs" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-churn-rate-24 + status: "ENABLED" + description: "The number of created new time series over last 24h is 4x times higher than + current number of active series.\n + This effect is known as Churn Rate.\n + High Churn Rate tightly connected with database performance and may + result in unexpected OOM's or slow queries." + arguments: + titleTemplate: "High churn rate 24hrs" + comparator: GT + failureState: "DEVIATING" + threshold: 1.0 + metric: + query: |- + sum(increase(stackstate_vm_new_timeseries_created_total[24h])) by (cluster_name, namespace, pod_name, container) + / sum(max_over_time(stackstate_vm_cache_entries{type="storage/hour_metric_ids"}[24h])*4) by (cluster_name, namespace, pod_name, container) + aliasTemplate: "Time series created" + unit: short + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + intervalSeconds: 90 + remediationHint: "The [cardinality explorer](https://docs.victoriametrics.com/#cardinality-explorer) of Victoria Metrics can help determine which metrics have the highest cardinality. These are likely sources of churn. Only known solution is to investigate the source of the metric and remove labels that change frequently." + + - _type: "Monitor" + name: "VictoriaMetrics - Too high slow insert rate" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-slow-insert-rate + arguments: + metric: + query: |- + sum(rate(stackstate_vm_slow_row_inserts_total[5m])) by (cluster_name, namespace, pod_name) + / + sum(rate(stackstate_vm_rows_inserted_total[5m])) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Percentage slow inserts" + comparator: GT + threshold: 0.05 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "High slow insert rate" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/slow-insert-rate.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "VictoriaMetrics - Labels limit exceeded on ingestion" + tags: + - victoria-metrics + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:victoria-metrics-labels-limit + arguments: + metric: + query: |- + sum(increase(stackstate_vm_metrics_with_dropped_labels_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Dropped labels" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Labels limit exceeded on ingestion" + intervalSeconds: 90 + remediationHint: !include services/victoria-metrics/labels-limit.md.hbs + status: "ENABLED" diff --git a/stackpacks/suse-observability-2/settings/services/victoria-metrics/presentation.sty b/stackpacks/suse-observability-2/settings/services/victoria-metrics/presentation.sty new file mode 100644 index 0000000..af22703 --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/victoria-metrics/presentation.sty @@ -0,0 +1,194 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability Victoria Metrics" + description: "ComponentPresentation for SUSE Observability Victoria Metrics" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-victoria-metrics + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:victoria-metrics-0", "app.kubernetes.io/name:victoria-metrics-1") + AND type IN ("statefulset") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - + tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - + sectionId: backup + title: Backup + order: 90 + metrics: + - metricId: victoria-metrics-backup-duration + - + sectionId: performance + title: Performance + order: 100 + metrics: + - metricId: victoria-metrics-storage + - metricId: victoria-metrics-churn-rate + - metricId: victoria-metrics-timeseries-dropped + - metricId: victoria-metrics-slow-inserts + - metricId: victoria-metrics-slow-queries + - + sectionId: limits + title: Limits + order: 80 + metrics: + - metricId: victoria-metrics-labels-limit + metrics: + - + metricId: victoria-metrics-backup-duration + name: VictoriaMetrics - Backup duration + description: Duration of VictoriaMetrics backups + metricQueries: + - expression: |- + increase(stackstate_supercronic_cron_execution_time_seconds_sum{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}]) > 0 + alias: "${kube_app_name}" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: victoria-metrics-churn-rate + name: VictoriaMetrics - Churn rate + description: "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality" + metricQueries: + - expression: |- + sum(rate(stackstate_vm_new_timeseries_created_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Time series created" + - expression: |- + sum(increase(stackstate_vm_new_timeseries_created_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[24h])) + alias: "Time series created over 24 hrs" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: victoria-metrics-slow-inserts + name: VictoriaMetrics - Slow inserts + description: "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. + The less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. + In general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values." + metricQueries: + - expression: |- + max(rate(stackstate_vm_slow_row_inserts_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name="${tags.singleValue('statefulset.kubernetes.io/pod-name')}" + }[${__rate_interval}]) / rate(stackstate_vm_rows_added_to_storage_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Percentage of slow inserts" + chart: + _type: TimeSeriesChart + unit: percentunit + - + metricId: victoria-metrics-slow-queries + name: VictoriaMetrics - Slow queries rate + description: "Slow queries rate according to `search.logSlowQueryDuration` flag, which is `5s` by default." + metricQueries: + - expression: |- + sum(rate(stackstate_vm_slow_queries_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Slow queries rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: victoria-metrics-labels-limit + name: VictoriaMetrics - Dropped timeseries + description: "VictoriaMetrics limits the number of labels per timeseries with `-maxLabelsPerTimeseries` command-line flag. + This prevents from ingesting metrics with too many labels. The value of `maxLabelsPerTimeseries` must be adjusted for your workload. + When limit is exceeded (graph is > 0) - the timeseries with too many labels are dropped, which results in missing metric data." + metricQueries: + - expression: |- + sum(increase(vm_rows_ignored_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-victoria-metrics-.*" + }[${__rate_interval}])) + alias: "Slow queries rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: victoria-metrics-storage + name: Volume usage + description: "Percentage of persistent volume capacity used by VictoriaMetrics." + chart: + _type: GaugeChart + unit: percentunit + decimalPlaces: 0 + max: 1 + calculation: last + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 0.9 + - color: "#FF7F00" + value: 0.8 + mode: absolute + metricQueries: + - expression: |- + max by (storage)(label_replace(max_over_time(kubernetes_kubelet_volume_stats_used_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m] + ) / max_over_time(kubernetes_kubelet_volume_stats_capacity_bytes{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + persistentvolumeclaim=~"server-volume-.*-(victoria-metrics-.+)"}[5m] + ), "storage","$1", "persistentvolumeclaim", "server-volume-.*-(victoria-metrics-.+)")) + alias: ${storage} + - + metricId: victoria-metrics-timeseries-dropped + name: Dropped timeseries in the past hour + description: >- + VictoriaMetrics limits the number of labels per timeseries with + `-maxLabelsPerTimeseries` command-line flag. + + This prevents from ingesting metrics with too many labels. The value + of `maxLabelsPerTimeseries` must be adjusted for your workload. + + When limit is exceeded (graph is > 0) - the timeseries with too many + labels are dropped, which results in missing metric data. + metricQueries: + - expression: |- + sum by (statefulset)(increase(stackstate_vm_rows_ignored_total{ + pod_name=~".*-victoria-metrics-.*", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}" + }[1h])) + alias: VictoriaMetrics + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 10 diff --git a/stackpacks/suse-observability-2/settings/services/vmagent/monitors.sty b/stackpacks/suse-observability-2/settings/services/vmagent/monitors.sty new file mode 100644 index 0000000..fd979fd --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/vmagent/monitors.sty @@ -0,0 +1,143 @@ +nodes: + - _type: "Monitor" + name: "Vmagent - Persistent queue is droppingData" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:vmagent-queue-dropping-data + arguments: + metric: + query: |- + sum(increase(stackstate_vm_persistentqueue_bytes_dropped_total[5m])) by (cluster_name, namespace, pod_name) + unit: "bytes(IEC)" + aliasTemplate: "Bytes dropped" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Persistent queue is dropping data" + intervalSeconds: 90 + remediationHint: !include services/vmagent/persistentqueue-dropping-data.md.hbs + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - RemoteWrite packets dropped" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:vmagent-remotewrite-packets-dropped + arguments: + metric: + query: |- + sum(increase(stackstate_vmagent_remotewrite_packets_dropped_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Packets dropped" + comparator: GT + threshold: 0 + failureState: "CRITICAL" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "RemoteWrite packets dropped" + intervalSeconds: 90 + remediationHint: "Vmagent drops the rejected by remote-write server data blocks. Check the logs to find the reason for rejects" + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Too many write errors" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:vmagent-errors + arguments: + metric: + query: |- + (sum(increase(stackstate_vm_ingestserver_request_errors_total[5m])) by (cluster_name, namespace, pod_name) + + + sum(increase(stackstate_vmagent_http_request_errors_total[5m])) by (cluster_name, namespace, pod_name)) + unit: "short" + aliasTemplate: "Error count" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Write errors" + intervalSeconds: 90 + remediationHint: "Vmagent responds with errors to write requests. Check the logs to find the reason for the errors" + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Too many remote write errors" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:vmagent-remotewrite-errors + arguments: + metric: + query: |- + sum(rate(stackstate_vmagent_remotewrite_retries_count_total[5m])) by (cluster_name, namespace, pod_name) + unit: "short" + aliasTemplate: "Error count" + comparator: GT + threshold: 0 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Remote write errors" + intervalSeconds: 90 + remediationHint: "Vmagent responds with errors to remote write requests. Check the logs to find the reason for the errors" + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Remote write connection is saturated" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:vmagent-remotewrite-connection-saturated + arguments: + metric: + query: |- + sum( + rate(stackstate_vmagent_remotewrite_send_duration_seconds_total[5m]) + / + stackstate_vmagent_remotewrite_queues + ) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Saturation %" + comparator: GT + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Remote write connection saturated" + intervalSeconds: 90 + remediationHint: "The remote write connection between vmagent and destination + is saturated by more than 90% and vmagent won't be able to keep up.\n + This usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase + the number of connections per each remote storage." + status: "ENABLED" + + - _type: "Monitor" + name: "Vmagent - Persistent queue for writes is saturated" + tags: + - vmagent + - suse-observability + function: urn:stackpack:common:monitor-function:threshold + identifier: urn:stackpack:suse-observability-2:shared:monitor:vmagent-persistent-queue-saturated + arguments: + metric: + query: |- + sum(rate(stackstate_vm_persistentqueue_write_duration_seconds_total[5m])) by (cluster_name, namespace, pod_name) + unit: "percentunit" + aliasTemplate: "Saturation %" + comparator: GT + threshold: 0.9 + failureState: "DEVIATING" + urnTemplate: "urn:kubernetes:/${cluster_name}:${namespace}:pod/${pod_name}" + titleTemplate: "Persistent queue writes saturated" + intervalSeconds: 90 + remediationHint: "Persistent queue writes for vmagent + are saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. + In this case, consider to decrease load on the vmagent or improve the disk throughput." + status: "ENABLED" \ No newline at end of file diff --git a/stackpacks/suse-observability-2/settings/services/vmagent/presentation.sty b/stackpacks/suse-observability-2/settings/services/vmagent/presentation.sty new file mode 100644 index 0000000..a4081cd --- /dev/null +++ b/stackpacks/suse-observability-2/settings/services/vmagent/presentation.sty @@ -0,0 +1,121 @@ +nodes: + - _type: ComponentPresentation + name: "SUSE Observability VM Agent" + description: "ComponentPresentation for SUSE Observability VM Agent" + identifier: urn:stackpack:suse-observability-2:component-presentation:service-vmagent + binding: + _type: ComponentPresentationQueryBinding + query: |- + label IN ("app.kubernetes.io/name:stackstate", "app.kubernetes.io/name:stackstate-k8s", "app.kubernetes.io/name:suse-observability") + and label IN ("app.kubernetes.io/component:vmagent") + and type in ("statefulset") + rank: + specificity: 10 + presentation: + metricPerspective: + tabs: + - tabId: suse-observability + title: SUSE Observability + order: 100 + sections: + - sectionId: vmagent + title: VMAgent + order: 100 + metrics: + - metricId: vmagent-write-duration + - metricId: vmagent-blocks-dropped + - metricId: vmagent-datapoints-rate + - metricId: vmagent-invalid-datapoints-rate + - metricId: vmagent-timeseries-dropped + metrics: + - + metricId: vmagent-write-duration + name: Persistent queue write saturation + description: "Shows saturation persistent queue for writes. If the threshold of 0.9sec is reached, then persistent is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput." + metricQueries: + - expression: |- + max(rate(stackstate_vm_persistentqueue_write_duration_seconds_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) + alias: "Write duration" + chart: + _type: TimeSeriesChart + unit: s + - + metricId: vmagent-blocks-dropped + name: Data blocks dropped + description: "Shows the rate of dropped data blocks in cases when remote storage replies with `400 Bad Request` and `409 Conflict` HTTP responses. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149" + metricQueries: + - expression: |- + sum(rate(stackstate_vmagent_remotewrite_packets_dropped_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) by(url) + alias: "Data blocks dropped" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: vmagent-datapoints-rate + name: Data points rate + description: "Shows the rate of parsed data points from write or scrape requests." + metricQueries: + - expression: |- + sum(rate(stackstate_vm_protoparser_rows_read_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) by(type) + alias: "Data points rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: vmagent-invalid-datapoints-rate + name: Invalid data points rate + description: "Tracks the rate of dropped invalid rows because of errors while unmarshaling write requests. The exact errors messages will be printed in logs." + metricQueries: + - expression: |- + sum(rate(stackstate_vm_rows_invalid_total{ + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}", + pod_name=~".*-vmagent-.*", + }[${__rate_interval}])) by(type) + alias: "Invalid data points rate" + chart: + _type: TimeSeriesChart + unit: short + - + metricId: vmagent-timeseries-dropped + name: Dropped timeseries in the past hour + description: >- + VictoriaMetrics limits the number of labels per timeseries with + `-maxLabelsPerTimeseries` command-line flag. + + This prevents from ingesting metrics with too many labels. The value + of `maxLabelsPerTimeseries` must be adjusted for your workload. + + When limit is exceeded (graph is > 0) - the timeseries with too many + labels are dropped, which results in missing metric data. + metricQueries: + - expression: |- + sum by (statefulset)(increase(stackstate_vm_rows_ignored_total{ + pod_name=~".*-vmagent-.*", + cluster_name="${tags.singleValue('cluster-name')}", + namespace="${tags.singleValue('namespace')}" + }[1h])) + alias: vmagent + chart: + _type: StatChart + unit: short + decimals: 0 + calculation: last + sparkline: true + thresholds: + defaultColor: "#33A02C" + steps: + - color: "#E31A1C" + value: 10 diff --git a/stackpacks/suse-observability-2/stackpack.yaml b/stackpacks/suse-observability-2/stackpack.yaml new file mode 100644 index 0000000..5174bfa --- /dev/null +++ b/stackpacks/suse-observability-2/stackpack.yaml @@ -0,0 +1,17 @@ +name: suse-observability-2 +version: "0.0.4" +schemaVersion: "2.0" +displayName: "SUSE Observability 2.0" +categories: [ "SUSE Observability" ] +logoUrl: !resource "logo.png" +overviewUrl: !resource "overview.md" +detailedOverviewUrl: !resource "detailed-overview.md" +configurationUrls: + NOT_INSTALLED: !resource "configuration.md" + PROVISIONING: !resource "provisioning.md" + WAITING_FOR_DATA: !resource "waitingfordata.md" + INSTALLED: !resource "enabled.md" + DEPROVISIONING: !resource "configuration.md" + ERROR: !resource "configuration.md" +provision: + companionStackPacks: []