From 7ceb0940dca371dd527e8ef7636c76cdcebf8980 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 30 Jul 2026 21:20:01 +0530 Subject: [PATCH 01/61] feat(helm): add HStore deployment chart --- .github/workflows/helm-chart-ci.yml | 82 ++ README.md | 11 +- helm/hugegraph/.helmignore | 33 + helm/hugegraph/Chart.yaml | 36 + helm/hugegraph/README.md | 396 +++++++++ helm/hugegraph/templates/NOTES.txt | 46 ++ helm/hugegraph/templates/_helpers.tpl | 283 +++++++ helm/hugegraph/templates/pd-pdb.yaml | 32 + .../templates/pd-service-client.yaml | 36 + .../templates/pd-service-headless.yaml | 41 + helm/hugegraph/templates/pd-statefulset.yaml | 158 ++++ .../templates/server-deployment.yaml | 242 ++++++ helm/hugegraph/templates/server-hpa.yaml | 40 + helm/hugegraph/templates/server-ingress.yaml | 53 ++ helm/hugegraph/templates/server-pdb.yaml | 33 + helm/hugegraph/templates/server-service.yaml | 41 + helm/hugegraph/templates/serviceaccount.yaml | 35 + helm/hugegraph/templates/store-pdb.yaml | 32 + .../templates/store-service-headless.yaml | 41 + .../templates/store-statefulset.yaml | 193 +++++ .../templates/tests/test-connection.yaml | 67 ++ .../testdata/values-pre-hardening.yaml | 128 +++ helm/hugegraph/values-cluster.yaml | 86 ++ helm/hugegraph/values-single.yaml | 40 + helm/hugegraph/values.schema.json | 765 ++++++++++++++++++ helm/hugegraph/values.yaml | 256 ++++++ 26 files changed, 3204 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/helm-chart-ci.yml create mode 100644 helm/hugegraph/.helmignore create mode 100644 helm/hugegraph/Chart.yaml create mode 100644 helm/hugegraph/README.md create mode 100644 helm/hugegraph/templates/NOTES.txt create mode 100644 helm/hugegraph/templates/_helpers.tpl create mode 100644 helm/hugegraph/templates/pd-pdb.yaml create mode 100644 helm/hugegraph/templates/pd-service-client.yaml create mode 100644 helm/hugegraph/templates/pd-service-headless.yaml create mode 100644 helm/hugegraph/templates/pd-statefulset.yaml create mode 100644 helm/hugegraph/templates/server-deployment.yaml create mode 100644 helm/hugegraph/templates/server-hpa.yaml create mode 100644 helm/hugegraph/templates/server-ingress.yaml create mode 100644 helm/hugegraph/templates/server-pdb.yaml create mode 100644 helm/hugegraph/templates/server-service.yaml create mode 100644 helm/hugegraph/templates/serviceaccount.yaml create mode 100644 helm/hugegraph/templates/store-pdb.yaml create mode 100644 helm/hugegraph/templates/store-service-headless.yaml create mode 100644 helm/hugegraph/templates/store-statefulset.yaml create mode 100644 helm/hugegraph/templates/tests/test-connection.yaml create mode 100644 helm/hugegraph/testdata/values-pre-hardening.yaml create mode 100644 helm/hugegraph/values-cluster.yaml create mode 100644 helm/hugegraph/values-single.yaml create mode 100644 helm/hugegraph/values.schema.json create mode 100644 helm/hugegraph/values.yaml diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml new file mode 100644 index 0000000000..a8ad116a48 --- /dev/null +++ b/.github/workflows/helm-chart-ci.yml @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: "Helm Chart CI" + +on: + push: + branches: [ master, 'release-*' ] + paths: [ 'helm/**', '.github/workflows/helm-chart-ci.yml' ] + pull_request: + paths: [ 'helm/**', '.github/workflows/helm-chart-ci.yml' ] + +permissions: + contents: read + +jobs: + lint-and-render: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: azure/setup-helm@v4 + with: + version: v3.16.2 + + - name: helm lint + run: | + helm lint helm/hugegraph + helm lint helm/hugegraph -f helm/hugegraph/values-single.yaml + helm lint helm/hugegraph -f helm/hugegraph/values-cluster.yaml + + - name: helm template + run: | + for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do + helm template ci helm/hugegraph $preset > /dev/null + done + + - name: reject invalid values + run: | + # each of these must fail; the schema and helpers are the contract + ! helm template ci helm/hugegraph --set pd.replicas=100 2>/dev/null + ! helm template ci helm/hugegraph --set pd.pdb.minAvailable=3 2>/dev/null + ! helm template ci helm/hugegraph --set server.hpa.enabled=true 2>/dev/null + ! helm template ci helm/hugegraph --set server.auth.enabled=true 2>/dev/null + + - name: kubeconform + shell: bash + run: | + set -o pipefail + curl -sSLo /tmp/kc.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz + tar -xzf /tmp/kc.tar.gz -C /tmp + for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do + helm template ci helm/hugegraph $preset | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 + done + + - name: legacy --reuse-values compatibility + run: | + # A release created before a field existed must still render. This has + # regressed five times, so it is guarded here rather than by review. + # `-f` is NOT equivalent: it merges over the new defaults, whereas + # --reuse-values discards them, so the fixture must become values.yaml. + cp -R helm/hugegraph /tmp/legacy + cp helm/hugegraph/testdata/values-pre-hardening.yaml /tmp/legacy/values.yaml + helm template legacy /tmp/legacy > /dev/null + helm install legacy /tmp/legacy --dry-run=client > /dev/null + + - name: helm package + run: helm package helm/hugegraph -d /tmp/chart diff --git a/README.md b/README.md index adf9792776..a876069740 100644 --- a/README.md +++ b/README.md @@ -207,9 +207,16 @@ For advanced Docker configurations, see: > > **Version Tags**: Use release tags (e.g., `1.7.0`) for stable deployments. The `latest` tag should only be used for testing or development. +### Option 2: Kubernetes with Helm + +The HStore Helm chart deploys HugeGraph PD, Store, and Server as a distributed +Kubernetes cluster. See the [chart documentation](helm/hugegraph/README.md) for +single-node and highly available presets, configuration, and +upgrade guidance. +
-Option 2: Download Binary Package +Option 3: Download Binary Package Download pre-built packages from the [Download Page](https://hugegraph.apache.org/docs/download/download/): @@ -242,7 +249,7 @@ For detailed instructions, see the [Binary Installation Guide](https://hugegraph
-Option 3: Build from Source +Option 4: Build from Source Build from source for development or customization: diff --git a/helm/hugegraph/.helmignore b/helm/hugegraph/.helmignore new file mode 100644 index 0000000000..8016b1b85d --- /dev/null +++ b/helm/hugegraph/.helmignore @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.swp +*.bak +*.tmp +*.orig +*~ +.idea/ +.vscode/ + +# Contributor tooling, if present in a working tree. Never ship it. +scripts/ +testdata/ + +# Never package a chart archive inside a chart. +*.tgz diff --git a/helm/hugegraph/Chart.yaml b/helm/hugegraph/Chart.yaml new file mode 100644 index 0000000000..0b7f7ccde1 --- /dev/null +++ b/helm/hugegraph/Chart.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v2 +name: hugegraph +description: Helm chart for Apache HugeGraph HStore cluster (PD + Store + Server) +type: application +version: 0.1.0 +appVersion: "latest" +kubeVersion: ">=1.23.0-0" +keywords: + - hugegraph + - graph + - hstore + - raft +home: https://hugegraph.apache.org/ +sources: + - https://github.com/apache/hugegraph +maintainers: + - name: HugeGraph Community + url: https://hugegraph.apache.org/ + email: dev@hugegraph.apache.org diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md new file mode 100644 index 0000000000..6d03f7677b --- /dev/null +++ b/helm/hugegraph/README.md @@ -0,0 +1,396 @@ +# HugeGraph HStore Helm Chart + +[Apache HugeGraph](https://hugegraph.apache.org/) - an open source, distributed graph database. + +## Documentation + +This chart deploys a distributed HugeGraph cluster - PD, Store, and Server - on +Kubernetes. For HugeGraph itself see . + +Note that this chart requires Helm 3. `--reset-then-reuse-values`, referenced +under Upgrading, requires Helm 3.14 or later. + +## Prerequisites Details + +* Kubernetes 1.23+ (the chart renders `autoscaling/v2` and `policy/v1`) +* PV support on the underlying infrastructure: a default StorageClass, or an + explicit `storageClassName` for PD and Store +* Sufficient memory for nine JVM processes in the default topology. + Insufficient memory causes OOM kills that surface as silent Raft failures + rather than as clear errors. + +## Chart Details + +| Component | Workload | Purpose | +|---|---|---| +| PD | StatefulSet + PVC | Placement driver; Raft group tracking Stores and partitions | +| Store | StatefulSet + PVC | Graph data storage (HStore) | +| Server | Deployment | Gremlin and REST query layer | + +A distributed HugeGraph cluster has a startup contract that this chart encodes +so operators do not have to: + +- **Server does not run `init-store`.** The chart injects + `HG_SERVER_INIT_STORE_ENABLED=false`, and the image's `init-store` exits when + `init_store.enabled=false`, after which Server registers with PD normally. + This matters because nothing serializes Server replicas: without the gate, + every replica would initialize the same backend concurrently. The chart + creates no init Job and does not set `HG_SERVER_SKIP_INIT`. Standalone + behavior is unchanged, because the option defaults to `true` when unset. +- **Store waits for PD quorum** in an init container before starting, so Store + never registers against an incomplete PD Raft group. +- **The Server startup probe allows at least 450 seconds.** The image may spend + 300 seconds waiting for storage and a further 120 seconds in the start + command. A lower configured `failureThreshold` is raised to this floor rather + than being rejected. +- **The image entrypoint keeps ownership of `PASSWORD` handling and + `auth.admin_pa`.** When authentication or a custom port or REST tuning is + configured, the chart's wrapper only ensures `usePD=true` and `pd.peers` are + present, then hands off to the image entrypoint. +- **Resource names reserve their suffix and StatefulSet ordinal before + truncation,** so a long release name cannot produce colliding or over-long + Pod and Service names, and PD/Store identities stay fixed when replicas + change. + +## Installing the Chart + +```bash +helm install hugegraph ./helm/hugegraph --namespace hugegraph --create-namespace +``` + +This deploys 3 PD + 3 Store + 3 Server, preserves the image's automatic JVM +sizing, and sets no resource requests or limits. Set resources before +production use. + +This first chart is version `0.1.0`. While the contribution is a draft, its +component image tags and `appVersion` track `latest` with pull policy `Always`. +Before stable publication, pin all three component tags and `appVersion` to the +next HugeGraph release and switch the component pull policies to +`IfNotPresent`. + +Verify the release: + +```bash +helm test hugegraph --namespace hugegraph +``` + +### Values Presets + +| File | Purpose | +|---|---| +| `values.yaml` | Default 3+3+3 topology | +| `values-single.yaml` | Single-node 1+1+1 example | +| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity | + +`values-cluster.yaml` is a production starting point, not a capacity +guarantee. Recalculate capacity for the graph size, traffic, failure budget, +node topology, and storage class before production use. + +## Upgrading the Chart + +```bash +helm upgrade hugegraph ./helm/hugegraph --namespace hugegraph --reuse-values +``` + +Every optional field stays optional, so a release created by an earlier +revision continues to render under `--reuse-values`. Note that `--reuse-values` +keeps the old release's values as the complete base, so a release created +before a field existed does **not** pick up its new default — including the +hardened `securityContext`, ServiceAccounts, and `terminationGracePeriodSeconds`. +Pod-level token mounting is the one exception: it is disabled unconditionally. +Use `-f` with your own values, or `--reset-then-reuse-values`, to adopt them. + +PD and Store resource names reserve room for their StatefulSet ordinal before truncation, so +identities stay fixed across replica changes and scaling never renames a +PersistentVolumeClaim. + +## Uninstalling the Chart + +```bash +helm uninstall hugegraph --namespace hugegraph +``` + +Helm does not remove PersistentVolumeClaims created by StatefulSets. Delete +them explicitly, and only when the data is no longer needed. + +## Configuration + +The following table lists the configurable parameters of the chart and their +default values. + +### Global + +| Parameter | Description | Default | +|---|---|---| +| `nameOverride` | Override the chart name in generated resource names | `""` | +| `fullnameOverride` | Override the full generated resource name | `""` | +| `imagePullSecrets` | Secrets used to pull the PD, Store, and Server images | `[]` | + +### PD + +| Parameter | Description | Default | +|---|---|---| +| `pd.replicas` | PD StatefulSet replicas. Maximum `99` | `3` | +| `pd.image.repository` | PD image repository | `hugegraph/pd` | +| `pd.image.tag` | PD image tag. Tracks the development image until the next release is pinned | `latest` | +| `pd.image.pullPolicy` | PD image pull policy | `Always` | +| `pd.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | +| `pd.ports.grpc` | PD gRPC port | `8686` | +| `pd.ports.rest` | PD REST port, also used by probes | `8620` | +| `pd.ports.raft` | PD Raft port | `8610` | +| `pd.dataPath` | PD data directory inside the container | `/hugegraph-pd/pd_data` | +| `pd.storage.size` | PD PersistentVolumeClaim size | `10Gi` | +| `pd.storage.storageClassName` | Empty uses the cluster default StorageClass | `""` | +| `pd.resources` | PD container resources. Set these for production | `{}` | +| `pd.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | +| `pd.securityContext` | Container-level securityContext. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | +| `pd.antiAffinity` | One of `required`, `preferred`, `disabled` | `required` | +| `pd.nodeSelector` | Node selector for pd Pods | `{}` | +| `pd.tolerations` | Tolerations for pd Pods | `[]` | +| `pd.affinity` | Raw affinity; overrides `pd.antiAffinity` when set | `{}` | +| `pd.topologySpreadConstraints` | Topology spread constraints for pd Pods | `[]` | +| `pd.priorityClassName` | PriorityClass for pd Pods | `""` | +| `pd.podAnnotations` | Extra annotations on pd Pods | `{}` | +| `pd.podLabels` | Extra labels on pd Pods | `{}` | +| `pd.extraEnv` | Extra environment variables for the PD container | `[]` | +| `pd.terminationGracePeriodSeconds` | Shutdown grace period | `300` | +| `pd.serviceAccount.create` | Create a ServiceAccount for pd | `true` | +| `pd.serviceAccount.name` | Use an existing ServiceAccount instead | `""` | +| `pd.serviceAccount.annotations` | Annotations on the created ServiceAccount | `{}` | +| `pd.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | +| `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | +| `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | +| `pd.probes.*.periodSeconds` | Probe interval | see `values.yaml` | +| `pd.probes.*.failureThreshold` | Probe failure threshold | see `values.yaml` | +| `pd.probes.*.timeoutSeconds` | Probe timeout. Defaults to `5` on readiness/liveness; Kubernetes would otherwise apply `1` | `5` | +| `pd.probes.*.initialDelaySeconds` | Optional probe start delay | unset | +| `pd.probes.*.successThreshold` | Optional probe success threshold | unset | + +### Store + +| Parameter | Description | Default | +|---|---|---| +| `store.replicas` | Store StatefulSet replicas. Maximum `99` | `3` | +| `store.image.repository` | Store image repository | `hugegraph/store` | +| `store.image.tag` | Store image tag. Tracks the development image until the next release is pinned | `latest` | +| `store.image.pullPolicy` | Store image pull policy | `Always` | +| `store.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | +| `store.ports.grpc` | Store gRPC port | `8500` | +| `store.ports.raft` | Store Raft port | `8510` | +| `store.ports.rest` | Store REST port | `8520` | +| `store.dataPath` | Store data directory | `/hugegraph-store/storage` | +| `store.storage.size` | Store PersistentVolumeClaim size | `50Gi` | +| `store.storage.storageClassName` | Empty uses the cluster default StorageClass | `""` | +| `store.resources` | Store container resources. Set these for production | `{}` | +| `store.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | +| `store.securityContext` | Container-level securityContext; also applied to the PD-quorum init container. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | +| `store.waitTimeoutSeconds` | Bound on the PD-quorum wait before the init container fails | `900` | +| `store.antiAffinity` | One of `required`, `preferred`, `disabled` | `required` | +| `store.nodeSelector` | Node selector for store Pods | `{}` | +| `store.tolerations` | Tolerations for store Pods | `[]` | +| `store.affinity` | Raw affinity; overrides `store.antiAffinity` when set | `{}` | +| `store.topologySpreadConstraints` | Topology spread constraints for store Pods | `[]` | +| `store.priorityClassName` | PriorityClass for store Pods | `""` | +| `store.podAnnotations` | Extra annotations on store Pods | `{}` | +| `store.podLabels` | Extra labels on store Pods | `{}` | +| `store.extraEnv` | Extra environment variables for the Store container | `[]` | +| `store.terminationGracePeriodSeconds` | Shutdown grace period | `300` | +| `store.serviceAccount.create` | Create a ServiceAccount for store | `true` | +| `store.serviceAccount.name` | Use an existing ServiceAccount instead | `""` | +| `store.serviceAccount.annotations` | Annotations on the created ServiceAccount | `{}` | +| `store.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | +| `store.pdb.enabled` | Create a PodDisruptionBudget for Store | `true` | +| `store.pdb.minAvailable` | Must be strictly less than `store.replicas`. No PDB is rendered when `store.replicas` is 1 | `2` | +| `store.waitImage` | Image for the PD-quorum init container | `curlimages/curl:8.5.0` | +| `store.waitResources` | Resources for the init container | `{}` | +| `store.probes.*` | Same probe keys as PD | see `values.yaml` | + +### Server + +| Parameter | Description | Default | +|---|---|---| +| `server.replicas` | Server Deployment replicas. Ignored when `server.hpa.enabled` | `3` | +| `server.image.repository` | Server image repository | `hugegraph/server` | +| `server.image.tag` | Server image tag. Tracks the development image until the next release is pinned | `latest` | +| `server.image.pullPolicy` | Server image pull policy | `Always` | +| `server.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | +| `server.port` | Server REST port, container port, and Service port | `8080` | +| `server.backend` | Storage backend | `hstore` | +| `server.resources` | Server resources. `requests.cpu` is required when HPA is enabled | `{}` | +| `server.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | +| `server.securityContext` | Container-level securityContext. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | +| `server.pdb.enabled` | Create a PodDisruptionBudget for Server. Off by default: Server holds no quorum | `false` | +| `server.pdb.minAvailable` | Must be strictly less than `server.replicas` | `2` | +| `server.antiAffinity` | One of `required`, `preferred`, `disabled`. Defaults to `preferred` rather than `required` because HPA may scale Server past the node count; set `required` when replicas always stay below it | `preferred` | +| `server.nodeSelector` | Node selector for server Pods | `{}` | +| `server.tolerations` | Tolerations for server Pods | `[]` | +| `server.affinity` | Raw affinity; overrides `server.antiAffinity` when set | `{}` | +| `server.topologySpreadConstraints` | Topology spread constraints for server Pods | `[]` | +| `server.priorityClassName` | PriorityClass for server Pods | `""` | +| `server.podAnnotations` | Extra annotations on server Pods | `{}` | +| `server.podLabels` | Extra labels on server Pods | `{}` | +| `server.extraEnv` | Extra environment variables for the Server container | `[]` | +| `server.terminationGracePeriodSeconds` | Shutdown grace period | `60` | +| `server.serviceAccount.create` | Create a ServiceAccount for server | `true` | +| `server.serviceAccount.name` | Use an existing ServiceAccount instead | `""` | +| `server.serviceAccount.annotations` | Annotations on the created ServiceAccount | `{}` | +| `server.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | +| `server.waitImage` | Image for the Helm test hook | `curlimages/curl:8.5.0` | +| `server.restServer.minFreeMemory` | Empty preserves the image default | `""` | +| `server.restServer.batchMaxWriteThreads` | Empty preserves the image default | `""` | +| `server.initStoreEnabled` | Must remain `false` for distributed HStore | `false` | +| `server.auth.enabled` | Enable admin authentication | `false` | +| `server.auth.existingSecret` | Required when auth is enabled; must contain key `password` | `""` | +| `server.ingress.enabled` | Create an Ingress for the Server Service | `false` | +| `server.ingress.className` | IngressClass name | `""` | +| `server.ingress.annotations` | Ingress annotations (cert-manager, nginx, ALB) | `{}` | +| `server.service.type` | Server Service type | `ClusterIP` | +| `server.service.annotations` | Server Service annotations | `{}` | +| `server.ingress.hosts` | Ingress hosts and paths | see `values.yaml` | +| `server.ingress.tls` | Ingress TLS configuration | `[]` | +| `server.hpa.enabled` | Create a HorizontalPodAutoscaler | `false` | +| `server.hpa.minReplicas` | HPA minimum replicas | `3` | +| `server.hpa.maxReplicas` | HPA maximum replicas | `10` | +| `server.hpa.targetCPUUtilizationPercentage` | HPA CPU utilization target | `70` | +| `server.probes.startup.failureThreshold` | Raised automatically so the budget is at least 450s | `90` | +| `server.probes.startup.periodSeconds` | Startup probe interval | `5` | +| `server.probes.*` | Same optional probe keys as PD | see `values.yaml` | + +When `server.hpa.enabled` is `true` the Deployment omits `spec.replicas`, so a +Helm upgrade does not overwrite the autoscaler's live replica count. Enabling +utilization-based HPA requires a strictly positive +`server.resources.requests.cpu`. + +Specify each parameter with `--set`, or supply a YAML file with `-f`: + +```bash +helm install hugegraph ./helm/hugegraph --set server.replicas=5 +``` + +`values.schema.json` and template helpers reject invalid input at render time, +before anything reaches the cluster: + +- Unknown keys and wrong types are rejected. +- `server.initStoreEnabled` must remain `false` for a distributed deployment. +- With authentication enabled, `server.auth.existingSecret` must name a Secret + containing a `password` key. With authentication disabled it must be empty, + so a configured but inactive Secret reference cannot be overlooked. A missing + Secret fails when Kubernetes configures the container; an empty `password` + fails in the Server startup wrapper. +- `server.hpa.minReplicas` must not exceed `maxReplicas`, and enabling + utilization-based HPA requires a strictly positive + `server.resources.requests.cpu`. +- `pdb.minAvailable` must be less than the matching `replicas`, so a + PodDisruptionBudget cannot permanently block node drains. +- `pd.replicas` and `store.replicas` are capped at 99. + +## Deep Dive + +### Connecting to the Cluster + +```bash +kubectl port-forward -n hugegraph svc/hugegraph-server 8080:8080 +curl http://127.0.0.1:8080/versions +curl http://127.0.0.1:8080/graphs +``` + +### Cluster Health + +| Component | Port | Purpose | +|------|-------------|---------| +| PD | `8686` | gRPC (Store and Server clients) | +| PD | `8620` | REST / health probes | +| PD | `8610` | Raft | +| Store | `8500` | gRPC | +| Store | `8510` | Raft | +| Store | `8520` | REST / health probes | +| Server | `8080` | Gremlin and REST API | + +All ports are configurable through `values.yaml`. Changing `server.port` updates +the listener, container port, and Service together. + +--- + +### Scaling + +PD and Store reserve the maximum StatefulSet ordinal in their resource names, +so scaling never renames a PersistentVolumeClaim or shifts a Pod identity. +Both are capped at 99 replicas. + +Server scales through `server.replicas`, or by enabling `server.hpa`. With HPA +enabled the Deployment omits `spec.replicas`, so a Helm upgrade does not +overwrite the autoscaler's live replica count. + +## Troubleshooting + +### Store Pods Stuck in `Init:0/1` + +The Store init container waits for PD to reach Raft quorum. Check PD first: + +```bash +kubectl get pods -l app.kubernetes.io/component=pd +kubectl logs -c wait-for-pd +``` + +The wait is bounded by `store.waitTimeoutSeconds` (default 900). On timeout the +init container exits with a message naming the peers it polled, so the failure +appears in `kubectl describe pod` instead of hanging silently. + +### PersistentVolumeClaims Stay `Pending` + +No default StorageClass, or the provisioner is unhealthy: + +```bash +kubectl get sc +kubectl get pvc -l app.kubernetes.io/instance= +kubectl -n get pods +``` + +### Server Ready but Queries Fail + +The Server readiness probe uses `/versions`, which can report ready before the +graph is fully able to serve index-backed queries. Confirm the graph is live: + +```bash +kubectl exec -c server -- curl -s localhost:8080/graphs +``` + +### Pods OOM Killed or Restarting + +The default `values.yaml` sets **no** resource requests or limits and preserves +the image's automatic JVM sizing. Set resources explicitly before production +use; see `values-cluster.yaml`. + +```bash +kubectl get pods -o wide +kubectl describe pod | grep -A5 "Last State" +``` + +### Release Name Too Long + +Helm itself rejects release names longer than 53 characters, before this chart +renders anything: + +``` +invalid release name ... the length must not be longer than 53 +``` + +Within that limit the chart is safe: resource names reserve their suffix and +StatefulSet ordinal before truncation, so every generated Service and Pod name +stays inside the 63-character DNS label limit, and PD/Store identities do not +shift when replicas change. Use `fullnameOverride` to shorten generated names +independently of the release name. + +--- + +## Limitations + +- No TLS, backups, Operator, multi-cluster support, automatic leader transfer, + or a complete monitoring stack. +- The published images run as root, so `runAsNonRoot` and + `readOnlyRootFilesystem` are not chart defaults. The container + `securityContext` does default to `allowPrivilegeEscalation: false`, + `capabilities.drop: [ALL]`, and `seccompProfile: RuntimeDefault`, which are + valid for a root image; `podSecurityContext` and `securityContext` are fully + configurable per component. +- `values-cluster.yaml` is a starting point, not a capacity guarantee. diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt new file mode 100644 index 0000000000..288c8d6f5f --- /dev/null +++ b/helm/hugegraph/templates/NOTES.txt @@ -0,0 +1,46 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ .Release.Name }} installed in namespace {{ .Release.Namespace }}. + + PD {{ .Values.pd.replicas }} replica(s) + Store {{ .Values.store.replicas }} replica(s) + Server {{ if .Values.server.hpa.enabled }}HPA {{ .Values.server.hpa.minReplicas }}-{{ .Values.server.hpa.maxReplicas }}{{ else }}{{ .Values.server.replicas }} replica(s){{ end }} + +Watch the cluster come up: + + kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -w + +PD reaches Raft quorum before Store registers, so Store Pods stay in Init until +PD is ready. + +Verify the release: + + helm test {{ .Release.Name }} --namespace {{ .Release.Namespace }} + +Reach the Server API: + + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.server.name" . }} {{ .Values.server.port }}:{{ .Values.server.port }} + curl http://127.0.0.1:{{ .Values.server.port }}/versions +{{- if not .Values.server.auth.enabled }} + +Authentication is disabled. Do not expose this release to untrusted networks. +{{- end }} +{{- if or (empty .Values.pd.resources) (empty .Values.store.resources) (empty .Values.server.resources) }} + +One or more components have no resource requests or limits. Set them before +production use; see values-cluster.yaml. +{{- end }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl new file mode 100644 index 0000000000..3f64b2b2b4 --- /dev/null +++ b/helm/hugegraph/templates/_helpers.tpl @@ -0,0 +1,283 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* +Expand the name of the chart. +*/}} +{{- define "hugegraph.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "hugegraph.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "hugegraph.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "hugegraph.labels" -}} +helm.sh/chart: {{ include "hugegraph.chart" . }} +{{ include "hugegraph.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "hugegraph.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hugegraph.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "hugegraph.pd.name" -}} +{{- printf "%s-pd" (include "hugegraph.fullname" . | trunc 57 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.pd.clientName" -}} +{{- printf "%s-pd-client" (include "hugegraph.fullname" . | trunc 53 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.store.name" -}} +{{- printf "%s-store" (include "hugegraph.fullname" . | trunc 54 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.server.name" -}} +{{- printf "%s-server" (include "hugegraph.fullname" . | trunc 56 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.test.name" -}} +{{- printf "%s-test-connection" (include "hugegraph.fullname" . | trunc 47 | trimSuffix "-") }} +{{- end }} + +{{/* +PD Raft peers list: pod-0.svc.ns.svc:8610,... +Uses short headless DNS (cluster.local optional) resolvable inside the namespace. +*/}} +{{- define "hugegraph.pd.raftPeersList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.pd.replicas -}} +{{- $name := include "hugegraph.pd.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.pd.ports.raft -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +PD gRPC peers for Store/Server. +*/}} +{{- define "hugegraph.pd.grpcPeersList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.pd.replicas -}} +{{- $name := include "hugegraph.pd.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.pd.ports.grpc -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +PD REST endpoints for Server storage-readiness checks. +*/}} +{{- define "hugegraph.pd.restPeersList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.pd.replicas -}} +{{- $name := include "hugegraph.pd.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.pd.ports.rest -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +Initial store list for PD bootstrap: store-0.svc.ns.svc:8500,... +*/}} +{{- define "hugegraph.store.initialStoreList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.store.replicas -}} +{{- $name := include "hugegraph.store.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.store.ports.grpc -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +First store REST endpoint for STORE_REST / wait-partition. +*/}} +{{- define "hugegraph.store.restPrimary" -}} +{{- $name := include "hugegraph.store.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- printf "%s-0.%s.%s.svc:%d" $name $name $ns (int .Values.store.ports.rest) -}} +{{- end }} + +{{/* +Quorum size: floor(replicas/2)+1 +*/}} +{{- define "hugegraph.pd.quorum" -}} +{{- add (div (int .Values.pd.replicas) 2) 1 -}} +{{- end }} + +{{/* +Render JAVA_OPTS only when explicitly configured. An empty value preserves the +image entrypoint's existing automatic JVM sizing behavior. +*/}} +{{- define "hugegraph.javaOptsEnv" -}} +{{- $javaOpts := default "" . -}} +{{- if ne (trim $javaOpts) "" -}} +- name: JAVA_OPTS + value: {{ $javaOpts | quote }} +{{- end -}} +{{- end }} + +{{/* +Keep the startup probe alive for the 300-second storage wait, the Server's +120-second start timeout, and 30 seconds of process overhead. Older stored +values remain accepted, but their rendered threshold is raised to this floor. +*/}} +{{- define "hugegraph.server.startupFailureThreshold" -}} +{{- $period := int .Values.server.probes.startup.periodSeconds -}} +{{- $configured := int .Values.server.probes.startup.failureThreshold -}} +{{- $minimum := div (add 449 $period) $period -}} +{{- max $configured $minimum -}} +{{- end }} + +{{/* +Optional probe tunables, emitted only when explicitly set. Kubernetes defaults +timeoutSeconds to 1 second, which a garbage-collection pause can exceed on a +loaded Server; operators need a supported way to raise it without forking the +chart. Only explicitly configured fields are rendered. +*/}} +{{- define "hugegraph.probeTuning" -}} +{{- if hasKey . "timeoutSeconds" }} +timeoutSeconds: {{ .timeoutSeconds }} +{{- end }} +{{- if hasKey . "initialDelaySeconds" }} +initialDelaySeconds: {{ .initialDelaySeconds }} +{{- end }} +{{- if hasKey . "successThreshold" }} +successThreshold: {{ .successThreshold }} +{{- end }} +{{- end }} + +{{/* +Resolve the ServiceAccount name for a component: an explicit name wins, +otherwise the generated one when create is true, otherwise "default". +*/}} +{{- define "hugegraph.serviceAccountName" -}} +{{- $sa := get .component "serviceAccount" | default dict -}} +{{- if get $sa "name" -}} +{{- get $sa "name" -}} +{{- else if (get $sa "create" | default false) -}} +{{- .name -}} +{{- else -}} +default +{{- end -}} +{{- end }} + +{{/* +Cross-field validation that JSON Schema draft-07 cannot express. +*/}} +{{- define "hugegraph.validateValues" -}} +{{- $networkPolicy := get .Values "networkPolicy" | default dict -}} +{{- if (get $networkPolicy "enabled" | default false) -}} +{{- fail "networkPolicy.enabled=true is unsupported because this chart does not implement NetworkPolicy resources" -}} +{{- end -}} +{{- if and .Values.server.hpa.enabled (gt (int .Values.server.hpa.minReplicas) (int .Values.server.hpa.maxReplicas)) -}} +{{- fail "server.hpa.minReplicas must be less than or equal to server.hpa.maxReplicas" -}} +{{- end -}} +{{- if .Values.server.hpa.enabled -}} +{{- $serverResources := .Values.server.resources | default dict -}} +{{- $serverRequests := get $serverResources "requests" | default dict -}} +{{- if not (hasKey $serverRequests "cpu") -}} +{{- fail "server.resources.requests.cpu is required when server.hpa.enabled=true" -}} +{{- end -}} +{{- $cpuRequest := trim (toString (get $serverRequests "cpu")) -}} +{{- if or (eq $cpuRequest "") (hasPrefix "-" $cpuRequest) (regexMatch "^[+]?((0+([.]0*)?)|([.]0+))(([KMGTPE]i)|[numkMGTPE]|[eE][+-]?[0-9]+)?$" $cpuRequest) -}} +{{- fail "server.resources.requests.cpu must be strictly positive when server.hpa.enabled=true" -}} +{{- end -}} +{{- end -}} +{{/* +Only validate minAvailable where a PDB is actually rendered. The pd/store PDB +templates require replicas > 1, so a single-replica release never creates one +and must not be failed for a value that has no effect. +*/}} +{{- if and .Values.pd.pdb.enabled (gt (int .Values.pd.replicas) 1) (ge (int .Values.pd.pdb.minAvailable) (int .Values.pd.replicas)) -}} +{{- fail "pd.pdb.minAvailable must be less than pd.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} +{{- end -}} +{{- if and .Values.store.pdb.enabled (gt (int .Values.store.replicas) 1) (ge (int .Values.store.pdb.minAvailable) (int .Values.store.replicas)) -}} +{{- fail "store.pdb.minAvailable must be less than store.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} +{{- end -}} +{{- $svc := get .Values.server "service" | default dict -}} +{{- if and (get $svc "nodePort") (not (has (get $svc "type" | default "ClusterIP") (list "NodePort" "LoadBalancer"))) -}} +{{- fail "server.service.nodePort requires server.service.type to be NodePort or LoadBalancer" -}} +{{- end -}} +{{- $serverPdb := get .Values.server "pdb" | default dict -}} +{{- if and (get $serverPdb "enabled" | default false) (gt (int .Values.server.replicas) 1) (ge (int (get $serverPdb "minAvailable" | default 1)) (int .Values.server.replicas)) -}} +{{- fail "server.pdb.minAvailable must be less than server.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} +{{- end -}} +{{- end }} + +{{/* +podAntiAffinity snippet for a component label key. +mode: required | preferred | disabled +*/}} +{{- define "hugegraph.antiAffinity" -}} +{{- $mode := .mode -}} +{{- $component := .component -}} +{{- $labels := .labels -}} +{{- if eq $mode "required" }} +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- toYaml $labels | nindent 12 }} + app.kubernetes.io/component: {{ $component }} + topologyKey: kubernetes.io/hostname +{{- else if eq $mode "preferred" }} +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + {{- toYaml $labels | nindent 14 }} + app.kubernetes.io/component: {{ $component }} + topologyKey: kubernetes.io/hostname +{{- end }} +{{- end }} diff --git a/helm/hugegraph/templates/pd-pdb.yaml b/helm/hugegraph/templates/pd-pdb.yaml new file mode 100644 index 0000000000..a0d93a098e --- /dev/null +++ b/helm/hugegraph/templates/pd-pdb.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if and .Values.pd.pdb.enabled (gt (int .Values.pd.replicas) 1) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "hugegraph.pd.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: pd +spec: + minAvailable: {{ .Values.pd.pdb.minAvailable }} + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: pd +{{- end }} diff --git a/helm/hugegraph/templates/pd-service-client.yaml b/helm/hugegraph/templates/pd-service-client.yaml new file mode 100644 index 0000000000..d40cc9325c --- /dev/null +++ b/helm/hugegraph/templates/pd-service-client.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hugegraph.pd.clientName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: pd +spec: + type: ClusterIP + selector: + {{- include "hugegraph.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: pd + ports: + - name: rest + port: {{ .Values.pd.ports.rest }} + targetPort: rest + - name: grpc + port: {{ .Values.pd.ports.grpc }} + targetPort: grpc diff --git a/helm/hugegraph/templates/pd-service-headless.yaml b/helm/hugegraph/templates/pd-service-headless.yaml new file mode 100644 index 0000000000..3b724f9dcf --- /dev/null +++ b/helm/hugegraph/templates/pd-service-headless.yaml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hugegraph.pd.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: pd +spec: + clusterIP: None + # Mandatory: Raft bootstrap needs DNS for not-yet-ready pods + publishNotReadyAddresses: true + selector: + {{- include "hugegraph.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: pd + ports: + - name: grpc + port: {{ .Values.pd.ports.grpc }} + targetPort: grpc + - name: rest + port: {{ .Values.pd.ports.rest }} + targetPort: rest + - name: raft + port: {{ .Values.pd.ports.raft }} + targetPort: raft diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml new file mode 100644 index 0000000000..01ef705006 --- /dev/null +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "hugegraph.pd.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: pd +spec: + serviceName: {{ include "hugegraph.pd.name" . }} + replicas: {{ .Values.pd.replicas }} + podManagementPolicy: Parallel + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: pd + template: + metadata: + labels: + {{- include "hugegraph.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: pd + {{- with .Values.pd.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} + {{- with .Values.pd.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + automountServiceAccountToken: {{ get (get .Values.pd "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} + serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.pd "name" (include "hugegraph.pd.name" .) ) }} + {{- with .Values.pd.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.pd.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.pd.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pd.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pd.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pd.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.pd.affinity }} + affinity: + {{- toYaml .Values.pd.affinity | nindent 8 }} + {{- else }} + {{- with include "hugegraph.antiAffinity" (dict "mode" .Values.pd.antiAffinity "component" "pd" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} + {{- end }} + containers: + - name: pd + image: "{{ .Values.pd.image.repository }}:{{ .Values.pd.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ .Values.pd.image.pullPolicy }} + {{- with .Values.pd.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: grpc + containerPort: {{ .Values.pd.ports.grpc }} + - name: rest + containerPort: {{ .Values.pd.ports.rest }} + - name: raft + containerPort: {{ .Values.pd.ports.raft }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: HG_PD_GRPC_HOST + value: "$(POD_NAME).{{ include "hugegraph.pd.name" . }}.$(NAMESPACE).svc" + - name: HG_PD_GRPC_PORT + value: {{ .Values.pd.ports.grpc | quote }} + - name: HG_PD_REST_PORT + value: {{ .Values.pd.ports.rest | quote }} + - name: HG_PD_RAFT_ADDRESS + value: "$(POD_NAME).{{ include "hugegraph.pd.name" . }}.$(NAMESPACE).svc:{{ .Values.pd.ports.raft }}" + - name: HG_PD_RAFT_PEERS_LIST + value: {{ include "hugegraph.pd.raftPeersList" . | quote }} + - name: HG_PD_INITIAL_STORE_LIST + value: {{ include "hugegraph.store.initialStoreList" . | quote }} + - name: HG_PD_INITIAL_STORE_COUNT + value: {{ .Values.store.replicas | quote }} + - name: HG_PD_DATA_PATH + value: {{ .Values.pd.dataPath | quote }} + {{- with .Values.pd.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} + {{- with include "hugegraph.javaOptsEnv" .Values.pd.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} + volumeMounts: + - name: pd-data + mountPath: {{ .Values.pd.dataPath }} + startupProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: {{ .Values.pd.probes.startup.failureThreshold }} + periodSeconds: {{ .Values.pd.probes.startup.periodSeconds }} + {{- with include "hugegraph.probeTuning" .Values.pd.probes.startup }}{{ . | trim | nindent 12 }}{{- end }} + readinessProbe: + httpGet: + path: /v1/health + port: rest + periodSeconds: {{ .Values.pd.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.pd.probes.readiness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.pd.probes.readiness }}{{ . | trim | nindent 12 }}{{- end }} + livenessProbe: + httpGet: + path: /v1/health + port: rest + periodSeconds: {{ .Values.pd.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.pd.probes.liveness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.pd.probes.liveness }}{{ . | trim | nindent 12 }}{{- end }} + {{- with .Values.pd.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: pd-data + spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.pd.storage.storageClassName }} + storageClassName: {{ .Values.pd.storage.storageClassName | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.pd.storage.size }} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml new file mode 100644 index 0000000000..6445c80c75 --- /dev/null +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -0,0 +1,242 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- include "hugegraph.validateValues" . }} +{{- $restServer := .Values.server.restServer | default dict }} +{{- $minFreeMemory := "" }} +{{- $batchMaxWriteThreads := "" }} +{{- if hasKey $restServer "minFreeMemory" }} +{{- $minFreeMemory = toString (get $restServer "minFreeMemory") }} +{{- end }} +{{- if hasKey $restServer "batchMaxWriteThreads" }} +{{- $batchMaxWriteThreads = toString (get $restServer "batchMaxWriteThreads") }} +{{- end }} +{{- $customPort := ne (int .Values.server.port) 8080 }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hugegraph.server.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + {{- if not .Values.server.hpa.enabled }} + replicas: {{ .Values.server.replicas }} + {{- end }} + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + template: + metadata: + labels: + {{- include "hugegraph.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: server + {{- with .Values.server.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} + {{- with .Values.server.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: {{ get (get .Values.server "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} + serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.server "name" (include "hugegraph.server.name" .) ) }} + {{- with .Values.server.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.server.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.server.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.server.affinity }} + affinity: + {{- toYaml .Values.server.affinity | nindent 8 }} + {{- else }} + {{- with include "hugegraph.antiAffinity" (dict "mode" (.Values.server.antiAffinity | default "preferred") "component" "server" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: server + image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ .Values.server.image.pullPolicy }} + {{- with .Values.server.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if or .Values.server.auth.enabled + $customPort + (ne $minFreeMemory "") + (ne $batchMaxWriteThreads "") }} + command: + - /usr/bin/dumb-init + - -- + - /bin/bash + - -c + args: + - | + set -euo pipefail + {{- if .Values.server.auth.enabled }} + : "${PASSWORD:?auth Secret key 'password' must not be empty}" + {{- end }} + CONF=./conf/rest-server.properties + TMP=$(mktemp) + {{- if .Values.server.auth.enabled }} + FOUND_USE_PD=false + FOUND_PD_PEERS=false + {{- end }} + {{- if $customPort }} + FOUND_RESTSERVER_URL=false + {{- end }} + {{- if ne $minFreeMemory "" }} + FOUND_MIN_FREE_MEMORY=false + {{- end }} + {{- if ne $batchMaxWriteThreads "" }} + FOUND_BATCH_MAX_WRITE_THREADS=false + {{- end }} + while IFS= read -r LINE || [[ -n "${LINE}" ]]; do + case "${LINE}" in + {{- if .Values.server.auth.enabled }} + usePD=*) + printf 'usePD=true\n' >>"${TMP}" + FOUND_USE_PD=true + ;; + pd.peers=*) + printf 'pd.peers=%s\n' "${HG_SERVER_PD_PEERS}" >>"${TMP}" + FOUND_PD_PEERS=true + ;; + {{- end }} + {{- if $customPort }} + restserver.url=*) + printf 'restserver.url=http://0.0.0.0:%s\n' \ + {{ .Values.server.port | quote }} >>"${TMP}" + FOUND_RESTSERVER_URL=true + ;; + {{- end }} + {{- if ne $minFreeMemory "" }} + restserver.min_free_memory=*) + printf 'restserver.min_free_memory=%s\n' \ + {{ $minFreeMemory | quote }} >>"${TMP}" + FOUND_MIN_FREE_MEMORY=true + ;; + {{- end }} + {{- if ne $batchMaxWriteThreads "" }} + batch.max_write_threads=*) + printf 'batch.max_write_threads=%s\n' \ + {{ $batchMaxWriteThreads | quote }} >>"${TMP}" + FOUND_BATCH_MAX_WRITE_THREADS=true + ;; + {{- end }} + *) + printf '%s\n' "${LINE}" >>"${TMP}" + ;; + esac + done <"${CONF}" + {{- if .Values.server.auth.enabled }} + if [[ "${FOUND_USE_PD}" == false ]]; then + printf 'usePD=true\n' >>"${TMP}" + fi + if [[ "${FOUND_PD_PEERS}" == false ]]; then + printf 'pd.peers=%s\n' "${HG_SERVER_PD_PEERS}" >>"${TMP}" + fi + {{- end }} + {{- if $customPort }} + if [[ "${FOUND_RESTSERVER_URL}" == false ]]; then + printf 'restserver.url=http://0.0.0.0:%s\n' \ + {{ .Values.server.port | quote }} >>"${TMP}" + fi + {{- end }} + {{- if ne $minFreeMemory "" }} + if [[ "${FOUND_MIN_FREE_MEMORY}" == false ]]; then + printf 'restserver.min_free_memory=%s\n' \ + {{ $minFreeMemory | quote }} >>"${TMP}" + fi + {{- end }} + {{- if ne $batchMaxWriteThreads "" }} + if [[ "${FOUND_BATCH_MAX_WRITE_THREADS}" == false ]]; then + printf 'batch.max_write_threads=%s\n' \ + {{ $batchMaxWriteThreads | quote }} >>"${TMP}" + fi + {{- end }} + chmod 600 "${TMP}" + mv "${TMP}" "${CONF}" + exec ./docker-entrypoint.sh + {{- end }} + ports: + - name: http + containerPort: {{ .Values.server.port }} + env: + - name: HG_SERVER_BACKEND + value: {{ .Values.server.backend | quote }} + - name: HG_SERVER_PD_PEERS + value: {{ include "hugegraph.pd.grpcPeersList" . | quote }} + - name: HG_SERVER_PD_REST_ENDPOINT + value: {{ include "hugegraph.pd.restPeersList" . | quote }} + - name: STORE_REST + value: {{ include "hugegraph.store.restPrimary" . | quote }} + - name: HG_SERVER_INIT_STORE_ENABLED + value: {{ .Values.server.initStoreEnabled | quote }} + {{- with .Values.server.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} + {{- with include "hugegraph.javaOptsEnv" .Values.server.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} + {{- if and .Values.server.auth.enabled .Values.server.auth.existingSecret }} + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.server.auth.existingSecret | quote }} + key: password + {{- end }} + startupProbe: + httpGet: + path: /versions + port: http + failureThreshold: {{ include "hugegraph.server.startupFailureThreshold" . }} + periodSeconds: {{ .Values.server.probes.startup.periodSeconds }} + {{- with include "hugegraph.probeTuning" .Values.server.probes.startup }}{{ . | trim | nindent 12 }}{{- end }} + readinessProbe: + httpGet: + path: /versions + port: http + periodSeconds: {{ .Values.server.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.server.probes.readiness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.server.probes.readiness }}{{ . | trim | nindent 12 }}{{- end }} + livenessProbe: + httpGet: + path: /versions + port: http + periodSeconds: {{ .Values.server.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.server.probes.liveness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.server.probes.liveness }}{{ . | trim | nindent 12 }}{{- end }} + {{- with .Values.server.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/helm/hugegraph/templates/server-hpa.yaml b/helm/hugegraph/templates/server-hpa.yaml new file mode 100644 index 0000000000..40e95c0d99 --- /dev/null +++ b/helm/hugegraph/templates/server-hpa.yaml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.server.hpa.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "hugegraph.server.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "hugegraph.server.name" . }} + minReplicas: {{ .Values.server.hpa.minReplicas }} + maxReplicas: {{ .Values.server.hpa.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.server.hpa.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/helm/hugegraph/templates/server-ingress.yaml b/helm/hugegraph/templates/server-ingress.yaml new file mode 100644 index 0000000000..08a8c6a7ea --- /dev/null +++ b/helm/hugegraph/templates/server-ingress.yaml @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.server.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "hugegraph.server.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: server + {{- with (get .Values.server.ingress "annotations") }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.server.ingress.className }} + ingressClassName: {{ .Values.server.ingress.className | quote }} + {{- end }} + {{- with .Values.server.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.server.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "hugegraph.server.name" $ }} + port: + number: {{ $.Values.server.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/hugegraph/templates/server-pdb.yaml b/helm/hugegraph/templates/server-pdb.yaml new file mode 100644 index 0000000000..3c1f0c4847 --- /dev/null +++ b/helm/hugegraph/templates/server-pdb.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- $pdb := get .Values.server "pdb" | default dict }} +{{- if and (get $pdb "enabled" | default false) (gt (int .Values.server.replicas) 1) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "hugegraph.server.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + minAvailable: {{ get $pdb "minAvailable" | default 1 }} + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server +{{- end }} diff --git a/helm/hugegraph/templates/server-service.yaml b/helm/hugegraph/templates/server-service.yaml new file mode 100644 index 0000000000..b4053dc4ba --- /dev/null +++ b/helm/hugegraph/templates/server-service.yaml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hugegraph.server.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: server + {{- $svc := get .Values.server "service" | default dict }} + {{- with (get $svc "annotations") }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ get $svc "type" | default "ClusterIP" }} + selector: + {{- include "hugegraph.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: server + ports: + - name: http + port: {{ .Values.server.port }} + targetPort: http + {{- with (get $svc "nodePort") }} + nodePort: {{ . }} + {{- end }} diff --git a/helm/hugegraph/templates/serviceaccount.yaml b/helm/hugegraph/templates/serviceaccount.yaml new file mode 100644 index 0000000000..9889414d4b --- /dev/null +++ b/helm/hugegraph/templates/serviceaccount.yaml @@ -0,0 +1,35 @@ +{{- /* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} +{{- range $component := list "pd" "store" "server" }} +{{- $values := index $.Values $component }} +{{- $sa := get $values "serviceAccount" | default dict }} +{{- if and (get $sa "create" | default false) (not (get $sa "name")) }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include (printf "hugegraph.%s.name" $component) $ }} + labels: + {{- include "hugegraph.labels" $ | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with (get $sa "annotations") }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ get $sa "automountServiceAccountToken" | default false }} +{{- end }} +{{- end }} diff --git a/helm/hugegraph/templates/store-pdb.yaml b/helm/hugegraph/templates/store-pdb.yaml new file mode 100644 index 0000000000..9fb1e536da --- /dev/null +++ b/helm/hugegraph/templates/store-pdb.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if and .Values.store.pdb.enabled (gt (int .Values.store.replicas) 1) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "hugegraph.store.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: store +spec: + minAvailable: {{ .Values.store.pdb.minAvailable }} + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: store +{{- end }} diff --git a/helm/hugegraph/templates/store-service-headless.yaml b/helm/hugegraph/templates/store-service-headless.yaml new file mode 100644 index 0000000000..4c82023b5d --- /dev/null +++ b/helm/hugegraph/templates/store-service-headless.yaml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hugegraph.store.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: store +spec: + clusterIP: None + # Mandatory: Raft / self-FQDN resolution before readiness + publishNotReadyAddresses: true + selector: + {{- include "hugegraph.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: store + ports: + - name: grpc + port: {{ .Values.store.ports.grpc }} + targetPort: grpc + - name: raft + port: {{ .Values.store.ports.raft }} + targetPort: raft + - name: rest + port: {{ .Values.store.ports.rest }} + targetPort: rest diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml new file mode 100644 index 0000000000..973ecacdff --- /dev/null +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -0,0 +1,193 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "hugegraph.store.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: store +spec: + serviceName: {{ include "hugegraph.store.name" . }} + replicas: {{ .Values.store.replicas }} + podManagementPolicy: Parallel + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: store + template: + metadata: + labels: + {{- include "hugegraph.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: store + {{- with .Values.store.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} + {{- with .Values.store.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + automountServiceAccountToken: {{ get (get .Values.store "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} + serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.store "name" (include "hugegraph.store.name" .) ) }} + {{- with .Values.store.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.store.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.store.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.store.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.store.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.store.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.store.affinity }} + affinity: + {{- toYaml .Values.store.affinity | nindent 8 }} + {{- else }} + {{- with include "hugegraph.antiAffinity" (dict "mode" .Values.store.antiAffinity "component" "store" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} + {{- end }} + initContainers: + - name: wait-for-pd + image: {{ .Values.store.waitImage | quote }} + {{- with .Values.store.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - sh + - -c + - | + set -eu + REQUIRED={{ include "hugegraph.pd.quorum" . }} + HEALTH_PEERS=$(echo "{{ include "hugegraph.pd.restPeersList" . }}" | tr ',' ' ') + TIMEOUT={{ .Values.store.waitTimeoutSeconds | default 900 }} + DEADLINE=$(( $(date +%s) + TIMEOUT )) + echo "Waiting for PD quorum (${REQUIRED}) among: ${HEALTH_PEERS}" + until [ "$( + ok=0 + for peer in ${HEALTH_PEERS}; do + if curl -fsS "http://${peer}/v1/health" >/dev/null 2>&1; then + ok=$((ok+1)) + fi + done + echo "$ok" + )" -ge "${REQUIRED}" ]; do + if [ "$(date +%s)" -ge "${DEADLINE}" ]; then + echo "Timed out after ${TIMEOUT}s waiting for PD quorum (${REQUIRED}) among: ${HEALTH_PEERS}" >&2 + echo "Check PD Pods: kubectl get pods -l app.kubernetes.io/component=pd" >&2 + exit 1 + fi + echo "Waiting for PD quorum..." + sleep 5 + done + echo "PD quorum reached." + {{- with .Values.store.waitResources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: store + image: "{{ .Values.store.image.repository }}:{{ .Values.store.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ .Values.store.image.pullPolicy }} + {{- with .Values.store.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: grpc + containerPort: {{ .Values.store.ports.grpc }} + - name: raft + containerPort: {{ .Values.store.ports.raft }} + - name: rest + containerPort: {{ .Values.store.ports.rest }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: HG_STORE_PD_ADDRESS + value: {{ include "hugegraph.pd.grpcPeersList" . | quote }} + - name: HG_STORE_GRPC_HOST + value: "$(POD_NAME).{{ include "hugegraph.store.name" . }}.$(NAMESPACE).svc" + - name: HG_STORE_GRPC_PORT + value: {{ .Values.store.ports.grpc | quote }} + - name: HG_STORE_REST_PORT + value: {{ .Values.store.ports.rest | quote }} + - name: HG_STORE_RAFT_ADDRESS + value: "$(POD_NAME).{{ include "hugegraph.store.name" . }}.$(NAMESPACE).svc:{{ .Values.store.ports.raft }}" + - name: HG_STORE_DATA_PATH + value: {{ .Values.store.dataPath | quote }} + {{- with .Values.store.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} + {{- with include "hugegraph.javaOptsEnv" .Values.store.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} + volumeMounts: + - name: store-data + mountPath: {{ .Values.store.dataPath }} + startupProbe: + httpGet: + path: /v1/health + port: rest + failureThreshold: {{ .Values.store.probes.startup.failureThreshold }} + periodSeconds: {{ .Values.store.probes.startup.periodSeconds }} + {{- with include "hugegraph.probeTuning" .Values.store.probes.startup }}{{ . | trim | nindent 12 }}{{- end }} + readinessProbe: + httpGet: + path: /v1/health + port: rest + periodSeconds: {{ .Values.store.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.store.probes.readiness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.store.probes.readiness }}{{ . | trim | nindent 12 }}{{- end }} + livenessProbe: + httpGet: + path: /v1/health + port: rest + periodSeconds: {{ .Values.store.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.store.probes.liveness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.store.probes.liveness }}{{ . | trim | nindent 12 }}{{- end }} + {{- with .Values.store.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: store-data + spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.store.storage.storageClassName }} + storageClassName: {{ .Values.store.storage.storageClassName | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.store.storage.size }} diff --git a/helm/hugegraph/templates/tests/test-connection.yaml b/helm/hugegraph/templates/tests/test-connection.yaml new file mode 100644 index 0000000000..b5c431b659 --- /dev/null +++ b/helm/hugegraph/templates/tests/test-connection.yaml @@ -0,0 +1,67 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Official Helm chart test hook — only runs with: helm test +# Not an install-time hook. Safe with --wait (no init Job in no-init design). +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "hugegraph.test.name" . }}" + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: curl + image: {{ .Values.server.waitImage | quote }} + {{- if .Values.server.auth.enabled }} + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.server.auth.existingSecret | quote }} + key: password + {{- end }} + command: + - sh + - -c + - | + set -eu + SVC="{{ include "hugegraph.server.name" . }}" + PORT="{{ .Values.server.port }}" + {{- if .Values.server.auth.enabled }} + AUTH_HEADER=$(printf 'admin:%s' "${PASSWORD}" | base64 | tr -d '\r\n') + request() { + printf 'header = "Authorization: Basic %s"\n' "${AUTH_HEADER}" | + curl --config - -fsS "$1" + } + {{- else }} + request() { + curl -fsS "$1" + } + {{- end }} + echo "helm test: GET http://${SVC}:${PORT}/versions" + request "http://${SVC}:${PORT}/versions" + echo + echo "helm test: GET http://${SVC}:${PORT}/graphs" + request "http://${SVC}:${PORT}/graphs" + echo + echo "helm test: OK" diff --git a/helm/hugegraph/testdata/values-pre-hardening.yaml b/helm/hugegraph/testdata/values-pre-hardening.yaml new file mode 100644 index 0000000000..d3d34cea1f --- /dev/null +++ b/helm/hugegraph/testdata/values-pre-hardening.yaml @@ -0,0 +1,128 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Values shape shipped before the production-hardening fields were added. +# validate-chart.sh installs this as the chart default values to emulate a +# Helm --reuse-values upgrade whose stored release values lack all new keys. + +fullnameOverride: "" +nameOverride: "" + +imagePullSecrets: [] + +pd: + replicas: 3 + image: + repository: hugegraph/pd + tag: latest + pullPolicy: IfNotPresent + ports: + grpc: 8686 + rest: 8620 + raft: 8610 + dataPath: /hugegraph-pd/pd_data + storage: + size: 10Gi + storageClassName: "" + resources: {} + antiAffinity: required + pdb: + enabled: true + minAvailable: 2 + probes: + startup: + failureThreshold: 30 + periodSeconds: 10 + readiness: + periodSeconds: 10 + failureThreshold: 3 + liveness: + periodSeconds: 20 + failureThreshold: 3 + +store: + replicas: 3 + image: + repository: hugegraph/store + tag: latest + pullPolicy: IfNotPresent + ports: + grpc: 8500 + raft: 8510 + rest: 8520 + dataPath: /hugegraph-store/storage + storage: + size: 50Gi + storageClassName: "" + resources: {} + antiAffinity: required + pdb: + enabled: true + minAvailable: 2 + waitImage: curlimages/curl:8.5.0 + probes: + startup: + failureThreshold: 40 + periodSeconds: 10 + readiness: + periodSeconds: 10 + failureThreshold: 3 + liveness: + periodSeconds: 20 + failureThreshold: 3 + +server: + replicas: 3 + image: + repository: hugegraph/server + tag: latest + pullPolicy: IfNotPresent + port: 8080 + backend: hstore + resources: {} + waitImage: curlimages/curl:8.5.0 + initStoreEnabled: false + auth: + enabled: false + existingSecret: "" + ingress: + enabled: false + className: "" + hosts: + - host: hugegraph.local + paths: + - path: / + pathType: Prefix + tls: [] + hpa: + enabled: false + minReplicas: 3 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + probes: + startup: + failureThreshold: 30 + periodSeconds: 5 + readiness: + periodSeconds: 10 + failureThreshold: 3 + liveness: + periodSeconds: 20 + failureThreshold: 3 + +networkPolicy: + enabled: false diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml new file mode 100644 index 0000000000..91ae09a9c6 --- /dev/null +++ b/helm/hugegraph/values-cluster.yaml @@ -0,0 +1,86 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Production starting point: 3 Server + 3 PD + 3 Store. +# Size this profile again for the real graph, traffic, and failure budget. +# Usage: helm install hg ./helm/hugegraph -f values-cluster.yaml + +pd: + replicas: 3 + javaOpts: >- + -Xms256m -Xmx512m + -XX:MaxMetaspaceSize=256m + -XX:MaxDirectMemorySize=256m + -XX:+UseContainerSupport + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + antiAffinity: required + pdb: + enabled: true + minAvailable: 2 + storage: + size: 10Gi + +store: + replicas: 3 + javaOpts: >- + -Xms512m -Xmx1024m + -XX:MaxMetaspaceSize=256m + -XX:MaxDirectMemorySize=512m + -XX:+UseContainerSupport + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "4" + memory: 4Gi + waitResources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 64Mi + antiAffinity: required + pdb: + enabled: true + minAvailable: 2 + storage: + size: 50Gi + +server: + replicas: 3 + javaOpts: >- + -Xms512m -Xmx1024m + -XX:MaxMetaspaceSize=256m + -XX:MaxDirectMemorySize=256m + -XX:+UseContainerSupport + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "4" + memory: 2Gi + hpa: + enabled: false diff --git a/helm/hugegraph/values-single.yaml b/helm/hugegraph/values-single.yaml new file mode 100644 index 0000000000..86dd71abd6 --- /dev/null +++ b/helm/hugegraph/values-single.yaml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Convenience preset: single-node / local kind-minikube development. +# Usage: helm install hg ./helm/hugegraph -f values-single.yaml + +pd: + replicas: 1 + antiAffinity: disabled + pdb: + enabled: false + storage: + size: 5Gi + +store: + replicas: 1 + antiAffinity: disabled + pdb: + enabled: false + storage: + size: 10Gi + +server: + replicas: 1 + hpa: + enabled: false diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json new file mode 100644 index 0000000000..23ff6f73d8 --- /dev/null +++ b/helm/hugegraph/values.schema.json @@ -0,0 +1,765 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "HugeGraph HStore Helm values", + "type": "object", + "required": [ + "pd", + "store", + "server" + ], + "properties": { + "fullnameOverride": { + "type": "string" + }, + "nameOverride": { + "type": "string" + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object" + } + }, + "pd": { + "$ref": "#/definitions/pd" + }, + "store": { + "$ref": "#/definitions/store" + }, + "server": { + "$ref": "#/definitions/server" + }, + "global": { + "type": "object" + }, + "networkPolicy": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + } + }, + "definitions": { + "image": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository", + "tag", + "pullPolicy" + ], + "properties": { + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } + } + }, + "resources": { + "type": "object", + "additionalProperties": false, + "properties": { + "requests": { + "$ref": "#/definitions/resourceList" + }, + "limits": { + "$ref": "#/definitions/resourceList" + } + } + }, + "resourceList": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number" + ] + }, + "properties": { + "cpu": { + "type": [ + "string", + "number" + ] + }, + "memory": { + "type": [ + "string", + "number" + ] + }, + "ephemeral-storage": { + "type": [ + "string", + "number" + ] + } + } + }, + "storage": { + "type": "object", + "additionalProperties": false, + "required": [ + "size", + "storageClassName" + ], + "properties": { + "size": { + "type": "string", + "minLength": 1 + }, + "storageClassName": { + "type": "string" + } + } + }, + "pdb": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "minAvailable" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "minAvailable": { + "type": "integer", + "minimum": 1 + } + } + }, + "probe": { + "type": "object", + "additionalProperties": false, + "required": [ + "periodSeconds", + "failureThreshold" + ], + "properties": { + "periodSeconds": { + "type": "integer", + "minimum": 1 + }, + "failureThreshold": { + "type": "integer", + "minimum": 1 + }, + "timeoutSeconds": { + "type": "integer", + "minimum": 1 + }, + "initialDelaySeconds": { + "type": "integer", + "minimum": 0 + }, + "successThreshold": { + "type": "integer", + "minimum": 1 + } + } + }, + "probes": { + "type": "object", + "additionalProperties": false, + "required": [ + "startup", + "readiness", + "liveness" + ], + "properties": { + "startup": { + "$ref": "#/definitions/probe" + }, + "readiness": { + "$ref": "#/definitions/probe" + }, + "liveness": { + "$ref": "#/definitions/probe" + } + } + }, + "ports": { + "type": "object", + "additionalProperties": false, + "required": [ + "grpc", + "rest", + "raft" + ], + "properties": { + "grpc": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "rest": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "raft": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + } + }, + "pd": { + "type": "object", + "additionalProperties": false, + "required": [ + "replicas", + "image", + "ports", + "dataPath", + "storage", + "resources", + "antiAffinity", + "pdb", + "probes" + ], + "properties": { + "replicas": { + "type": "integer", + "minimum": 1, + "maximum": 99 + }, + "image": { + "$ref": "#/definitions/image" + }, + "javaOpts": { + "type": "string" + }, + "ports": { + "$ref": "#/definitions/ports" + }, + "dataPath": { + "type": "string", + "minLength": 1 + }, + "storage": { + "$ref": "#/definitions/storage" + }, + "resources": { + "$ref": "#/definitions/resources" + }, + "antiAffinity": { + "type": "string", + "enum": [ + "required", + "preferred", + "disabled" + ] + }, + "pdb": { + "$ref": "#/definitions/pdb" + }, + "probes": { + "$ref": "#/definitions/probes" + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "topologySpreadConstraints": { + "type": "array" + }, + "priorityClassName": { + "type": "string" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "extraEnv": { + "type": "array" + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 0 + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "annotations": { + "type": "object" + }, + "automountServiceAccountToken": { + "type": "boolean" + } + } + } + } + }, + "store": { + "type": "object", + "additionalProperties": false, + "required": [ + "replicas", + "image", + "ports", + "dataPath", + "storage", + "resources", + "antiAffinity", + "pdb", + "waitImage", + "probes" + ], + "properties": { + "replicas": { + "type": "integer", + "minimum": 1, + "maximum": 99 + }, + "image": { + "$ref": "#/definitions/image" + }, + "javaOpts": { + "type": "string" + }, + "ports": { + "$ref": "#/definitions/ports" + }, + "dataPath": { + "type": "string", + "minLength": 1 + }, + "storage": { + "$ref": "#/definitions/storage" + }, + "resources": { + "$ref": "#/definitions/resources" + }, + "antiAffinity": { + "type": "string", + "enum": [ + "required", + "preferred", + "disabled" + ] + }, + "pdb": { + "$ref": "#/definitions/pdb" + }, + "waitImage": { + "type": "string", + "minLength": 1 + }, + "waitResources": { + "$ref": "#/definitions/resources" + }, + "probes": { + "$ref": "#/definitions/probes" + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "waitTimeoutSeconds": { + "type": "integer", + "minimum": 1 + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "topologySpreadConstraints": { + "type": "array" + }, + "priorityClassName": { + "type": "string" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "extraEnv": { + "type": "array" + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 0 + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "annotations": { + "type": "object" + }, + "automountServiceAccountToken": { + "type": "boolean" + } + } + } + } + }, + "server": { + "type": "object", + "additionalProperties": false, + "required": [ + "replicas", + "image", + "port", + "backend", + "resources", + "waitImage", + "initStoreEnabled", + "auth", + "ingress", + "hpa", + "probes" + ], + "properties": { + "replicas": { + "type": "integer", + "minimum": 1 + }, + "image": { + "$ref": "#/definitions/image" + }, + "javaOpts": { + "type": "string" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "backend": { + "type": "string", + "const": "hstore" + }, + "resources": { + "$ref": "#/definitions/resources" + }, + "waitImage": { + "type": "string", + "minLength": 1, + "description": "Image used by the Helm test hook" + }, + "waitResources": { + "$ref": "#/definitions/resources" + }, + "restServer": { + "type": "object", + "additionalProperties": false, + "properties": { + "minFreeMemory": { + "oneOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "string", + "const": "" + } + ] + }, + "batchMaxWriteThreads": { + "oneOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "string", + "const": "" + } + ] + } + } + }, + "initStoreEnabled": { + "type": "boolean", + "const": false + }, + "auth": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "existingSecret" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "existingSecret": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "properties": { + "existingSecret": { + "minLength": 1 + } + } + } + }, + { + "if": { + "properties": { + "enabled": { + "const": false + } + } + }, + "then": { + "properties": { + "existingSecret": { + "const": "" + } + } + } + } + ] + }, + "ingress": { + "type": "object", + "required": [ + "enabled", + "className", + "hosts", + "tls" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "className": { + "type": "string" + }, + "hosts": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "host", + "paths" + ], + "properties": { + "host": { + "type": "string", + "minLength": 1 + }, + "paths": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "pathType" + ], + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string", + "enum": [ + "Exact", + "Prefix", + "ImplementationSpecific" + ] + } + } + } + } + } + } + }, + "tls": { + "type": "array", + "items": { + "type": "object" + } + }, + "annotations": { + "type": "object" + } + } + }, + "hpa": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "minReplicas", + "maxReplicas", + "targetCPUUtilizationPercentage" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "minReplicas": { + "type": "integer", + "minimum": 1 + }, + "maxReplicas": { + "type": "integer", + "minimum": 1 + }, + "targetCPUUtilizationPercentage": { + "type": "integer", + "minimum": 1, + "maximum": 100 + } + } + }, + "probes": { + "$ref": "#/definitions/probes" + }, + "antiAffinity": { + "type": "string", + "enum": [ + "required", + "preferred", + "disabled" + ] + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "pdb": { + "$ref": "#/definitions/pdb" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "topologySpreadConstraints": { + "type": "array" + }, + "priorityClassName": { + "type": "string" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "extraEnv": { + "type": "array" + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 0 + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "annotations": { + "type": "object" + }, + "automountServiceAccountToken": { + "type": "boolean" + } + } + }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ] + }, + "annotations": { + "type": "object" + }, + "nodePort": { + "type": [ + "integer", + "null" + ], + "minimum": 30000, + "maximum": 32767 + } + } + } + } + } + } +} diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml new file mode 100644 index 0000000000..a43ae7b98b --- /dev/null +++ b/helm/hugegraph/values.yaml @@ -0,0 +1,256 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default values for HugeGraph HStore Helm chart. +# See values-single.yaml and values-cluster.yaml for presets. + +fullnameOverride: "" +nameOverride: "" + +imagePullSecrets: [] + +pd: + replicas: 3 + image: + repository: hugegraph/pd + # The draft tracks latest until the next HugeGraph release tag is available. + # Pin the release tag and switch to IfNotPresent before stable publication. + tag: latest + pullPolicy: Always + # Empty preserves the image entrypoint's automatic JVM sizing. + javaOpts: "" + ports: + grpc: 8686 + rest: 8620 + raft: 8610 + dataPath: /hugegraph-pd/pd_data + storage: + size: 10Gi + storageClassName: "" + resources: {} + # Pod-level and container-level securityContext. Empty by default because the + # published images run as root; set these to satisfy a restricted namespace. + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + # required | preferred | disabled + antiAffinity: required + # Scheduling. `affinity` takes precedence over the antiAffinity preset. + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + podAnnotations: {} + podLabels: {} + # Extra environment variables appended to the pd container. + extraEnv: [] + # Gives a JVM with an on-disk store time to shut down cleanly on drain. + terminationGracePeriodSeconds: 300 + serviceAccount: + create: true + name: "" + annotations: {} + # This chart makes no Kubernetes API calls, so no token is mounted. + automountServiceAccountToken: false + pdb: + enabled: true + minAvailable: 2 + # Startup can take a while during Raft bootstrap + probes: + startup: + failureThreshold: 30 + periodSeconds: 10 + timeoutSeconds: 5 + readiness: + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + liveness: + periodSeconds: 20 + failureThreshold: 3 + timeoutSeconds: 5 + +store: + replicas: 3 + image: + repository: hugegraph/store + # The draft tracks latest until the next HugeGraph release tag is available. + # Pin the release tag and switch to IfNotPresent before stable publication. + tag: latest + pullPolicy: Always + # Empty preserves the image entrypoint's automatic JVM sizing. + javaOpts: "" + ports: + grpc: 8500 + raft: 8510 + rest: 8520 + dataPath: /hugegraph-store/storage + storage: + size: 50Gi + storageClassName: "" + resources: {} + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + antiAffinity: required + # Scheduling. `affinity` takes precedence over the antiAffinity preset. + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + podAnnotations: {} + podLabels: {} + # Extra environment variables appended to the store container. + extraEnv: [] + # Gives a JVM with an on-disk store time to shut down cleanly on drain. + terminationGracePeriodSeconds: 300 + serviceAccount: + create: true + name: "" + annotations: {} + # This chart makes no Kubernetes API calls, so no token is mounted. + automountServiceAccountToken: false + pdb: + enabled: true + minAvailable: 2 + waitImage: curlimages/curl:8.5.0 + # Bound the PD-quorum wait so a cluster that never reaches quorum fails + # visibly instead of sitting in Init:0/1 forever. + waitTimeoutSeconds: 900 + # Optional bounds for the PD-quorum wait init container. + waitResources: {} + probes: + startup: + failureThreshold: 40 + periodSeconds: 10 + timeoutSeconds: 5 + readiness: + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + liveness: + periodSeconds: 20 + failureThreshold: 3 + timeoutSeconds: 5 + +server: + replicas: 3 + image: + repository: hugegraph/server + # The draft tracks latest until the next HugeGraph release tag is available. + # Pin the release tag and switch to IfNotPresent before stable publication. + tag: latest + pullPolicy: Always + # Empty preserves the image entrypoint's automatic JVM sizing. + javaOpts: "" + port: 8080 + backend: hstore + resources: {} + # Server is stateless and may scale past the node count via HPA, so the + # default only prefers spreading. Use "required" when replicas are always + # fewer than schedulable nodes. + # required | preferred | disabled + antiAffinity: preferred + # Scheduling. `affinity` takes precedence over the antiAffinity preset. + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + podAnnotations: {} + podLabels: {} + # Extra environment variables appended to the server container. + extraEnv: [] + # Gives a JVM with an on-disk store time to shut down cleanly on drain. + terminationGracePeriodSeconds: 60 + serviceAccount: + create: true + name: "" + annotations: {} + # This chart makes no Kubernetes API calls, so no token is mounted. + automountServiceAccountToken: false + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + # Server has no quorum to preserve, so no PodDisruptionBudget by default. + pdb: + enabled: false + minAvailable: 2 + # Image used by the Helm test hook. + waitImage: curlimages/curl:8.5.0 + restServer: + # Empty preserves the image's restserver.min_free_memory default. + minFreeMemory: "" + # Empty preserves the image's batch.max_write_threads default. + batchMaxWriteThreads: "" + # Distributed HStore: the init-store gate must be explicitly false, so that + # concurrent Server replicas never initialize the same backend. No + # HG_SERVER_SKIP_INIT and no init Job. + initStoreEnabled: false + auth: + enabled: false + # Secret must contain key "password" + existingSecret: "" + service: + type: ClusterIP + annotations: {} + ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: hugegraph.local + paths: + - path: / + pathType: Prefix + tls: [] + hpa: + enabled: false + minReplicas: 3 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + probes: + startup: + # 450s covers the 300s storage wait and Server process startup. + failureThreshold: 90 + periodSeconds: 5 + timeoutSeconds: 5 + readiness: + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + liveness: + periodSeconds: 20 + failureThreshold: 3 + timeoutSeconds: 5 + +# No init Job; see README.md for the HStore initialization contract. +# Install with: helm install ... --wait (no --wait-for-jobs) From 96e664f4db079ad6203fea183852052eaaf81239 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 31 Jul 2026 12:08:37 +0530 Subject: [PATCH 02/61] fix(ci): keep legacy Helm validation cluster-independent --- .github/workflows/helm-chart-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index a8ad116a48..a8d93c4601 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -76,7 +76,7 @@ jobs: cp -R helm/hugegraph /tmp/legacy cp helm/hugegraph/testdata/values-pre-hardening.yaml /tmp/legacy/values.yaml helm template legacy /tmp/legacy > /dev/null - helm install legacy /tmp/legacy --dry-run=client > /dev/null + helm template legacy /tmp/legacy --is-upgrade > /dev/null - name: helm package run: helm package helm/hugegraph -d /tmp/chart From f852f36e6519164dbb8ad8017ae0f5aafceb3bd0 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 31 Jul 2026 13:41:19 +0530 Subject: [PATCH 03/61] fix(helm): address chart review feedback --- .github/workflows/helm-chart-ci.yml | 8 ++++++++ helm/hugegraph/README.md | 5 +++-- helm/hugegraph/templates/NOTES.txt | 5 +++++ helm/hugegraph/templates/_helpers.tpl | 16 ++++++++++++++-- helm/hugegraph/templates/server-ingress.yaml | 2 +- helm/hugegraph/templates/server-pdb.yaml | 3 ++- .../templates/tests/test-connection.yaml | 16 ++++++++++++++++ helm/hugegraph/values.schema.json | 12 ++++-------- helm/hugegraph/values.yaml | 4 +++- 9 files changed, 56 insertions(+), 15 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index a8d93c4601..0291b4d6ba 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -32,6 +32,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: azure/setup-helm@v4 with: @@ -55,6 +57,12 @@ jobs: ! helm template ci helm/hugegraph --set pd.replicas=100 2>/dev/null ! helm template ci helm/hugegraph --set pd.pdb.minAvailable=3 2>/dev/null ! helm template ci helm/hugegraph --set server.hpa.enabled=true 2>/dev/null + ! helm template ci helm/hugegraph \ + --set server.hpa.enabled=true \ + --set server.hpa.minReplicas=2 \ + --set server.resources.requests.cpu=100m \ + --set server.pdb.enabled=true \ + --set server.pdb.minAvailable=2 2>/dev/null ! helm template ci helm/hugegraph --set server.auth.enabled=true 2>/dev/null - name: kubeconform diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 6d03f7677b..b68a102893 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -220,7 +220,7 @@ default values. | `server.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | | `server.securityContext` | Container-level securityContext. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | | `server.pdb.enabled` | Create a PodDisruptionBudget for Server. Off by default: Server holds no quorum | `false` | -| `server.pdb.minAvailable` | Must be strictly less than `server.replicas` | `2` | +| `server.pdb.minAvailable` | Must be less than `server.hpa.minReplicas` when HPA is enabled, otherwise less than `server.replicas` | `2` | | `server.antiAffinity` | One of `required`, `preferred`, `disabled`. Defaults to `preferred` rather than `required` because HPA may scale Server past the node count; set `required` when replicas always stay below it | `preferred` | | `server.nodeSelector` | Node selector for server Pods | `{}` | | `server.tolerations` | Tolerations for server Pods | `[]` | @@ -236,6 +236,7 @@ default values. | `server.serviceAccount.annotations` | Annotations on the created ServiceAccount | `{}` | | `server.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | | `server.waitImage` | Image for the Helm test hook | `curlimages/curl:8.5.0` | +| `server.testResources` | Resources for the Helm test hook container | `{}` | | `server.restServer.minFreeMemory` | Empty preserves the image default | `""` | | `server.restServer.batchMaxWriteThreads` | Empty preserves the image default | `""` | | `server.initStoreEnabled` | Must remain `false` for distributed HStore | `false` | @@ -371,7 +372,7 @@ kubectl describe pod | grep -A5 "Last State" Helm itself rejects release names longer than 53 characters, before this chart renders anything: -``` +```text invalid release name ... the length must not be longer than 53 ``` diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 288c8d6f5f..a0d410951d 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -34,7 +34,12 @@ Verify the release: Reach the Server API: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.server.name" . }} {{ .Values.server.port }}:{{ .Values.server.port }} +{{- if .Values.server.auth.enabled }} + PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ .Values.server.auth.existingSecret }} -o jsonpath='{.data.password}' | base64 --decode)" + curl --user "admin:${PASSWORD}" http://127.0.0.1:{{ .Values.server.port }}/versions +{{- else }} curl http://127.0.0.1:{{ .Values.server.port }}/versions +{{- end }} {{- if not .Values.server.auth.enabled }} Authentication is disabled. Do not expose this release to untrusted networks. diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 3f64b2b2b4..cc9a24b59f 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -208,6 +208,17 @@ default {{- end -}} {{- end }} +{{/* +The minimum Server replica count that a PDB must remain valid against. +*/}} +{{- define "hugegraph.server.replicaFloor" -}} +{{- if .Values.server.hpa.enabled -}} +{{- .Values.server.hpa.minReplicas -}} +{{- else -}} +{{- .Values.server.replicas -}} +{{- end -}} +{{- end }} + {{/* Cross-field validation that JSON Schema draft-07 cannot express. */}} @@ -246,8 +257,9 @@ and must not be failed for a value that has no effect. {{- fail "server.service.nodePort requires server.service.type to be NodePort or LoadBalancer" -}} {{- end -}} {{- $serverPdb := get .Values.server "pdb" | default dict -}} -{{- if and (get $serverPdb "enabled" | default false) (gt (int .Values.server.replicas) 1) (ge (int (get $serverPdb "minAvailable" | default 1)) (int .Values.server.replicas)) -}} -{{- fail "server.pdb.minAvailable must be less than server.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} +{{- $serverReplicaFloor := include "hugegraph.server.replicaFloor" . | int -}} +{{- if and (get $serverPdb "enabled" | default false) (gt $serverReplicaFloor 1) (ge (int (get $serverPdb "minAvailable" | default 1)) $serverReplicaFloor) -}} +{{- fail "server.pdb.minAvailable must be less than the active Server replica floor (server.hpa.minReplicas when HPA is enabled, otherwise server.replicas), otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} {{- end }} diff --git a/helm/hugegraph/templates/server-ingress.yaml b/helm/hugegraph/templates/server-ingress.yaml index 08a8c6a7ea..ad473fde73 100644 --- a/helm/hugegraph/templates/server-ingress.yaml +++ b/helm/hugegraph/templates/server-ingress.yaml @@ -41,7 +41,7 @@ spec: http: paths: {{- range .paths }} - - path: {{ .path }} + - path: {{ .path | quote }} pathType: {{ .pathType }} backend: service: diff --git a/helm/hugegraph/templates/server-pdb.yaml b/helm/hugegraph/templates/server-pdb.yaml index 3c1f0c4847..7fb4826aec 100644 --- a/helm/hugegraph/templates/server-pdb.yaml +++ b/helm/hugegraph/templates/server-pdb.yaml @@ -16,7 +16,8 @@ # {{- $pdb := get .Values.server "pdb" | default dict }} -{{- if and (get $pdb "enabled" | default false) (gt (int .Values.server.replicas) 1) }} +{{- $replicaFloor := include "hugegraph.server.replicaFloor" . | int }} +{{- if and (get $pdb "enabled" | default false) (gt $replicaFloor 1) }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: diff --git a/helm/hugegraph/templates/tests/test-connection.yaml b/helm/hugegraph/templates/tests/test-connection.yaml index b5c431b659..beef5c1d47 100644 --- a/helm/hugegraph/templates/tests/test-connection.yaml +++ b/helm/hugegraph/templates/tests/test-connection.yaml @@ -28,10 +28,26 @@ metadata: "helm.sh/hook": test "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: + automountServiceAccountToken: false restartPolicy: Never + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: curl image: {{ .Values.server.waitImage | quote }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 101 + runAsNonRoot: true + runAsUser: 100 + {{- with .Values.server.testResources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.server.auth.enabled }} env: - name: PASSWORD diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 23ff6f73d8..7c2185c5be 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -31,14 +31,6 @@ }, "global": { "type": "object" - }, - "networkPolicy": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - } - } } }, "definitions": { @@ -576,6 +568,7 @@ }, "ingress": { "type": "object", + "additionalProperties": false, "required": [ "enabled", "className", @@ -734,6 +727,9 @@ } } }, + "testResources": { + "type": "object" + }, "service": { "type": "object", "additionalProperties": false, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index a43ae7b98b..83bb1f3c74 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -185,7 +185,7 @@ server: podLabels: {} # Extra environment variables appended to the server container. extraEnv: [] - # Gives a JVM with an on-disk store time to shut down cleanly on drain. + # Allows in-flight requests to drain before the Server is stopped. terminationGracePeriodSeconds: 60 serviceAccount: create: true @@ -206,6 +206,8 @@ server: minAvailable: 2 # Image used by the Helm test hook. waitImage: curlimages/curl:8.5.0 + # Optional resources for the Helm test hook container. + testResources: {} restServer: # Empty preserves the image's restserver.min_free_memory default. minFreeMemory: "" From b468b54ee3c7bd370a9ffd5dd1a5782145f9a342 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 31 Jul 2026 13:49:40 +0530 Subject: [PATCH 04/61] fix(helm): validate test hook resources --- helm/hugegraph/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 7c2185c5be..56c1284bca 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -728,7 +728,7 @@ } }, "testResources": { - "type": "object" + "$ref": "#/definitions/resources" }, "service": { "type": "object", From 7f2457aebe8e9ceb546c837fdeba507ff475934b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 3 Aug 2026 17:12:50 +0530 Subject: [PATCH 05/61] feat(helm): add optional Hubble UI and fix auth bootstrap with init-store disabled The Server wrapper now writes auth.admin_pa from the auth Secret alongside usePD and pd.peers, so an auth-enabled release keeps its configured admin password with init_store.enabled=false instead of silently falling back to the public default. The Secret value is rejected when it contains properties-parser metacharacters that would inject config lines or store a different password than the Secret holds. The new hubble component deploys the Hubble UI as a single-replica Deployment with pd and direct wiring modes, optional Ingress and H2 persistence, schema validation, render-time guards, docs, and CI coverage. PD-meta installs (auth enabled, or Hubble in pd mode) also announce the Server client Service URL to PD via server.urls_to_pd and server.deploy_in_k8s so discovery clients receive a resolvable address instead of the 0.0.0.0 default, and the Hubble wrapper writes server.host so current images bind all interfaces. Because current Hubble images authenticate their login against the cluster, rendering Hubble without server.auth fails unless explicitly overridden. The CI invalid-value step now fails on every case rather than only its last line, and positive renders cover both Hubble modes. Validated against a composition of master 1716c774 plus the current heads of #3119 (edf07d0f), #3126 (b40c42fb), and #3130 (198de19e): fresh auth-enabled installs reach Ready with zero restarts, the admin credential comes from the Secret while unauthenticated and default-password requests get 401, and Hubble logs in with the Secret credential and reads cluster metadata through PD discovery, with its H2 metadata persisted on the PVC. --- .github/workflows/helm-chart-ci.yml | 63 +++- helm/hugegraph/README.md | 112 ++++++- helm/hugegraph/templates/NOTES.txt | 26 ++ helm/hugegraph/templates/_helpers.tpl | 62 ++++ .../templates/hubble-deployment.yaml | 268 ++++++++++++++++ helm/hugegraph/templates/hubble-ingress.yaml | 54 ++++ helm/hugegraph/templates/hubble-pvc.yaml | 41 +++ helm/hugegraph/templates/hubble-service.yaml | 43 +++ .../templates/server-deployment.yaml | 76 ++++- helm/hugegraph/templates/serviceaccount.yaml | 6 +- helm/hugegraph/values-cluster.yaml | 18 ++ helm/hugegraph/values.schema.json | 292 +++++++++++++----- helm/hugegraph/values.yaml | 90 +++++- 13 files changed, 1051 insertions(+), 100 deletions(-) create mode 100644 helm/hugegraph/templates/hubble-deployment.yaml create mode 100644 helm/hugegraph/templates/hubble-ingress.yaml create mode 100644 helm/hugegraph/templates/hubble-pvc.yaml create mode 100644 helm/hugegraph/templates/hubble-service.yaml diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 0291b4d6ba..b052c86bd7 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -50,20 +50,55 @@ jobs: for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do helm template ci helm/hugegraph $preset > /dev/null done + # Positive coverage for every hubble wrapper branch: pd mode with + # auth, direct mode with a custom port, and a TLS-less ingress with + # the explicit opt-in. + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set server.auth.enabled=true \ + --set server.auth.existingSecret=ci-auth > /dev/null + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set hubble.allowWithoutServerAuth=true \ + --set hubble.mode=direct \ + --set hubble.port=9090 \ + --set hubble.persistence.enabled=true \ + --set hubble.ingress.enabled=true \ + --set hubble.ingress.allowPlainHttp=true > /dev/null - name: reject invalid values run: | - # each of these must fail; the schema and helpers are the contract - ! helm template ci helm/hugegraph --set pd.replicas=100 2>/dev/null - ! helm template ci helm/hugegraph --set pd.pdb.minAvailable=3 2>/dev/null - ! helm template ci helm/hugegraph --set server.hpa.enabled=true 2>/dev/null - ! helm template ci helm/hugegraph \ + # Each case must fail to render; the schema and helpers are the + # contract. `! cmd` alone is NOT enforced under `set -e` (errexit + # ignores inverted commands), so every case checks explicitly. + must_fail() { + if helm template ci helm/hugegraph "$@" >/dev/null 2>&1; then + echo "expected render failure for: $*" >&2 + exit 1 + fi + } + must_fail --set pd.replicas=100 + must_fail --set pd.pdb.minAvailable=3 + must_fail --set server.hpa.enabled=true + must_fail \ --set server.hpa.enabled=true \ --set server.hpa.minReplicas=2 \ --set server.resources.requests.cpu=100m \ --set server.pdb.enabled=true \ - --set server.pdb.minAvailable=2 2>/dev/null - ! helm template ci helm/hugegraph --set server.auth.enabled=true 2>/dev/null + --set server.pdb.minAvailable=2 + must_fail --set server.auth.enabled=true + must_fail --set hubble.enabled=true + A="--set hubble.enabled=true --set hubble.allowWithoutServerAuth=true" + must_fail $A --set hubble.port=0 + must_fail $A \ + --set hubble.persistence.enabled=true \ + --set hubble.persistence.size="" + must_fail $A --set hubble.service.nodePort=30080 + must_fail $A --set hubble.mode=bogus + must_fail $A --set hubble.image.tag="" + must_fail $A --set hubble.ingress.enabled=true + must_fail --set server.ingress.enabled=true \ + --set server.ingress.allowPlainHttp=true - name: kubeconform shell: bash @@ -74,6 +109,20 @@ jobs: for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do helm template ci helm/hugegraph $preset | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 done + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set hubble.allowWithoutServerAuth=true \ + --set hubble.mode=direct \ + --set hubble.port=9090 \ + --set hubble.persistence.enabled=true \ + --set hubble.ingress.enabled=true \ + --set hubble.ingress.allowPlainHttp=true \ + | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set server.auth.enabled=true \ + --set server.auth.existingSecret=ci-auth \ + | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 - name: legacy --reuse-values compatibility run: | diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index b68a102893..c2c78693fe 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -43,10 +43,18 @@ so operators do not have to: 300 seconds waiting for storage and a further 120 seconds in the start command. A lower configured `failureThreshold` is raised to this floor rather than being rejected. -- **The image entrypoint keeps ownership of `PASSWORD` handling and - `auth.admin_pa`.** When authentication or a custom port or REST tuning is - configured, the chart's wrapper only ensures `usePD=true` and `pd.peers` are - present, then hands off to the image entrypoint. +- **The wrapper writes `auth.admin_pa` from the auth Secret.** With + `init_store.enabled=false` the admin credential is created on the PD startup + path from `auth.admin_pa`, not from the Docker `PASSWORD` stdin path. When + authentication is enabled, the chart's wrapper therefore writes + `auth.admin_pa` from the mounted Secret alongside `usePD=true` and + `pd.peers`, then hands off to the image entrypoint. Two caveats: + `auth.admin_pa` applies only when the admin is first created, so changing the + Secret does not rotate an existing cluster's password, and the value lands in + `rest-server.properties` inside the container (file mode 600). Because the + Java properties parser reinterprets them, the Secret value must not contain + newlines, carriage returns, or backslashes; the wrapper refuses to start if + it does. - **Resource names reserve their suffix and StatefulSet ordinal before truncation,** so a long release name cannot produce colliding or over-long Pod and Service names, and PD/Store identities stay fixed when replicas @@ -80,7 +88,7 @@ helm test hugegraph --namespace hugegraph |---|---| | `values.yaml` | Default 3+3+3 topology | | `values-single.yaml` | Single-node 1+1+1 example | -| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity | +| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity; Hubble stays opt-in because the preset does not enable authentication | `values-cluster.yaml` is a production starting point, not a capacity guarantee. Recalculate capacity for the graph size, traffic, failure budget, @@ -262,6 +270,82 @@ Helm upgrade does not overwrite the autoscaler's live replica count. Enabling utilization-based HPA requires a strictly positive `server.resources.requests.cpu`. +### Hubble (optional UI) + +Set `hubble.enabled=true` to deploy [HugeGraph Hubble](https://hugegraph.apache.org/docs/quickstart/toolchain/hugegraph-hubble/), +the web UI for graph management, schema browsing, Gremlin queries, and the +cluster operations view. `hubble.mode` selects the wiring. In the default +`pd` mode the chart points `pd.peers` at the PD gRPC peers, `pd.server` at +the PD client Service REST port, and the Store metrics allow-list at the +Store REST endpoints, so the cluster view works without manual wiring; the +Server is additionally configured to register its client Service URL with PD +(see below). In `direct` mode Hubble only receives `server.direct_url` +pointing at the Server client Service; there is no PD discovery and no +operations view. Everything else in `hugegraph-hubble.properties` keeps the +image default. + +Enabling `pd`-mode Hubble switches the Server into PD meta mode (`usePD`, +`server.urls_to_pd`, `server.deploy_in_k8s`) so that PD can hand Hubble a +resolvable Server address; auth-enabled installs already run in this mode. +On an existing release this change rolls the Server Deployment once. The +Store allow-list is computed from `store.replicas` at render time, so scale +Store with `helm upgrade`, not `kubectl scale`, or the list goes stale until +the next upgrade. + +Hubble is one replica by design: it keeps UI connection metadata, including +any graph credentials entered in the UI, in an embedded per-instance H2 +database. Enable `hubble.persistence` to keep that metadata across Pod +replacement; the chart then redirects the H2 location into the mounted +volume through Spring's environment binding. The Deployment uses the +`Recreate` strategy so two Hubble instances never attach the same database. +The PVC is kept on `helm uninstall` (delete it explicitly to discard the +stored metadata), `size` and `storageClassName` apply at install time only, +and a non-root `podSecurityContext` needs a matching `fsGroup` so H2 can +write the volume. + +**Enable `server.auth` when using Hubble.** Current Hubble images gate the +UI behind a login that authenticates against the cluster; with server +authentication disabled the login cannot complete (the server rejects +`/auth/login` with "Unconfigured authenticator"), so Hubble is only useful +on an auth-enabled deployment, where the admin credential from +`server.auth.existingSecret` logs in. The chart therefore refuses to render +`hubble.enabled=true` without `server.auth` unless +`hubble.allowWithoutServerAuth=true` explicitly overrides it for images +whose login does not need cluster authentication. + +**Hubble serves plain HTTP.** Reach it with `kubectl port-forward` or behind +an HTTPS-terminating Ingress; never expose the port directly to an untrusted +network. An Ingress without `tls` is rejected at render time unless +`hubble.ingress.allowPlainHttp=true` explicitly accepts plain HTTP for a +trusted network. + +| Parameter | Description | Default | +|---|---|---| +| `hubble.enabled` | Deploy the Hubble UI. Requires `server.auth` (see below) | `false` | +| `hubble.mode` | `pd` discovers the cluster through PD and enables the operations view; `direct` talks to the Server client Service only | `pd` | +| `hubble.allowWithoutServerAuth` | Renders Hubble without `server.auth`, for future images whose login does not require cluster authentication | `false` | +| `hubble.image.repository` | Hubble image repository | `hugegraph/hubble` | +| `hubble.image.tag` | Hubble image tag. Tracks the development image until the next release is pinned | `latest` | +| `hubble.image.pullPolicy` | Hubble image pull policy | `Always` | +| `hubble.port` | Hubble HTTP port, container port, and Service port | `8088` | +| `hubble.persistence.enabled` | Persist UI connection metadata in a PVC | `false` | +| `hubble.persistence.size` | PVC size | `1Gi` | +| `hubble.persistence.storageClassName` | Empty uses the cluster default StorageClass | `""` | +| `hubble.resources` | Hubble resources | `{}` | +| `hubble.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | +| `hubble.securityContext` | Container-level securityContext, hardened like the other components | see `values.yaml` | +| `hubble.service.type` | Hubble Service type | `ClusterIP` | +| `hubble.service.annotations` | Hubble Service annotations | `{}` | +| `hubble.service.nodePort` | Requires a `NodePort` or `LoadBalancer` Service type | unset | +| `hubble.ingress.*` | Same Ingress keys as `server.ingress.*`, plus `allowPlainHttp`, which applies to the Hubble Ingress only | `enabled: false` | +| `hubble.serviceAccount.*` | Same ServiceAccount keys as the other components | `create: true` | +| `hubble.nodeSelector` / `tolerations` / `affinity` / `topologySpreadConstraints` | Scheduling controls | unset | +| `hubble.priorityClassName` | PriorityClass for the hubble Pod | `""` | +| `hubble.podAnnotations` / `hubble.podLabels` | Extra Pod metadata | `{}` | +| `hubble.extraEnv` | Extra environment variables for the hubble container | `[]` | +| `hubble.terminationGracePeriodSeconds` | Shutdown grace period | `30` | +| `hubble.probes.*` | Same probe keys as PD; startup and readiness check `/actuator/health`, liveness is a TCP check | see `values.yaml` | + Specify each parameter with `--set`, or supply a YAML file with `-f`: ```bash @@ -284,6 +368,16 @@ before anything reaches the cluster: - `pdb.minAvailable` must be less than the matching `replicas`, so a PodDisruptionBudget cannot permanently block node drains. - `pd.replicas` and `store.replicas` are capped at 99. +- `hubble.port` must be a valid port, `hubble.persistence.size` must be + non-empty, and `hubble.service.nodePort` requires a `NodePort` or + `LoadBalancer` Service type. +- `hubble.image.tag` must be non-empty (the chart `appVersion` tracks the + Server release, not Hubble), and a Hubble Ingress without `tls` is rejected + unless `hubble.ingress.allowPlainHttp=true`. +- `hubble.enabled` without `server.auth.enabled` is rejected unless + `hubble.allowWithoutServerAuth=true`, and setting + `server.ingress.allowPlainHttp` is rejected because the plain-HTTP opt-in + applies to the Hubble Ingress only. ## Deep Dive @@ -395,3 +489,11 @@ independently of the release name. valid for a root image; `podSecurityContext` and `securityContext` are fully configurable per component. - `values-cluster.yaml` is a starting point, not a capacity guarantee. +- The auth Secret sets the admin password only at first creation via + `auth.admin_pa`; the chart cannot rotate an existing cluster's admin + password. +- Hubble is single-replica, serves plain HTTP, requires `server.auth` to be + enabled for its login to complete, and keeps UI connection metadata, + including any graph credentials entered in the UI, in an embedded H2 + database that is lost on Pod replacement unless `hubble.persistence` is + enabled. diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index a0d410951d..72b8f54cf9 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -14,11 +14,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} +{{- $hubbleEnabled := get (get .Values "hubble" | default dict) "enabled" | default false }} {{ .Release.Name }} installed in namespace {{ .Release.Namespace }}. PD {{ .Values.pd.replicas }} replica(s) Store {{ .Values.store.replicas }} replica(s) Server {{ if .Values.server.hpa.enabled }}HPA {{ .Values.server.hpa.minReplicas }}-{{ .Values.server.hpa.maxReplicas }}{{ else }}{{ .Values.server.replicas }} replica(s){{ end }} +{{- if $hubbleEnabled }} + Hubble 1 replica (UI) +{{- end }} Watch the cluster come up: @@ -40,6 +44,28 @@ Reach the Server API: {{- else }} curl http://127.0.0.1:{{ .Values.server.port }}/versions {{- end }} +{{- if $hubbleEnabled }} + +Reach the Hubble UI: + + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.hubble.name" . }} {{ .Values.hubble.port }}:{{ .Values.hubble.port }} + + then open http://127.0.0.1:{{ .Values.hubble.port }} + +{{- if not .Values.server.auth.enabled }} + +Server authentication is disabled, so the Hubble login cannot complete on +this release: current Hubble images authenticate against the cluster. +Enable server.auth to use Hubble. +{{- end }} + +Hubble serves plain HTTP. Reach it through port-forward or an +HTTPS-terminating Ingress; never expose it directly to an untrusted network. +{{- if not .Values.hubble.persistence.enabled }} +Hubble persistence is disabled: UI connection metadata is lost when the +Hubble Pod is replaced. Graph data is unaffected. +{{- end }} +{{- end }} {{- if not .Values.server.auth.enabled }} Authentication is disabled. Do not expose this release to untrusted networks. diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index cc9a24b59f..c32d261fcd 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -70,6 +70,14 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- printf "%s-server" (include "hugegraph.fullname" . | trunc 56 | trimSuffix "-") }} {{- end }} +{{- define "hugegraph.hubble.name" -}} +{{- printf "%s-hubble" (include "hugegraph.fullname" . | trunc 56 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.hubble.dataName" -}} +{{- printf "%s-hubble-data" (include "hugegraph.fullname" . | trunc 51 | trimSuffix "-") }} +{{- end }} + {{- define "hugegraph.test.name" -}} {{- printf "%s-test-connection" (include "hugegraph.fullname" . | trunc 47 | trimSuffix "-") }} {{- end }} @@ -144,6 +152,38 @@ First store REST endpoint for STORE_REST / wait-partition. {{- printf "%s-0.%s.%s.svc:%d" $name $name $ns (int .Values.store.ports.rest) -}} {{- end }} +{{/* +Server REST URL reached through the client Service. Announced to PD via +server.urls_to_pd so PD-discovered clients (Hubble) get a resolvable address +instead of the in-pod 0.0.0.0 default. +*/}} +{{- define "hugegraph.server.clientUrl" -}} +{{- printf "http://%s.%s.svc:%d" (include "hugegraph.server.name" .) .Release.Namespace (int .Values.server.port) -}} +{{- end }} + +{{/* +PD REST endpoint reached through the client Service, for Hubble's pd.server. +*/}} +{{- define "hugegraph.pd.restClientEndpoint" -}} +{{- printf "%s.%s.svc:%d" (include "hugegraph.pd.clientName" .) .Release.Namespace (int .Values.pd.ports.rest) -}} +{{- end }} + +{{/* +Store REST origins in Hubble's bracketed allow-list form: +[http://store-0.svc.ns.svc:8520,...] +*/}} +{{- define "hugegraph.store.restOriginsList" -}} +{{- $origins := list -}} +{{- $replicas := int .Values.store.replicas -}} +{{- $name := include "hugegraph.store.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.store.ports.rest -}} +{{- range $i := until $replicas -}} + {{- $origins = append $origins (printf "http://%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- printf "[%s]" (join "," $origins) -}} +{{- end }} + {{/* Quorum size: floor(replicas/2)+1 */}} @@ -261,6 +301,28 @@ and must not be failed for a value that has no effect. {{- if and (get $serverPdb "enabled" | default false) (gt $serverReplicaFloor 1) (ge (int (get $serverPdb "minAvailable" | default 1)) $serverReplicaFloor) -}} {{- fail "server.pdb.minAvailable must be less than the active Server replica floor (server.hpa.minReplicas when HPA is enabled, otherwise server.replicas), otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} +{{- $serverIngress := get .Values.server "ingress" | default dict -}} +{{- if hasKey $serverIngress "allowPlainHttp" -}} +{{- fail "server.ingress.allowPlainHttp has no effect; the plain-HTTP opt-in applies to hubble.ingress only" -}} +{{- end -}} +{{- $hubble := get .Values "hubble" | default dict -}} +{{- if get $hubble "enabled" | default false -}} +{{- if and (not .Values.server.auth.enabled) (not (get $hubble "allowWithoutServerAuth" | default false)) -}} +{{- fail "hubble.enabled requires server.auth: current Hubble images authenticate against the cluster and cannot complete their login on an auth-less deployment. Enable server.auth, or set hubble.allowWithoutServerAuth=true for images that support it" -}} +{{- end -}} +{{- $hubbleSvc := get $hubble "service" | default dict -}} +{{- if and (get $hubbleSvc "nodePort") (not (has (get $hubbleSvc "type" | default "ClusterIP") (list "NodePort" "LoadBalancer"))) -}} +{{- fail "hubble.service.nodePort requires hubble.service.type to be NodePort or LoadBalancer" -}} +{{- end -}} +{{- $hubbleImage := get $hubble "image" | default dict -}} +{{- if eq (trim (get $hubbleImage "tag" | default "")) "" -}} +{{- fail "hubble.image.tag must not be empty: the chart appVersion tracks the Server release, not Hubble, so there is no meaningful fallback" -}} +{{- end -}} +{{- $hubbleIngress := get $hubble "ingress" | default dict -}} +{{- if and (get $hubbleIngress "enabled" | default false) (empty (get $hubbleIngress "tls")) (not (get $hubbleIngress "allowPlainHttp" | default false)) -}} +{{- fail "hubble.ingress.enabled without tls publishes the plain-HTTP, unauthenticated Hubble UI; configure hubble.ingress.tls, or set hubble.ingress.allowPlainHttp=true to accept that on a trusted network" -}} +{{- end -}} +{{- end -}} {{- end }} {{/* diff --git a/helm/hugegraph/templates/hubble-deployment.yaml b/helm/hugegraph/templates/hubble-deployment.yaml new file mode 100644 index 0000000000..d094db4a45 --- /dev/null +++ b/helm/hugegraph/templates/hubble-deployment.yaml @@ -0,0 +1,268 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if get (get .Values "hubble" | default dict) "enabled" | default false }} +{{- include "hugegraph.validateValues" . }} +{{- $customPort := ne (int .Values.hubble.port) 8088 }} +{{- $mode := .Values.hubble.mode | default "pd" }} +{{- if not (has $mode (list "pd" "direct")) }} +{{- fail "hubble.mode must be one of: pd, direct" }} +{{- end }} +{{- $pdMode := eq $mode "pd" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hugegraph.hubble.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: hubble +spec: + # Hubble keeps its connection metadata in an embedded per-instance H2 + # database, so this Deployment is fixed at one replica and replaces the Pod + # instead of rolling, which also keeps a persistent volume single-attached. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: hubble + template: + metadata: + labels: + {{- include "hugegraph.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: hubble + {{- with .Values.hubble.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} + {{- with .Values.hubble.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: {{ get (get .Values.hubble "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} + serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.hubble "name" (include "hugegraph.hubble.name" .) ) }} + {{- with .Values.hubble.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.hubble.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.hubble.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hubble.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hubble.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hubble.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.hubble.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: hubble + image: "{{ .Values.hubble.image.repository }}:{{ .Values.hubble.image.tag }}" + imagePullPolicy: {{ .Values.hubble.image.pullPolicy }} + {{- with .Values.hubble.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + # The Hubble image reads conf/hugegraph-hubble.properties only; it + # has no environment mapping for these keys, so the wrapper rewrites + # them before handing off to the image's own start script. The image + # ships no dumb-init; start-hubble.sh -f execs java, which then + # receives signals directly. + command: + - /bin/bash + - -c + args: + - | + set -euo pipefail + CONF=./conf/hugegraph-hubble.properties + if [[ ! -r "${CONF}" ]]; then + echo "missing or unreadable ${CONF}; refusing to write a stub config" >&2 + exit 1 + fi + TMP=$(mktemp) + FOUND_PD_ENABLED=false + {{- if $pdMode }} + FOUND_PD_PEERS=false + FOUND_PD_SERVER=false + FOUND_STORE_TARGETS=false + {{- else }} + FOUND_DIRECT_URL=false + {{- end }} + FOUND_SERVER_HOST=false + {{- if $customPort }} + FOUND_HUBBLE_PORT=false + FOUND_SERVER_PORT=false + {{- end }} + while IFS= read -r LINE || [[ -n "${LINE}" ]]; do + case "${LINE}" in + pd.enabled=*) + printf 'pd.enabled={{ $pdMode }}\n' >>"${TMP}" + FOUND_PD_ENABLED=true + ;; + {{- if $pdMode }} + pd.peers=*) + printf 'pd.peers=%s\n' "${HG_HUBBLE_PD_PEERS}" >>"${TMP}" + FOUND_PD_PEERS=true + ;; + pd.server=*) + printf 'pd.server=%s\n' "${HG_HUBBLE_PD_SERVER}" >>"${TMP}" + FOUND_PD_SERVER=true + ;; + operations.store.allowed_targets=*) + printf 'operations.store.allowed_targets=%s\n' \ + "${HG_HUBBLE_STORE_TARGETS}" >>"${TMP}" + FOUND_STORE_TARGETS=true + ;; + {{- else }} + server.direct_url=*) + printf 'server.direct_url=%s\n' "${HG_HUBBLE_SERVER_URL}" >>"${TMP}" + FOUND_DIRECT_URL=true + ;; + {{- end }} + server.host=*) + printf 'server.host=0.0.0.0\n' >>"${TMP}" + FOUND_SERVER_HOST=true + ;; + {{- if $customPort }} + hubble.port=*) + printf 'hubble.port=%s\n' {{ .Values.hubble.port | quote }} >>"${TMP}" + FOUND_HUBBLE_PORT=true + ;; + server.port=*) + printf 'server.port=%s\n' {{ .Values.hubble.port | quote }} >>"${TMP}" + FOUND_SERVER_PORT=true + ;; + {{- end }} + *) + printf '%s\n' "${LINE}" >>"${TMP}" + ;; + esac + done <"${CONF}" + if [[ "${FOUND_PD_ENABLED}" == false ]]; then + printf 'pd.enabled={{ $pdMode }}\n' >>"${TMP}" + fi + {{- if $pdMode }} + if [[ "${FOUND_PD_PEERS}" == false ]]; then + printf 'pd.peers=%s\n' "${HG_HUBBLE_PD_PEERS}" >>"${TMP}" + fi + if [[ "${FOUND_PD_SERVER}" == false ]]; then + printf 'pd.server=%s\n' "${HG_HUBBLE_PD_SERVER}" >>"${TMP}" + fi + if [[ "${FOUND_STORE_TARGETS}" == false ]]; then + printf 'operations.store.allowed_targets=%s\n' \ + "${HG_HUBBLE_STORE_TARGETS}" >>"${TMP}" + fi + {{- else }} + if [[ "${FOUND_DIRECT_URL}" == false ]]; then + printf 'server.direct_url=%s\n' "${HG_HUBBLE_SERVER_URL}" >>"${TMP}" + fi + {{- end }} + # Current Hubble binds server.host, which defaults to + # localhost; the shipped conf's hubble.host=0.0.0.0 line is + # legacy, is ignored by current images, and is preserved + # verbatim above for images that still read it. + if [[ "${FOUND_SERVER_HOST}" == false ]]; then + printf 'server.host=0.0.0.0\n' >>"${TMP}" + fi + {{- if $customPort }} + if [[ "${FOUND_HUBBLE_PORT}" == false ]]; then + printf 'hubble.port=%s\n' {{ .Values.hubble.port | quote }} >>"${TMP}" + fi + if [[ "${FOUND_SERVER_PORT}" == false ]]; then + printf 'server.port=%s\n' {{ .Values.hubble.port | quote }} >>"${TMP}" + fi + {{- end }} + chmod 600 "${TMP}" + mv "${TMP}" "${CONF}" + exec ./bin/start-hubble.sh -f + ports: + - name: http + containerPort: {{ .Values.hubble.port }} + env: + {{- if $pdMode }} + - name: HG_HUBBLE_PD_PEERS + value: {{ include "hugegraph.pd.grpcPeersList" . | quote }} + - name: HG_HUBBLE_PD_SERVER + value: {{ include "hugegraph.pd.restClientEndpoint" . | quote }} + - name: HG_HUBBLE_STORE_TARGETS + value: {{ include "hugegraph.store.restOriginsList" . | quote }} + {{- else }} + - name: HG_HUBBLE_SERVER_URL + value: {{ include "hugegraph.server.clientUrl" . | quote }} + {{- end }} + {{- if .Values.hubble.persistence.enabled }} + # Hubble's H2 location is fixed to ./db inside the image classpath + # config; Spring Boot's environment binding is the supported way to + # point it into the mounted volume. + - name: SPRING_DATASOURCE_URL + value: "jdbc:h2:file:/hubble-data/db;DB_CLOSE_ON_EXIT=FALSE" + {{- end }} + {{- with .Values.hubble.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} + {{- if .Values.hubble.persistence.enabled }} + volumeMounts: + - name: data + mountPath: /hubble-data + {{- end }} + startupProbe: + httpGet: + path: /actuator/health + port: http + failureThreshold: {{ .Values.hubble.probes.startup.failureThreshold }} + periodSeconds: {{ .Values.hubble.probes.startup.periodSeconds }} + {{- with include "hugegraph.probeTuning" .Values.hubble.probes.startup }}{{ . | trim | nindent 12 }}{{- end }} + readinessProbe: + httpGet: + path: /actuator/health + port: http + periodSeconds: {{ .Values.hubble.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.hubble.probes.readiness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.hubble.probes.readiness }}{{ . | trim | nindent 12 }}{{- end }} + # TCP for liveness: /actuator/health aggregates datasource health, + # and a transiently slow persistent volume must not get a healthy + # JVM killed (Recreate + RWO makes an overlapping restart worse). + livenessProbe: + tcpSocket: + port: http + periodSeconds: {{ .Values.hubble.probes.liveness.periodSeconds }} + failureThreshold: {{ .Values.hubble.probes.liveness.failureThreshold }} + {{- with include "hugegraph.probeTuning" .Values.hubble.probes.liveness }}{{ . | trim | nindent 12 }}{{- end }} + {{- with .Values.hubble.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.hubble.persistence.enabled }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "hugegraph.hubble.dataName" . }} + {{- end }} +{{- end }} diff --git a/helm/hugegraph/templates/hubble-ingress.yaml b/helm/hugegraph/templates/hubble-ingress.yaml new file mode 100644 index 0000000000..845730619d --- /dev/null +++ b/helm/hugegraph/templates/hubble-ingress.yaml @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- $hubble := get .Values "hubble" | default dict }} +{{- if and (get $hubble "enabled" | default false) .Values.hubble.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "hugegraph.hubble.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: hubble + {{- with (get .Values.hubble.ingress "annotations") }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.hubble.ingress.className }} + ingressClassName: {{ .Values.hubble.ingress.className | quote }} + {{- end }} + {{- with .Values.hubble.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.hubble.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path | quote }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "hugegraph.hubble.name" $ }} + port: + number: {{ $.Values.hubble.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/hugegraph/templates/hubble-pvc.yaml b/helm/hugegraph/templates/hubble-pvc.yaml new file mode 100644 index 0000000000..8143fab844 --- /dev/null +++ b/helm/hugegraph/templates/hubble-pvc.yaml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- $hubble := get .Values "hubble" | default dict }} +{{- if and (get $hubble "enabled" | default false) .Values.hubble.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "hugegraph.hubble.dataName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: hubble + # Survive helm uninstall, matching the PD/Store volumeClaimTemplates + # behavior; delete the PVC explicitly to discard the stored connection + # metadata and credentials. + annotations: + helm.sh/resource-policy: keep +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.hubble.persistence.size }} + {{- if .Values.hubble.persistence.storageClassName }} + storageClassName: {{ .Values.hubble.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/helm/hugegraph/templates/hubble-service.yaml b/helm/hugegraph/templates/hubble-service.yaml new file mode 100644 index 0000000000..53eed8ec4a --- /dev/null +++ b/helm/hugegraph/templates/hubble-service.yaml @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if get (get .Values "hubble" | default dict) "enabled" | default false }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hugegraph.hubble.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: hubble + {{- $svc := get .Values.hubble "service" | default dict }} + {{- with (get $svc "annotations") }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ get $svc "type" | default "ClusterIP" }} + selector: + {{- include "hugegraph.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: hubble + ports: + - name: http + port: {{ .Values.hubble.port }} + targetPort: http + {{- with (get $svc "nodePort") }} + nodePort: {{ . }} + {{- end }} +{{- end }} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 6445c80c75..2d9b1c333a 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -26,6 +26,16 @@ {{- $batchMaxWriteThreads = toString (get $restServer "batchMaxWriteThreads") }} {{- end }} {{- $customPort := ne (int .Values.server.port) 8080 }} +{{- $hubble := get .Values "hubble" | default dict }} +{{- $hubblePdMode := and (get $hubble "enabled" | default false) (ne (get $hubble "mode" | default "pd") "direct") }} +{{/* +PD meta mode (usePD) is required on two paths: the built-in authenticator +creates the admin on the PD startup path, and PD-mode Hubble discovers the +Server through the service URL the Server registers with PD. Both need the +same four properties, so they share one condition. +*/}} +{{- $pdMeta := or .Values.server.auth.enabled $hubblePdMode }} +{{- $wrapper := or $pdMeta $customPort (ne $minFreeMemory "") (ne $batchMaxWriteThreads "") }} apiVersion: apps/v1 kind: Deployment metadata: @@ -94,10 +104,7 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or .Values.server.auth.enabled - $customPort - (ne $minFreeMemory "") - (ne $batchMaxWriteThreads "") }} + {{- if $wrapper }} command: - /usr/bin/dumb-init - -- @@ -108,12 +115,34 @@ spec: set -euo pipefail {{- if .Values.server.auth.enabled }} : "${PASSWORD:?auth Secret key 'password' must not be empty}" + # The value is written into a Java properties file, whose parser + # treats CR as a line terminator, unescapes backslashes, and + # skips whitespace after the separator, so these shapes would + # inject config lines or silently store a password that differs + # from the Secret. + case "${PASSWORD}" in + *$'\n'* | *$'\r'* | *\\* | ' '* | $'\t'*) + echo "auth Secret key 'password' must not contain newlines," \ + "carriage returns, or backslashes, or start with" \ + "whitespace" >&2 + exit 1 + ;; + esac {{- end }} CONF=./conf/rest-server.properties + if [[ ! -r "${CONF}" ]]; then + echo "missing or unreadable ${CONF}; refusing to write a stub config" >&2 + exit 1 + fi TMP=$(mktemp) - {{- if .Values.server.auth.enabled }} + {{- if $pdMeta }} FOUND_USE_PD=false FOUND_PD_PEERS=false + FOUND_URLS_TO_PD=false + FOUND_DEPLOY_IN_K8S=false + {{- end }} + {{- if .Values.server.auth.enabled }} + FOUND_AUTH_ADMIN_PA=false {{- end }} {{- if $customPort }} FOUND_RESTSERVER_URL=false @@ -126,7 +155,7 @@ spec: {{- end }} while IFS= read -r LINE || [[ -n "${LINE}" ]]; do case "${LINE}" in - {{- if .Values.server.auth.enabled }} + {{- if $pdMeta }} usePD=*) printf 'usePD=true\n' >>"${TMP}" FOUND_USE_PD=true @@ -135,6 +164,20 @@ spec: printf 'pd.peers=%s\n' "${HG_SERVER_PD_PEERS}" >>"${TMP}" FOUND_PD_PEERS=true ;; + server.urls_to_pd=*) + printf 'server.urls_to_pd=%s\n' "${HG_SERVER_URLS_TO_PD}" >>"${TMP}" + FOUND_URLS_TO_PD=true + ;; + server.deploy_in_k8s=*) + printf 'server.deploy_in_k8s=true\n' >>"${TMP}" + FOUND_DEPLOY_IN_K8S=true + ;; + {{- end }} + {{- if .Values.server.auth.enabled }} + auth.admin_pa=*) + printf 'auth.admin_pa=%s\n' "${PASSWORD}" >>"${TMP}" + FOUND_AUTH_ADMIN_PA=true + ;; {{- end }} {{- if $customPort }} restserver.url=*) @@ -162,13 +205,28 @@ spec: ;; esac done <"${CONF}" - {{- if .Values.server.auth.enabled }} + {{- if $pdMeta }} if [[ "${FOUND_USE_PD}" == false ]]; then printf 'usePD=true\n' >>"${TMP}" fi if [[ "${FOUND_PD_PEERS}" == false ]]; then printf 'pd.peers=%s\n' "${HG_SERVER_PD_PEERS}" >>"${TMP}" fi + # PD hands this URL to discovery clients such as Hubble. The + # k8s branch is taken only when server.deploy_in_k8s is true; + # otherwise the announcement falls back to restserver.url, + # whose 0.0.0.0 is never resolvable from another Pod. + if [[ "${FOUND_URLS_TO_PD}" == false ]]; then + printf 'server.urls_to_pd=%s\n' "${HG_SERVER_URLS_TO_PD}" >>"${TMP}" + fi + if [[ "${FOUND_DEPLOY_IN_K8S}" == false ]]; then + printf 'server.deploy_in_k8s=true\n' >>"${TMP}" + fi + {{- end }} + {{- if .Values.server.auth.enabled }} + if [[ "${FOUND_AUTH_ADMIN_PA}" == false ]]; then + printf 'auth.admin_pa=%s\n' "${PASSWORD}" >>"${TMP}" + fi {{- end }} {{- if $customPort }} if [[ "${FOUND_RESTSERVER_URL}" == false ]]; then @@ -206,6 +264,10 @@ spec: value: {{ include "hugegraph.store.restPrimary" . | quote }} - name: HG_SERVER_INIT_STORE_ENABLED value: {{ .Values.server.initStoreEnabled | quote }} + {{- if $pdMeta }} + - name: HG_SERVER_URLS_TO_PD + value: {{ include "hugegraph.server.clientUrl" . | quote }} + {{- end }} {{- with .Values.server.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} {{- with include "hugegraph.javaOptsEnv" .Values.server.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} {{- if and .Values.server.auth.enabled .Values.server.auth.existingSecret }} diff --git a/helm/hugegraph/templates/serviceaccount.yaml b/helm/hugegraph/templates/serviceaccount.yaml index 9889414d4b..0dbf4c302c 100644 --- a/helm/hugegraph/templates/serviceaccount.yaml +++ b/helm/hugegraph/templates/serviceaccount.yaml @@ -14,7 +14,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -}} -{{- range $component := list "pd" "store" "server" }} +{{- $components := list "pd" "store" "server" }} +{{- if get (get .Values "hubble" | default dict) "enabled" | default false }} +{{- $components = append $components "hubble" }} +{{- end }} +{{- range $component := $components }} {{- $values := index $.Values $component }} {{- $sa := get $values "serviceAccount" | default dict }} {{- if and (get $sa "create" | default false) (not (get $sa "name")) }} diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index 91ae09a9c6..b3d37b2ced 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -84,3 +84,21 @@ server: memory: 2Gi hpa: enabled: false + +# The optional Hubble UI is not enabled here: this preset does not enable +# server authentication, and current Hubble images cannot complete their +# login against an auth-less cluster. Enable it together with server.auth, +# starting from: +# +# hubble: +# enabled: true +# persistence: +# enabled: true +# size: 1Gi +# resources: +# requests: +# cpu: 250m +# memory: 768Mi +# limits: +# cpu: "1" +# memory: 1536Mi diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 56c1284bca..9ad8d96de7 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -31,6 +31,9 @@ }, "global": { "type": "object" + }, + "hubble": { + "$ref": "#/definitions/hubble" } }, "definitions": { @@ -567,72 +570,7 @@ ] }, "ingress": { - "type": "object", - "additionalProperties": false, - "required": [ - "enabled", - "className", - "hosts", - "tls" - ], - "properties": { - "enabled": { - "type": "boolean" - }, - "className": { - "type": "string" - }, - "hosts": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "host", - "paths" - ], - "properties": { - "host": { - "type": "string", - "minLength": 1 - }, - "paths": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "path", - "pathType" - ], - "properties": { - "path": { - "type": "string" - }, - "pathType": { - "type": "string", - "enum": [ - "Exact", - "Prefix", - "ImplementationSpecific" - ] - } - } - } - } - } - } - }, - "tls": { - "type": "array", - "items": { - "type": "object" - } - }, - "annotations": { - "type": "object" - } - } + "$ref": "#/definitions/ingress" }, "hpa": { "type": "object", @@ -731,29 +669,225 @@ "$ref": "#/definitions/resources" }, "service": { + "$ref": "#/definitions/service" + } + } + }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ] + }, + "annotations": { + "type": "object" + }, + "nodePort": { + "type": [ + "integer", + "null" + ], + "minimum": 30000, + "maximum": 32767 + } + } + }, + "ingress": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "className", + "hosts", + "tls" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "className": { + "type": "string" + }, + "hosts": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "host", + "paths" + ], + "properties": { + "host": { + "type": "string", + "minLength": 1 + }, + "paths": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "pathType" + ], + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string", + "enum": [ + "Exact", + "Prefix", + "ImplementationSpecific" + ] + } + } + } + } + } + } + }, + "tls": { + "type": "array", + "items": { + "type": "object" + } + }, + "annotations": { + "type": "object" + }, + "allowPlainHttp": { + "type": "boolean" + } + } + }, + "hubble": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "mode", + "image", + "port", + "persistence", + "resources", + "service", + "ingress", + "probes" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "mode": { + "type": "string", + "enum": [ + "pd", + "direct" + ] + }, + "allowWithoutServerAuth": { + "type": "boolean" + }, + "image": { + "$ref": "#/definitions/image" + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "persistence": { "type": "object", "additionalProperties": false, + "required": [ + "enabled", + "size", + "storageClassName" + ], "properties": { - "type": { + "enabled": { + "type": "boolean" + }, + "size": { "type": "string", - "enum": [ - "ClusterIP", - "NodePort", - "LoadBalancer" - ] + "minLength": 1 + }, + "storageClassName": { + "type": "string" + } + } + }, + "resources": { + "$ref": "#/definitions/resources" + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "topologySpreadConstraints": { + "type": "array" + }, + "priorityClassName": { + "type": "string" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "extraEnv": { + "type": "array" + }, + "terminationGracePeriodSeconds": { + "type": "integer", + "minimum": 0 + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" }, "annotations": { "type": "object" }, - "nodePort": { - "type": [ - "integer", - "null" - ], - "minimum": 30000, - "maximum": 32767 + "automountServiceAccountToken": { + "type": "boolean" } } + }, + "service": { + "$ref": "#/definitions/service" + }, + "ingress": { + "$ref": "#/definitions/ingress" + }, + "probes": { + "$ref": "#/definitions/probes" } } } diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 83bb1f3c74..88142c23dd 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -219,7 +219,9 @@ server: initStoreEnabled: false auth: enabled: false - # Secret must contain key "password" + # Secret must contain key "password" (no newlines). Applied as + # auth.admin_pa when the admin account is first created; changing the + # Secret later does not rotate an existing cluster's password. existingSecret: "" service: type: ClusterIP @@ -254,5 +256,91 @@ server: failureThreshold: 3 timeoutSeconds: 5 +# Optional Hubble UI. Serves plain HTTP; keep it on a ClusterIP Service or +# behind an HTTPS-terminating Ingress, never directly on an untrusted network. +hubble: + enabled: false + # pd: discover the cluster through PD (enables the cluster operations view). + # direct: talk to the Server client Service only, without PD discovery. + # Current Hubble images require server.auth to be enabled: the UI login + # authenticates against the cluster, so rendering fails otherwise unless + # allowWithoutServerAuth explicitly overrides for images that support it. + mode: pd + allowWithoutServerAuth: false + image: + repository: hugegraph/hubble + # The draft tracks latest until the next HugeGraph release tag is available. + # Pin the release tag and switch to IfNotPresent before stable publication. + tag: latest + pullPolicy: Always + port: 8088 + # Hubble keeps UI connection metadata, including any graph credentials + # entered in the UI, in an embedded per-instance H2 database, so the + # Deployment is fixed at a single replica (pointing SPRING_DATASOURCE_URL + # at an external database via extraEnv is not a supported configuration). + # Without persistence that metadata is lost on Pod replacement; graph data + # is unaffected. size and storageClassName apply at install time only, and + # the PVC is kept on helm uninstall. + persistence: + enabled: false + size: 1Gi + storageClassName: "" + resources: {} + # When persistence is enabled and the pod runs as non-root, set a matching + # fsGroup here so H2 can write /hubble-data. + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + podAnnotations: {} + podLabels: {} + # Extra environment variables appended to the hubble container. + extraEnv: [] + terminationGracePeriodSeconds: 30 + serviceAccount: + create: true + name: "" + annotations: {} + # This chart makes no Kubernetes API calls, so no token is mounted. + automountServiceAccountToken: false + service: + type: ClusterIP + annotations: {} + ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: hubble.local + paths: + - path: / + pathType: Prefix + tls: [] + # Hubble serves plain HTTP, so an Ingress without tls is rejected at + # render time unless this is explicitly set to true for a trusted + # network. + allowPlainHttp: false + probes: + startup: + failureThreshold: 30 + periodSeconds: 5 + timeoutSeconds: 5 + readiness: + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + liveness: + periodSeconds: 20 + failureThreshold: 3 + timeoutSeconds: 5 + # No init Job; see README.md for the HStore initialization contract. # Install with: helm install ... --wait (no --wait-for-jobs) From ad42c4601a6fa9272da3b63f5affe9390d12156e Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 3 Aug 2026 17:20:57 +0530 Subject: [PATCH 06/61] fix(helm): enforce the PD PDB quorum floor and reject chart-managed extraEnv overrides A PD PodDisruptionBudget with minAvailable below floor(replicas/2)+1 permits voluntary evictions that leave PD without a Raft majority, so the render now requires the quorum floor in addition to the existing blocks-all-drains upper bound. With 2 replicas no valid budget exists; the README documents the even/odd contract. extraEnv entries render after the chart-owned variables and Kubernetes lets the last duplicate win, so a duplicate name could silently override a validated contract such as HG_SERVER_INIT_STORE_ENABLED=false. Each component's extraEnv now rejects its chart-managed variable names. Boundary and negative render cases for both rules are part of the CI invalid-value step. --- .github/workflows/helm-chart-ci.yml | 14 ++++++++++++++ helm/hugegraph/README.md | 8 ++++++++ helm/hugegraph/templates/_helpers.tpl | 22 ++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index b052c86bd7..a632a9de69 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -99,6 +99,20 @@ jobs: must_fail $A --set hubble.ingress.enabled=true must_fail --set server.ingress.enabled=true \ --set server.ingress.allowPlainHttp=true + # PD PDB must keep the Raft majority: floor(replicas/2)+1 + must_fail --set pd.replicas=5 --set pd.pdb.minAvailable=2 + must_fail --set pd.replicas=4 --set pd.pdb.minAvailable=2 + helm template ci helm/hugegraph \ + --set pd.replicas=5 --set pd.pdb.minAvailable=3 > /dev/null + helm template ci helm/hugegraph \ + --set pd.replicas=5 --set pd.pdb.minAvailable=4 > /dev/null + # extraEnv must not override chart-managed variables + must_fail --set 'server.extraEnv[0].name=HG_SERVER_INIT_STORE_ENABLED' \ + --set 'server.extraEnv[0].value=true' + must_fail --set 'pd.extraEnv[0].name=HG_PD_RAFT_PEERS_LIST' \ + --set 'pd.extraEnv[0].value=x' + must_fail --set 'store.extraEnv[0].name=HG_STORE_PD_ADDRESS' \ + --set 'store.extraEnv[0].value=x' - name: kubeconform shell: bash diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index c2c78693fe..323be460ed 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -367,6 +367,14 @@ before anything reaches the cluster: `server.resources.requests.cpu`. - `pdb.minAvailable` must be less than the matching `replicas`, so a PodDisruptionBudget cannot permanently block node drains. +- `pd.pdb.minAvailable` must also be at least the PD Raft majority, + `floor(replicas/2)+1`, so the budget cannot permit evictions that drop PD + below quorum. With 2 PD replicas no valid budget exists (the majority is + the whole membership); disable the PD PDB or use an odd replica count. +- `extraEnv` must not set chart-managed variable names (for example + `HG_SERVER_INIT_STORE_ENABLED` or the PD/Store identity and topology + variables): entries render after the chart-owned variables and the last + duplicate wins, so an override would silently bypass a validated contract. - `pd.replicas` and `store.replicas` are capped at 99. - `hubble.port` must be a valid port, `hubble.persistence.size` must be non-empty, and `hubble.service.nodePort` requires a `NodePort` or diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index c32d261fcd..5b077bf3b3 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -289,6 +289,9 @@ and must not be failed for a value that has no effect. {{- if and .Values.pd.pdb.enabled (gt (int .Values.pd.replicas) 1) (ge (int .Values.pd.pdb.minAvailable) (int .Values.pd.replicas)) -}} {{- fail "pd.pdb.minAvailable must be less than pd.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} +{{- if and .Values.pd.pdb.enabled (gt (int .Values.pd.replicas) 1) (lt (int .Values.pd.pdb.minAvailable) (include "hugegraph.pd.quorum" . | int)) -}} +{{- fail "pd.pdb.minAvailable must be at least the PD Raft majority, floor(replicas/2)+1, otherwise the budget permits voluntary evictions that drop PD below quorum" -}} +{{- end -}} {{- if and .Values.store.pdb.enabled (gt (int .Values.store.replicas) 1) (ge (int .Values.store.pdb.minAvailable) (int .Values.store.replicas)) -}} {{- fail "store.pdb.minAvailable must be less than store.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} @@ -305,6 +308,25 @@ and must not be failed for a value that has no effect. {{- if hasKey $serverIngress "allowPlainHttp" -}} {{- fail "server.ingress.allowPlainHttp has no effect; the plain-HTTP opt-in applies to hubble.ingress only" -}} {{- end -}} +{{/* +extraEnv entries render after the chart-owned variables and Kubernetes lets +the last duplicate win, so a duplicate name would silently override a +validated contract (for example re-enabling init-store across Server +replicas). Reserved names are rejected instead. +*/}} +{{- $reservedEnv := dict + "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "JAVA_OPTS") + "store" (list "HG_STORE_PD_ADDRESS" "HG_STORE_GRPC_HOST" "HG_STORE_GRPC_PORT" "HG_STORE_REST_PORT" "HG_STORE_RAFT_ADDRESS" "HG_STORE_DATA_PATH" "JAVA_OPTS") + "server" (list "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "JAVA_OPTS") + "hubble" (list "HG_HUBBLE_PD_PEERS" "HG_HUBBLE_PD_SERVER" "HG_HUBBLE_STORE_TARGETS" "HG_HUBBLE_SERVER_URL" "SPRING_DATASOURCE_URL") -}} +{{- range $component, $reserved := $reservedEnv -}} +{{- $componentValues := get $.Values $component | default dict -}} +{{- range $entry := get $componentValues "extraEnv" | default list -}} +{{- if has (get $entry "name") $reserved -}} +{{- fail (printf "%s.extraEnv must not set the chart-managed variable %s" $component (get $entry "name")) -}} +{{- end -}} +{{- end -}} +{{- end -}} {{- $hubble := get .Values "hubble" | default dict -}} {{- if get $hubble "enabled" | default false -}} {{- if and (not .Values.server.auth.enabled) (not (get $hubble "allowWithoutServerAuth" | default false)) -}} From 25c426d2b0bf801b7c6e01b8541a62de85805080 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 3 Aug 2026 22:40:33 +0530 Subject: [PATCH 07/61] ci(helm): fix workflow startup failure The run failed at startup because azure/setup-helm@v4 is not on the ASF-approved actions allowlist; install the pinned helm release from the official tarball in a plain run step instead. --- .github/workflows/helm-chart-ci.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index a632a9de69..20105e104c 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -35,9 +35,13 @@ jobs: with: persist-credentials: false - - uses: azure/setup-helm@v4 - with: - version: v3.16.2 + # azure/setup-helm is not on the ASF-approved actions allowlist, which + # fails the workflow at startup; install the pinned release directly. + - name: install helm + run: | + curl -fsSL https://get.helm.sh/helm-v3.16.2-linux-amd64.tar.gz | tar -xz -C /tmp + sudo install -m 0755 /tmp/linux-amd64/helm /usr/local/bin/helm + helm version - name: helm lint run: | @@ -48,6 +52,7 @@ jobs: - name: helm template run: | for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do + # shellcheck disable=SC2086 # $preset intentionally splits into flags helm template ci helm/hugegraph $preset > /dev/null done # Positive coverage for every hubble wrapper branch: pd mode with @@ -88,15 +93,15 @@ jobs: --set server.pdb.minAvailable=2 must_fail --set server.auth.enabled=true must_fail --set hubble.enabled=true - A="--set hubble.enabled=true --set hubble.allowWithoutServerAuth=true" - must_fail $A --set hubble.port=0 - must_fail $A \ + A=(--set hubble.enabled=true --set hubble.allowWithoutServerAuth=true) + must_fail "${A[@]}" --set hubble.port=0 + must_fail "${A[@]}" \ --set hubble.persistence.enabled=true \ --set hubble.persistence.size="" - must_fail $A --set hubble.service.nodePort=30080 - must_fail $A --set hubble.mode=bogus - must_fail $A --set hubble.image.tag="" - must_fail $A --set hubble.ingress.enabled=true + must_fail "${A[@]}" --set hubble.service.nodePort=30080 + must_fail "${A[@]}" --set hubble.mode=bogus + must_fail "${A[@]}" --set hubble.image.tag="" + must_fail "${A[@]}" --set hubble.ingress.enabled=true must_fail --set server.ingress.enabled=true \ --set server.ingress.allowPlainHttp=true # PD PDB must keep the Raft majority: floor(replicas/2)+1 @@ -121,6 +126,7 @@ jobs: curl -sSLo /tmp/kc.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz tar -xzf /tmp/kc.tar.gz -C /tmp for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do + # shellcheck disable=SC2086 # $preset intentionally splits into flags helm template ci helm/hugegraph $preset | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 done helm template ci helm/hugegraph \ From c2f61c221df62478ed866b1fb63bbad990d15a3d Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 4 Aug 2026 01:08:39 +0530 Subject: [PATCH 08/61] feat(helm): default partition sharding to store-level HA and soften scheduling defaults Fresh installs now seed PD with partition.default-shard-count derived from the Store count (3 when store.replicas is at least 3, otherwise 1, matching PD's odd-only constraint and its 2-to-1 clamp), so a default 3-Store deployment gets store-level HA instead of the image default of one shard replica per partition. The seed is delivered as -D system properties prepended into the PD JAVA_OPTS ahead of pd.javaOpts, preserving the start script's automatic heap sizing; it applies at first bootstrap only, after which PD metadata is authoritative, all documented together with the resulting initial partition count change. pd.antiAffinity and store.antiAffinity default to preferred so the chart schedules on clusters with fewer nodes than replicas; values-cluster.yaml keeps required for both, NOTES warns when PD quorum members may co-locate, and the README documents the upgrade implications. The Disaster Recovery documentation describes what current PD builds actually do: the scheduled patrol only marks silent stores Offline, shard reconciliation and tombstone processing run only via the manual /v1/task/patrolPartitions endpoint, and the pd.patrol-interval and store.max-down-time properties are bound but never read, which is why the chart does not expose them. Periodic leader balancing and recovery metrics are referenced as upstream feature requests. extraEnv now also rejects JAVA_OPTIONS for pd, store, and server, because the start scripts drop the chart-managed JAVA_OPTS entirely when it is set. Schema accepts numeric strings for the new keys, values-file integers at or above one million no longer fail as scientific notation, and CI asserts the rendered -D content, covers both shard-count validation messages, and kubeconforms the sharded render. --- .github/workflows/helm-chart-ci.yml | 29 +++ helm/hugegraph/README.md | 181 ++++++++++++++++++- helm/hugegraph/templates/NOTES.txt | 8 + helm/hugegraph/templates/_helpers.tpl | 115 +++++++++++- helm/hugegraph/templates/pd-statefulset.yaml | 2 +- helm/hugegraph/values-cluster.yaml | 2 + helm/hugegraph/values.schema.json | 24 +++ helm/hugegraph/values.yaml | 37 +++- 8 files changed, 380 insertions(+), 18 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 20105e104c..c4d8794abe 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -70,6 +70,19 @@ jobs: --set hubble.persistence.enabled=true \ --set hubble.ingress.enabled=true \ --set hubble.ingress.allowPlainHttp=true > /dev/null + # PD sharding knobs rendered as -D system properties, including + # the values-file shape where the count arrives as a numeric string + helm template ci helm/hugegraph \ + --set pd.partition.defaultShardCount=3 \ + --set pd.partition.storeMaxShardCount=12 > /dev/null + helm template ci helm/hugegraph \ + --set-string pd.partition.defaultShardCount=3 > /dev/null + # The derived shard count must land in the PD JAVA_OPTS verbatim: + # 3 on the default topology, 1 on the single-node preset + helm template ci helm/hugegraph \ + | grep -qF 'value: "-Dpartition.default-shard-count=3"' + helm template ci helm/hugegraph -f helm/hugegraph/values-single.yaml \ + | grep -qF 'value: "-Dpartition.default-shard-count=1"' - name: reject invalid values run: | @@ -111,6 +124,14 @@ jobs: --set pd.replicas=5 --set pd.pdb.minAvailable=3 > /dev/null helm template ci helm/hugegraph \ --set pd.replicas=5 --set pd.pdb.minAvailable=4 > /dev/null + # PD sharding knobs: empty or a positive integer; an explicit + # shard count must be odd and stay within the store count + must_fail --set pd.partition.defaultShardCount=0 + must_fail --set pd.partition.defaultShardCount=-1 + must_fail --set-string pd.partition.defaultShardCount=abc + must_fail --set pd.partition.defaultShardCount=2 + must_fail --set pd.partition.defaultShardCount=5 + must_fail --set pd.partition.storeMaxShardCount=0 # extraEnv must not override chart-managed variables must_fail --set 'server.extraEnv[0].name=HG_SERVER_INIT_STORE_ENABLED' \ --set 'server.extraEnv[0].value=true' @@ -118,6 +139,10 @@ jobs: --set 'pd.extraEnv[0].value=x' must_fail --set 'store.extraEnv[0].name=HG_STORE_PD_ADDRESS' \ --set 'store.extraEnv[0].value=x' + # JAVA_OPTIONS is reserved: a preset value makes the start scripts + # skip auto heap sizing and drop the chart's JAVA_OPTS entirely + must_fail --set 'pd.extraEnv[0].name=JAVA_OPTIONS' \ + --set 'pd.extraEnv[0].value=-Xmx1g' - name: kubeconform shell: bash @@ -143,6 +168,10 @@ jobs: --set server.auth.enabled=true \ --set server.auth.existingSecret=ci-auth \ | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 + helm template ci helm/hugegraph \ + --set pd.partition.defaultShardCount=3 \ + --set pd.partition.storeMaxShardCount=12 \ + | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 - name: legacy --reuse-values compatibility run: | diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 323be460ed..16884212a6 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -70,6 +70,19 @@ This deploys 3 PD + 3 Store + 3 Server, preserves the image's automatic JVM sizing, and sets no resource requests or limits. Set resources before production use. +The default anti-affinity for `pd`, `store`, and `server` is `preferred` +(Server always was; Hubble has no anti-affinity knob because it is +single-replica by design), so the chart schedules even on clusters with +fewer nodes than replicas. Production should pin `pd.antiAffinity` and +`store.antiAffinity` to `required`, as `values-cluster.yaml` does, so one +node failure cannot take out the PD quorum or co-locate shard replicas; see +Scheduling below. + +A fresh install seeds PD with a partition shard count of 3 when +`store.replicas` is at least 3, and 1 otherwise, instead of the image +default of 1. The seed applies at first bootstrap only; see Partition +Sharding below. + This first chart is version `0.1.0`. While the contribution is a draft, its component image tags and `appVersion` track `latest` with pull policy `Always`. Before stable publication, pin all three component tags and `appVersion` to the @@ -86,9 +99,9 @@ helm test hugegraph --namespace hugegraph | File | Purpose | |---|---| -| `values.yaml` | Default 3+3+3 topology | +| `values.yaml` | Default 3+3+3 topology with preferred anti-affinity, so it schedules on clusters of any node count | | `values-single.yaml` | Single-node 1+1+1 example | -| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity; Hubble stays opt-in because the preset does not enable authentication | +| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity for PD and Store; Hubble stays opt-in because the preset does not enable authentication | `values-cluster.yaml` is a production starting point, not a capacity guarantee. Recalculate capacity for the graph size, traffic, failure budget, @@ -103,10 +116,27 @@ helm upgrade hugegraph ./helm/hugegraph --namespace hugegraph --reuse-values Every optional field stays optional, so a release created by an earlier revision continues to render under `--reuse-values`. Note that `--reuse-values` keeps the old release's values as the complete base, so a release created -before a field existed does **not** pick up its new default — including the +before a field existed does **not** pick up its new default, including the hardened `securityContext`, ServiceAccounts, and `terminationGracePeriodSeconds`. -Pod-level token mounting is the one exception: it is disabled unconditionally. Use `-f` with your own values, or `--reset-then-reuse-values`, to adopt them. +That rule covers values-sourced defaults only; the asymmetry is that +template-derived settings **are** applied even under `--reuse-values`, +because they are computed at render time from whatever values are in effect. +Pod-level token mounting (disabled unconditionally) and the derived +`-Dpartition.default-shard-count` in the PD `JAVA_OPTS` are the current +cases. On an already-initialized cluster the seeded shard count is inert +either way; see Partition Sharding. + +Upgrading an existing release to this chart version rolls the PD StatefulSet +once: PD Pods now always carry a `JAVA_OPTS` environment variable with the +chart-derived partition properties, where previous versions set the variable +only when `pd.javaOpts` was non-empty. + +The `pd.antiAffinity` and `store.antiAffinity` defaults changed from +`required` to `preferred` in this version. `--reuse-values` keeps the old +effective value, but installs that relied on the old `required` default +while supplying their own values files must now pin `antiAffinity: required` +explicitly. PD and Store resource names reserve room for their StatefulSet ordinal before truncation, so identities stay fixed across replica changes and scaling never renames a @@ -142,7 +172,9 @@ default values. | `pd.image.repository` | PD image repository | `hugegraph/pd` | | `pd.image.tag` | PD image tag. Tracks the development image until the next release is pinned | `latest` | | `pd.image.pullPolicy` | PD image pull policy | `Always` | -| `pd.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | +| `pd.javaOpts` | Extra JVM flags, rendered after the chart-derived `-D` properties below so an explicit duplicate here wins. The image's automatic heap sizing is preserved unless heap flags are set | `""` | +| `pd.partition.defaultShardCount` | Shard replicas per partition, seeded into PD's persisted config at first bootstrap only; inert on an initialized cluster (see Partition Sharding). Empty derives 3 when `store.replicas` is at least 3, else 1. An explicit value must be odd and must not exceed `store.replicas` | `""` | +| `pd.partition.storeMaxShardCount` | Maximum shards per Store, seeded at first bootstrap only. Also fixes the initial partition count, `store.replicas x storeMaxShardCount / shardCount` (see Partition Sharding). Empty preserves the image default of `12` | `""` | | `pd.ports.grpc` | PD gRPC port | `8686` | | `pd.ports.rest` | PD REST port, also used by probes | `8620` | | `pd.ports.raft` | PD Raft port | `8610` | @@ -152,7 +184,7 @@ default values. | `pd.resources` | PD container resources. Set these for production | `{}` | | `pd.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | | `pd.securityContext` | Container-level securityContext. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | -| `pd.antiAffinity` | One of `required`, `preferred`, `disabled` | `required` | +| `pd.antiAffinity` | One of `required`, `preferred`, `disabled`. `preferred` schedules on clusters with fewer nodes than replicas; production should use `required` so one node failure cannot take out the PD quorum | `preferred` | | `pd.nodeSelector` | Node selector for pd Pods | `{}` | | `pd.tolerations` | Tolerations for pd Pods | `[]` | | `pd.affinity` | Raw affinity; overrides `pd.antiAffinity` when set | `{}` | @@ -193,7 +225,7 @@ default values. | `store.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | | `store.securityContext` | Container-level securityContext; also applied to the PD-quorum init container. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | | `store.waitTimeoutSeconds` | Bound on the PD-quorum wait before the init container fails | `900` | -| `store.antiAffinity` | One of `required`, `preferred`, `disabled` | `required` | +| `store.antiAffinity` | One of `required`, `preferred`, `disabled`. `preferred` schedules on clusters with fewer nodes than replicas; production should use `required` so one node failure cannot co-locate shard replicas | `preferred` | | `store.nodeSelector` | Node selector for store Pods | `{}` | | `store.tolerations` | Tolerations for store Pods | `[]` | | `store.affinity` | Raw affinity; overrides `store.antiAffinity` when set | `{}` | @@ -375,7 +407,15 @@ before anything reaches the cluster: `HG_SERVER_INIT_STORE_ENABLED` or the PD/Store identity and topology variables): entries render after the chart-owned variables and the last duplicate wins, so an override would silently bypass a validated contract. + `JAVA_OPTS` and `JAVA_OPTIONS` are reserved for the same reason: the + component start scripts skip automatic heap sizing and drop the chart's + `JAVA_OPTS` flags entirely when `JAVA_OPTIONS` arrives preset. - `pd.replicas` and `store.replicas` are capped at 99. +- `pd.partition.defaultShardCount` and `pd.partition.storeMaxShardCount` + must each be empty or a positive integer. An explicit shard count must + also be odd (PD's config API rejects even values, and PD clamps 2 to 1) + and must not exceed `store.replicas`, past which PD would silently clamp + it to the live store count. - `hubble.port` must be a valid port, `hubble.persistence.size` must be non-empty, and `hubble.service.nodePort` requires a `NodePort` or `LoadBalancer` Service type. @@ -414,6 +454,107 @@ the listener, container port, and Service together. --- +### Scheduling + +Every component (`pd`, `store`, `server`, `hubble`) exposes the full set of +scheduling controls: `nodeSelector`, `tolerations`, `affinity`, +`topologySpreadConstraints`, and `priorityClassName`. For example, pinning +Store to labeled nodes is just: + +```yaml +store: + nodeSelector: + hugegraph/role: storage +``` + +`antiAffinity` (`required` | `preferred` | `disabled`) renders a hostname +pod-anti-affinity preset for `pd`, `store`, and `server`; Hubble has no +`antiAffinity` key because it is single-replica by design. Setting a raw +`affinity` replaces the preset entirely. All three default to `preferred` +(Server always did; the pd and store defaults changed from `required`), so +the chart schedules on clusters with fewer nodes than replicas (including +single-node development clusters). The trade: `preferred` lets the +scheduler co-locate replicas under node pressure, so a single node failure +can then take more than one PD or Store replica with it. Production +clusters with enough nodes should pin `pd.antiAffinity` and +`store.antiAffinity` to `required`, as `values-cluster.yaml` does. + +### Partition Sharding + +A fresh install seeds PD's persisted configuration with a partition shard +count of 3 when `store.replicas` is at least 3, and 1 otherwise. Without +this the PD image's `conf/application.yml` would pin +`partition.default-shard-count` to 1, leaving chart-deployed clusters +without store-level HA. The derivation never produces 2 because PD clamps a +shard count of 2 to 1: two shards cannot elect a leader. + +The chart renders the setting as `-Dpartition.default-shard-count` in the PD +container's `JAVA_OPTS`; system properties outrank the shipped config file, +and the PD start script appends `JAVA_OPTS` after its automatically computed +heap flags, so the image's JVM auto-sizing is unaffected. + +**The seed applies at first bootstrap only.** PD persists the shard count +into its own metadata the first time it starts with empty storage, and from +then on the stored value is authoritative: every PD leader change re-reads +it from storage, overwriting whatever the `-D` flag says. Changing +`pd.partition.defaultShardCount` later, or scaling `store.replicas` across +the derivation boundary, therefore has **no** effect on an initialized +cluster. Nor is the value frozen at partition creation: PD reconciles +existing shard groups toward the stored value whenever a partition patrol +runs. To change the shard count of a running cluster, use PD's own config +API (which accepts only odd values not exceeding the live store count) and +then trigger `GET /v1/task/patrolPartitions`; expect shard-group +reallocation when the counts differ. + +The shard count also fixes the initial partition count: +`store.replicas x storeMaxShardCount / shardCount`, computed once at +bootstrap. With the image's `store-max-shard-count` default of 12, the +derived shard count moves a default 3-store install from 36 partitions +(shard count 1) to 12 (shard count 3). Set +`pd.partition.storeMaxShardCount` higher to compensate when more partitions +are wanted; it is likewise seeded at first bootstrap only. + +An explicit `pd.partition.defaultShardCount` must be odd and at most +`store.replicas`. The chart rejects other values at render time: PD would +silently clamp a value above the live store count, clamp 2 to 1, and reject +even values at its config API, so an accepted render would not mean an +honored setting. + +### Disaster Recovery + +What PD automates on current builds is narrow. A scheduled patrol runs on a +hardcoded 60-second cadence and only marks Stores that stopped sending +heartbeats as `Offline`; it does not touch partitions. There is **no +automatic re-replication**: re-placing the replicas of a lost Store, +reconciling shard groups against the stored shard count, and processing +tombstoned Stores all run only when a partition patrol is triggered +explicitly. PD's configuration binds `pd.patrol-interval` and +`store.max-down-time` keys, but no code path on current builds reads +either, which is why this chart does not expose them. + +Recovery and rebalancing are operator-triggered. PD exposes REST triggers, +reachable through the PD client Service: + +```bash +kubectl port-forward -n hugegraph svc/hugegraph-pd-client 8620:8620 +curl http://127.0.0.1:8620/v1/task/patrolPartitions # reconcile shard groups, process tombstoned Stores +curl http://127.0.0.1:8620/v1/task/balanceLeaders # spread Raft leaders +curl http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data +``` + +Run `patrolPartitions` after replacing a Store that is not coming back, +`balancePartitions` once the cluster is stable again, and `balanceLeaders` +after restarts that skewed leader placement. + +Periodic balancing and shard-sync progress metrics do not exist upstream +yet and are out of scope for this chart. Periodic leader balancing is +tracked in +[apache/hugegraph#3135](https://github.com/apache/hugegraph/issues/3135); +disaster-recovery metrics are tracked in +[apache/hugegraph#3136](https://github.com/apache/hugegraph/issues/3136). + +--- + ### Scaling PD and Store reserve the maximum StatefulSet ordinal in their resource names, @@ -458,6 +599,25 @@ graph is fully able to serve index-backed queries. Confirm the graph is live: kubectl exec -c server -- curl -s localhost:8080/graphs ``` +### Queries Fail with "Could not rebind" Right After Creating a Graph + +Creating a graph returns before every Server replica has opened and bound it. +The creating Server writes the graph to PD metadata and registers its own +Gremlin binding; the other replicas converge independently through a PD watch +plus a local graph open. Until they do, a Gremlin query routed through the +load-balanced Service to a not-yet-converged replica fails with a 400 error +such as `Could not rebind [g]`. This is upstream behavior, not a chart +setting. Mitigations: + +- Retry with backoff in the client; the window normally closes in seconds. +- Use sticky routing (or `kubectl port-forward` to one Pod) for + create-then-verify flows, so follow-up queries hit the creating replica. +- Poll `/graphs` on each replica until the new graph appears everywhere + before opening query traffic. + +The underlying fix, orchestrating graph creation through PD, is upstream +work. + ### Pods OOM Killed or Restarting The default `values.yaml` sets **no** resource requests or limits and preserves @@ -489,7 +649,12 @@ independently of the release name. ## Limitations - No TLS, backups, Operator, multi-cluster support, automatic leader transfer, - or a complete monitoring stack. + or a complete monitoring stack. Store recovery is manual on current builds: + re-replication after Store loss, leader balancing, and partition + rebalancing run only when triggered (see Disaster Recovery); periodic + balancing and shard-sync metrics are upstream feature work. +- Newly created graphs are visible on all Server replicas only after a short + propagation window (see Troubleshooting: "Could not rebind"). - The published images run as root, so `runAsNonRoot` and `readOnlyRootFilesystem` are not chart defaults. The container `securityContext` does default to `allowPrivilegeEscalation: false`, diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 72b8f54cf9..cd555837e4 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -70,6 +70,14 @@ Hubble Pod is replaced. Graph data is unaffected. Authentication is disabled. Do not expose this release to untrusted networks. {{- end }} +{{- if and (gt (int .Values.pd.replicas) 1) (ne (get .Values.pd "antiAffinity" | default "") "required") (empty (get .Values.pd "affinity")) }} + +pd.antiAffinity is not "required", so the scheduler may co-locate PD quorum +members on one node and a single node loss can take down the PD quorum. The +PD PodDisruptionBudget only limits voluntary disruption (drains, evictions), +not node failure. On clusters with at least {{ .Values.pd.replicas }} nodes, +set pd.antiAffinity=required; see values-cluster.yaml. +{{- end }} {{- if or (empty .Values.pd.resources) (empty .Values.store.resources) (empty .Values.server.resources) }} One or more components have no resource requests or limits. Set them before diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 5b077bf3b3..a7993b7e9b 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -203,6 +203,80 @@ image entrypoint's existing automatic JVM sizing behavior. {{- end -}} {{- end }} +{{/* +String form of a possibly-absent scalar value, preserving zero. sprig's +`default` treats 0 as unset, which would let a zero slip past the named +validation below, so absence is detected explicitly instead. Numbers from a +values file arrive as float64, whose toString switches to scientific +notation at 1e6 or higher (1000000 becomes "1e+06"), so integral float64 +values are formatted without an exponent. Non-integral floats keep their +raw form on purpose: the schema already rejects them, and the raw form +fails the named validation instead of being silently rounded. +*/}} +{{- define "hugegraph.optionalScalar" -}} +{{- if not (kindIs "invalid" .) -}} +{{- if and (kindIs "float64" .) (eq (floor .) .) -}}{{- printf "%.0f" . -}}{{- else -}}{{- trim (toString .) -}}{{- end -}} +{{- end -}} +{{- end }} + +{{/* +Effective PD JAVA_OPTS: chart-derived -D system properties, then pd.javaOpts. + +The -D route is the grounded override mechanism: the PD image's +docker-entrypoint.sh forwards JAVA_OPTS via `-j` into +bin/start-hugegraph-pd.sh, which places it on the java command line ahead of +-Dspring.config.location, and Spring system properties outrank the shipped +conf/application.yml (which pins partition.default-shard-count to 1). + +The shard count is always derived, so PD Pods always carry a JAVA_OPTS +variable, which shadows the PD image's `ENV JAVA_OPTS` default +(-XX:MaxRAMPercentage=50, -XX:+UseContainerSupport, -XshowSettings:vm). +That is acceptable: the start script always computes explicit -Xms/-Xmx +heap flags when the separate JAVA_OPTIONS variable is unset, which makes +MaxRAMPercentage moot, and only the -XshowSettings:vm startup diagnostics +are lost. + +JVM auto-sizing is preserved, verified against the PD dist start script: +`-j` lands in USER_OPTION, while the automatic heap sizing branch is gated on +the separate JAVA_OPTIONS variable and appends USER_OPTION after the computed +-Xms/-Xmx flags. A JAVA_OPTS holding only -D flags therefore still gets +automatic heap sizing, and heap flags in pd.javaOpts land later on the +command line, so they win. The derived -D flags come first for the same +reason: an explicit duplicate in pd.javaOpts overrides them. + +Both -D properties seed PD's persisted config at first bootstrap only: +ConfigService.loadConfig persists them when no stored config exists, and +every leader change re-reads the stored values (updatePDConfig), so on an +initialized cluster the flags are inert and the authoritative values live +in PD metadata, changeable only through PD's own config API. PD reconciles +existing shard groups toward the stored value when a partition patrol is +triggered (TaskScheduleService reallocShards). The empty-value derivation +is 3 when store.replicas is at least 3, else 1, because PD clamps a shard +count of 2 to 1 (two shards cannot elect a leader) and its config API +accepts only odd values. store-max-shard-count is rendered only when set, +keeping the image default. All lookups tolerate absent keys so releases +stored before these values existed keep rendering under --reuse-values. +*/}} +{{- define "hugegraph.pd.effectiveJavaOpts" -}} +{{- $pd := .Values.pd -}} +{{- $partition := get $pd "partition" | default dict -}} +{{- $flags := list -}} +{{- $shardCount := include "hugegraph.optionalScalar" (get $partition "defaultShardCount") -}} +{{- if eq $shardCount "" -}} +{{- $shardCount = ternary "3" "1" (ge (int .Values.store.replicas) 3) -}} +{{- end -}} +{{- $flags = append $flags (printf "-Dpartition.default-shard-count=%s" $shardCount) -}} +{{- $maxShard := include "hugegraph.optionalScalar" (get $partition "storeMaxShardCount") -}} +{{- if ne $maxShard "" -}} +{{- $flags = append $flags (printf "-Dpartition.store-max-shard-count=%s" $maxShard) -}} +{{- end -}} +{{- $userOpts := trim (get $pd "javaOpts" | default "") -}} +{{- if ne $userOpts "" -}} +{{- $flags = append $flags $userOpts -}} +{{- end -}} +{{- join " " $flags -}} +{{- end }} + {{/* Keep the startup probe alive for the 300-second storage wait, the Server's 120-second start timeout, and 30 seconds of process overhead. Older stored @@ -290,11 +364,38 @@ and must not be failed for a value that has no effect. {{- fail "pd.pdb.minAvailable must be less than pd.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} {{- if and .Values.pd.pdb.enabled (gt (int .Values.pd.replicas) 1) (lt (int .Values.pd.pdb.minAvailable) (include "hugegraph.pd.quorum" . | int)) -}} -{{- fail "pd.pdb.minAvailable must be at least the PD Raft majority, floor(replicas/2)+1, otherwise the budget permits voluntary evictions that drop PD below quorum" -}} +{{- fail "pd.pdb.minAvailable must be at least the PD Raft majority, floor(replicas/2)+1, otherwise the budget permits voluntary disruptions that drop PD below quorum. Note a PDB only limits voluntary disruption such as drains and evictions; it cannot protect quorum from node failure" -}} {{- end -}} {{- if and .Values.store.pdb.enabled (gt (int .Values.store.replicas) 1) (ge (int .Values.store.pdb.minAvailable) (int .Values.store.replicas)) -}} {{- fail "store.pdb.minAvailable must be less than store.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} +{{/* +PD -D system properties must be empty or a positive integer. The schema +enforces the types; these checks add a named failure for zero, negative, and +nonsense values, and check an explicit shard count against PD's real +constraints: PD's config API accepts only odd shard counts, PD clamps a +count of 2 to 1 (two shards cannot elect a leader), and PD clamps the +effective count to the number of live stores. All lookups tolerate absent +keys for releases stored before the values existed. +*/}} +{{- $pdPartition := get .Values.pd "partition" | default dict -}} +{{- $pdProps := dict + "pd.partition.defaultShardCount" (include "hugegraph.optionalScalar" (get $pdPartition "defaultShardCount")) + "pd.partition.storeMaxShardCount" (include "hugegraph.optionalScalar" (get $pdPartition "storeMaxShardCount")) -}} +{{- range $label, $raw := $pdProps -}} +{{- if and (ne $raw "") (or (not (regexMatch "^[0-9]+$" $raw)) (eq (int $raw) 0)) -}} +{{- fail (printf "%s must be empty or a positive integer" $label) -}} +{{- end -}} +{{- end -}} +{{- $explicitShards := include "hugegraph.optionalScalar" (get $pdPartition "defaultShardCount") -}} +{{- if regexMatch "^[1-9][0-9]*$" $explicitShards -}} +{{- if eq (mod (int $explicitShards) 2) 0 -}} +{{- fail "pd.partition.defaultShardCount must be odd: PD's config API rejects even shard counts, and PD clamps a bootstrap value of 2 to 1 because two shards cannot elect a leader" -}} +{{- end -}} +{{- if gt (int $explicitShards) (int .Values.store.replicas) -}} +{{- fail "pd.partition.defaultShardCount is greater than store.replicas: PD would clamp the effective shard count to the number of live stores, so the extra replicas would silently never be placed. Raise store.replicas or lower the shard count" -}} +{{- end -}} +{{- end -}} {{- $svc := get .Values.server "service" | default dict -}} {{- if and (get $svc "nodePort") (not (has (get $svc "type" | default "ClusterIP") (list "NodePort" "LoadBalancer"))) -}} {{- fail "server.service.nodePort requires server.service.type to be NodePort or LoadBalancer" -}} @@ -312,12 +413,16 @@ and must not be failed for a value that has no effect. extraEnv entries render after the chart-owned variables and Kubernetes lets the last duplicate win, so a duplicate name would silently override a validated contract (for example re-enabling init-store across Server -replicas). Reserved names are rejected instead. +replicas). Reserved names are rejected instead. JAVA_OPTIONS is reserved +for pd, store, and server because each component's start script skips its +automatic heap sizing and drops the chart's JAVA_OPTS flags entirely when +JAVA_OPTIONS arrives preset in the environment (verified in +start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). */}} {{- $reservedEnv := dict - "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "JAVA_OPTS") - "store" (list "HG_STORE_PD_ADDRESS" "HG_STORE_GRPC_HOST" "HG_STORE_GRPC_PORT" "HG_STORE_REST_PORT" "HG_STORE_RAFT_ADDRESS" "HG_STORE_DATA_PATH" "JAVA_OPTS") - "server" (list "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "JAVA_OPTS") + "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") + "store" (list "HG_STORE_PD_ADDRESS" "HG_STORE_GRPC_HOST" "HG_STORE_GRPC_PORT" "HG_STORE_REST_PORT" "HG_STORE_RAFT_ADDRESS" "HG_STORE_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") + "server" (list "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "JAVA_OPTS" "JAVA_OPTIONS") "hubble" (list "HG_HUBBLE_PD_PEERS" "HG_HUBBLE_PD_SERVER" "HG_HUBBLE_STORE_TARGETS" "HG_HUBBLE_SERVER_URL" "SPRING_DATASOURCE_URL") -}} {{- range $component, $reserved := $reservedEnv -}} {{- $componentValues := get $.Values $component | default dict -}} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 01ef705006..e16ff53b2d 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -116,7 +116,7 @@ spec: - name: HG_PD_DATA_PATH value: {{ .Values.pd.dataPath | quote }} {{- with .Values.pd.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} - {{- with include "hugegraph.javaOptsEnv" .Values.pd.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} + {{- with include "hugegraph.javaOptsEnv" (include "hugegraph.pd.effectiveJavaOpts" .) }}{{ . | trim | nindent 12 }}{{- end }} volumeMounts: - name: pd-data mountPath: {{ .Values.pd.dataPath }} diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index b3d37b2ced..98202e5f16 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -33,6 +33,7 @@ pd: limits: cpu: "2" memory: 2Gi + # Pinned to "required": production must not co-locate PD quorum members. antiAffinity: required pdb: enabled: true @@ -61,6 +62,7 @@ store: limits: cpu: 250m memory: 64Mi + # Pinned to "required": production must not co-locate shard replicas. antiAffinity: required pdb: enabled: true diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 9ad8d96de7..fa99e12a41 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -37,6 +37,18 @@ } }, "definitions": { + "optionalPositiveInteger": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^([1-9][0-9]*)?$" + } + ] + }, "image": { "type": "object", "additionalProperties": false, @@ -240,6 +252,18 @@ "javaOpts": { "type": "string" }, + "partition": { + "type": "object", + "additionalProperties": false, + "properties": { + "defaultShardCount": { + "$ref": "#/definitions/optionalPositiveInteger" + }, + "storeMaxShardCount": { + "$ref": "#/definitions/optionalPositiveInteger" + } + } + }, "ports": { "$ref": "#/definitions/ports" }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 88142c23dd..b13a190e50 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -31,8 +31,32 @@ pd: # Pin the release tag and switch to IfNotPresent before stable publication. tag: latest pullPolicy: Always - # Empty preserves the image entrypoint's automatic JVM sizing. + # Empty preserves the image entrypoint's automatic JVM sizing. The chart + # renders its partition settings below as -D system properties ahead of + # this value, so flags placed here win on conflict. javaOpts: "" + # Partition sharding, rendered as -D system properties that outrank the + # image's conf/application.yml. PD copies both values into its own + # persisted metadata at first bootstrap only; from then on the stored + # values are authoritative, so changing these later (or scaling + # store.replicas across the derivation boundary) has no effect on an + # initialized cluster. Post-bootstrap changes go through PD's config API + # and take effect on existing shard groups only when a partition patrol + # is triggered; see the README's Partition Sharding section. + partition: + # Shard replicas per partition, seeding PD at first bootstrap only. + # Empty derives 3 when store.replicas is at least 3, else 1, so a fresh + # multi-store install gets store-level HA instead of the image default + # of 1. The derivation skips 2 because PD clamps a shard count of 2 to + # 1 (two shards cannot elect a leader). An explicit value must be odd + # and must not exceed store.replicas. + defaultShardCount: "" + # Maximum shards per store, seeding PD at first bootstrap only. Also + # fixes the initial partition count: store.replicas x this value / + # shard count. The derived shard count of 3 gives the default topology + # 12 partitions instead of 36; raise this value to compensate when more + # partitions are wanted. Empty preserves the image default of 12. + storeMaxShardCount: "" ports: grpc: 8686 rest: 8620 @@ -51,8 +75,10 @@ pd: drop: ["ALL"] seccompProfile: type: RuntimeDefault - # required | preferred | disabled - antiAffinity: required + # required | preferred | disabled. "preferred" schedules on clusters with + # fewer nodes than replicas; production should use "required" (see + # values-cluster.yaml) so one node failure cannot take out PD quorum. + antiAffinity: preferred # Scheduling. `affinity` takes precedence over the antiAffinity preset. nodeSelector: {} tolerations: [] @@ -115,7 +141,10 @@ store: drop: ["ALL"] seccompProfile: type: RuntimeDefault - antiAffinity: required + # required | preferred | disabled. "preferred" schedules on clusters with + # fewer nodes than replicas; production should use "required" (see + # values-cluster.yaml) so one node failure cannot co-locate shard replicas. + antiAffinity: preferred # Scheduling. `affinity` takes precedence over the antiAffinity preset. nodeSelector: {} tolerations: [] From 6237186756756120b32fc88145042f3a5552a82b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 13 Aug 2026 14:57:52 +0530 Subject: [PATCH 09/61] docs(helm): note #3138 phase-1 fix for CreateGraph race (#3137) Document that the creating Server is consistent at HTTP 200 after #3138, while cross-replica convergence and PD-owned creation remain open upstream. --- helm/hugegraph/README.md | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 16884212a6..ca6b8d4bb2 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -601,22 +601,30 @@ kubectl exec -c server -- curl -s localhost:8080/graphs ### Queries Fail with "Could not rebind" Right After Creating a Graph -Creating a graph returns before every Server replica has opened and bound it. -The creating Server writes the graph to PD metadata and registers its own -Gremlin binding; the other replicas converge independently through a PD watch -plus a local graph open. Until they do, a Gremlin query routed through the -load-balanced Service to a not-yet-converged replica fails with a 400 error -such as `Could not rebind [g]`. This is upstream behavior, not a chart -setting. Mitigations: +**Update (2026-08-12):** [#3138](https://github.com/apache/hugegraph/pull/3138) +merged on `master` and closes Phase 1 of +[#3137](https://github.com/apache/hugegraph/issues/3137). The Server that +handles `CreateGraph` now waits for its own Gremlin binding before returning +HTTP 200, so create-then-query on the **same** Server (or sticky routing to +that Pod) is reliable. + +Other Server replicas still converge independently through a PD metadata +watch plus a local graph open. Until they finish, a Gremlin query routed +through the load-balanced Service to a not-yet-converged replica can still +fail with a 400 error such as `Could not rebind [g]`. This is upstream +behavior, not a chart setting. Mitigations for multi-replica load-balanced +deployments: - Retry with backoff in the client; the window normally closes in seconds. - Use sticky routing (or `kubectl port-forward` to one Pod) for - create-then-verify flows, so follow-up queries hit the creating replica. + create-then-verify flows. - Poll `/graphs` on each replica until the new graph appears everywhere before opening query traffic. -The underlying fix, orchestrating graph creation through PD, is upstream -work. +Cluster-wide readiness and PD-owned graph creation remain tracked in +[#3137](https://github.com/apache/hugegraph/issues/3137) (Phase 2: +[#3139](https://github.com/apache/hugegraph/pull/3139); Phase 3: PD +orchestration). ### Pods OOM Killed or Restarting @@ -653,8 +661,11 @@ independently of the release name. re-replication after Store loss, leader balancing, and partition rebalancing run only when triggered (see Disaster Recovery); periodic balancing and shard-sync metrics are upstream feature work. -- Newly created graphs are visible on all Server replicas only after a short - propagation window (see Troubleshooting: "Could not rebind"). +- After [#3138](https://github.com/apache/hugegraph/pull/3138), the creating + Server is consistent at HTTP 200; other replicas may still lag for a short + window on load-balanced installs (see Troubleshooting: "Could not rebind"; + [#3137](https://github.com/apache/hugegraph/issues/3137) stays open for + cluster-wide and PD-owned creation). - The published images run as root, so `runAsNonRoot` and `readOnlyRootFilesystem` are not chart defaults. The container `securityContext` does default to `allowPrivilegeEscalation: false`, From cb7eee168db4bccc1513bcc971a3b18de8f92b57 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 13 Aug 2026 18:28:23 +0530 Subject: [PATCH 10/61] fix(helm): enable PD metadata by default --- .github/workflows/helm-chart-ci.yml | 8 ++++++ helm/hugegraph/README.md | 26 ++++++++++++------- .../templates/server-deployment.yaml | 11 +++----- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index c4d8794abe..fd9d0a7860 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -83,6 +83,14 @@ jobs: | grep -qF 'value: "-Dpartition.default-shard-count=3"' helm template ci helm/hugegraph -f helm/hugegraph/values-single.yaml \ | grep -qF 'value: "-Dpartition.default-shard-count=1"' + # The stock distributed install must put every Server replica on + # the shared PD graph catalog, even without auth or Hubble. + DEFAULT=$(helm template ci helm/hugegraph) + grep -qF "printf 'usePD=true\\n'" <<<"$DEFAULT" + grep -qF 'printf '\''pd.peers=%s\\n'\'' "${HG_SERVER_PD_PEERS}"' \ + <<<"$DEFAULT" + grep -qF 'value: "ci-hugegraph-pd-0.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-1.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-2.ci-hugegraph-pd.default.svc:8686"' \ + <<<"$DEFAULT" - name: reject invalid values run: | diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index ca6b8d4bb2..3dad756129 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -37,6 +37,12 @@ so operators do not have to: every replica would initialize the same backend concurrently. The chart creates no init Job and does not set `HG_SERVER_SKIP_INIT`. Standalone behavior is unchanged, because the option defaults to `true` when unset. +- **Every Server uses PD for graph metadata.** The startup wrapper always writes + `usePD=true` and the chart-derived `pd.peers` into + `rest-server.properties`, so all Server replicas share the graph catalog + through PD. It also registers the Server client Service URL for Kubernetes + discovery. This is required for distributed HStore and does not make a local + RocksDB backend shared across replicas. - **Store waits for PD quorum** in an init container before starting, so Store never registers against an incomplete PD Raft group. - **The Server startup probe allows at least 450 seconds.** The image may spend @@ -123,9 +129,10 @@ That rule covers values-sourced defaults only; the asymmetry is that template-derived settings **are** applied even under `--reuse-values`, because they are computed at render time from whatever values are in effect. Pod-level token mounting (disabled unconditionally) and the derived -`-Dpartition.default-shard-count` in the PD `JAVA_OPTS` are the current -cases. On an already-initialized cluster the seeded shard count is inert -either way; see Partition Sharding. +`-Dpartition.default-shard-count` in the PD `JAVA_OPTS`, plus the Server's +enforced PD metadata mode, are the current cases. The PD metadata change rolls +the Server Deployment. On an already-initialized cluster the seeded shard +count is inert either way; see Partition Sharding. Upgrading an existing release to this chart version rolls the PD StatefulSet once: PD Pods now always carry a `JAVA_OPTS` environment variable with the @@ -316,13 +323,12 @@ pointing at the Server client Service; there is no PD discovery and no operations view. Everything else in `hugegraph-hubble.properties` keeps the image default. -Enabling `pd`-mode Hubble switches the Server into PD meta mode (`usePD`, -`server.urls_to_pd`, `server.deploy_in_k8s`) so that PD can hand Hubble a -resolvable Server address; auth-enabled installs already run in this mode. -On an existing release this change rolls the Server Deployment once. The -Store allow-list is computed from `store.replicas` at render time, so scale -Store with `helm upgrade`, not `kubectl scale`, or the list goes stale until -the next upgrade. +The chart always runs the Server in PD meta mode (`usePD`, `pd.peers`, +`server.urls_to_pd`, `server.deploy_in_k8s`). In `pd` mode, Hubble uses that +registration so PD can hand it a resolvable Server address. The Store +allow-list is computed from `store.replicas` at render time, so scale Store +with `helm upgrade`, not `kubectl scale`, or the list goes stale until the next +upgrade. Hubble is one replica by design: it keeps UI connection metadata, including any graph credentials entered in the UI, in an embedded per-instance H2 diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 2d9b1c333a..c26b307128 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -26,15 +26,12 @@ {{- $batchMaxWriteThreads = toString (get $restServer "batchMaxWriteThreads") }} {{- end }} {{- $customPort := ne (int .Values.server.port) 8080 }} -{{- $hubble := get .Values "hubble" | default dict }} -{{- $hubblePdMode := and (get $hubble "enabled" | default false) (ne (get $hubble "mode" | default "pd") "direct") }} {{/* -PD meta mode (usePD) is required on two paths: the built-in authenticator -creates the admin on the PD startup path, and PD-mode Hubble discovers the -Server through the service URL the Server registers with PD. Both need the -same four properties, so they share one condition. +Distributed HStore requires every Server replica to share graph metadata +through PD. The same registration properties let PD-mode Hubble discover the +Server and let the built-in authenticator create the admin on the PD path. */}} -{{- $pdMeta := or .Values.server.auth.enabled $hubblePdMode }} +{{- $pdMeta := true }} {{- $wrapper := or $pdMeta $customPort (ne $minFreeMemory "") (ne $batchMaxWriteThreads "") }} apiVersion: apps/v1 kind: Deployment From a9c64b39fbce878c5b583162e03e3127377741bd Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 13 Aug 2026 18:42:52 +0530 Subject: [PATCH 11/61] ci(helm): fix PD peers render assertion --- .github/workflows/helm-chart-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index fd9d0a7860..fc46e823a7 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -87,7 +87,7 @@ jobs: # the shared PD graph catalog, even without auth or Hubble. DEFAULT=$(helm template ci helm/hugegraph) grep -qF "printf 'usePD=true\\n'" <<<"$DEFAULT" - grep -qF 'printf '\''pd.peers=%s\\n'\'' "${HG_SERVER_PD_PEERS}"' \ + grep -qF "printf 'pd.peers=%s\\n' \"\${HG_SERVER_PD_PEERS}\"" \ <<<"$DEFAULT" grep -qF 'value: "ci-hugegraph-pd-0.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-1.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-2.ci-hugegraph-pd.default.svc:8686"' \ <<<"$DEFAULT" From 13a5614228e66786d5faf5e88eb75ee2f84a0ba9 Mon Sep 17 00:00:00 2001 From: bitflicker64 Date: Sat, 15 Aug 2026 20:13:47 +0530 Subject: [PATCH 12/61] feat(helm): share JWT auth.token_secret across Server replicas Multi-replica Server pods must use one JWT signing key or Hubble login fails behind the Service; chart-manage or BYO via server.auth.tokenSecret. --- helm/hugegraph/README.md | 6 +++ helm/hugegraph/templates/_helpers.tpl | 40 ++++++++++++++++++- .../templates/server-auth-token-secret.yaml | 32 +++++++++++++++ .../templates/server-deployment.yaml | 7 ++++ helm/hugegraph/values.schema.json | 17 ++++++++ helm/hugegraph/values.yaml | 7 ++++ 6 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 helm/hugegraph/templates/server-auth-token-secret.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 3dad756129..6ebb797c25 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -289,6 +289,8 @@ default values. | `server.initStoreEnabled` | Must remain `false` for distributed HStore | `false` | | `server.auth.enabled` | Enable admin authentication | `false` | | `server.auth.existingSecret` | Required when auth is enabled; must contain key `password` | `""` | +| `server.auth.tokenSecret.existingSecret` | BYO Secret for the JWT signing key (`auth.token_secret`); empty creates a kept release-auth-token Secret | `""` | +| `server.auth.tokenSecret.key` | Key inside the JWT signing Secret | `token_secret` | | `server.ingress.enabled` | Create an Ingress for the Server Service | `false` | | `server.ingress.className` | IngressClass name | `""` | | `server.ingress.annotations` | Ingress annotations (cert-manager, nginx, ALB) | `{}` | @@ -682,6 +684,10 @@ independently of the release name. - The auth Secret sets the admin password only at first creation via `auth.admin_pa`; the chart cannot rotate an existing cluster's admin password. +- With authentication enabled, every Server replica must share one JWT + signing key. The chart injects `HG_SERVER_AUTH_TOKEN_SECRET` from + `server.auth.tokenSecret` (chart-managed by default) so Hubble login + stays stable behind a multi-replica Service. - Hubble is single-replica, serves plain HTTP, requires `server.auth` to be enabled for its login to complete, and keeps UI connection metadata, including any graph credentials entered in the UI, in an embedded H2 diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index a7993b7e9b..da9af6239a 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -82,6 +82,44 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- printf "%s-test-connection" (include "hugegraph.fullname" . | trunc 47 | trimSuffix "-") }} {{- end }} + +{{/* +Resolve the JWT signing Secret. User-provided tokenSecret.existingSecret wins; +otherwise use a stable chart-managed name so every Server replica shares one key. +*/}} +{{- define "hugegraph.server.authTokenSecretName" -}} +{{- $auth := get .Values.server "auth" | default dict -}} +{{- $token := get $auth "tokenSecret" | default dict -}} +{{- $existing := get $token "existingSecret" | default "" -}} +{{- if $existing -}} +{{- $existing -}} +{{- else -}} +{{- printf "%s-auth-token" (.Release.Name | trunc 51 | trimSuffix "-") -}} +{{- end -}} +{{- end }} + +{{- define "hugegraph.server.authTokenSecretKey" -}} +{{- $auth := get .Values.server "auth" | default dict -}} +{{- $token := get $auth "tokenSecret" | default dict -}} +{{- get $token "key" | default "token_secret" -}} +{{- end }} + +{{/* +Return the already-encoded JWT signing secret when present; otherwise generate +32 random bytes (base64). Lookup keeps multi-replica Server pods and upgrades +on the same signing key. +*/}} +{{- define "hugegraph.server.authTokenSecretValue" -}} +{{- $name := include "hugegraph.server.authTokenSecretName" . -}} +{{- $key := include "hugegraph.server.authTokenSecretKey" . -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace $name -}} +{{- if and $secret (hasKey $secret "data") (hasKey (get $secret "data") $key) -}} +{{- get (get $secret "data") $key -}} +{{- else -}} +{{- randAlphaNum 32 | b64enc -}} +{{- end -}} +{{- end }} + {{/* PD Raft peers list: pod-0.svc.ns.svc:8610,... Uses short headless DNS (cluster.local optional) resolvable inside the namespace. @@ -422,7 +460,7 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- $reservedEnv := dict "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") "store" (list "HG_STORE_PD_ADDRESS" "HG_STORE_GRPC_HOST" "HG_STORE_GRPC_PORT" "HG_STORE_REST_PORT" "HG_STORE_RAFT_ADDRESS" "HG_STORE_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") - "server" (list "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "JAVA_OPTS" "JAVA_OPTIONS") + "server" (list "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "HG_SERVER_AUTH_TOKEN_SECRET" "JAVA_OPTS" "JAVA_OPTIONS") "hubble" (list "HG_HUBBLE_PD_PEERS" "HG_HUBBLE_PD_SERVER" "HG_HUBBLE_STORE_TARGETS" "HG_HUBBLE_SERVER_URL" "SPRING_DATASOURCE_URL") -}} {{- range $component, $reserved := $reservedEnv -}} {{- $componentValues := get $.Values $component | default dict -}} diff --git a/helm/hugegraph/templates/server-auth-token-secret.yaml b/helm/hugegraph/templates/server-auth-token-secret.yaml new file mode 100644 index 0000000000..3581b44b1a --- /dev/null +++ b/helm/hugegraph/templates/server-auth-token-secret.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- $auth := get .Values.server "auth" | default dict -}} +{{- $token := get $auth "tokenSecret" | default dict -}} +{{- if and (get $auth "enabled" | default false) (not (get $token "existingSecret" | default "")) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "hugegraph.server.authTokenSecretName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{ include "hugegraph.server.authTokenSecretKey" . }}: {{ include "hugegraph.server.authTokenSecretValue" . | quote }} +{{- end }} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index c26b307128..314f69daf5 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -274,6 +274,13 @@ spec: name: {{ .Values.server.auth.existingSecret | quote }} key: password {{- end }} + {{- if .Values.server.auth.enabled }} + - name: HG_SERVER_AUTH_TOKEN_SECRET + valueFrom: + secretKeyRef: + name: {{ include "hugegraph.server.authTokenSecretName" . | quote }} + key: {{ include "hugegraph.server.authTokenSecretKey" . | quote }} + {{- end }} startupProbe: httpGet: path: /versions diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index fa99e12a41..a6220cfec1 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -556,6 +556,23 @@ }, "existingSecret": { "type": "string" + }, + "tokenSecret": { + "type": "object", + "additionalProperties": false, + "required": [ + "existingSecret", + "key" + ], + "properties": { + "existingSecret": { + "type": "string" + }, + "key": { + "type": "string", + "minLength": 1 + } + } } }, "allOf": [ diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index b13a190e50..e5454f1081 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -252,6 +252,13 @@ server: # auth.admin_pa when the admin account is first created; changing the # Secret later does not rotate an existing cluster's password. existingSecret: "" + # JWT signing key for auth.token_secret / HG_SERVER_AUTH_TOKEN_SECRET. + # Must be identical on every Server replica or Hubble login fails behind + # the Service. Empty existingSecret: chart creates a kept Secret (lookup + # keeps the value across upgrades). BYO with existingSecret + key. + tokenSecret: + existingSecret: "" + key: token_secret service: type: ClusterIP annotations: {} From fef4a498cec631b27a9520380163fb09da84a0d8 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 15 Aug 2026 20:32:30 +0530 Subject: [PATCH 13/61] feat(helm): auto-generate kept admin password Secret When auth is enabled without existingSecret, create a stable release-admin Secret so operators are not forced to pre-create credentials. --- .github/workflows/helm-chart-ci.yml | 2 +- helm/hugegraph/README.md | 13 +++++--- helm/hugegraph/templates/NOTES.txt | 3 +- helm/hugegraph/templates/_helpers.tpl | 33 +++++++++++++++++++ .../templates/server-deployment.yaml | 6 ++-- helm/hugegraph/templates/server-secret.yaml | 31 +++++++++++++++++ .../templates/tests/test-connection.yaml | 2 +- helm/hugegraph/values.schema.json | 25 +++++++++++--- helm/hugegraph/values.yaml | 4 ++- 9 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 helm/hugegraph/templates/server-secret.yaml diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index fc46e823a7..7f50bcb8a5 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -112,7 +112,7 @@ jobs: --set server.resources.requests.cpu=100m \ --set server.pdb.enabled=true \ --set server.pdb.minAvailable=2 - must_fail --set server.auth.enabled=true + must_fail --set server.auth.enabled=true --set server.auth.autoGenerateSecret=false must_fail --set hubble.enabled=true A=(--set hubble.enabled=true --set hubble.allowWithoutServerAuth=true) must_fail "${A[@]}" --set hubble.port=0 diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 6ebb797c25..74af440168 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -288,7 +288,8 @@ default values. | `server.restServer.batchMaxWriteThreads` | Empty preserves the image default | `""` | | `server.initStoreEnabled` | Must remain `false` for distributed HStore | `false` | | `server.auth.enabled` | Enable admin authentication | `false` | -| `server.auth.existingSecret` | Required when auth is enabled; must contain key `password` | `""` | +| `server.auth.autoGenerateSecret` | Create and keep a random release-admin Secret when `existingSecret` is empty | `true` | +| `server.auth.existingSecret` | Use a pre-created Secret instead; it must contain key `password` and takes priority | `""` | | `server.auth.tokenSecret.existingSecret` | BYO Secret for the JWT signing key (`auth.token_secret`); empty creates a kept release-auth-token Secret | `""` | | `server.auth.tokenSecret.key` | Key inside the JWT signing Secret | `token_secret` | | `server.ingress.enabled` | Create an Ingress for the Server Service | `false` | @@ -348,7 +349,8 @@ UI behind a login that authenticates against the cluster; with server authentication disabled the login cannot complete (the server rejects `/auth/login` with "Unconfigured authenticator"), so Hubble is only useful on an auth-enabled deployment, where the admin credential from -`server.auth.existingSecret` logs in. The chart therefore refuses to render +`server.auth.existingSecret` or the chart-managed admin Secret logs in. +The chart therefore refuses to render `hubble.enabled=true` without `server.auth` unless `hubble.allowWithoutServerAuth=true` explicitly overrides it for images whose login does not need cluster authentication. @@ -397,9 +399,10 @@ before anything reaches the cluster: - Unknown keys and wrong types are rejected. - `server.initStoreEnabled` must remain `false` for a distributed deployment. -- With authentication enabled, `server.auth.existingSecret` must name a Secret - containing a `password` key. With authentication disabled it must be empty, - so a configured but inactive Secret reference cannot be overlooked. A missing +- With authentication enabled, either `server.auth.existingSecret` must name a + Secret containing a `password` key, or `server.auth.autoGenerateSecret` must + be true. With authentication disabled `existingSecret` must be empty, so a + configured but inactive Secret reference cannot be overlooked. A missing Secret fails when Kubernetes configures the container; an empty `password` fails in the Server startup wrapper. - `server.hpa.minReplicas` must not exceed `maxReplicas`, and enabling diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index cd555837e4..2d2eb0cc4f 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -39,7 +39,8 @@ Reach the Server API: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.server.name" . }} {{ .Values.server.port }}:{{ .Values.server.port }} {{- if .Values.server.auth.enabled }} - PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ .Values.server.auth.existingSecret }} -o jsonpath='{.data.password}' | base64 --decode)" + PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.server.authSecretName" . }} -o jsonpath='{.data.password}' | base64 --decode)" + echo "Admin password: ${PASSWORD}" curl --user "admin:${PASSWORD}" http://127.0.0.1:{{ .Values.server.port }}/versions {{- else }} curl http://127.0.0.1:{{ .Values.server.port }}/versions diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index da9af6239a..c033af5db9 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -82,6 +82,35 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- printf "%s-test-connection" (include "hugegraph.fullname" . | trunc 47 | trimSuffix "-") }} {{- end }} +{{/* +Resolve the Server authentication Secret. A user-provided Secret always wins; +otherwise use a stable chart-managed name so the generated Secret can survive +uninstall and be reused by a later install of the same release. +*/}} +{{- define "hugegraph.server.authSecretName" -}} +{{- $auth := get .Values.server "auth" | default dict -}} +{{- $existingSecret := get $auth "existingSecret" | default "" -}} +{{- if $existingSecret -}} +{{- $existingSecret -}} +{{- else -}} +{{- printf "%s-admin" (.Release.Name | trunc 55 | trimSuffix "-") -}} +{{- end -}} +{{- end }} + +{{/* +Return the generated Secret's already-encoded password when it exists. The +lookup keeps upgrades from rotating the administrator credential; a first +install gets a random password. This helper is only used for chart-managed +Secrets, never for an external existingSecret. +*/}} +{{- define "hugegraph.server.authSecretPassword" -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authSecretName" .) -}} +{{- if and $secret (hasKey $secret "data") (hasKey (get $secret "data") "password") -}} +{{- get (get $secret "data") "password" -}} +{{- else -}} +{{- randAlphaNum 32 | b64enc -}} +{{- end -}} +{{- end }} {{/* Resolve the JWT signing Secret. User-provided tokenSecret.existingSecret wins; @@ -488,6 +517,10 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- fail "hubble.ingress.enabled without tls publishes the plain-HTTP, unauthenticated Hubble UI; configure hubble.ingress.tls, or set hubble.ingress.allowPlainHttp=true to accept that on a trusted network" -}} {{- end -}} {{- end -}} +{{- $auth := get .Values.server "auth" | default dict -}} +{{- if and (get $auth "enabled" | default false) (not (get $auth "existingSecret" | default "")) (not (get $auth "autoGenerateSecret" | default false)) -}} +{{- fail "server.auth requires existingSecret when autoGenerateSecret=false" -}} +{{- end -}} {{- end }} {{/* diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 314f69daf5..d31f2254df 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -267,14 +267,12 @@ spec: {{- end }} {{- with .Values.server.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} {{- with include "hugegraph.javaOptsEnv" .Values.server.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} - {{- if and .Values.server.auth.enabled .Values.server.auth.existingSecret }} + {{- if .Values.server.auth.enabled }} - name: PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.server.auth.existingSecret | quote }} + name: {{ include "hugegraph.server.authSecretName" . | quote }} key: password - {{- end }} - {{- if .Values.server.auth.enabled }} - name: HG_SERVER_AUTH_TOKEN_SECRET valueFrom: secretKeyRef: diff --git a/helm/hugegraph/templates/server-secret.yaml b/helm/hugegraph/templates/server-secret.yaml new file mode 100644 index 0000000000..9557723597 --- /dev/null +++ b/helm/hugegraph/templates/server-secret.yaml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- $auth := get .Values.server "auth" | default dict -}} +{{- if and (get $auth "enabled" | default false) (get $auth "autoGenerateSecret" | default false) (not (get $auth "existingSecret" | default "")) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "hugegraph.server.authSecretName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + password: {{ include "hugegraph.server.authSecretPassword" . | quote }} +{{- end }} diff --git a/helm/hugegraph/templates/tests/test-connection.yaml b/helm/hugegraph/templates/tests/test-connection.yaml index beef5c1d47..432a82e182 100644 --- a/helm/hugegraph/templates/tests/test-connection.yaml +++ b/helm/hugegraph/templates/tests/test-connection.yaml @@ -53,7 +53,7 @@ spec: - name: PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.server.auth.existingSecret | quote }} + name: {{ include "hugegraph.server.authSecretName" . | quote }} key: password {{- end }} command: diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index a6220cfec1..bca6562ee1 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -554,6 +554,9 @@ "enabled": { "type": "boolean" }, + "autoGenerateSecret": { + "type": "boolean" + }, "existingSecret": { "type": "string" }, @@ -585,11 +588,25 @@ } }, "then": { - "properties": { - "existingSecret": { - "minLength": 1 + "anyOf": [ + { + "properties": { + "existingSecret": { + "minLength": 1 + } + } + }, + { + "properties": { + "autoGenerateSecret": { + "const": true + }, + "existingSecret": { + "const": "" + } + } } - } + ] } }, { diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index e5454f1081..62c44ab952 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -248,9 +248,11 @@ server: initStoreEnabled: false auth: enabled: false + autoGenerateSecret: true # Secret must contain key "password" (no newlines). Applied as # auth.admin_pa when the admin account is first created; changing the - # Secret later does not rotate an existing cluster's password. + # Secret later does not rotate an existing cluster's password. When empty, + # the chart creates a kept release-admin Secret. existingSecret: "" # JWT signing key for auth.token_secret / HG_SERVER_AUTH_TOKEN_SECRET. # Must be identical on every Server replica or Hubble login fails behind From 54cb7d24c5320097bbfaffdedf2765b1111d291b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 15 Aug 2026 21:29:25 +0530 Subject: [PATCH 14/61] feat(helm): enable auth by default with Hubble opt-in Default installs get a chart-managed admin Secret; leave Hubble off so API-only clusters stay lean, and enable the UI with one flag when wanted. --- .github/workflows/helm-chart-ci.yml | 10 +++- helm/hugegraph/README.md | 73 +++++++++++++++++++---------- helm/hugegraph/values-cluster.yaml | 14 ++++-- helm/hugegraph/values-single.yaml | 4 ++ helm/hugegraph/values.yaml | 4 +- 5 files changed, 74 insertions(+), 31 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 7f50bcb8a5..c25d89a436 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -84,13 +84,17 @@ jobs: helm template ci helm/hugegraph -f helm/hugegraph/values-single.yaml \ | grep -qF 'value: "-Dpartition.default-shard-count=1"' # The stock distributed install must put every Server replica on - # the shared PD graph catalog, even without auth or Hubble. + # the shared PD graph catalog, even without Hubble. Auth is on by + # default, so the chart-managed admin Secret must render too. DEFAULT=$(helm template ci helm/hugegraph) grep -qF "printf 'usePD=true\\n'" <<<"$DEFAULT" grep -qF "printf 'pd.peers=%s\\n' \"\${HG_SERVER_PD_PEERS}\"" \ <<<"$DEFAULT" grep -qF 'value: "ci-hugegraph-pd-0.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-1.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-2.ci-hugegraph-pd.default.svc:8686"' \ <<<"$DEFAULT" + grep -qE '^kind: Secret$' <<<"$DEFAULT" + grep -qF 'name: ci-admin' <<<"$DEFAULT" + grep -qF 'name: ci-auth-token' <<<"$DEFAULT" - name: reject invalid values run: | @@ -113,7 +117,9 @@ jobs: --set server.pdb.enabled=true \ --set server.pdb.minAvailable=2 must_fail --set server.auth.enabled=true --set server.auth.autoGenerateSecret=false - must_fail --set hubble.enabled=true + # Auth defaults to on, so Hubble alone is valid; refuse Hubble only + # when authentication is explicitly disabled. + must_fail --set hubble.enabled=true --set server.auth.enabled=false A=(--set hubble.enabled=true --set hubble.allowWithoutServerAuth=true) must_fail "${A[@]}" --set hubble.port=0 must_fail "${A[@]}" \ diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 74af440168..93c2898b2d 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -76,6 +76,28 @@ This deploys 3 PD + 3 Store + 3 Server, preserves the image's automatic JVM sizing, and sets no resource requests or limits. Set resources before production use. +**Authentication is enabled by default.** The chart creates a kept Secret +named `-admin` (for example `hugegraph-admin`) with a random +password unless `server.auth.existingSecret` points at a pre-created Secret. +Read the password and exercise the API: + +```bash +PASSWORD="$(kubectl get secret -n hugegraph hugegraph-admin \ + -o jsonpath='{.data.password}' | base64 --decode)" +kubectl port-forward -n hugegraph svc/hugegraph-server 8080:8080 +curl --user "admin:${PASSWORD}" http://127.0.0.1:8080/versions +``` + +**Hubble is not installed by default.** Enable the optional UI after install: + +```bash +helm upgrade --install hugegraph ./helm/hugegraph --namespace hugegraph \ + --set hubble.enabled=true +``` + +Auth is already on, so that single flag is enough. Login uses the same admin +credential from the chart-managed (or BYO) Secret. + The default anti-affinity for `pd`, `store`, and `server` is `preferred` (Server always was; Hubble has no anti-affinity knob because it is single-replica by design), so the chart schedules even on clusters with @@ -105,9 +127,9 @@ helm test hugegraph --namespace hugegraph | File | Purpose | |---|---| -| `values.yaml` | Default 3+3+3 topology with preferred anti-affinity, so it schedules on clusters of any node count | -| `values-single.yaml` | Single-node 1+1+1 example | -| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity for PD and Store; Hubble stays opt-in because the preset does not enable authentication | +| `values.yaml` | Default 3+3+3 topology with preferred anti-affinity, authentication on, and Hubble off | +| `values-single.yaml` | Single-node 1+1+1 example with authentication on | +| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity for PD and Store; authentication on, Hubble still opt-in | `values-cluster.yaml` is a production starting point, not a capacity guarantee. Recalculate capacity for the graph size, traffic, failure budget, @@ -287,7 +309,7 @@ default values. | `server.restServer.minFreeMemory` | Empty preserves the image default | `""` | | `server.restServer.batchMaxWriteThreads` | Empty preserves the image default | `""` | | `server.initStoreEnabled` | Must remain `false` for distributed HStore | `false` | -| `server.auth.enabled` | Enable admin authentication | `false` | +| `server.auth.enabled` | Enable admin authentication | `true` | | `server.auth.autoGenerateSecret` | Create and keep a random release-admin Secret when `existingSecret` is empty | `true` | | `server.auth.existingSecret` | Use a pre-created Secret instead; it must contain key `password` and takes priority | `""` | | `server.auth.tokenSecret.existingSecret` | BYO Secret for the JWT signing key (`auth.token_secret`); empty creates a kept release-auth-token Secret | `""` | @@ -316,7 +338,11 @@ utilization-based HPA requires a strictly positive Set `hubble.enabled=true` to deploy [HugeGraph Hubble](https://hugegraph.apache.org/docs/quickstart/toolchain/hugegraph-hubble/), the web UI for graph management, schema browsing, Gremlin queries, and the -cluster operations view. `hubble.mode` selects the wiring. In the default +cluster operations view. A default install leaves Hubble off so API-only +clusters stay lean; authentication is already on, so enabling the UI is a +single flag (see Installing above). Login uses the admin credential from +`server.auth.existingSecret` or the chart-managed `-admin` Secret. +`hubble.mode` selects the wiring. In the default `pd` mode the chart points `pd.peers` at the PD gRPC peers, `pd.server` at the PD client Service REST port, and the Store metrics allow-list at the Store REST endpoints, so the cluster view works without manual wiring; the @@ -344,16 +370,13 @@ stored metadata), `size` and `storageClassName` apply at install time only, and a non-root `podSecurityContext` needs a matching `fsGroup` so H2 can write the volume. -**Enable `server.auth` when using Hubble.** Current Hubble images gate the -UI behind a login that authenticates against the cluster; with server -authentication disabled the login cannot complete (the server rejects -`/auth/login` with "Unconfigured authenticator"), so Hubble is only useful -on an auth-enabled deployment, where the admin credential from -`server.auth.existingSecret` or the chart-managed admin Secret logs in. -The chart therefore refuses to render -`hubble.enabled=true` without `server.auth` unless -`hubble.allowWithoutServerAuth=true` explicitly overrides it for images -whose login does not need cluster authentication. +**Current Hubble images still require `server.auth`.** The UI login +authenticates against the cluster; with authentication explicitly disabled +the login cannot complete (the server rejects `/auth/login` with +"Unconfigured authenticator"). The chart therefore refuses to render +`hubble.enabled=true` when `server.auth.enabled=false` unless +`hubble.allowWithoutServerAuth=true` overrides it for images whose login +does not need cluster authentication. **Hubble serves plain HTTP.** Reach it with `kubectl port-forward` or behind an HTTPS-terminating Ingress; never expose the port directly to an untrusted @@ -443,9 +466,11 @@ before anything reaches the cluster: ### Connecting to the Cluster ```bash +PASSWORD="$(kubectl get secret -n hugegraph hugegraph-admin \ + -o jsonpath='{.data.password}' | base64 --decode)" kubectl port-forward -n hugegraph svc/hugegraph-server 8080:8080 -curl http://127.0.0.1:8080/versions -curl http://127.0.0.1:8080/graphs +curl --user "admin:${PASSWORD}" http://127.0.0.1:8080/versions +curl --user "admin:${PASSWORD}" http://127.0.0.1:8080/graphs ``` ### Cluster Health @@ -684,13 +709,13 @@ independently of the release name. valid for a root image; `podSecurityContext` and `securityContext` are fully configurable per component. - `values-cluster.yaml` is a starting point, not a capacity guarantee. -- The auth Secret sets the admin password only at first creation via - `auth.admin_pa`; the chart cannot rotate an existing cluster's admin - password. -- With authentication enabled, every Server replica must share one JWT - signing key. The chart injects `HG_SERVER_AUTH_TOKEN_SECRET` from - `server.auth.tokenSecret` (chart-managed by default) so Hubble login - stays stable behind a multi-replica Service. +- Authentication is on by default. The auth Secret sets the admin password + only at first creation via `auth.admin_pa`; the chart cannot rotate an + existing cluster's admin password. +- Every Server replica must share one JWT signing key. The chart injects + `HG_SERVER_AUTH_TOKEN_SECRET` from `server.auth.tokenSecret` + (chart-managed by default) so Hubble login stays stable behind a + multi-replica Service. - Hubble is single-replica, serves plain HTTP, requires `server.auth` to be enabled for its login to complete, and keeps UI connection metadata, including any graph credentials entered in the UI, in an embedded H2 diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index 98202e5f16..bc67ffe042 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -86,11 +86,17 @@ server: memory: 2Gi hpa: enabled: false + # Auth is on by default (chart-managed admin Secret). Pin it here so a + # production overlay cannot accidentally drop authentication. + auth: + enabled: true + autoGenerateSecret: true + existingSecret: "" -# The optional Hubble UI is not enabled here: this preset does not enable -# server authentication, and current Hubble images cannot complete their -# login against an auth-less cluster. Enable it together with server.auth, -# starting from: +# The optional Hubble UI is not enabled here: authentication is already on +# by default, so turn Hubble on with --set hubble.enabled=true (or the +# snippet below) when the browser UI is wanted. Current Hubble images still +# refuse to render if server.auth is explicitly disabled. # # hubble: # enabled: true diff --git a/helm/hugegraph/values-single.yaml b/helm/hugegraph/values-single.yaml index 86dd71abd6..985a1f0bd6 100644 --- a/helm/hugegraph/values-single.yaml +++ b/helm/hugegraph/values-single.yaml @@ -38,3 +38,7 @@ server: replicas: 1 hpa: enabled: false + auth: + enabled: true + autoGenerateSecret: true + existingSecret: "" diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 62c44ab952..0e7f5cdb1a 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -247,7 +247,9 @@ server: # HG_SERVER_SKIP_INIT and no init Job. initStoreEnabled: false auth: - enabled: false + # On by default: a chart-managed release-admin Secret supplies the password + # unless existingSecret is set. Set enabled=false only for trusted networks. + enabled: true autoGenerateSecret: true # Secret must contain key "password" (no newlines). Applied as # auth.admin_pa when the admin account is first created; changing the From ba03f7263bad828b29ca2d876a6ef5debc710e2c Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 15 Aug 2026 22:04:43 +0530 Subject: [PATCH 15/61] feat(helm): nest auth admin/token value vs Secret keys Split server.auth into admin and token blocks so inline credentials (password/value) are distinct from Kubernetes Secret refs (existingSecret), matching the clearer values API for operators. --- .github/workflows/helm-chart-ci.yml | 6 +- helm/hugegraph/README.md | 33 +++-- helm/hugegraph/templates/NOTES.txt | 2 +- helm/hugegraph/templates/_helpers.tpl | 58 ++++++-- .../templates/server-auth-token-secret.yaml | 4 +- .../templates/server-deployment.yaml | 2 +- helm/hugegraph/templates/server-secret.yaml | 5 +- .../templates/tests/test-connection.yaml | 2 +- .../testdata/values-pre-hardening.yaml | 11 +- helm/hugegraph/values-cluster.yaml | 6 +- helm/hugegraph/values-single.yaml | 6 +- helm/hugegraph/values.schema.json | 138 +++++++++++++++--- helm/hugegraph/values.yaml | 29 ++-- 13 files changed, 225 insertions(+), 77 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index c25d89a436..5768de8815 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -61,7 +61,7 @@ jobs: helm template ci helm/hugegraph \ --set hubble.enabled=true \ --set server.auth.enabled=true \ - --set server.auth.existingSecret=ci-auth > /dev/null + --set server.auth.admin.existingSecret=ci-auth > /dev/null helm template ci helm/hugegraph \ --set hubble.enabled=true \ --set hubble.allowWithoutServerAuth=true \ @@ -116,7 +116,7 @@ jobs: --set server.resources.requests.cpu=100m \ --set server.pdb.enabled=true \ --set server.pdb.minAvailable=2 - must_fail --set server.auth.enabled=true --set server.auth.autoGenerateSecret=false + must_fail --set server.auth.enabled=true --set server.auth.admin.autoGenerate=false --set server.auth.token.autoGenerate=false # Auth defaults to on, so Hubble alone is valid; refuse Hubble only # when authentication is explicitly disabled. must_fail --set hubble.enabled=true --set server.auth.enabled=false @@ -180,7 +180,7 @@ jobs: helm template ci helm/hugegraph \ --set hubble.enabled=true \ --set server.auth.enabled=true \ - --set server.auth.existingSecret=ci-auth \ + --set server.auth.admin.existingSecret=ci-auth \ | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 helm template ci helm/hugegraph \ --set pd.partition.defaultShardCount=3 \ diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 93c2898b2d..5f5b6c6085 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -78,7 +78,7 @@ production use. **Authentication is enabled by default.** The chart creates a kept Secret named `-admin` (for example `hugegraph-admin`) with a random -password unless `server.auth.existingSecret` points at a pre-created Secret. +password unless `server.auth.admin.existingSecret` points at a pre-created Secret. Read the password and exercise the API: ```bash @@ -310,10 +310,14 @@ default values. | `server.restServer.batchMaxWriteThreads` | Empty preserves the image default | `""` | | `server.initStoreEnabled` | Must remain `false` for distributed HStore | `false` | | `server.auth.enabled` | Enable admin authentication | `true` | -| `server.auth.autoGenerateSecret` | Create and keep a random release-admin Secret when `existingSecret` is empty | `true` | -| `server.auth.existingSecret` | Use a pre-created Secret instead; it must contain key `password` and takes priority | `""` | -| `server.auth.tokenSecret.existingSecret` | BYO Secret for the JWT signing key (`auth.token_secret`); empty creates a kept release-auth-token Secret | `""` | -| `server.auth.tokenSecret.key` | Key inside the JWT signing Secret | `token_secret` | +| `server.auth.admin.password` | Optional inline admin password; prefer a Secret in shared clusters | `""` | +| `server.auth.admin.existingSecret` | Pre-created Secret name (key defaults to `password`); takes priority | `""` | +| `server.auth.admin.key` | Key inside the admin password Secret | `password` | +| `server.auth.admin.autoGenerate` | Create and keep a random release-admin Secret when password and existingSecret are empty | `true` | +| `server.auth.token.value` | Optional inline JWT signing key; prefer a Secret in shared clusters | `""` | +| `server.auth.token.existingSecret` | Pre-created Secret for the JWT signing key (`auth.token_secret`) | `""` | +| `server.auth.token.key` | Key inside the JWT signing Secret | `token_secret` | +| `server.auth.token.autoGenerate` | Create and keep a random release-auth-token Secret when value and existingSecret are empty | `true` | | `server.ingress.enabled` | Create an Ingress for the Server Service | `false` | | `server.ingress.className` | IngressClass name | `""` | | `server.ingress.annotations` | Ingress annotations (cert-manager, nginx, ALB) | `{}` | @@ -341,7 +345,7 @@ the web UI for graph management, schema browsing, Gremlin queries, and the cluster operations view. A default install leaves Hubble off so API-only clusters stay lean; authentication is already on, so enabling the UI is a single flag (see Installing above). Login uses the admin credential from -`server.auth.existingSecret` or the chart-managed `-admin` Secret. +`server.auth.admin.existingSecret` or the chart-managed `-admin` Secret. `hubble.mode` selects the wiring. In the default `pd` mode the chart points `pd.peers` at the PD gRPC peers, `pd.server` at the PD client Service REST port, and the Store metrics allow-list at the @@ -422,12 +426,15 @@ before anything reaches the cluster: - Unknown keys and wrong types are rejected. - `server.initStoreEnabled` must remain `false` for a distributed deployment. -- With authentication enabled, either `server.auth.existingSecret` must name a - Secret containing a `password` key, or `server.auth.autoGenerateSecret` must - be true. With authentication disabled `existingSecret` must be empty, so a - configured but inactive Secret reference cannot be overlooked. A missing - Secret fails when Kubernetes configures the container; an empty `password` - fails in the Server startup wrapper. +- With authentication enabled, either `server.auth.admin.existingSecret` must + name a Secret containing the configured key (default `password`), or + `server.auth.admin.password` must be set, or `server.auth.admin.autoGenerate` + must be true. The same shape applies to `server.auth.token` (`existingSecret` + / `value` / `autoGenerate`). With authentication disabled, + `admin.existingSecret`, `admin.password`, `token.existingSecret`, and + `token.value` must be empty, so a configured but inactive Secret reference + cannot be overlooked. A missing Secret fails when Kubernetes configures the + container; an empty `password` fails in the Server startup wrapper. - `server.hpa.minReplicas` must not exceed `maxReplicas`, and enabling utilization-based HPA requires a strictly positive `server.resources.requests.cpu`. @@ -713,7 +720,7 @@ independently of the release name. only at first creation via `auth.admin_pa`; the chart cannot rotate an existing cluster's admin password. - Every Server replica must share one JWT signing key. The chart injects - `HG_SERVER_AUTH_TOKEN_SECRET` from `server.auth.tokenSecret` + `HG_SERVER_AUTH_TOKEN_SECRET` from `server.auth.token` (chart-managed by default) so Hubble login stays stable behind a multi-replica Service. - Hubble is single-replica, serves plain HTTP, requires `server.auth` to be diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 2d2eb0cc4f..0c0f0e407e 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -39,7 +39,7 @@ Reach the Server API: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.server.name" . }} {{ .Values.server.port }}:{{ .Values.server.port }} {{- if .Values.server.auth.enabled }} - PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.server.authSecretName" . }} -o jsonpath='{.data.password}' | base64 --decode)" + PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.server.authSecretName" . }} -o jsonpath='{.data.{{ include "hugegraph.server.authSecretKey" . }}}' | base64 --decode)" echo "Admin password: ${PASSWORD}" curl --user "admin:${PASSWORD}" http://127.0.0.1:{{ .Values.server.port }}/versions {{- else }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index c033af5db9..5e10b129b3 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -89,7 +89,8 @@ uninstall and be reused by a later install of the same release. */}} {{- define "hugegraph.server.authSecretName" -}} {{- $auth := get .Values.server "auth" | default dict -}} -{{- $existingSecret := get $auth "existingSecret" | default "" -}} +{{- $admin := get $auth "admin" | default dict -}} +{{- $existingSecret := get $admin "existingSecret" | default "" -}} {{- if $existingSecret -}} {{- $existingSecret -}} {{- else -}} @@ -97,28 +98,41 @@ uninstall and be reused by a later install of the same release. {{- end -}} {{- end }} +{{- define "hugegraph.server.authSecretKey" -}} +{{- $auth := get .Values.server "auth" | default dict -}} +{{- $admin := get $auth "admin" | default dict -}} +{{- get $admin "key" | default "password" -}} +{{- end }} + {{/* -Return the generated Secret's already-encoded password when it exists. The -lookup keeps upgrades from rotating the administrator credential; a first -install gets a random password. This helper is only used for chart-managed -Secrets, never for an external existingSecret. +Return the chart-managed admin password (base64). Inline admin.password wins +on first write; otherwise lookup keeps upgrades from rotating a generated +credential. Never used for an external existingSecret. */}} {{- define "hugegraph.server.authSecretPassword" -}} +{{- $auth := get .Values.server "auth" | default dict -}} +{{- $admin := get $auth "admin" | default dict -}} +{{- $password := get $admin "password" | default "" -}} +{{- if $password -}} +{{- $password | b64enc -}} +{{- else -}} +{{- $key := include "hugegraph.server.authSecretKey" . -}} {{- $secret := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authSecretName" .) -}} -{{- if and $secret (hasKey $secret "data") (hasKey (get $secret "data") "password") -}} -{{- get (get $secret "data") "password" -}} +{{- if and $secret (hasKey $secret "data") (hasKey (get $secret "data") $key) -}} +{{- get (get $secret "data") $key -}} {{- else -}} {{- randAlphaNum 32 | b64enc -}} {{- end -}} +{{- end -}} {{- end }} {{/* -Resolve the JWT signing Secret. User-provided tokenSecret.existingSecret wins; +Resolve the JWT signing Secret. User-provided token.existingSecret wins; otherwise use a stable chart-managed name so every Server replica shares one key. */}} {{- define "hugegraph.server.authTokenSecretName" -}} {{- $auth := get .Values.server "auth" | default dict -}} -{{- $token := get $auth "tokenSecret" | default dict -}} +{{- $token := get $auth "token" | default dict -}} {{- $existing := get $token "existingSecret" | default "" -}} {{- if $existing -}} {{- $existing -}} @@ -129,16 +143,22 @@ otherwise use a stable chart-managed name so every Server replica shares one key {{- define "hugegraph.server.authTokenSecretKey" -}} {{- $auth := get .Values.server "auth" | default dict -}} -{{- $token := get $auth "tokenSecret" | default dict -}} +{{- $token := get $auth "token" | default dict -}} {{- get $token "key" | default "token_secret" -}} {{- end }} {{/* -Return the already-encoded JWT signing secret when present; otherwise generate -32 random bytes (base64). Lookup keeps multi-replica Server pods and upgrades -on the same signing key. +Return the chart-managed JWT signing secret (base64). Inline token.value wins +on first write; otherwise lookup keeps multi-replica pods and upgrades on the +same signing key. */}} {{- define "hugegraph.server.authTokenSecretValue" -}} +{{- $auth := get .Values.server "auth" | default dict -}} +{{- $token := get $auth "token" | default dict -}} +{{- $value := get $token "value" | default "" -}} +{{- if $value -}} +{{- $value | b64enc -}} +{{- else -}} {{- $name := include "hugegraph.server.authTokenSecretName" . -}} {{- $key := include "hugegraph.server.authTokenSecretKey" . -}} {{- $secret := lookup "v1" "Secret" .Release.Namespace $name -}} @@ -147,6 +167,7 @@ on the same signing key. {{- else -}} {{- randAlphaNum 32 | b64enc -}} {{- end -}} +{{- end -}} {{- end }} {{/* @@ -518,8 +539,15 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- end -}} {{- end -}} {{- $auth := get .Values.server "auth" | default dict -}} -{{- if and (get $auth "enabled" | default false) (not (get $auth "existingSecret" | default "")) (not (get $auth "autoGenerateSecret" | default false)) -}} -{{- fail "server.auth requires existingSecret when autoGenerateSecret=false" -}} +{{- $admin := get $auth "admin" | default dict -}} +{{- $token := get $auth "token" | default dict -}} +{{- if get $auth "enabled" | default false -}} +{{- if and (not (get $admin "existingSecret" | default "")) (not (get $admin "password" | default "")) (not (get $admin "autoGenerate" | default false)) -}} +{{- fail "server.auth.admin requires existingSecret, password, or autoGenerate=true when auth is enabled" -}} +{{- end -}} +{{- if and (not (get $token "existingSecret" | default "")) (not (get $token "value" | default "")) (not (get $token "autoGenerate" | default false)) -}} +{{- fail "server.auth.token requires existingSecret, value, or autoGenerate=true when auth is enabled" -}} +{{- end -}} {{- end -}} {{- end }} diff --git a/helm/hugegraph/templates/server-auth-token-secret.yaml b/helm/hugegraph/templates/server-auth-token-secret.yaml index 3581b44b1a..f416e313de 100644 --- a/helm/hugegraph/templates/server-auth-token-secret.yaml +++ b/helm/hugegraph/templates/server-auth-token-secret.yaml @@ -16,8 +16,8 @@ # {{- $auth := get .Values.server "auth" | default dict -}} -{{- $token := get $auth "tokenSecret" | default dict -}} -{{- if and (get $auth "enabled" | default false) (not (get $token "existingSecret" | default "")) }} +{{- $token := get $auth "token" | default dict -}} +{{- if and (get $auth "enabled" | default false) (not (get $token "existingSecret" | default "")) (or (get $token "value" | default "") (get $token "autoGenerate" | default false)) }} apiVersion: v1 kind: Secret metadata: diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index d31f2254df..e7722ea48f 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -272,7 +272,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "hugegraph.server.authSecretName" . | quote }} - key: password + key: {{ include "hugegraph.server.authSecretKey" . | quote }} - name: HG_SERVER_AUTH_TOKEN_SECRET valueFrom: secretKeyRef: diff --git a/helm/hugegraph/templates/server-secret.yaml b/helm/hugegraph/templates/server-secret.yaml index 9557723597..c25f9a72b7 100644 --- a/helm/hugegraph/templates/server-secret.yaml +++ b/helm/hugegraph/templates/server-secret.yaml @@ -16,7 +16,8 @@ # {{- $auth := get .Values.server "auth" | default dict -}} -{{- if and (get $auth "enabled" | default false) (get $auth "autoGenerateSecret" | default false) (not (get $auth "existingSecret" | default "")) }} +{{- $admin := get $auth "admin" | default dict -}} +{{- if and (get $auth "enabled" | default false) (not (get $admin "existingSecret" | default "")) (or (get $admin "password" | default "") (get $admin "autoGenerate" | default false)) }} apiVersion: v1 kind: Secret metadata: @@ -27,5 +28,5 @@ metadata: helm.sh/resource-policy: keep type: Opaque data: - password: {{ include "hugegraph.server.authSecretPassword" . | quote }} + {{ include "hugegraph.server.authSecretKey" . }}: {{ include "hugegraph.server.authSecretPassword" . | quote }} {{- end }} diff --git a/helm/hugegraph/templates/tests/test-connection.yaml b/helm/hugegraph/templates/tests/test-connection.yaml index 432a82e182..1176538a91 100644 --- a/helm/hugegraph/templates/tests/test-connection.yaml +++ b/helm/hugegraph/templates/tests/test-connection.yaml @@ -54,7 +54,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "hugegraph.server.authSecretName" . | quote }} - key: password + key: {{ include "hugegraph.server.authSecretKey" . | quote }} {{- end }} command: - sh diff --git a/helm/hugegraph/testdata/values-pre-hardening.yaml b/helm/hugegraph/testdata/values-pre-hardening.yaml index d3d34cea1f..990f5fe7e0 100644 --- a/helm/hugegraph/testdata/values-pre-hardening.yaml +++ b/helm/hugegraph/testdata/values-pre-hardening.yaml @@ -98,7 +98,16 @@ server: initStoreEnabled: false auth: enabled: false - existingSecret: "" + admin: + password: "" + existingSecret: "" + key: password + autoGenerate: true + token: + value: "" + existingSecret: "" + key: token_secret + autoGenerate: true ingress: enabled: false className: "" diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index bc67ffe042..0518468666 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -90,8 +90,10 @@ server: # production overlay cannot accidentally drop authentication. auth: enabled: true - autoGenerateSecret: true - existingSecret: "" + admin: + autoGenerate: true + token: + autoGenerate: true # The optional Hubble UI is not enabled here: authentication is already on # by default, so turn Hubble on with --set hubble.enabled=true (or the diff --git a/helm/hugegraph/values-single.yaml b/helm/hugegraph/values-single.yaml index 985a1f0bd6..92f4acf4d5 100644 --- a/helm/hugegraph/values-single.yaml +++ b/helm/hugegraph/values-single.yaml @@ -40,5 +40,7 @@ server: enabled: false auth: enabled: true - autoGenerateSecret: true - existingSecret: "" + admin: + autoGenerate: true + token: + autoGenerate: true diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index bca6562ee1..777055451f 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -548,32 +548,60 @@ "additionalProperties": false, "required": [ "enabled", - "existingSecret" + "admin", + "token" ], "properties": { "enabled": { "type": "boolean" }, - "autoGenerateSecret": { - "type": "boolean" - }, - "existingSecret": { - "type": "string" + "admin": { + "type": "object", + "additionalProperties": false, + "required": [ + "password", + "existingSecret", + "key", + "autoGenerate" + ], + "properties": { + "password": { + "type": "string" + }, + "existingSecret": { + "type": "string" + }, + "key": { + "type": "string", + "minLength": 1 + }, + "autoGenerate": { + "type": "boolean" + } + } }, - "tokenSecret": { + "token": { "type": "object", "additionalProperties": false, "required": [ + "value", "existingSecret", - "key" + "key", + "autoGenerate" ], "properties": { + "value": { + "type": "string" + }, "existingSecret": { "type": "string" }, "key": { "type": "string", "minLength": 1 + }, + "autoGenerate": { + "type": "boolean" } } } @@ -588,25 +616,70 @@ } }, "then": { - "anyOf": [ - { - "properties": { - "existingSecret": { - "minLength": 1 + "properties": { + "admin": { + "anyOf": [ + { + "properties": { + "existingSecret": { + "minLength": 1 + } + } + }, + { + "properties": { + "password": { + "minLength": 1 + }, + "existingSecret": { + "const": "" + } + } + }, + { + "properties": { + "autoGenerate": { + "const": true + }, + "existingSecret": { + "const": "" + } + } } - } + ] }, - { - "properties": { - "autoGenerateSecret": { - "const": true + "token": { + "anyOf": [ + { + "properties": { + "existingSecret": { + "minLength": 1 + } + } }, - "existingSecret": { - "const": "" + { + "properties": { + "value": { + "minLength": 1 + }, + "existingSecret": { + "const": "" + } + } + }, + { + "properties": { + "autoGenerate": { + "const": true + }, + "existingSecret": { + "const": "" + } + } } - } + ] } - ] + } } }, { @@ -619,8 +692,25 @@ }, "then": { "properties": { - "existingSecret": { - "const": "" + "admin": { + "properties": { + "existingSecret": { + "const": "" + }, + "password": { + "const": "" + } + } + }, + "token": { + "properties": { + "existingSecret": { + "const": "" + }, + "value": { + "const": "" + } + } } } } diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 0e7f5cdb1a..684720a64a 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -248,21 +248,30 @@ server: initStoreEnabled: false auth: # On by default: a chart-managed release-admin Secret supplies the password - # unless existingSecret is set. Set enabled=false only for trusted networks. + # unless admin.existingSecret or admin.password is set. Set enabled=false + # only for trusted networks. enabled: true - autoGenerateSecret: true - # Secret must contain key "password" (no newlines). Applied as - # auth.admin_pa when the admin account is first created; changing the - # Secret later does not rotate an existing cluster's password. When empty, - # the chart creates a kept release-admin Secret. - existingSecret: "" + # Admin password: inline value vs Kubernetes Secret name are separate keys. + # Priority: existingSecret > password > autoGenerate. + admin: + # Optional plaintext password (prefer existingSecret in shared clusters). + password: "" + # Pre-created Secret name. Must contain key below; chart does not manage it. + existingSecret: "" + key: password + # When existingSecret and password are empty, create a kept release-admin Secret. + autoGenerate: true # JWT signing key for auth.token_secret / HG_SERVER_AUTH_TOKEN_SECRET. # Must be identical on every Server replica or Hubble login fails behind - # the Service. Empty existingSecret: chart creates a kept Secret (lookup - # keeps the value across upgrades). BYO with existingSecret + key. - tokenSecret: + # the Service. Priority: existingSecret > value > autoGenerate. + token: + # Optional plaintext signing key (prefer existingSecret in shared clusters). + value: "" + # Pre-created Secret name. Chart does not manage it. existingSecret: "" key: token_secret + # When existingSecret and value are empty, create a kept release-auth-token Secret. + autoGenerate: true service: type: ClusterIP annotations: {} From f1b6e3a4bc00d3af8487c4688d6468c7a575265a Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sun, 16 Aug 2026 22:44:57 +0530 Subject: [PATCH 16/61] feat(helm): support outside Hubble PD discovery via advertiseUrl Let Server register a reachable URL with PD and optionally expose the PD client Service, so standalone Hubble can discover the cluster without in-cluster DNS. Fail closed when advertiseUrl is set without PD meta mode. --- helm/hugegraph/README.md | 101 ++++++++++++++++++ helm/hugegraph/templates/NOTES.txt | 14 +++ helm/hugegraph/templates/_helpers.tpl | 29 +++++ .../templates/pd-service-client.yaml | 13 ++- .../templates/server-deployment.yaml | 8 +- helm/hugegraph/values.schema.json | 37 +++++++ helm/hugegraph/values.yaml | 18 ++++ 7 files changed, 216 insertions(+), 4 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 5f5b6c6085..fafd08b05f 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -321,6 +321,7 @@ default values. | `server.ingress.enabled` | Create an Ingress for the Server Service | `false` | | `server.ingress.className` | IngressClass name | `""` | | `server.ingress.annotations` | Ingress annotations (cert-manager, nginx, ALB) | `{}` | +| `server.advertiseUrl` | Absolute Server URL registered with PD (`server.urls_to_pd`). Empty uses the in-cluster Service URL | `""` | | `server.service.type` | Server Service type | `ClusterIP` | | `server.service.annotations` | Server Service annotations | `{}` | | `server.ingress.hosts` | Ingress hosts and paths | see `values.yaml` | @@ -338,8 +339,108 @@ Helm upgrade does not overwrite the autoscaler's live replica count. Enabling utilization-based HPA requires a strictly positive `server.resources.requests.cpu`. + +### Reaching Hubble (pick one path) + +Most people should stop at **1**. Use **2** only if Hubble must run outside +the cluster. Use **3** only if that outside Hubble must discover Server through +PD. Store Operations metrics from outside the cluster are out of scope here. + +#### 1. In-cluster Hubble (recommended) + +Set `hubble.enabled=true` (off by default so API-only clusters stay lean). The +chart wires PD/Server for you. + +Open the UI with one port-forward: + +```bash +kubectl -n port-forward svc/-hugegraph-hubble 8088:8088 +``` + +Then open `http://127.0.0.1:8088`. For a shared environment, expose Hubble with +`hubble.service.type` NodePort/LoadBalancer or `hubble.ingress` instead of +port-forward. Log in with the chart admin password from the NOTES / admin +Secret. + +This is the average-user path: no Docker, no advertise URL, no PD peer list. + +#### 2. Outside Hubble, direct Server URL (simple external) + +Use this when Hubble runs on a host or VM outside the cluster, and you only +need graph / schema / data / Gremlin (not PD discovery). + +1. Leave in-chart Hubble off (`hubble.enabled=false`). +2. Expose Server (`server.service.type` NodePort/LoadBalancer, or Ingress). +3. Run a standalone Hubble image with `pd.enabled=false` and + `server.direct_url` set to that reachable Server URL (match Server auth). +4. Open the standalone Hubble port in a browser (or SSH tunnel to it). + +Example property fragment for the standalone process: + +```properties +pd.enabled=false +server.direct_url=http://: +``` + +Mount the file at `/hubble/conf/hugegraph-hubble.properties` inside the +official image (workdir is `/hubble`). One Server URL is enough; you do not +need to expose PD. + +#### 3. Outside Hubble, PD discovery (advanced) + +Use this when an outside Hubble must ask PD for the Server address. + +In-cluster names such as `*.svc` are not reachable from outside. The chart +helps with two knobs: advertise a reachable Server URL to PD, and expose the +PD client Service. + +Keep **`server.auth.enabled=true`**. With `hubble.enabled=false`, auth-on is +what enables PD meta mode so the chart actually writes `server.urls_to_pd` +(and therefore honors `server.advertiseUrl`). Auth-off plus `advertiseUrl` +fails at render time. + +1. Leave in-chart Hubble off and keep `server.auth.enabled=true`. +2. Expose Server and set `server.advertiseUrl` to the absolute `http(s)://` + URL outside Hubble will use after discovery. The chart registers it via + `server.urls_to_pd` instead of the in-cluster Service URL. +3. Expose PD (`pd.service.type` NodePort/LoadBalancer) so Hubble can dial PD + REST and gRPC. +4. Run standalone Hubble with `pd.enabled=true` and `pd.peers` / `pd.server` + pointed at those external PD addresses. Mount config at + `/hubble/conf/hugegraph-hubble.properties`. + +Example property fragment: + +```properties +pd.enabled=true +pd.peers=: +pd.server=: +``` + +Trade-off: when `server.advertiseUrl` is set, PD returns that same URL to +every discovery client, including an in-cluster Hubble. Leave it empty for the +default in-cluster path. + +Local quick test (cluster and Hubble on the same machine): port-forward Server +`8080` and PD client `8620`/`8686`, set +`server.advertiseUrl=http://127.0.0.1:8080`, run standalone Hubble with +`--network host` and the PD properties above, then open Hubble on `8088` +(or SSH `-L 8088:127.0.0.1:8088` from a laptop). + +| Parameter | Description | Default | +|---|---|---| +| `server.advertiseUrl` | Absolute Server URL registered with PD for discovery clients. Empty uses the in-cluster Server Service URL | `""` | +| `pd.service.type` | PD client Service type (`ClusterIP`, `NodePort`, `LoadBalancer`) | `ClusterIP` | +| `pd.service.annotations` | Annotations on the PD client Service | `{}` | +| `pd.service.restNodePort` | Optional fixed NodePort for PD REST; requires NodePort/LoadBalancer | unset | +| `pd.service.grpcNodePort` | Optional fixed NodePort for PD gRPC; requires NodePort/LoadBalancer | unset | + ### Hubble (optional UI) +How to open Hubble (in-cluster vs outside) is under +[Reaching Hubble](#reaching-hubble-pick-one-path) above. This section covers +chart wiring and parameters. + Set `hubble.enabled=true` to deploy [HugeGraph Hubble](https://hugegraph.apache.org/docs/quickstart/toolchain/hugegraph-hubble/), the web UI for graph management, schema browsing, Gremlin queries, and the cluster operations view. A default install leaves Hubble off so API-only diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 0c0f0e407e..02ad09bafc 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -71,6 +71,20 @@ Hubble Pod is replaced. Graph data is unaffected. Authentication is disabled. Do not expose this release to untrusted networks. {{- end }} +{{- $advertiseUrl := trim (default "" .Values.server.advertiseUrl) }} +{{- if $advertiseUrl }} + +Outside PD discovery: Server registers this URL with PD: + + {{ $advertiseUrl }} + +Keep server.auth.enabled=true so this registration stays active. Expose the +PD client Service (pd.service.type=NodePort or LoadBalancer) and point a +standalone Hubble at that PD endpoint with pd.enabled=true. Graph UI works +via PD discovery; Store Operations metrics are unchanged and still use +in-cluster Store URLs unless configured separately. +{{- end }} + {{- if and (gt (int .Values.pd.replicas) 1) (ne (get .Values.pd "antiAffinity" | default "") "required") (empty (get .Values.pd "affinity")) }} pd.antiAffinity is not "required", so the scheduler may co-locate PD quorum diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 5e10b129b3..8f0d707a3f 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -249,6 +249,20 @@ instead of the in-pod 0.0.0.0 default. {{- printf "http://%s.%s.svc:%d" (include "hugegraph.server.name" .) .Release.Namespace (int .Values.server.port) -}} {{- end }} +{{/* +URL registered with PD (server.urls_to_pd / HG_SERVER_URLS_TO_PD). +server.advertiseUrl wins when set so outside PD-mode Hubble receives a +reachable address; otherwise the in-cluster Server Service URL is used. +*/}} +{{- define "hugegraph.server.urlsToPd" -}} +{{- $advertise := trim (default "" .Values.server.advertiseUrl) -}} +{{- if $advertise -}} +{{- $advertise -}} +{{- else -}} +{{- include "hugegraph.server.clientUrl" . -}} +{{- end -}} +{{- end }} + {{/* PD REST endpoint reached through the client Service, for Hubble's pd.server. */}} @@ -488,6 +502,21 @@ keys for releases stored before the values existed. {{- if and (get $svc "nodePort") (not (has (get $svc "type" | default "ClusterIP") (list "NodePort" "LoadBalancer"))) -}} {{- fail "server.service.nodePort requires server.service.type to be NodePort or LoadBalancer" -}} {{- end -}} +{{- $advertiseUrl := trim (default "" .Values.server.advertiseUrl) -}} +{{- if and $advertiseUrl (not (or (hasPrefix "http://" $advertiseUrl) (hasPrefix "https://" $advertiseUrl))) -}} +{{- fail "server.advertiseUrl must be an absolute http:// or https:// URL when set" -}} +{{- end -}} +{{- $hubbleForAdvertise := get .Values "hubble" | default dict -}} +{{- $hubblePdModeForAdvertise := and (get $hubbleForAdvertise "enabled" | default false) (ne (get $hubbleForAdvertise "mode" | default "pd") "direct") -}} +{{- $pdMetaForAdvertise := or .Values.server.auth.enabled $hubblePdModeForAdvertise -}} +{{- if and $advertiseUrl (not $pdMetaForAdvertise) -}} +{{- fail "server.advertiseUrl requires server.auth.enabled=true (or in-chart hubble.enabled with hubble.mode=pd), because only then does the chart register server.urls_to_pd with PD" -}} +{{- end -}} +{{- $pdSvc := get .Values.pd "service" | default dict -}} +{{- $pdSvcType := get $pdSvc "type" | default "ClusterIP" -}} +{{- if and (or (get $pdSvc "restNodePort") (get $pdSvc "grpcNodePort")) (not (has $pdSvcType (list "NodePort" "LoadBalancer"))) -}} +{{- fail "pd.service.restNodePort and pd.service.grpcNodePort require pd.service.type to be NodePort or LoadBalancer" -}} +{{- end -}} {{- $serverPdb := get .Values.server "pdb" | default dict -}} {{- $serverReplicaFloor := include "hugegraph.server.replicaFloor" . | int -}} {{- if and (get $serverPdb "enabled" | default false) (gt $serverReplicaFloor 1) (ge (int (get $serverPdb "minAvailable" | default 1)) $serverReplicaFloor) -}} diff --git a/helm/hugegraph/templates/pd-service-client.yaml b/helm/hugegraph/templates/pd-service-client.yaml index d40cc9325c..a25108b511 100644 --- a/helm/hugegraph/templates/pd-service-client.yaml +++ b/helm/hugegraph/templates/pd-service-client.yaml @@ -15,6 +15,7 @@ # limitations under the License. # +{{- $svc := get .Values.pd "service" | default dict }} apiVersion: v1 kind: Service metadata: @@ -22,8 +23,12 @@ metadata: labels: {{- include "hugegraph.labels" . | nindent 4 }} app.kubernetes.io/component: pd + {{- with (get $svc "annotations") }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: - type: ClusterIP + type: {{ get $svc "type" | default "ClusterIP" }} selector: {{- include "hugegraph.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: pd @@ -31,6 +36,12 @@ spec: - name: rest port: {{ .Values.pd.ports.rest }} targetPort: rest + {{- with (get $svc "restNodePort") }} + nodePort: {{ . }} + {{- end }} - name: grpc port: {{ .Values.pd.ports.grpc }} targetPort: grpc + {{- with (get $svc "grpcNodePort") }} + nodePort: {{ . }} + {{- end }} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index e7722ea48f..467382535e 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -209,8 +209,10 @@ spec: if [[ "${FOUND_PD_PEERS}" == false ]]; then printf 'pd.peers=%s\n' "${HG_SERVER_PD_PEERS}" >>"${TMP}" fi - # PD hands this URL to discovery clients such as Hubble. The - # k8s branch is taken only when server.deploy_in_k8s is true; + # PD hands this URL to discovery clients such as Hubble. + # HG_SERVER_URLS_TO_PD uses server.advertiseUrl when set, + # otherwise the in-cluster Server Service URL. The k8s + # branch is taken only when server.deploy_in_k8s is true; # otherwise the announcement falls back to restserver.url, # whose 0.0.0.0 is never resolvable from another Pod. if [[ "${FOUND_URLS_TO_PD}" == false ]]; then @@ -263,7 +265,7 @@ spec: value: {{ .Values.server.initStoreEnabled | quote }} {{- if $pdMeta }} - name: HG_SERVER_URLS_TO_PD - value: {{ include "hugegraph.server.clientUrl" . | quote }} + value: {{ include "hugegraph.server.urlsToPd" . | quote }} {{- end }} {{- with .Values.server.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} {{- with include "hugegraph.javaOptsEnv" .Values.server.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 777055451f..77b6b146b7 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -341,6 +341,39 @@ "type": "boolean" } } + }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ] + }, + "annotations": { + "type": "object" + }, + "restNodePort": { + "type": [ + "integer", + "null" + ], + "minimum": 30000, + "maximum": 32767 + }, + "grpcNodePort": { + "type": [ + "integer", + "null" + ], + "minimum": 30000, + "maximum": 32767 + } + } } } }, @@ -818,6 +851,10 @@ }, "service": { "$ref": "#/definitions/service" + }, + "advertiseUrl": { + "type": "string", + "description": "URL registered with PD via server.urls_to_pd. Empty uses the in-cluster Server Service URL. Set to an externally reachable URL for outside Hubble PD discovery." } } }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 684720a64a..8999ddfbd8 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -97,6 +97,15 @@ pd: annotations: {} # This chart makes no Kubernetes API calls, so no token is mounted. automountServiceAccountToken: false + # Client Service for PD REST/gRPC. Defaults to ClusterIP for in-cluster + # consumers (Server, Store, in-chart Hubble). Set type to NodePort or + # LoadBalancer when an outside Hubble must reach PD for discovery. + service: + type: ClusterIP + annotations: {} + # Optional fixed NodePorts; require service.type NodePort or LoadBalancer. + restNodePort: + grpcNodePort: pdb: enabled: true minAvailable: 2 @@ -272,6 +281,15 @@ server: key: token_secret # When existingSecret and value are empty, create a kept release-auth-token Secret. autoGenerate: true + # URL announced to PD via server.urls_to_pd for discovery clients such as + # Hubble. Empty keeps the in-cluster Server Service URL + # (http://-server..svc:). Set this to a URL reachable + # from outside the cluster (NodePort, LoadBalancer, or Ingress) when an + # external Hubble uses PD mode; PD will hand that address back to Hubble. + # Trade-off: in-cluster discovery clients then receive the same URL, so it + # must also be reachable from inside the cluster (or keep Hubble in-cluster + # and leave this empty). + advertiseUrl: "" service: type: ClusterIP annotations: {} From 4c4eb90955035ee1fb3c12837b85ed05dcc45a3b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 22 Aug 2026 19:50:26 +0530 Subject: [PATCH 17/61] fix(helm): gate PD startup on peer DNS and harden chart defaults PD resolves its raft peer hostnames once at boot and freezes the result as the RPC allowlist. With podManagementPolicy: Parallel a PD can start before its peers A records are published, freeze a partial allowlist, and permanently block the elected leader, wedging a fresh install. Add a wait-for-pd-dns init container mirroring the store wait-for-pd gate: hold the pd container until every peer hostname resolves, bounded by pd.waitTimeoutSeconds (values: pd.waitEnabled escape hatch, waitImage, waitTimeoutSeconds, waitResources, schema-enforced). busybox nslookup ignores resolv.conf search domains, so the gate derives the cluster domain and retries the FQDN. The PD headless Service publishes not-ready addresses, so records appear at IP assignment and the gate cannot deadlock. First-boot mitigation only: a PD rescheduled with a new pod IP still requires the image-side allowlist refresh, noted under Limitations. Hardening and docs in the same pass: - explicit updateStrategy and persistentVolumeClaimRetentionPolicy (Retain/Retain; honored on K8s >=1.27) on the PD and Store StatefulSets, value-driven and schema-typed - checksum/auth pod annotation on the Server Deployment so rotating an existingSecret rolls Server pods on the next upgrade; hashes Secret names, keys, and resourceVersion, never Secret data. Best-effort: template-only pipelines render a constant, and the first upgrade after a fresh install rolls Server once - NOTES.txt no longer echoes the admin password - README: release-name assumption note, port-forward instead of exec curl in troubleshooting, Upgrading notes for the one-time PD and Server rollouts this version causes, and Limitations entries for the image-side PD allowlist and PD management REST issues (chart 0.1.1) --- helm/hugegraph/Chart.yaml | 2 +- helm/hugegraph/README.md | 38 +++++++++- helm/hugegraph/templates/NOTES.txt | 1 - helm/hugegraph/templates/_helpers.tpl | 28 +++++++ helm/hugegraph/templates/pd-statefulset.yaml | 61 +++++++++++++++ .../templates/server-deployment.yaml | 9 ++- .../templates/store-statefulset.yaml | 8 ++ helm/hugegraph/values.schema.json | 76 +++++++++++++++++++ helm/hugegraph/values.yaml | 30 ++++++++ 9 files changed, 246 insertions(+), 7 deletions(-) diff --git a/helm/hugegraph/Chart.yaml b/helm/hugegraph/Chart.yaml index 0b7f7ccde1..90b97375eb 100644 --- a/helm/hugegraph/Chart.yaml +++ b/helm/hugegraph/Chart.yaml @@ -19,7 +19,7 @@ apiVersion: v2 name: hugegraph description: Helm chart for Apache HugeGraph HStore cluster (PD + Store + Server) type: application -version: 0.1.0 +version: 0.1.1 appVersion: "latest" kubeVersion: ">=1.23.0-0" keywords: diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index fafd08b05f..d708460938 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -76,6 +76,10 @@ This deploys 3 PD + 3 Store + 3 Server, preserves the image's automatic JVM sizing, and sets no resource requests or limits. Set resources before production use. +The command examples in this document assume the release is named +`hugegraph`. With a different release name, substitute the release-prefixed +resource names (`kubectl get svc,secret -n ` lists them). + **Authentication is enabled by default.** The chart creates a kept Secret named `-admin` (for example `hugegraph-admin`) with a random password unless `server.auth.admin.existingSecret` points at a pre-created Secret. @@ -111,7 +115,7 @@ A fresh install seeds PD with a partition shard count of 3 when default of 1. The seed applies at first bootstrap only; see Partition Sharding below. -This first chart is version `0.1.0`. While the contribution is a draft, its +This chart is at an early 0.1.x version. While the contribution is a draft, its component image tags and `appVersion` track `latest` with pull policy `Always`. Before stable publication, pin all three component tags and `appVersion` to the next HugeGraph release and switch the component pull policies to @@ -141,6 +145,21 @@ node topology, and storage class before production use. helm upgrade hugegraph ./helm/hugegraph --namespace hugegraph --reuse-values ``` +Upgrading to 0.1.1 from an earlier revision rolls two workloads once: + +- **PD** restarts one pod at a time because the Pod template gains the + `wait-for-pd-dns` init container. On current images a restarted PD returns + with a new Pod IP that peers holding older allowlists may reject (see + Limitations). If PDs log `Blocked connection` after the roll, delete all PD + pods at once — the DNS gate makes the parallel cold start deterministic. + For a maintenance-window upgrade, set `pd.updateStrategy.type=OnDelete` + and restart the pods yourself. +- **Server** rolls because the Pod template gains the `checksum/auth` + annotation, and once more on the first upgrade after a fresh install, when + the checksum first observes the install-created Secrets. Template-only + pipelines (`helm template`, GitOps renderers) never see live Secrets, so + there the annotation is a constant and Secret rotation does not roll pods. + Every optional field stays optional, so a release created by an earlier revision continues to render under `--reuse-values`. Note that `--reuse-values` keeps the old release's values as the complete base, so a release created @@ -737,10 +756,12 @@ kubectl -n get pods ### Server Ready but Queries Fail The Server readiness probe uses `/versions`, which can report ready before the -graph is fully able to serve index-backed queries. Confirm the graph is live: +graph is fully able to serve index-backed queries. Confirm the graph is live +(the server image does not ship `curl`, so probe through a port-forward): ```bash -kubectl exec -c server -- curl -s localhost:8080/graphs +kubectl port-forward -n hugegraph svc/hugegraph-server 8080:8080 +curl -s --user "admin:${PASSWORD}" http://127.0.0.1:8080/graphs ``` ### Queries Fail with "Could not rebind" Right After Creating a Graph @@ -800,6 +821,17 @@ independently of the release name. ## Limitations +- PD builds its raft RPC allowlist by resolving peer hostnames once at + startup. The chart gates PD start on all peer DNS names resolving + (`wait-for-pd-dns`), which makes first boot deterministic, but a PD Pod + rescheduled to a new IP later can still be rejected by peers until they + restart and re-resolve; a dynamic refresh is upstream image work. +- The PD management REST endpoints (`/v1/members`, `/v1/stores`) reject + requests on current images (`invalid service name`), and unauthenticated + GETs return HTTP 200 with an `Unauthorized` JSON body. Until that is + resolved upstream, the operator-triggered Disaster Recovery flow below may + be unavailable; rely on `helm test`, Pod readiness, and Server APIs for + health checks. - No TLS, backups, Operator, multi-cluster support, automatic leader transfer, or a complete monitoring stack. Store recovery is manual on current builds: re-replication after Store loss, leader balancing, and partition diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 02ad09bafc..bbba9e24f1 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -40,7 +40,6 @@ Reach the Server API: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.server.name" . }} {{ .Values.server.port }}:{{ .Values.server.port }} {{- if .Values.server.auth.enabled }} PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.server.authSecretName" . }} -o jsonpath='{.data.{{ include "hugegraph.server.authSecretKey" . }}}' | base64 --decode)" - echo "Admin password: ${PASSWORD}" curl --user "admin:${PASSWORD}" http://127.0.0.1:{{ .Values.server.port }}/versions {{- else }} curl http://127.0.0.1:{{ .Values.server.port }}/versions diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 8f0d707a3f..27274ede88 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -216,6 +216,34 @@ PD REST endpoints for Server storage-readiness checks. {{- join "," $peers -}} {{- end }} +{{/* +PD peer hostnames without port, for the PD-start DNS gate init container: +pod-0...svc,pod-1...svc,... +Derived from hugegraph.pd.raftPeersList so it tracks pd.replicas. +*/}} +{{- define "hugegraph.pd.peerHostsList" -}} +{{- regexReplaceAll ":[0-9]+" (include "hugegraph.pd.raftPeersList" .) "" -}} +{{- end }} + +{{/* +Checksum for the Server pod template so rotating the referenced auth Secrets +rolls Server pods. Hashes Secret names, keys, and metadata.resourceVersion - +never Secret data - so the annotation carries no credential-derived material. +Lookup-based and therefore best-effort: plain `helm template` (and +template-only GitOps renderers) see no live Secrets and emit a constant; the +first upgrade after a fresh install rolls Server once as the checksum picks +up the Secrets created by that install; out-of-band rotation of an +existingSecret applies on the next `helm upgrade`. +*/}} +{{- define "hugegraph.server.authChecksum" -}} +{{- $parts := list (include "hugegraph.server.authSecretName" .) (include "hugegraph.server.authSecretKey" .) (include "hugegraph.server.authTokenSecretName" .) (include "hugegraph.server.authTokenSecretKey" .) -}} +{{- $admin := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authSecretName" .) -}} +{{- if $admin -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $admin) -}}{{- end -}} +{{- $token := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authTokenSecretName" .) -}} +{{- if $token -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $token) -}}{{- end -}} +{{- join "|" $parts | sha256sum -}} +{{- end }} + {{/* Initial store list for PD bootstrap: store-0.svc.ns.svc:8500,... */}} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index e16ff53b2d..4c595999b6 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -26,6 +26,14 @@ spec: serviceName: {{ include "hugegraph.pd.name" . }} replicas: {{ .Values.pd.replicas }} podManagementPolicy: Parallel + {{- with .Values.pd.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.pd.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: + {{- toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: {{- include "hugegraph.selectorLabels" . | nindent 6 }} @@ -75,6 +83,59 @@ spec: {{- else }} {{- with include "hugegraph.antiAffinity" (dict "mode" .Values.pd.antiAffinity "component" "pd" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} {{- end }} + {{- if .Values.pd.waitEnabled }} + initContainers: + # Gate PD start on all peer DNS names resolving. PD builds its raft RPC + # allowlist by resolving peer hostnames once at boot; with a Parallel + # podManagementPolicy a PD can start before its peers' A records are + # published and freeze a partial allowlist that blocks the leader. The + # PD headless Service sets publishNotReadyAddresses: true, so peer + # records appear as soon as pods get IPs (before readiness), keeping + # this gate free of a circular wait. + - name: wait-for-pd-dns + image: {{ .Values.pd.waitImage | quote }} + {{- with .Values.pd.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - sh + - -c + - | + set -eu + PEERS=$(echo "{{ include "hugegraph.pd.peerHostsList" . }}" | tr ',' ' ') + TIMEOUT={{ .Values.pd.waitTimeoutSeconds | default 300 }} + DEADLINE=$(( $(date +%s) + TIMEOUT )) + # busybox nslookup ignores resolv.conf search domains, so the + # short ...svc names NXDOMAIN even when published. + # Derive the cluster domain and retry against the FQDN. + CLUSTER_DOMAIN=$(awk '/^search/ {for(i=2;i<=NF;i++) if ($i ~ /^svc\./) {sub(/^svc\./,"",$i); print $i; exit}}' /etc/resolv.conf 2>/dev/null || true) + resolves() { + nslookup "$1" >/dev/null 2>&1 && return 0 + if [ -n "${CLUSTER_DOMAIN}" ]; then + nslookup "$1.${CLUSTER_DOMAIN}" >/dev/null 2>&1 && return 0 + fi + nslookup "$1.cluster.local" >/dev/null 2>&1 + } + echo "Waiting for all PD peer DNS to resolve: ${PEERS}" + for host in ${PEERS}; do + until resolves "${host}"; do + if [ "$(date +%s)" -ge "${DEADLINE}" ]; then + echo "Timed out after ${TIMEOUT}s waiting for PD peer DNS: ${host}" >&2 + echo "Check PD Pods: kubectl get pods -l app.kubernetes.io/component=pd" >&2 + exit 1 + fi + echo "Waiting for DNS: ${host}" + sleep 3 + done + echo "Resolved: ${host}" + done + echo "All PD peer DNS resolved." + {{- with .Values.pd.waitResources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} containers: - name: pd image: "{{ .Values.pd.image.repository }}:{{ .Values.pd.image.tag | default $.Chart.AppVersion }}" diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 467382535e..dee189cfbb 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -54,9 +54,14 @@ spec: {{- include "hugegraph.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: server {{- with .Values.server.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} - {{- with .Values.server.podAnnotations }} + {{- if or .Values.server.auth.enabled .Values.server.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} + {{- if .Values.server.auth.enabled }} + # Rolls Server pods when the resolved auth Secrets change, so rotating + # an existingSecret takes effect without a manual restart. + checksum/auth: {{ include "hugegraph.server.authChecksum" . | quote }} + {{- end }} + {{- with .Values.server.podAnnotations }}{{ toYaml . | nindent 8 }}{{- end }} {{- end }} spec: automountServiceAccountToken: {{ get (get .Values.server "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index 973ecacdff..5edf9517d0 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -26,6 +26,14 @@ spec: serviceName: {{ include "hugegraph.store.name" . }} replicas: {{ .Values.store.replicas }} podManagementPolicy: Parallel + {{- with .Values.store.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.store.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: + {{- toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: {{- include "hugegraph.selectorLabels" . | nindent 6 }} diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 77b6b146b7..e8bdb33d8d 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -87,6 +87,55 @@ } } }, + "updateStrategy": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "RollingUpdate", + "OnDelete" + ] + }, + "rollingUpdate": { + "type": "object", + "additionalProperties": false, + "properties": { + "partition": { + "type": "integer", + "minimum": 0 + }, + "maxUnavailable": { + "type": [ + "integer", + "string" + ] + } + } + } + } + }, + "persistentVolumeClaimRetentionPolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "whenDeleted": { + "type": "string", + "enum": [ + "Retain", + "Delete" + ] + }, + "whenScaled": { + "type": "string", + "enum": [ + "Retain", + "Delete" + ] + } + } + }, "resourceList": { "type": "object", "additionalProperties": { @@ -238,6 +287,7 @@ "resources", "antiAffinity", "pdb", + "waitImage", "probes" ], "properties": { @@ -252,6 +302,26 @@ "javaOpts": { "type": "string" }, + "waitEnabled": { + "type": "boolean" + }, + "waitImage": { + "type": "string", + "minLength": 1 + }, + "waitResources": { + "$ref": "#/definitions/resources" + }, + "waitTimeoutSeconds": { + "type": "integer", + "minimum": 1 + }, + "updateStrategy": { + "$ref": "#/definitions/updateStrategy" + }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy" + }, "partition": { "type": "object", "additionalProperties": false, @@ -448,6 +518,12 @@ "type": "integer", "minimum": 1 }, + "updateStrategy": { + "$ref": "#/definitions/updateStrategy" + }, + "persistentVolumeClaimRetentionPolicy": { + "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy" + }, "nodeSelector": { "type": "object" }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 8999ddfbd8..398e3a9272 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -109,6 +109,28 @@ pd: pdb: enabled: true minAvailable: 2 + # Escape hatch for the peer-DNS gate below. Disable only when pod DNS + # cannot resolve the headless Service names in any form the gate tries; + # disabling re-exposes the first-boot allowlist race. + waitEnabled: true + # The gate needs an nslookup whose exit code reflects NXDOMAIN; busybox + # older than 1.30 exits 0 on failure and would make the gate a no-op. + waitImage: curlimages/curl:8.5.0 + # Bound the peer-DNS wait so a cluster whose PD DNS never publishes fails + # visibly instead of sitting in Init:0/1 forever. + waitTimeoutSeconds: 300 + # Optional bounds for the PD-start DNS-gate init container. + waitResources: {} + # Explicit rollout strategy instead of the implicit StatefulSet default. + updateStrategy: + type: RollingUpdate + # Keep graph metadata PVCs across delete and scale-down; data outlives the + # workload by default. Set to Delete for disposable environments. Honored + # on Kubernetes >=1.27 (or with the StatefulSetAutoDeletePVC feature gate); + # older API servers drop the field, which matches Retain behavior anyway. + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain # Startup can take a while during Raft bootstrap probes: startup: @@ -181,6 +203,14 @@ store: waitTimeoutSeconds: 900 # Optional bounds for the PD-quorum wait init container. waitResources: {} + # Explicit rollout strategy instead of the implicit StatefulSet default. + updateStrategy: + type: RollingUpdate + # Keep graph data PVCs across delete and scale-down; data outlives the + # workload by default. Set to Delete for disposable environments. + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain probes: startup: failureThreshold: 40 From 6217578b203f80cba419e649b1df3a3da7e03fb7 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 24 Aug 2026 01:19:00 +0530 Subject: [PATCH 18/61] fix(helm): keep the chart at 0.1.0 on this branch This PR introduces the chart, so there is no published 0.1.0 to bump from and the version should stay at the initial one until a release happens. Reword the Upgrading note for the same reason: it described rolls that happen when upgrading "from an earlier revision", which does not exist here. State the forward-looking behavior instead, that any Pod template change rolls that workload, and keep the PD allowlist caveat and the first-upgrade Server roll, both of which still apply. --- helm/hugegraph/Chart.yaml | 2 +- helm/hugegraph/README.md | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/helm/hugegraph/Chart.yaml b/helm/hugegraph/Chart.yaml index 90b97375eb..0b7f7ccde1 100644 --- a/helm/hugegraph/Chart.yaml +++ b/helm/hugegraph/Chart.yaml @@ -19,7 +19,7 @@ apiVersion: v2 name: hugegraph description: Helm chart for Apache HugeGraph HStore cluster (PD + Store + Server) type: application -version: 0.1.1 +version: 0.1.0 appVersion: "latest" kubeVersion: ">=1.23.0-0" keywords: diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index d708460938..3064078fdd 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -145,20 +145,21 @@ node topology, and storage class before production use. helm upgrade hugegraph ./helm/hugegraph --namespace hugegraph --reuse-values ``` -Upgrading to 0.1.1 from an earlier revision rolls two workloads once: - -- **PD** restarts one pod at a time because the Pod template gains the - `wait-for-pd-dns` init container. On current images a restarted PD returns - with a new Pod IP that peers holding older allowlists may reject (see - Limitations). If PDs log `Blocked connection` after the roll, delete all PD - pods at once — the DNS gate makes the parallel cold start deterministic. - For a maintenance-window upgrade, set `pd.updateStrategy.type=OnDelete` - and restart the pods yourself. -- **Server** rolls because the Pod template gains the `checksum/auth` - annotation, and once more on the first upgrade after a fresh install, when - the checksum first observes the install-created Secrets. Template-only - pipelines (`helm template`, GitOps renderers) never see live Secrets, so - there the annotation is a constant and Secret rotation does not roll pods. +Any upgrade that changes a Pod template rolls that workload once. Two cases +are worth knowing about in advance: + +- **PD** restarts one pod at a time whenever its Pod template changes. On + current images a restarted PD returns with a new Pod IP that peers holding + older allowlists may reject (see Limitations). If PDs log + `Blocked connection` after a roll, delete all PD pods at once — the + `wait-for-pd-dns` gate makes the parallel cold start deterministic. For a + maintenance-window upgrade, set `pd.updateStrategy.type=OnDelete` and + restart the pods yourself. +- **Server** rolls once on the first `helm upgrade` after a fresh install, + when the `checksum/auth` annotation first observes the install-created + Secrets. Template-only pipelines (`helm template`, GitOps renderers) never + see live Secrets, so there the annotation is a constant and Secret rotation + does not roll pods. Every optional field stays optional, so a release created by an earlier revision continues to render under `--reuse-values`. Note that `--reuse-values` From 770aa98a617131ea9b11d555ef6c288e610da713 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 24 Aug 2026 01:23:12 +0530 Subject: [PATCH 19/61] fix(helm): keep pd.waitImage optional for --reuse-values releases The lint-and-render legacy guard caught a regression: pd.waitImage was added to the pd required list, so a release created before the field existed failed schema validation under --reuse-values. Make the field optional and default the image in the template instead, matching how waitTimeoutSeconds already degrades. Also flip the gate condition to "not explicitly disabled" so those same legacy releases pick the DNS gate up on upgrade rather than silently losing the first-boot protection because waitEnabled is absent from their values. --- helm/hugegraph/templates/pd-statefulset.yaml | 4 ++-- helm/hugegraph/values.schema.json | 1 - helm/hugegraph/values.yaml | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 4c595999b6..d0e20d08cd 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -83,7 +83,7 @@ spec: {{- else }} {{- with include "hugegraph.antiAffinity" (dict "mode" .Values.pd.antiAffinity "component" "pd" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} {{- end }} - {{- if .Values.pd.waitEnabled }} + {{- if ne .Values.pd.waitEnabled false }} initContainers: # Gate PD start on all peer DNS names resolving. PD builds its raft RPC # allowlist by resolving peer hostnames once at boot; with a Parallel @@ -93,7 +93,7 @@ spec: # records appear as soon as pods get IPs (before readiness), keeping # this gate free of a circular wait. - name: wait-for-pd-dns - image: {{ .Values.pd.waitImage | quote }} + image: {{ .Values.pd.waitImage | default "curlimages/curl:8.5.0" | quote }} {{- with .Values.pd.securityContext }} securityContext: {{- toYaml . | nindent 12 }} diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index e8bdb33d8d..fdfc09b2a1 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -287,7 +287,6 @@ "resources", "antiAffinity", "pdb", - "waitImage", "probes" ], "properties": { diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 398e3a9272..d473e99f52 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -115,6 +115,8 @@ pd: waitEnabled: true # The gate needs an nslookup whose exit code reflects NXDOMAIN; busybox # older than 1.30 exits 0 on failure and would make the gate a no-op. + # Optional in the schema and defaulted in the template so releases created + # before this field existed keep rendering under --reuse-values. waitImage: curlimages/curl:8.5.0 # Bound the peer-DNS wait so a cluster whose PD DNS never publishes fails # visibly instead of sitting in Init:0/1 forever. From 9734fcfa4b25c4933300f015adeeb727a2598fc6 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 25 Aug 2026 13:16:06 +0530 Subject: [PATCH 20/61] fix(helm): drop the stale server.advertiseUrl auth requirement The guard rejected advertiseUrl unless auth was on or in-chart Hubble ran in pd mode, on the premise that only then does the chart register server.urls_to_pd. That premise stopped being true when the chart moved every Server to PD meta mode unconditionally: server-deployment.yaml sets $pdMeta := true, so urls_to_pd is always written. Rendering with advertiseUrl set, auth off and Hubble off now emits HG_SERVER_URLS_TO_PD with the configured URL, which the guard had been refusing to produce. The URL-format guard (absolute http:// or https:// only) is unchanged. Correct the README passage that documented the removed requirement. --- helm/hugegraph/README.md | 8 +++----- helm/hugegraph/templates/_helpers.tpl | 6 ------ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 3064078fdd..891f679683 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -414,12 +414,10 @@ In-cluster names such as `*.svc` are not reachable from outside. The chart helps with two knobs: advertise a reachable Server URL to PD, and expose the PD client Service. -Keep **`server.auth.enabled=true`**. With `hubble.enabled=false`, auth-on is -what enables PD meta mode so the chart actually writes `server.urls_to_pd` -(and therefore honors `server.advertiseUrl`). Auth-off plus `advertiseUrl` -fails at render time. +The chart always registers `server.urls_to_pd` with PD, so `server.advertiseUrl` +is honored whenever it is set. -1. Leave in-chart Hubble off and keep `server.auth.enabled=true`. +1. Leave in-chart Hubble off if the bundled UI is not wanted. 2. Expose Server and set `server.advertiseUrl` to the absolute `http(s)://` URL outside Hubble will use after discovery. The chart registers it via `server.urls_to_pd` instead of the in-cluster Service URL. diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 27274ede88..7c3b7c4fa1 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -534,12 +534,6 @@ keys for releases stored before the values existed. {{- if and $advertiseUrl (not (or (hasPrefix "http://" $advertiseUrl) (hasPrefix "https://" $advertiseUrl))) -}} {{- fail "server.advertiseUrl must be an absolute http:// or https:// URL when set" -}} {{- end -}} -{{- $hubbleForAdvertise := get .Values "hubble" | default dict -}} -{{- $hubblePdModeForAdvertise := and (get $hubbleForAdvertise "enabled" | default false) (ne (get $hubbleForAdvertise "mode" | default "pd") "direct") -}} -{{- $pdMetaForAdvertise := or .Values.server.auth.enabled $hubblePdModeForAdvertise -}} -{{- if and $advertiseUrl (not $pdMetaForAdvertise) -}} -{{- fail "server.advertiseUrl requires server.auth.enabled=true (or in-chart hubble.enabled with hubble.mode=pd), because only then does the chart register server.urls_to_pd with PD" -}} -{{- end -}} {{- $pdSvc := get .Values.pd "service" | default dict -}} {{- $pdSvcType := get $pdSvc "type" | default "ClusterIP" -}} {{- if and (or (get $pdSvc "restNodePort") (get $pdSvc "grpcNodePort")) (not (has $pdSvcType (list "NodePort" "LoadBalancer"))) -}} From d97c7597ac2d39eaccef068cd5f8a6b4b22e5591 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 25 Aug 2026 17:57:28 +0530 Subject: [PATCH 21/61] fix(helm): disable PD raft IP whitelist in-cluster; drop the DNS gate PD resolves its raft peer allowlist once at boot, so under Kubernetes a peer whose Pod IP was unpublished at that moment, or changes later, is rejected: a first-boot race that could wedge an install, and a permanent rejection after any PD reschedule. The upstream raft.ip-whitelist.enabled switch fixes this at the source, leaving peer authentication to Kubernetes-level controls. Render -Draft.ip-whitelist.enabled=false into PD's derived JAVA_OPTS via the new pd.raftIpWhitelistEnabled value (default false, schema-typed; images predating the switch ignore the property), and remove the interim wait-for-pd-dns init container with its pd.wait* values, schema entries, and peer-hosts helper. The gate only ever mitigated the first-boot half of the problem. Verified on Kubernetes with a PD image carrying the switch: two deterministic installs, PD follower/leader crash, PD and Store majority loss, leader network partition, PVC reattach, rolling restart under write load, auth on every Server replica, a 0.1.2-to-this upgrade, Secret rotation, and a zero-override install followed through the documented first-run steps. Zero "Blocked connection" and zero "Could not resolve allowlist entry" lines across the entire campaign; the pod-IP recycle case that previously failed reproducibly now passes. Also update the chart CI's JAVA_OPTS assertions, which matched a quote-terminated shard-count prefix and therefore broke once the whitelist flag was appended, and bring the README in line: the Limitations and Upgrading sections described the removed init container, and the values table had no row for the new setting. --- .github/workflows/helm-chart-ci.yml | 9 ++-- helm/hugegraph/README.md | 33 +++++++----- helm/hugegraph/templates/_helpers.tpl | 18 +++---- helm/hugegraph/templates/pd-statefulset.yaml | 53 -------------------- helm/hugegraph/values.schema.json | 13 +---- helm/hugegraph/values.yaml | 21 +++----- 6 files changed, 43 insertions(+), 104 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 5768de8815..63826fd936 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -77,12 +77,13 @@ jobs: --set pd.partition.storeMaxShardCount=12 > /dev/null helm template ci helm/hugegraph \ --set-string pd.partition.defaultShardCount=3 > /dev/null - # The derived shard count must land in the PD JAVA_OPTS verbatim: - # 3 on the default topology, 1 on the single-node preset + # The derived shard count and the raft whitelist switch must land in + # the PD JAVA_OPTS verbatim: shard count 3 on the default topology, + # 1 on the single-node preset, whitelist disabled in both. helm template ci helm/hugegraph \ - | grep -qF 'value: "-Dpartition.default-shard-count=3"' + | grep -qF 'value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false"' helm template ci helm/hugegraph -f helm/hugegraph/values-single.yaml \ - | grep -qF 'value: "-Dpartition.default-shard-count=1"' + | grep -qF 'value: "-Dpartition.default-shard-count=1 -Draft.ip-whitelist.enabled=false"' # The stock distributed install must put every Server replica on # the shared PD graph catalog, even without Hubble. Auth is on by # default, so the chart-managed admin Secret must render too. diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 891f679683..c69cfed781 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -148,13 +148,15 @@ helm upgrade hugegraph ./helm/hugegraph --namespace hugegraph --reuse-values Any upgrade that changes a Pod template rolls that workload once. Two cases are worth knowing about in advance: -- **PD** restarts one pod at a time whenever its Pod template changes. On - current images a restarted PD returns with a new Pod IP that peers holding - older allowlists may reject (see Limitations). If PDs log - `Blocked connection` after a roll, delete all PD pods at once — the - `wait-for-pd-dns` gate makes the parallel cold start deterministic. For a - maintenance-window upgrade, set `pd.updateStrategy.type=OnDelete` and - restart the pods yourself. +- **PD** restarts one pod at a time whenever its Pod template changes, which + includes adopting the `-Draft.ip-whitelist.enabled=false` setting described + under Limitations. With a PD image that carries the upstream whitelist + switch this roll is uneventful. On an older image the whitelist stays + active, and a restarted PD returning on a new Pod IP may be rejected by + peers holding stale allowlists; if PDs log `Blocked connection` after a + roll, delete all PD pods at once so they cold-start together and + re-resolve. For a maintenance-window upgrade, set + `pd.updateStrategy.type=OnDelete` and restart the pods yourself. - **Server** rolls once on the first `helm upgrade` after a fresh install, when the `checksum/auth` annotation first observes the install-created Secrets. Template-only pipelines (`helm template`, GitOps renderers) never @@ -222,6 +224,7 @@ default values. | `pd.image.tag` | PD image tag. Tracks the development image until the next release is pinned | `latest` | | `pd.image.pullPolicy` | PD image pull policy | `Always` | | `pd.javaOpts` | Extra JVM flags, rendered after the chart-derived `-D` properties below so an explicit duplicate here wins. The image's automatic heap sizing is preserved unless heap flags are set | `""` | +| `pd.raftIpWhitelistEnabled` | Enable PD's raft peer IP whitelist. Off in-cluster because PD resolves peers once at boot; requires a PD image carrying the upstream switch | `false` | | `pd.partition.defaultShardCount` | Shard replicas per partition, seeded into PD's persisted config at first bootstrap only; inert on an initialized cluster (see Partition Sharding). Empty derives 3 when `store.replicas` is at least 3, else 1. An explicit value must be odd and must not exceed `store.replicas` | `""` | | `pd.partition.storeMaxShardCount` | Maximum shards per Store, seeded at first bootstrap only. Also fixes the initial partition count, `store.replicas x storeMaxShardCount / shardCount` (see Partition Sharding). Empty preserves the image default of `12` | `""` | | `pd.ports.grpc` | PD gRPC port | `8686` | @@ -820,11 +823,17 @@ independently of the release name. ## Limitations -- PD builds its raft RPC allowlist by resolving peer hostnames once at - startup. The chart gates PD start on all peer DNS names resolving - (`wait-for-pd-dns`), which makes first boot deterministic, but a PD Pod - rescheduled to a new IP later can still be rejected by peers until they - restart and re-resolve; a dynamic refresh is upstream image work. +- PD's raft IP whitelist resolves peer hostnames to IPs once at startup, + which under Kubernetes can block peers whose pod IPs were unpublished at + that moment or change later. The chart therefore disables the whitelist + in-cluster via the upstream `raft.ip-whitelist.enabled` switch, leaving + peer authentication to Kubernetes-level controls. Setting + `pd.raftIpWhitelistEnabled=true` restores the image default along with + its one-shot resolution semantics — bring-up races and pod-IP-change + rejections included — at the operator's own risk. PD images that predate + the switch ignore the flag and keep the whitelist active, so they remain + exposed to those failure modes; use images built from a source tree that + includes the switch. - The PD management REST endpoints (`/v1/members`, `/v1/stores`) reject requests on current images (`invalid service name`), and unauthenticated GETs return HTTP 200 with an `Unauthorized` JSON body. Until that is diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 7c3b7c4fa1..7b528a61c5 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -216,15 +216,6 @@ PD REST endpoints for Server storage-readiness checks. {{- join "," $peers -}} {{- end }} -{{/* -PD peer hostnames without port, for the PD-start DNS gate init container: -pod-0...svc,pod-1...svc,... -Derived from hugegraph.pd.raftPeersList so it tracks pd.replicas. -*/}} -{{- define "hugegraph.pd.peerHostsList" -}} -{{- regexReplaceAll ":[0-9]+" (include "hugegraph.pd.raftPeersList" .) "" -}} -{{- end }} - {{/* Checksum for the Server pod template so rotating the referenced auth Secrets rolls Server pods. Hashes Secret names, keys, and metadata.resourceVersion - @@ -386,6 +377,13 @@ count of 2 to 1 (two shards cannot elect a leader) and its config API accepts only odd values. store-max-shard-count is rendered only when set, keeping the image default. All lookups tolerate absent keys so releases stored before these values existed keep rendering under --reuse-values. + +raft.ip-whitelist.enabled is always rendered, default false: PD resolves its +raft peer allowlist once at boot, which under Kubernetes blocks peers whose +pod IPs were unpublished at that moment or change later, so the switch is +off in-cluster per the upstream design and k8s auth owns that layer. Images +without the property ignore the flag. Set pd.raftIpWhitelistEnabled=true to +restore the image default. */}} {{- define "hugegraph.pd.effectiveJavaOpts" -}} {{- $pd := .Values.pd -}} @@ -400,6 +398,8 @@ stored before these values existed keep rendering under --reuse-values. {{- if ne $maxShard "" -}} {{- $flags = append $flags (printf "-Dpartition.store-max-shard-count=%s" $maxShard) -}} {{- end -}} +{{- $ipWhitelist := ternary "true" "false" (eq (get $pd "raftIpWhitelistEnabled" | toString) "true") -}} +{{- $flags = append $flags (printf "-Draft.ip-whitelist.enabled=%s" $ipWhitelist) -}} {{- $userOpts := trim (get $pd "javaOpts" | default "") -}} {{- if ne $userOpts "" -}} {{- $flags = append $flags $userOpts -}} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index d0e20d08cd..5b4ac07f55 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -83,59 +83,6 @@ spec: {{- else }} {{- with include "hugegraph.antiAffinity" (dict "mode" .Values.pd.antiAffinity "component" "pd" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} {{- end }} - {{- if ne .Values.pd.waitEnabled false }} - initContainers: - # Gate PD start on all peer DNS names resolving. PD builds its raft RPC - # allowlist by resolving peer hostnames once at boot; with a Parallel - # podManagementPolicy a PD can start before its peers' A records are - # published and freeze a partial allowlist that blocks the leader. The - # PD headless Service sets publishNotReadyAddresses: true, so peer - # records appear as soon as pods get IPs (before readiness), keeping - # this gate free of a circular wait. - - name: wait-for-pd-dns - image: {{ .Values.pd.waitImage | default "curlimages/curl:8.5.0" | quote }} - {{- with .Values.pd.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - command: - - sh - - -c - - | - set -eu - PEERS=$(echo "{{ include "hugegraph.pd.peerHostsList" . }}" | tr ',' ' ') - TIMEOUT={{ .Values.pd.waitTimeoutSeconds | default 300 }} - DEADLINE=$(( $(date +%s) + TIMEOUT )) - # busybox nslookup ignores resolv.conf search domains, so the - # short ...svc names NXDOMAIN even when published. - # Derive the cluster domain and retry against the FQDN. - CLUSTER_DOMAIN=$(awk '/^search/ {for(i=2;i<=NF;i++) if ($i ~ /^svc\./) {sub(/^svc\./,"",$i); print $i; exit}}' /etc/resolv.conf 2>/dev/null || true) - resolves() { - nslookup "$1" >/dev/null 2>&1 && return 0 - if [ -n "${CLUSTER_DOMAIN}" ]; then - nslookup "$1.${CLUSTER_DOMAIN}" >/dev/null 2>&1 && return 0 - fi - nslookup "$1.cluster.local" >/dev/null 2>&1 - } - echo "Waiting for all PD peer DNS to resolve: ${PEERS}" - for host in ${PEERS}; do - until resolves "${host}"; do - if [ "$(date +%s)" -ge "${DEADLINE}" ]; then - echo "Timed out after ${TIMEOUT}s waiting for PD peer DNS: ${host}" >&2 - echo "Check PD Pods: kubectl get pods -l app.kubernetes.io/component=pd" >&2 - exit 1 - fi - echo "Waiting for DNS: ${host}" - sleep 3 - done - echo "Resolved: ${host}" - done - echo "All PD peer DNS resolved." - {{- with .Values.pd.waitResources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} containers: - name: pd image: "{{ .Values.pd.image.repository }}:{{ .Values.pd.image.tag | default $.Chart.AppVersion }}" diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index fdfc09b2a1..593a1c8d12 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -301,20 +301,9 @@ "javaOpts": { "type": "string" }, - "waitEnabled": { + "raftIpWhitelistEnabled": { "type": "boolean" }, - "waitImage": { - "type": "string", - "minLength": 1 - }, - "waitResources": { - "$ref": "#/definitions/resources" - }, - "waitTimeoutSeconds": { - "type": "integer", - "minimum": 1 - }, "updateStrategy": { "$ref": "#/definitions/updateStrategy" }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index d473e99f52..86b46c0119 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -109,20 +109,13 @@ pd: pdb: enabled: true minAvailable: 2 - # Escape hatch for the peer-DNS gate below. Disable only when pod DNS - # cannot resolve the headless Service names in any form the gate tries; - # disabling re-exposes the first-boot allowlist race. - waitEnabled: true - # The gate needs an nslookup whose exit code reflects NXDOMAIN; busybox - # older than 1.30 exits 0 on failure and would make the gate a no-op. - # Optional in the schema and defaulted in the template so releases created - # before this field existed keep rendering under --reuse-values. - waitImage: curlimages/curl:8.5.0 - # Bound the peer-DNS wait so a cluster whose PD DNS never publishes fails - # visibly instead of sitting in Init:0/1 forever. - waitTimeoutSeconds: 300 - # Optional bounds for the PD-start DNS-gate init container. - waitResources: {} + # PD's raft IP whitelist resolves peers once at boot, which under + # Kubernetes blocks peers whose pod IPs were not yet published or change + # later. Off by default in-cluster (upstream switch + # raft.ip-whitelist.enabled); k8s network policy/auth owns that layer. + # Ignored by images that predate the switch. Set true to restore the + # image default. + raftIpWhitelistEnabled: false # Explicit rollout strategy instead of the implicit StatefulSet default. updateStrategy: type: RollingUpdate From 1ca41013140d862894be32b2604cea1de2c4d16c Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 29 Aug 2026 19:20:35 +0530 Subject: [PATCH 22/61] fix(helm): guard short auth tokens, bound test hook, allow digest pinning Setting server.auth.token.value shorter than 32 bytes passed every chart check and then failed at runtime: all Server pods crash looped on the entrypoint assertion and helm install --wait timed out with no indication of the cause. Add a minLength to the schema so the value is rejected at render time with a precise path. The default values set no container resources, so each JVM sizes its heap against total node memory rather than a cgroup limit. On a multi-node cluster where several pods share a node the heaps oversubscribe it. Note the cluster preset in the README and warn from NOTES.txt. Images could only be referenced as repository:tag, so a release could not pin an immutable digest. Add an optional image.digest per component that takes precedence over the tag. Give the Helm test hook bounded default resources so it is admissible on quota-managed namespaces. --- helm/hugegraph/README.md | 8 ++++++ helm/hugegraph/templates/NOTES.txt | 6 +++++ helm/hugegraph/templates/_helpers.tpl | 15 +++++++++++ .../templates/hubble-deployment.yaml | 2 +- helm/hugegraph/templates/pd-statefulset.yaml | 2 +- .../templates/server-deployment.yaml | 2 +- .../templates/store-statefulset.yaml | 2 +- helm/hugegraph/values.schema.json | 15 ++++++++++- helm/hugegraph/values.yaml | 26 ++++++++++++++++++- 9 files changed, 72 insertions(+), 6 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index c69cfed781..b14e6a6ef6 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -823,6 +823,14 @@ independently of the release name. ## Limitations +- The default values set no container resources, so every pod is QoS class + BestEffort and each JVM sizes its heap against total NODE memory rather than + a cgroup limit. That is fine for a single-node or development install, but on + a multi-node cluster where several pods share a node the heaps oversubscribe + it and pods abort. A measured example: on 7.6 GB workers the default install + gave PD `-Xmx3299m` and, with three to four pods per node, never converged. + Use `values-cluster.yaml`, or set your own `resources`, for any multi-node + deployment. - PD's raft IP whitelist resolves peer hostnames to IPs once at startup, which under Kubernetes can block peers whose pod IPs were unpublished at that moment or change later. The chart therefore disables the whitelist diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index bbba9e24f1..b24a3a826b 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -97,3 +97,9 @@ set pd.antiAffinity=required; see values-cluster.yaml. One or more components have no resource requests or limits. Set them before production use; see values-cluster.yaml. {{- end }} + +{{- if not .Values.pd.resources }} +WARNING: no resources are set, so every pod is BestEffort and each JVM sizes its +heap against total node memory. On a multi-node cluster this oversubscribes the +nodes and pods may abort. Use values-cluster.yaml or set resources explicitly. +{{- end }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 7b528a61c5..f998eb684d 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -632,3 +632,18 @@ affinity: topologyKey: kubernetes.io/hostname {{- end }} {{- end }} + +{{/* +Render a container image reference. An explicit image.digest pins immutably and wins +over tag; otherwise fall back to tag, then to the chart appVersion. Takes a dict of +(image, appVersion). +*/}} +{{- define "hugegraph.image" -}} +{{- $img := .image -}} +{{- $digest := trim (get $img "digest" | default "") -}} +{{- if ne $digest "" -}} +{{- printf "%s@%s" $img.repository $digest -}} +{{- else -}} +{{- printf "%s:%s" $img.repository (default .appVersion $img.tag) -}} +{{- end -}} +{{- end }} diff --git a/helm/hugegraph/templates/hubble-deployment.yaml b/helm/hugegraph/templates/hubble-deployment.yaml index d094db4a45..b8645cb56f 100644 --- a/helm/hugegraph/templates/hubble-deployment.yaml +++ b/helm/hugegraph/templates/hubble-deployment.yaml @@ -86,7 +86,7 @@ spec: {{- end }} containers: - name: hubble - image: "{{ .Values.hubble.image.repository }}:{{ .Values.hubble.image.tag }}" + image: {{ include "hugegraph.image" (dict "image" .Values.hubble.image "appVersion" $.Chart.AppVersion) | quote }} imagePullPolicy: {{ .Values.hubble.image.pullPolicy }} {{- with .Values.hubble.securityContext }} securityContext: diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 5b4ac07f55..1318741280 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -85,7 +85,7 @@ spec: {{- end }} containers: - name: pd - image: "{{ .Values.pd.image.repository }}:{{ .Values.pd.image.tag | default $.Chart.AppVersion }}" + image: {{ include "hugegraph.image" (dict "image" .Values.pd.image "appVersion" $.Chart.AppVersion) | quote }} imagePullPolicy: {{ .Values.pd.image.pullPolicy }} {{- with .Values.pd.securityContext }} securityContext: diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index dee189cfbb..75c1ea6707 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -100,7 +100,7 @@ spec: {{- end }} containers: - name: server - image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag | default $.Chart.AppVersion }}" + image: {{ include "hugegraph.image" (dict "image" .Values.server.image "appVersion" $.Chart.AppVersion) | quote }} imagePullPolicy: {{ .Values.server.image.pullPolicy }} {{- with .Values.server.securityContext }} securityContext: diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index 5edf9517d0..9ec7d8e7e7 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -124,7 +124,7 @@ spec: {{- end }} containers: - name: store - image: "{{ .Values.store.image.repository }}:{{ .Values.store.image.tag | default $.Chart.AppVersion }}" + image: {{ include "hugegraph.image" (dict "image" .Values.store.image "appVersion" $.Chart.AppVersion) | quote }} imagePullPolicy: {{ .Values.store.image.pullPolicy }} {{- with .Values.store.securityContext }} securityContext: diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 593a1c8d12..1a2ba207c0 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -72,6 +72,10 @@ "IfNotPresent", "Never" ] + }, + "digest": { + "type": "string", + "description": "Optional immutable image digest, for example sha256:... Wins over tag when set." } } }, @@ -688,7 +692,16 @@ ], "properties": { "value": { - "type": "string" + "type": "string", + "description": "JWT signing key. Either empty, which defers to existingSecret or autoGenerate, or at least 32 characters: the Server entrypoint rejects a shorter key and the pods CrashLoop.", + "anyOf": [ + { + "maxLength": 0 + }, + { + "minLength": 32 + } + ] }, "existingSecret": { "type": "string" diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 86b46c0119..f0fe21bc91 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -30,6 +30,10 @@ pd: # The draft tracks latest until the next HugeGraph release tag is available. # Pin the release tag and switch to IfNotPresent before stable publication. tag: latest + # Optional immutable digest, for example sha256:abc... When set it wins over + # tag and the image is pulled by digest, which is what a release gate should + # assert instead of a mutable tag. + digest: "" pullPolicy: Always # Empty preserves the image entrypoint's automatic JVM sizing. The chart # renders its partition settings below as -D system properties ahead of @@ -148,6 +152,10 @@ store: # The draft tracks latest until the next HugeGraph release tag is available. # Pin the release tag and switch to IfNotPresent before stable publication. tag: latest + # Optional immutable digest, for example sha256:abc... When set it wins over + # tag and the image is pulled by digest, which is what a release gate should + # assert instead of a mutable tag. + digest: "" pullPolicy: Always # Empty preserves the image entrypoint's automatic JVM sizing. javaOpts: "" @@ -227,6 +235,10 @@ server: # The draft tracks latest until the next HugeGraph release tag is available. # Pin the release tag and switch to IfNotPresent before stable publication. tag: latest + # Optional immutable digest, for example sha256:abc... When set it wins over + # tag and the image is pulled by digest, which is what a release gate should + # assert instead of a mutable tag. + digest: "" pullPolicy: Always # Empty preserves the image entrypoint's automatic JVM sizing. javaOpts: "" @@ -270,7 +282,15 @@ server: # Image used by the Helm test hook. waitImage: curlimages/curl:8.5.0 # Optional resources for the Helm test hook container. - testResources: {} + # Resources for the Helm test hook container. Bounded by default so the hook + # cannot run unlimited on a restricted or quota-managed namespace. + testResources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 250m + memory: 64Mi restServer: # Empty preserves the image's restserver.min_free_memory default. minFreeMemory: "" @@ -364,6 +384,10 @@ hubble: # The draft tracks latest until the next HugeGraph release tag is available. # Pin the release tag and switch to IfNotPresent before stable publication. tag: latest + # Optional immutable digest, for example sha256:abc... When set it wins over + # tag and the image is pulled by digest, which is what a release gate should + # assert instead of a mutable tag. + digest: "" pullPolicy: Always port: 8088 # Hubble keeps UI connection metadata, including any graph credentials From 481f38dc2cc3cf80d389c16f9a74891a0ca77f5c Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 29 Aug 2026 22:26:18 +0530 Subject: [PATCH 23/61] test(helm): add helm-unittest suites The chart shipped no unit tests. Cover the behaviour that fails silently: the PD raft whitelist flag and derived JAVA_OPTS, image digest pinning, the Server auth checksum annotation and Secret wiring, update strategy and PVC retention, quorum arithmetic at one three and five replicas, the test hook resource bounds, and every validateValues guard asserted against its real error message. --- .../tests/auth_token_length_test.yaml | 38 ++++++++ helm/hugegraph/tests/image_digest_test.yaml | 57 +++++++++++ helm/hugegraph/tests/pd_javaopts_test.yaml | 52 ++++++++++ .../tests/server_auth_checksum_test.yaml | 66 +++++++++++++ .../tests/statefulset_hardening_test.yaml | 51 ++++++++++ .../tests/test_hook_resources_test.yaml | 45 +++++++++ .../hugegraph/tests/topology_quorum_test.yaml | 95 +++++++++++++++++++ .../hugegraph/tests/validate_values_test.yaml | 88 +++++++++++++++++ 8 files changed, 492 insertions(+) create mode 100644 helm/hugegraph/tests/auth_token_length_test.yaml create mode 100644 helm/hugegraph/tests/image_digest_test.yaml create mode 100644 helm/hugegraph/tests/pd_javaopts_test.yaml create mode 100644 helm/hugegraph/tests/server_auth_checksum_test.yaml create mode 100644 helm/hugegraph/tests/statefulset_hardening_test.yaml create mode 100644 helm/hugegraph/tests/test_hook_resources_test.yaml create mode 100644 helm/hugegraph/tests/topology_quorum_test.yaml create mode 100644 helm/hugegraph/tests/validate_values_test.yaml diff --git a/helm/hugegraph/tests/auth_token_length_test.yaml b/helm/hugegraph/tests/auth_token_length_test.yaml new file mode 100644 index 0000000000..0ca92a38c5 --- /dev/null +++ b/helm/hugegraph/tests/auth_token_length_test.yaml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Auth token length guard +templates: + - server-deployment.yaml +tests: + - it: accepts an empty token value, which defers to autoGenerate + asserts: + - hasDocuments: + count: 1 + + - it: accepts a signing key of at least 32 characters + set: + server.auth.token.value: ktestJwtSigningKey2026xyzABCDEFGH12345678 + asserts: + - hasDocuments: + count: 1 + + - it: rejects a signing key shorter than 32 characters + set: + server.auth.token.value: tooshortkey + asserts: + - failedTemplate: {} diff --git a/helm/hugegraph/tests/image_digest_test.yaml b/helm/hugegraph/tests/image_digest_test.yaml new file mode 100644 index 0000000000..da518908bd --- /dev/null +++ b/helm/hugegraph/tests/image_digest_test.yaml @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Image reference and digest pinning +tests: + - it: renders repository and tag when no digest is set + template: pd-statefulset.yaml + set: + pd.image.tag: pinned-for-test + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/pd:pinned-for-test + + - it: pins by digest when one is supplied, ignoring the tag + template: pd-statefulset.yaml + set: + pd.image.digest: sha256:43999a5ccda34883a9e4e458e25cb903ce34adbc66782d9f4a5260d68b2b5a82 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/pd@sha256:43999a5ccda34883a9e4e458e25cb903ce34adbc66782d9f4a5260d68b2b5a82 + - notMatchRegex: + path: spec.template.spec.containers[0].image + pattern: "helm-dev" + + - it: supports digest pinning on store as well + template: store-statefulset.yaml + set: + store.image.digest: sha256:458d77a18542e8a2f7980b075f2fda32026d582200a8983921354398467ff860 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^hugegraph/store@sha256:458d77a1" + + - it: supports digest pinning on server as well + template: server-deployment.yaml + set: + server.image.digest: sha256:30f99c6b9ab605accf96e9c179434b13495547f532dcb89d840231c54f1cc101 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: "^hugegraph/server@sha256:30f99c6b" diff --git a/helm/hugegraph/tests/pd_javaopts_test.yaml b/helm/hugegraph/tests/pd_javaopts_test.yaml new file mode 100644 index 0000000000..396d293094 --- /dev/null +++ b/helm/hugegraph/tests/pd_javaopts_test.yaml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: PD JAVA_OPTS derivation +templates: + - pd-statefulset.yaml +tests: + - it: disables the raft IP whitelist in-cluster by default + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: JAVA_OPTS + value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false" + + - it: re-enables the whitelist when the operator opts in + set: + pd.raftIpWhitelistEnabled: true + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value + pattern: "-Draft\\.ip-whitelist\\.enabled=true" + + - it: derives shard count 1 when store replicas are below 3 + set: + store.replicas: 1 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value + pattern: "-Dpartition\\.default-shard-count=1" + + - it: appends operator javaOpts after the derived flags + set: + pd.javaOpts: "-Xmx2g" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value + pattern: "-Draft\\.ip-whitelist\\.enabled=false -Xmx2g$" diff --git a/helm/hugegraph/tests/server_auth_checksum_test.yaml b/helm/hugegraph/tests/server_auth_checksum_test.yaml new file mode 100644 index 0000000000..b25b6a67c8 --- /dev/null +++ b/helm/hugegraph/tests/server_auth_checksum_test.yaml @@ -0,0 +1,66 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Server auth checksum and secret wiring +templates: + - server-deployment.yaml +tests: + - it: stamps a checksum/auth annotation on the pod template + asserts: + - matchRegex: + path: spec.template.metadata.annotations["checksum/auth"] + pattern: "^[a-f0-9]{64}$" + + - it: keeps a valid checksum when an external admin Secret is supplied + set: + server.auth.admin.existingSecret: ext-admin-secret + server.auth.admin.key: password + asserts: + - matchRegex: + path: spec.template.metadata.annotations["checksum/auth"] + pattern: "^[a-f0-9]{64}$" + + - it: never places credential material in pod metadata + set: + server.auth.admin.password: sup3rs3cr3tpw + server.auth.token.value: t0k3nvalu3PADDINGtoREACH32charsMIN + asserts: + - notMatchRegex: + path: spec.template.metadata.annotations["checksum/auth"] + pattern: "(sup3rs3cr3tpw|t0k3nvalu3PADDING)" + + - it: sources the admin password from a Secret, never a literal value + set: + server.auth.admin.password: sup3rs3cr3tpw + asserts: + - exists: + path: spec.template.spec.containers[0].env[?(@.name=="PASSWORD")].valueFrom.secretKeyRef + - notExists: + path: spec.template.spec.containers[0].env[?(@.name=="PASSWORD")].value + + - it: points the admin Secret ref at an operator-supplied Secret when set + set: + server.auth.admin.existingSecret: ext-admin-secret + asserts: + - equal: + path: spec.template.spec.containers[0].env[?(@.name=="PASSWORD")].valueFrom.secretKeyRef.name + value: ext-admin-secret + + - it: shares one JWT signing key across replicas via a Secret ref + asserts: + - exists: + path: spec.template.spec.containers[0].env[?(@.name=="HG_SERVER_AUTH_TOKEN_SECRET")].valueFrom.secretKeyRef diff --git a/helm/hugegraph/tests/statefulset_hardening_test.yaml b/helm/hugegraph/tests/statefulset_hardening_test.yaml new file mode 100644 index 0000000000..aecbdda335 --- /dev/null +++ b/helm/hugegraph/tests/statefulset_hardening_test.yaml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: StatefulSet update and PVC retention hardening +templates: + - pd-statefulset.yaml + - store-statefulset.yaml +tests: + - it: declares an explicit updateStrategy + asserts: + - equal: + path: spec.updateStrategy.type + value: RollingUpdate + + - it: retains PVCs on delete and scale-down + asserts: + - equal: + path: spec.persistentVolumeClaimRetentionPolicy.whenDeleted + value: Retain + - equal: + path: spec.persistentVolumeClaimRetentionPolicy.whenScaled + value: Retain + + - it: uses Parallel pod management for raft bring-up + asserts: + - equal: + path: spec.podManagementPolicy + value: Parallel + + - it: honours an OnDelete updateStrategy override + set: + pd.updateStrategy.type: OnDelete + store.updateStrategy.type: OnDelete + asserts: + - equal: + path: spec.updateStrategy.type + value: OnDelete diff --git a/helm/hugegraph/tests/test_hook_resources_test.yaml b/helm/hugegraph/tests/test_hook_resources_test.yaml new file mode 100644 index 0000000000..f57334d8a0 --- /dev/null +++ b/helm/hugegraph/tests/test_hook_resources_test.yaml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Helm test hook is resource bounded +templates: + - tests/test-connection.yaml +tests: + - it: bounds the test hook by default so quota-managed namespaces accept it + asserts: + - exists: + path: spec.containers[0].resources.limits.cpu + - exists: + path: spec.containers[0].resources.limits.memory + - exists: + path: spec.containers[0].resources.requests.cpu + - exists: + path: spec.containers[0].resources.requests.memory + + - it: lets an operator override the hook resources + set: + server.testResources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi + asserts: + - equal: + path: spec.containers[0].resources.limits.cpu + value: 500m diff --git a/helm/hugegraph/tests/topology_quorum_test.yaml b/helm/hugegraph/tests/topology_quorum_test.yaml new file mode 100644 index 0000000000..c375cde5ea --- /dev/null +++ b/helm/hugegraph/tests/topology_quorum_test.yaml @@ -0,0 +1,95 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: PD topology, quorum math and PDB gating +tests: + - it: points the PD StatefulSet at its own headless Service + template: pd-statefulset.yaml + asserts: + - matchRegex: + path: spec.serviceName + pattern: "-hugegraph-pd$" + + - it: builds a DNS-based raft peer list that tracks pd.replicas + template: pd-statefulset.yaml + set: + pd.replicas: 3 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="HG_PD_RAFT_PEERS_LIST")].value + pattern: "pd-0\\..*svc.*pd-1\\..*svc.*pd-2\\..*svc" + + - it: shrinks the raft peer list for a single-node install + template: pd-statefulset.yaml + set: + pd.replicas: 1 + pd.partition.defaultShardCount: 1 + store.replicas: 1 + asserts: + - notMatchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="HG_PD_RAFT_PEERS_LIST")].value + pattern: "pd-1\\." + + - it: waits for a majority of 2 when PD has 3 replicas + template: store-statefulset.yaml + set: + pd.replicas: 3 + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: "REQUIRED=2" + + - it: waits for a majority of 3 when PD has 5 replicas + template: store-statefulset.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: "REQUIRED=3" + + - it: waits for a majority of 1 when PD is single-node + template: store-statefulset.yaml + set: + pd.replicas: 1 + pd.partition.defaultShardCount: 1 + store.replicas: 1 + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: "REQUIRED=1" + + - it: emits no PodDisruptionBudget for a single-replica PD + template: pd-pdb.yaml + set: + pd.replicas: 1 + pd.partition.defaultShardCount: 1 + store.replicas: 1 + asserts: + - hasDocuments: + count: 0 + + - it: publishes not-ready addresses so peer DNS resolves before readiness + template: pd-service-headless.yaml + asserts: + - equal: + path: spec.publishNotReadyAddresses + value: true + - equal: + path: spec.clusterIP + value: None diff --git a/helm/hugegraph/tests/validate_values_test.yaml b/helm/hugegraph/tests/validate_values_test.yaml new file mode 100644 index 0000000000..f0700982fc --- /dev/null +++ b/helm/hugegraph/tests/validate_values_test.yaml @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: validateValues guard rails +templates: + - server-deployment.yaml +tests: + - it: rejects a PDB that would let PD drop below raft majority + set: + pd.pdb.minAvailable: 1 + asserts: + - failedTemplate: + errorPattern: "pd.pdb.minAvailable must be at least the PD Raft majority" + + - it: rejects a PDB that would permanently block drains + set: + pd.pdb.minAvailable: 3 + asserts: + - failedTemplate: + errorPattern: "must be less than pd.replicas" + + - it: rejects an even shard count + set: + pd.partition.defaultShardCount: 2 + asserts: + - failedTemplate: + errorPattern: "must be odd" + + - it: rejects a shard count above store.replicas + set: + pd.partition.defaultShardCount: 5 + asserts: + - failedTemplate: + errorPattern: "greater than store.replicas" + + - it: rejects operator overrides of the chart-managed JAVA_OPTIONS + set: + server.extraEnv: + - name: JAVA_OPTIONS + value: "-Xmx1g" + asserts: + - failedTemplate: + errorPattern: "must not set the chart-managed variable JAVA_OPTIONS" + + - it: rejects networkPolicy.enabled because the chart ships no policies + set: + networkPolicy.enabled: true + asserts: + - failedTemplate: + errorPattern: "does not implement NetworkPolicy resources" + + - it: rejects Hubble without Server auth + set: + server.auth.enabled: false + hubble.enabled: true + asserts: + - failedTemplate: + errorPattern: "hubble.enabled requires server.auth" + + - it: rejects a plain-HTTP Hubble Ingress + set: + hubble.ingress.enabled: true + hubble.enabled: true + asserts: + - failedTemplate: + errorPattern: "publishes the plain-HTTP, unauthenticated Hubble UI" + + - it: rejects an empty Hubble image tag + set: + hubble.image.tag: "" + hubble.enabled: true + asserts: + - failedTemplate: + errorPattern: "hubble.image.tag must not be empty" From e767965e20858dafc42be2f4e3d07a6e3e080fcf Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 29 Aug 2026 22:26:18 +0530 Subject: [PATCH 24/61] docs(helm): document digest and token rules, fix two stale references Record the values that had no documentation: the optional image.digest for each component, and the 32 byte minimum on server.auth.token.value that the schema now enforces. Add the credential lifecycle that operators hit first. The chart-managed auth Secret survives uninstall and is reused by a later install of the same release name, and helm template cannot read an existing Secret so the password it prints is only a render-time placeholder. Add a worked example for supplying an admin Secret instead of letting the chart generate one. Correct two stale statements: the Limitations note pointed to the Disaster Recovery section as being below when it is above, and the pinning advice said three component tags when there are four, since Hubble also tracks a mutable tag. --- helm/hugegraph/README.md | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index b14e6a6ef6..4b52d831d1 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -83,6 +83,20 @@ resource names (`kubectl get svc,secret -n ` lists them). **Authentication is enabled by default.** The chart creates a kept Secret named `-admin` (for example `hugegraph-admin`) with a random password unless `server.auth.admin.existingSecret` points at a pre-created Secret. +To manage the credential yourself, create the Secret before installing and set +`server.auth.admin.existingSecret`. It always takes priority, and the chart does +not overwrite or manage that Secret: + +```bash +kubectl -n hugegraph create secret generic my-hugegraph-admin \ + --from-literal=password='CHANGE_ME' +``` + +Then add `--set-string server.auth.admin.existingSecret=my-hugegraph-admin` to +the install command. The Secret must contain a `password` key with no newlines, +carriage returns, backslashes, or leading whitespace. The JWT signing key uses +the same shape under `server.auth.token` (`value`, `existingSecret`, +`autoGenerate`), and its value must be at least 32 bytes. Read the password and exercise the API: ```bash @@ -117,7 +131,8 @@ Sharding below. This chart is at an early 0.1.x version. While the contribution is a draft, its component image tags and `appVersion` track `latest` with pull policy `Always`. -Before stable publication, pin all three component tags and `appVersion` to the +Before stable publication, pin all four component tags (PD, Store, Server and +Hubble) and `appVersion` to the next HugeGraph release and switch the component pull policies to `IfNotPresent`. @@ -202,6 +217,13 @@ helm uninstall hugegraph --namespace hugegraph Helm does not remove PersistentVolumeClaims created by StatefulSets. Delete them explicitly, and only when the data is no longer needed. +The chart-managed authentication Secret is kept on uninstall and reused by a +later install of the same release name. Do not delete it unless you intend to +manage the password separately. `helm template` and client-side dry runs cannot +read an existing Secret, so the password they generate is only a render-time +placeholder; a live install or upgrade reuses the existing Secret when Helm has +permission to read it. + ## Configuration The following table lists the configurable parameters of the chart and their @@ -222,6 +244,7 @@ default values. | `pd.replicas` | PD StatefulSet replicas. Maximum `99` | `3` | | `pd.image.repository` | PD image repository | `hugegraph/pd` | | `pd.image.tag` | PD image tag. Tracks the development image until the next release is pinned | `latest` | +| `pd.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `pd.image.pullPolicy` | PD image pull policy | `Always` | | `pd.javaOpts` | Extra JVM flags, rendered after the chart-derived `-D` properties below so an explicit duplicate here wins. The image's automatic heap sizing is preserved unless heap flags are set | `""` | | `pd.raftIpWhitelistEnabled` | Enable PD's raft peer IP whitelist. Off in-cluster because PD resolves peers once at boot; requires a PD image carrying the upstream switch | `false` | @@ -265,6 +288,7 @@ default values. | `store.replicas` | Store StatefulSet replicas. Maximum `99` | `3` | | `store.image.repository` | Store image repository | `hugegraph/store` | | `store.image.tag` | Store image tag. Tracks the development image until the next release is pinned | `latest` | +| `store.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `store.image.pullPolicy` | Store image pull policy | `Always` | | `store.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | | `store.ports.grpc` | Store gRPC port | `8500` | @@ -304,6 +328,7 @@ default values. | `server.replicas` | Server Deployment replicas. Ignored when `server.hpa.enabled` | `3` | | `server.image.repository` | Server image repository | `hugegraph/server` | | `server.image.tag` | Server image tag. Tracks the development image until the next release is pinned | `latest` | +| `server.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `server.image.pullPolicy` | Server image pull policy | `Always` | | `server.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | | `server.port` | Server REST port, container port, and Service port | `8080` | @@ -337,7 +362,7 @@ default values. | `server.auth.admin.existingSecret` | Pre-created Secret name (key defaults to `password`); takes priority | `""` | | `server.auth.admin.key` | Key inside the admin password Secret | `password` | | `server.auth.admin.autoGenerate` | Create and keep a random release-admin Secret when password and existingSecret are empty | `true` | -| `server.auth.token.value` | Optional inline JWT signing key; prefer a Secret in shared clusters | `""` | +| `server.auth.token.value` | Optional inline JWT signing key, minimum 32 bytes; prefer a Secret in shared clusters | `""` | | `server.auth.token.existingSecret` | Pre-created Secret for the JWT signing key (`auth.token_secret`) | `""` | | `server.auth.token.key` | Key inside the JWT signing Secret | `token_secret` | | `server.auth.token.autoGenerate` | Create and keep a random release-auth-token Secret when value and existingSecret are empty | `true` | @@ -517,6 +542,7 @@ trusted network. | `hubble.allowWithoutServerAuth` | Renders Hubble without `server.auth`, for future images whose login does not require cluster authentication | `false` | | `hubble.image.repository` | Hubble image repository | `hugegraph/hubble` | | `hubble.image.tag` | Hubble image tag. Tracks the development image until the next release is pinned | `latest` | +| `hubble.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `hubble.image.pullPolicy` | Hubble image pull policy | `Always` | | `hubble.port` | Hubble HTTP port, container port, and Service port | `8088` | | `hubble.persistence.enabled` | Persist UI connection metadata in a PVC | `false` | @@ -837,15 +863,15 @@ independently of the release name. in-cluster via the upstream `raft.ip-whitelist.enabled` switch, leaving peer authentication to Kubernetes-level controls. Setting `pd.raftIpWhitelistEnabled=true` restores the image default along with - its one-shot resolution semantics — bring-up races and pod-IP-change - rejections included — at the operator's own risk. PD images that predate + its one-shot resolution semantics (bring-up races and pod-IP-change + rejections included) at the operator's own risk. PD images that predate the switch ignore the flag and keep the whitelist active, so they remain exposed to those failure modes; use images built from a source tree that includes the switch. - The PD management REST endpoints (`/v1/members`, `/v1/stores`) reject requests on current images (`invalid service name`), and unauthenticated GETs return HTTP 200 with an `Unauthorized` JSON body. Until that is - resolved upstream, the operator-triggered Disaster Recovery flow below may + resolved upstream, the operator-triggered Disaster Recovery flow above may be unavailable; rely on `helm test`, Pod readiness, and Server APIs for health checks. - No TLS, backups, Operator, multi-cluster support, automatic leader transfer, From 6f3ba544b077e6137e93aba359777479d804908b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 2 Sep 2026 09:19:52 +0530 Subject: [PATCH 25/61] fix(helm): announce each Server Pod IP to PD for replica discovery Every Server replica registered the shared client Service URL with PD, so the discovery registry collapsed three replicas into one logical entry and PD-discovered clients such as Hubble saw a single Server. The registration is a lease renewed by a per-instance heartbeat, so a replaced Pod's entry ages out on its own; announcing each Pod's own IP keeps the replica list truthful without leaving permanent stale entries. Empty server.advertiseUrl now announces the Pod IP through a POD_IP downward-API variable. Setting advertiseUrl keeps its meaning as the shared external endpoint for an outside Hubble. POD_IP joins the reserved environment names so extraEnv cannot shadow it. Ports c2d086de from helm-dev and adds a unit suite covering the fieldRef, the default URL shape, the advertiseUrl override and the reserved name. --- helm/hugegraph/README.md | 10 ++-- helm/hugegraph/templates/_helpers.tpl | 11 ++--- .../templates/server-deployment.yaml | 4 ++ .../tests/server_discovery_test.yaml | 49 +++++++++++++++++++ helm/hugegraph/values.yaml | 11 ++--- 5 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 helm/hugegraph/tests/server_discovery_test.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 4b52d831d1..d2d7e4710b 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -369,7 +369,7 @@ default values. | `server.ingress.enabled` | Create an Ingress for the Server Service | `false` | | `server.ingress.className` | IngressClass name | `""` | | `server.ingress.annotations` | Ingress annotations (cert-manager, nginx, ALB) | `{}` | -| `server.advertiseUrl` | Absolute Server URL registered with PD (`server.urls_to_pd`). Empty uses the in-cluster Service URL | `""` | +| `server.advertiseUrl` | Absolute Server URL registered with PD (`server.urls_to_pd`). Empty registers each Server Pod IP for in-cluster discovery | `""` | | `server.service.type` | Server Service type | `ClusterIP` | | `server.service.annotations` | Server Service annotations | `{}` | | `server.ingress.hosts` | Ingress hosts and paths | see `values.yaml` | @@ -463,9 +463,7 @@ pd.peers=: pd.server=: ``` -Trade-off: when `server.advertiseUrl` is set, PD returns that same URL to -every discovery client, including an in-cluster Hubble. Leave it empty for the -default in-cluster path. +Trade-off: when `server.advertiseUrl` is set, every Server replica registers that same logical URL and PD returns it to every discovery client, including an in-cluster Hubble. Leave it empty for the default in-cluster path, where each Server Pod registers its own IP and Hubble can retain the replica list. Local quick test (cluster and Hubble on the same machine): port-forward Server `8080` and PD client `8620`/`8686`, set @@ -475,7 +473,7 @@ Local quick test (cluster and Hubble on the same machine): port-forward Server | Parameter | Description | Default | |---|---|---| -| `server.advertiseUrl` | Absolute Server URL registered with PD for discovery clients. Empty uses the in-cluster Server Service URL | `""` | +| `server.advertiseUrl` | Absolute Server URL registered with PD for discovery clients. Empty registers each Server Pod IP for in-cluster discovery | `""` | | `pd.service.type` | PD client Service type (`ClusterIP`, `NodePort`, `LoadBalancer`) | `ClusterIP` | | `pd.service.annotations` | Annotations on the PD client Service | `{}` | | `pd.service.restNodePort` | Optional fixed NodePort for PD REST; requires NodePort/LoadBalancer | unset | @@ -497,7 +495,7 @@ single flag (see Installing above). Login uses the admin credential from `pd` mode the chart points `pd.peers` at the PD gRPC peers, `pd.server` at the PD client Service REST port, and the Store metrics allow-list at the Store REST endpoints, so the cluster view works without manual wiring; the -Server is additionally configured to register its client Service URL with PD +Server is additionally configured to register each Server Pod IP with PD (see below). In `direct` mode Hubble only receives `server.direct_url` pointing at the Server client Service; there is no PD discovery and no operations view. Everything else in `hugegraph-hubble.properties` keeps the diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index f998eb684d..eeea380bbd 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -260,9 +260,7 @@ First store REST endpoint for STORE_REST / wait-partition. {{- end }} {{/* -Server REST URL reached through the client Service. Announced to PD via -server.urls_to_pd so PD-discovered clients (Hubble) get a resolvable address -instead of the in-pod 0.0.0.0 default. +Server REST URL reached through the client Service. */}} {{- define "hugegraph.server.clientUrl" -}} {{- printf "http://%s.%s.svc:%d" (include "hugegraph.server.name" .) .Release.Namespace (int .Values.server.port) -}} @@ -270,15 +268,14 @@ instead of the in-pod 0.0.0.0 default. {{/* URL registered with PD (server.urls_to_pd / HG_SERVER_URLS_TO_PD). -server.advertiseUrl wins when set so outside PD-mode Hubble receives a -reachable address; otherwise the in-cluster Server Service URL is used. +server.advertiseUrl wins when set so outside PD-mode Hubble receives a reachable address; otherwise each Server Pod announces its own Pod IP so PD discovery preserves the replica list for in-cluster clients. */}} {{- define "hugegraph.server.urlsToPd" -}} {{- $advertise := trim (default "" .Values.server.advertiseUrl) -}} {{- if $advertise -}} {{- $advertise -}} {{- else -}} -{{- include "hugegraph.server.clientUrl" . -}} +{{- printf "http://$(POD_IP):%d" (int .Values.server.port) -}} {{- end -}} {{- end }} @@ -561,7 +558,7 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- $reservedEnv := dict "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") "store" (list "HG_STORE_PD_ADDRESS" "HG_STORE_GRPC_HOST" "HG_STORE_GRPC_PORT" "HG_STORE_REST_PORT" "HG_STORE_RAFT_ADDRESS" "HG_STORE_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") - "server" (list "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "HG_SERVER_AUTH_TOKEN_SECRET" "JAVA_OPTS" "JAVA_OPTIONS") + "server" (list "POD_IP" "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "HG_SERVER_AUTH_TOKEN_SECRET" "JAVA_OPTS" "JAVA_OPTIONS") "hubble" (list "HG_HUBBLE_PD_PEERS" "HG_HUBBLE_PD_SERVER" "HG_HUBBLE_STORE_TARGETS" "HG_HUBBLE_SERVER_URL" "SPRING_DATASOURCE_URL") -}} {{- range $component, $reserved := $reservedEnv -}} {{- $componentValues := get $.Values $component | default dict -}} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 75c1ea6707..2c06d8be23 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -258,6 +258,10 @@ spec: - name: http containerPort: {{ .Values.server.port }} env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: HG_SERVER_BACKEND value: {{ .Values.server.backend | quote }} - name: HG_SERVER_PD_PEERS diff --git a/helm/hugegraph/tests/server_discovery_test.yaml b/helm/hugegraph/tests/server_discovery_test.yaml new file mode 100644 index 0000000000..63aca4db3d --- /dev/null +++ b/helm/hugegraph/tests/server_discovery_test.yaml @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Server replica discovery announced to PD +templates: + - server-deployment.yaml +tests: + - it: injects the pod IP through the downward API + asserts: + - equal: + path: spec.template.spec.containers[0].env[?(@.name=="POD_IP")].valueFrom.fieldRef.fieldPath + value: status.podIP + + - it: announces each pod's own address to PD by default + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="HG_SERVER_URLS_TO_PD")].value + pattern: "^http://\\$\\(POD_IP\\):8080$" + + - it: announces the shared advertiseUrl instead when one is set + set: + server.advertiseUrl: http://graph.example.com:30080 + asserts: + - equal: + path: spec.template.spec.containers[0].env[?(@.name=="HG_SERVER_URLS_TO_PD")].value + value: http://graph.example.com:30080 + + - it: refuses an operator extraEnv that would shadow POD_IP + set: + server.extraEnv: + - name: POD_IP + value: 10.0.0.1 + asserts: + - failedTemplate: + errorPattern: "POD_IP" diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index f0fe21bc91..87660cc28e 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -326,14 +326,9 @@ server: key: token_secret # When existingSecret and value are empty, create a kept release-auth-token Secret. autoGenerate: true - # URL announced to PD via server.urls_to_pd for discovery clients such as - # Hubble. Empty keeps the in-cluster Server Service URL - # (http://-server..svc:). Set this to a URL reachable - # from outside the cluster (NodePort, LoadBalancer, or Ingress) when an - # external Hubble uses PD mode; PD will hand that address back to Hubble. - # Trade-off: in-cluster discovery clients then receive the same URL, so it - # must also be reachable from inside the cluster (or keep Hubble in-cluster - # and leave this empty). + # URL announced to PD via server.urls_to_pd for discovery clients such as Hubble. + # Empty announces each Server Pod IP so in-cluster clients retain the full replica list. + # Set this to a URL reachable from outside the cluster (NodePort, LoadBalancer, or Ingress) when an external Hubble uses PD mode; all replicas then announce that shared logical endpoint. advertiseUrl: "" service: type: ClusterIP From c768f9e232e6a1fcb0f2e7c9a7aa6926f65d5de3 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 2 Sep 2026 14:08:29 +0530 Subject: [PATCH 26/61] docs(helm): correct the PD REST auth limitation and fix the recovery commands The Limitations entry said the PD management endpoints reject requests on current images, so the Disaster Recovery flow might be unavailable. Measuring it shows the opposite. PD compares the Basic-auth username against a fixed internal set and never reads the password, so any password, including an empty one, is accepted for those names. The endpoints are effectively unauthenticated rather than unusable, which is a caveat about exposure, not availability. That also makes the documented recovery commands wrong. Written without a credential they answer HTTP 200 with an Unauthorized body and the task never runs, so an operator mid-incident sees success and gets nothing. Add the credential the endpoints actually require and say why the password is empty. Also record that success, refusal and a missing credential all return HTTP 200, so the status code carries no signal for health checks. --- helm/hugegraph/README.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index d2d7e4710b..4e91608c2a 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -726,11 +726,16 @@ reachable through the PD client Service: ```bash kubectl port-forward -n hugegraph svc/hugegraph-pd-client 8620:8620 -curl http://127.0.0.1:8620/v1/task/patrolPartitions # reconcile shard groups, process tombstoned Stores -curl http://127.0.0.1:8620/v1/task/balanceLeaders # spread Raft leaders -curl http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data +curl -u hg: http://127.0.0.1:8620/v1/task/patrolPartitions # reconcile shard groups, process tombstoned Stores +curl -u hg: http://127.0.0.1:8620/v1/task/balanceLeaders # spread Raft leaders +curl -u hg: http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data ``` +The `-u hg:` credential is required. Without it these endpoints answer HTTP 200 +with an `Unauthorized` body and the task does not run, so a recovery attempt +looks successful while doing nothing. The password is empty on purpose: current +PD images check only the service name. See Limitations. + Run `patrolPartitions` after replacing a Store that is not coming back, `balancePartitions` once the cluster is stable again, and `balanceLeaders` after restarts that skewed leader placement. @@ -866,12 +871,16 @@ independently of the release name. the switch ignore the flag and keep the whitelist active, so they remain exposed to those failure modes; use images built from a source tree that includes the switch. -- The PD management REST endpoints (`/v1/members`, `/v1/stores`) reject - requests on current images (`invalid service name`), and unauthenticated - GETs return HTTP 200 with an `Unauthorized` JSON body. Until that is - resolved upstream, the operator-triggered Disaster Recovery flow above may - be unavailable; rely on `helm test`, Pod readiness, and Server APIs for - health checks. +- The PD management REST endpoints (`/v1/members`, `/v1/stores`, + `/v1/task/*`) authenticate on service name only. Current images compare the + Basic-auth username against a fixed internal set (`hg`, `store`, `hubble`, + `vermeer`) and do not validate the password at all, so any password, + including an empty one, is accepted for those names while every other name + is refused. Treat these endpoints as unauthenticated: keep the PD client + Service on ClusterIP and do not expose it. All three outcomes, success, + refusal, and a missing credential, return HTTP 200 with the result in the + body, so the status code carries no signal and no health check should key on + it. The Disaster Recovery calls above therefore need `-u hg:` to run. - No TLS, backups, Operator, multi-cluster support, automatic leader transfer, or a complete monitoring stack. Store recovery is manual on current builds: re-replication after Store loss, leader balancing, and partition From 7d9d084ed188c7a12ce749987ad519527db455fd Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 2 Sep 2026 20:16:21 +0530 Subject: [PATCH 27/61] feat(helm): make PD readiness and Store wait paths configurable, document health vs ready PD's /v1/health answers 200 as soon as the REST listener is up and never consults raft, so every PD and Store probe and the Store init container's PD wait count listeners, not quorum members (apache/hugegraph#3183). The fix, apache/hugegraph#3185, adds /v1/ready from 1.8.0. - pd.readinessPath and store.waitPath, both defaulting to /v1/health, so the switch to /v1/ready is a values change made with the 1.8.0 pin; the schema rejects paths without a leading slash - README: Limitations entries for the liveness-only health endpoint and for the 45 second discovery lease (measured 30 to 35 seconds); the Store wait is described as a PD wait rather than a quorum wait; the Server now registers its Pod IP, not the Service URL - NOTES and the init container messages no longer claim a quorum - tests: pd_readiness_path_test.yaml, five cases --- helm/hugegraph/README.md | 46 ++++++++--- helm/hugegraph/templates/NOTES.txt | 4 +- helm/hugegraph/templates/pd-statefulset.yaml | 2 +- .../templates/store-statefulset.yaml | 11 +-- .../tests/pd_readiness_path_test.yaml | 77 +++++++++++++++++++ helm/hugegraph/values.schema.json | 10 +++ helm/hugegraph/values.yaml | 21 ++++- 7 files changed, 151 insertions(+), 20 deletions(-) create mode 100644 helm/hugegraph/tests/pd_readiness_path_test.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 4e91608c2a..06b9942d1d 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -40,11 +40,14 @@ so operators do not have to: - **Every Server uses PD for graph metadata.** The startup wrapper always writes `usePD=true` and the chart-derived `pd.peers` into `rest-server.properties`, so all Server replicas share the graph catalog - through PD. It also registers the Server client Service URL for Kubernetes - discovery. This is required for distributed HStore and does not make a local - RocksDB backend shared across replicas. -- **Store waits for PD quorum** in an init container before starting, so Store - never registers against an incomplete PD Raft group. + through PD. It also registers each Server Pod IP with PD for in-cluster + discovery (`server.advertiseUrl` replaces that with one shared URL). This is + required for distributed HStore and does not make a local RocksDB backend + shared across replicas. +- **Store waits for PD** in an init container before starting: a majority of + the PD peers must answer `store.waitPath`. The default `/v1/health` proves + each PD's listener is up, not that a raft quorum exists; see Limitations for + the `/v1/ready` switch. - **The Server startup probe allows at least 450 seconds.** The image may spend 300 seconds waiting for storage and a further 120 seconds in the start command. A lower configured `failureThreshold` is raised to this floor rather @@ -275,6 +278,7 @@ default values. | `pd.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | +| `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/health` is liveness only; set `/v1/ready` on PD images from 1.8.0 that carry it, never on older images | `/v1/health` | | `pd.probes.*.periodSeconds` | Probe interval | see `values.yaml` | | `pd.probes.*.failureThreshold` | Probe failure threshold | see `values.yaml` | | `pd.probes.*.timeoutSeconds` | Probe timeout. Defaults to `5` on readiness/liveness; Kubernetes would otherwise apply `1` | `5` | @@ -299,8 +303,9 @@ default values. | `store.storage.storageClassName` | Empty uses the cluster default StorageClass | `""` | | `store.resources` | Store container resources. Set these for production | `{}` | | `store.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | -| `store.securityContext` | Container-level securityContext; also applied to the PD-quorum init container. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | -| `store.waitTimeoutSeconds` | Bound on the PD-quorum wait before the init container fails | `900` | +| `store.securityContext` | Container-level securityContext; also applied to the PD wait init container. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | +| `store.waitPath` | Path the init container polls on each PD peer; a majority must answer 2xx. `/v1/health` counts listeners; `/v1/ready` counts quorum members but exists only on PD images from 1.8.0 | `/v1/health` | +| `store.waitTimeoutSeconds` | Bound on the PD wait before the init container fails | `900` | | `store.antiAffinity` | One of `required`, `preferred`, `disabled`. `preferred` schedules on clusters with fewer nodes than replicas; production should use `required` so one node failure cannot co-locate shard replicas | `preferred` | | `store.nodeSelector` | Node selector for store Pods | `{}` | | `store.tolerations` | Tolerations for store Pods | `[]` | @@ -317,7 +322,7 @@ default values. | `store.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | | `store.pdb.enabled` | Create a PodDisruptionBudget for Store | `true` | | `store.pdb.minAvailable` | Must be strictly less than `store.replicas`. No PDB is rendered when `store.replicas` is 1 | `2` | -| `store.waitImage` | Image for the PD-quorum init container | `curlimages/curl:8.5.0` | +| `store.waitImage` | Image for the PD wait init container | `curlimages/curl:8.5.0` | | `store.waitResources` | Resources for the init container | `{}` | | `store.probes.*` | Same probe keys as PD | see `values.yaml` | @@ -763,7 +768,8 @@ overwrite the autoscaler's live replica count. ### Store Pods Stuck in `Init:0/1` -The Store init container waits for PD to reach Raft quorum. Check PD first: +The Store init container waits for a majority of PD peers to answer +`store.waitPath`. Check PD first: ```bash kubectl get pods -l app.kubernetes.io/component=pd @@ -881,6 +887,28 @@ independently of the release name. refusal, and a missing credential, return HTTP 200 with the result in the body, so the status code carries no signal and no health check should key on it. The Disaster Recovery calls above therefore need `-u hg:` to run. +- PD's `/v1/health` is liveness only: it answers 200 as soon as the REST + listener is up and never consults raft. Measured on a 3-PD install with two + PDs deleted, the survivor logged `Raft lost leader` within a second and kept + answering 200 while quorum-dependent calls failed. Every PD and Store probe + in this chart, and the Store init container's PD wait, key on that endpoint, + so a listening-but-leaderless PD passes readiness and the wait counts + listeners rather than quorum members. Tracked upstream as + [#3183](https://github.com/apache/hugegraph/issues/3183); the fix, + [#3185](https://github.com/apache/hugegraph/pull/3185), adds an + unauthenticated `/v1/ready` that answers 503 without a raft leader, plus + raft gauges, from 1.8.0. On such an image set `pd.readinessPath=/v1/ready` + and `store.waitPath=/v1/ready`; keep startup and liveness on `/v1/health` + so a PD that merely lost its leader is not restarted. Do not set either + path on an older image: it does not exist there, the PD never turns Ready + and Stores never leave Init. +- Server discovery is a lease. Each Server re-registers its Pod IP with PD + every 15 seconds and PD drops an entry after three missed heartbeats, so a + replaced or evicted Server can stay in PD's list for up to 45 seconds after + it stops (measured 30 to 35 seconds on a live rollout). Hubble's cluster + view and other discovery clients may show that stale address for the + duration; application traffic is unaffected because it reaches Servers + through the Service, which drops the Pod immediately. - No TLS, backups, Operator, multi-cluster support, automatic leader transfer, or a complete monitoring stack. Store recovery is manual on current builds: re-replication after Store loss, leader balancing, and partition diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index b24a3a826b..9bd81648c3 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -28,8 +28,8 @@ Watch the cluster come up: kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -w -PD reaches Raft quorum before Store registers, so Store Pods stay in Init until -PD is ready. +Store Pods stay in Init until a majority of PD peers answer store.waitPath +(/v1/health by default, which proves the listener is up, not a raft quorum). Verify the release: diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 1318741280..b7575eebe4 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -137,7 +137,7 @@ spec: {{- with include "hugegraph.probeTuning" .Values.pd.probes.startup }}{{ . | trim | nindent 12 }}{{- end }} readinessProbe: httpGet: - path: /v1/health + path: {{ .Values.pd.readinessPath | default "/v1/health" }} port: rest periodSeconds: {{ .Values.pd.probes.readiness.periodSeconds }} failureThreshold: {{ .Values.pd.probes.readiness.failureThreshold }} diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index 9ec7d8e7e7..10a0934079 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -99,25 +99,26 @@ spec: HEALTH_PEERS=$(echo "{{ include "hugegraph.pd.restPeersList" . }}" | tr ',' ' ') TIMEOUT={{ .Values.store.waitTimeoutSeconds | default 900 }} DEADLINE=$(( $(date +%s) + TIMEOUT )) - echo "Waiting for PD quorum (${REQUIRED}) among: ${HEALTH_PEERS}" + WAIT_PATH={{ .Values.store.waitPath | default "/v1/health" | quote }} + echo "Waiting for ${REQUIRED} PD peers to answer ${WAIT_PATH} among: ${HEALTH_PEERS}" until [ "$( ok=0 for peer in ${HEALTH_PEERS}; do - if curl -fsS "http://${peer}/v1/health" >/dev/null 2>&1; then + if curl -fsS "http://${peer}${WAIT_PATH}" >/dev/null 2>&1; then ok=$((ok+1)) fi done echo "$ok" )" -ge "${REQUIRED}" ]; do if [ "$(date +%s)" -ge "${DEADLINE}" ]; then - echo "Timed out after ${TIMEOUT}s waiting for PD quorum (${REQUIRED}) among: ${HEALTH_PEERS}" >&2 + echo "Timed out after ${TIMEOUT}s waiting for ${REQUIRED} PD peers to answer ${WAIT_PATH} among: ${HEALTH_PEERS}" >&2 echo "Check PD Pods: kubectl get pods -l app.kubernetes.io/component=pd" >&2 exit 1 fi - echo "Waiting for PD quorum..." + echo "Waiting for PD peers..." sleep 5 done - echo "PD quorum reached." + echo "Enough PD peers answered ${WAIT_PATH}." {{- with .Values.store.waitResources }} resources: {{- toYaml . | nindent 12 }} diff --git a/helm/hugegraph/tests/pd_readiness_path_test.yaml b/helm/hugegraph/tests/pd_readiness_path_test.yaml new file mode 100644 index 0000000000..6042dec259 --- /dev/null +++ b/helm/hugegraph/tests/pd_readiness_path_test.yaml @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: PD readiness path and Store wait path +tests: + - it: keeps every PD probe on /v1/health by default + template: pd-statefulset.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /v1/health + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /v1/health + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /v1/health + + - it: moves only the PD readiness probe when pd.readinessPath is set + template: pd-statefulset.yaml + set: + pd.readinessPath: /v1/ready + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /v1/ready + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /v1/health + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /v1/health + + - it: polls /v1/health on each PD peer in the Store wait by default + template: store-statefulset.yaml + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'WAIT_PATH="/v1/health"' + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'curl -fsS "http://\${peer}\${WAIT_PATH}"' + + - it: polls store.waitPath on each PD peer when set + template: store-statefulset.yaml + set: + store.waitPath: /v1/ready + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'WAIT_PATH="/v1/ready"' + - notMatchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'v1/health' + + - it: leaves the Store's own probes on /v1/health when store.waitPath is set + template: store-statefulset.yaml + set: + store.waitPath: /v1/ready + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /v1/health diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 1a2ba207c0..ffc1935054 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -436,6 +436,11 @@ "maximum": 32767 } } + }, + "readinessPath": { + "type": "string", + "pattern": "^/", + "description": "HTTP path for the PD readinessProbe; /v1/health (default) or /v1/ready on PD images that have it" } } }, @@ -560,6 +565,11 @@ "type": "boolean" } } + }, + "waitPath": { + "type": "string", + "pattern": "^/", + "description": "HTTP path the Store init container polls on each PD peer; /v1/health (default) or /v1/ready on PD images that have it" } } }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 87660cc28e..caf04cdd20 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -131,6 +131,15 @@ pd: whenDeleted: Retain whenScaled: Retain # Startup can take a while during Raft bootstrap + # HTTP path the PD readinessProbe hits. /v1/health answers 200 as soon as + # the REST listener is up and never consults raft, so a PD outside the + # quorum still passes readiness. PD images from 1.8.0 on add /v1/ready + # (apache/hugegraph#3185), which answers 503 without a raft leader; switch + # this to /v1/ready only on such an image, an older image has no such + # path and the PD never turns Ready. Startup and liveness stay on + # /v1/health regardless, so a PD that merely lost its leader is not + # restarted. + readinessPath: /v1/health probes: startup: failureThreshold: 30 @@ -201,10 +210,16 @@ store: enabled: true minAvailable: 2 waitImage: curlimages/curl:8.5.0 - # Bound the PD-quorum wait so a cluster that never reaches quorum fails - # visibly instead of sitting in Init:0/1 forever. + # HTTP path the init container polls on every PD peer; a majority must + # answer 2xx before the Store starts. /v1/health proves the listener is + # up, not a raft quorum. On PD images that carry /v1/ready (1.8.0 on, + # apache/hugegraph#3185) set /v1/ready so the count means quorum members; + # on older images that path does not exist and Stores never leave Init. + waitPath: /v1/health + # Bound the PD wait so a cluster whose PDs never come up fails visibly + # instead of sitting in Init:0/1 forever. waitTimeoutSeconds: 900 - # Optional bounds for the PD-quorum wait init container. + # Optional bounds for the PD wait init container. waitResources: {} # Explicit rollout strategy instead of the implicit StatefulSet default. updateStrategy: From 535bc8b1f139599f19452bffd6783bd04958da17 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 2 Sep 2026 20:31:23 +0530 Subject: [PATCH 28/61] chore(helm): mark the /v1/ready switch as a TODO on both path values Points at apache/hugegraph#3185 and says the defaults flip with the 1.8.0 image pin, so the change is not lost once that PR merges. --- helm/hugegraph/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index caf04cdd20..d20be1e7dd 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -139,6 +139,8 @@ pd: # path and the PD never turns Ready. Startup and liveness stay on # /v1/health regardless, so a PD that merely lost its leader is not # restarted. + # TODO(apache/hugegraph#3185): change the default to /v1/ready in the same + # commit that pins the image tag to 1.8.0 or newer. readinessPath: /v1/health probes: startup: @@ -215,6 +217,8 @@ store: # up, not a raft quorum. On PD images that carry /v1/ready (1.8.0 on, # apache/hugegraph#3185) set /v1/ready so the count means quorum members; # on older images that path does not exist and Stores never leave Init. + # TODO(apache/hugegraph#3185): change the default to /v1/ready in the same + # commit that pins the image tag to 1.8.0 or newer. waitPath: /v1/health # Bound the PD wait so a cluster whose PDs never come up fails visibly # instead of sitting in Init:0/1 forever. From 0a50f6fe40e96cf6388e1c156c1b51f764164466 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 5 Sep 2026 12:53:39 +0530 Subject: [PATCH 29/61] feat(helm): wire the PD REST secret into PD, Server and Hubble PD images from 1.8.0 (apache/hugegraph#3189) check the Basic-auth password of every management call against auth.secret-key and refuse to start without one. The chart now keeps that value in a kept release-pd-auth Secret, or in pd.auth.existingSecret, and hands it to the three readers: PD as HG_PD_AUTH_SECRET_KEY, the Server storage wait as PD_AUTH_PASSWORD, and Hubble as operations.pd.password written into its properties file by the existing wrapper. A checksum/pd-auth annotation on the three Pod templates rolls them when the Secret changes; the Server annotations block is now rendered unconditionally for it. Priority and lookup semantics mirror server.auth.token. Older images ignore the password, so the wiring is harmless on the images the draft currently tracks. The values schema requires one of existingSecret, value or autoGenerate and refuses newlines, carriage returns and backslashes in an inline value, since it lands in a Java properties file; the template guard repeats the first rule for values that bypass the schema. The three chart-managed variables join the reserved extraEnv lists. README: Chart Details bullet, four parameter rows, Disaster Recovery calls carry the secret, and the Limitations bullet separates the 1.7.0 behaviour from 1.8.0. NOTES prints how to read the secret. New suite pd_auth_secret_test.yaml, 9 tests; 58 in total. Lint on three presets; renders 16 objects by default and 19 with Hubble. Measured on a kind cluster with images built from master plus #3185, #3187 and #3189: the Secret is created, PD starts with the variable, the Server storage wait passes with the credential, and Hubble lists all nine nodes. --- helm/hugegraph/README.md | 60 ++++++-- helm/hugegraph/templates/NOTES.txt | 5 + helm/hugegraph/templates/_helpers.tpl | 65 +++++++- .../templates/hubble-deployment.yaml | 30 +++- helm/hugegraph/templates/pd-auth-secret.yaml | 31 ++++ helm/hugegraph/templates/pd-statefulset.yaml | 12 +- .../templates/server-deployment.yaml | 11 +- helm/hugegraph/tests/pd_auth_secret_test.yaml | 139 ++++++++++++++++++ helm/hugegraph/values.schema.json | 57 +++++++ helm/hugegraph/values.yaml | 19 +++ 10 files changed, 403 insertions(+), 26 deletions(-) create mode 100644 helm/hugegraph/templates/pd-auth-secret.yaml create mode 100644 helm/hugegraph/tests/pd_auth_secret_test.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 06b9942d1d..270ef1a906 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -48,6 +48,15 @@ so operators do not have to: the PD peers must answer `store.waitPath`. The default `/v1/health` proves each PD's listener is up, not that a raft quorum exists; see Limitations for the `/v1/ready` switch. +- **One PD REST secret, three readers.** PD images from 1.8.0 + ([#3189](https://github.com/apache/hugegraph/pull/3189)) check the Basic-auth + password of every management call against `auth.secret-key` and refuse to + start without one. The chart keeps that value in a release-pd-auth Secret + (or `pd.auth.existingSecret`) and hands it to PD as `HG_PD_AUTH_SECRET_KEY`, + to the Server storage wait as `PD_AUTH_PASSWORD`, and to Hubble as + `operations.pd.password`; a `checksum/pd-auth` annotation rolls all three + when the Secret changes. Older images ignore the password, so the wiring is + harmless on them. - **The Server startup probe allows at least 450 seconds.** The image may spend 300 seconds waiting for storage and a further 120 seconds in the start command. A lower configured `failureThreshold` is raised to this floor rather @@ -180,6 +189,15 @@ are worth knowing about in advance: Secrets. Template-only pipelines (`helm template`, GitOps renderers) never see live Secrets, so there the annotation is a constant and Secret rotation does not roll pods. +- **PD and Hubble** roll once on the first `helm upgrade` after a fresh + install as well, when the `checksum/pd-auth` annotation first observes the + install-created PD REST Secret (same mechanism as the Server annotation + above; measured on a kind cluster: PD, Server and Hubble replaced, Store + untouched). A PD roll is a raft rolling restart, one pod at a time; for a + maintenance-window upgrade set `pd.updateStrategy.type=OnDelete` and + restart the PD pods yourself. Rotating the PD REST Secret later rolls the + same three workloads together, which keeps their copies of the secret in + step. Every optional field stays optional, so a release created by an earlier revision continues to render under `--reuse-values`. Note that `--reuse-values` @@ -279,6 +297,10 @@ default values. | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | | `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/health` is liveness only; set `/v1/ready` on PD images from 1.8.0 that carry it, never on older images | `/v1/health` | +| `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. No newlines, carriage returns, or backslashes | `""` | +| `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it | `""` | +| `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | +| `pd.auth.autoGenerate` | Create and keep a random release-pd-auth Secret when `value` and `existingSecret` are empty | `true` | | `pd.probes.*.periodSeconds` | Probe interval | see `values.yaml` | | `pd.probes.*.failureThreshold` | Probe failure threshold | see `values.yaml` | | `pd.probes.*.timeoutSeconds` | Probe timeout. Defaults to `5` on readiness/liveness; Kubernetes would otherwise apply `1` | `5` | @@ -731,15 +753,19 @@ reachable through the PD client Service: ```bash kubectl port-forward -n hugegraph svc/hugegraph-pd-client 8620:8620 -curl -u hg: http://127.0.0.1:8620/v1/task/patrolPartitions # reconcile shard groups, process tombstoned Stores -curl -u hg: http://127.0.0.1:8620/v1/task/balanceLeaders # spread Raft leaders -curl -u hg: http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data +PD_SECRET="$(kubectl -n hugegraph get secret hugegraph-pd-auth \ + -o jsonpath='{.data.secret-key}' | base64 --decode)" +curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/patrolPartitions # reconcile shard groups, process tombstoned Stores +curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balanceLeaders # spread Raft leaders +curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data ``` -The `-u hg:` credential is required. Without it these endpoints answer HTTP 200 -with an `Unauthorized` body and the task does not run, so a recovery attempt -looks successful while doing nothing. The password is empty on purpose: current -PD images check only the service name. See Limitations. +The credential is required. PD images from 1.8.0 answer 401 without it; older +images answer HTTP 200 with an `Unauthorized` body and the task does not run, +so a recovery attempt looks successful while doing nothing. On those older +images the password is not checked, so `-u hg:` alone also works. The Secret +name follows the release (`-pd-auth`) unless `pd.auth.existingSecret` +is set. See Limitations. Run `patrolPartitions` after replacing a Store that is not coming back, `balancePartitions` once the cluster is stable again, and `balanceLeaders` @@ -878,15 +904,19 @@ independently of the release name. exposed to those failure modes; use images built from a source tree that includes the switch. - The PD management REST endpoints (`/v1/members`, `/v1/stores`, - `/v1/task/*`) authenticate on service name only. Current images compare the - Basic-auth username against a fixed internal set (`hg`, `store`, `hubble`, - `vermeer`) and do not validate the password at all, so any password, - including an empty one, is accepted for those names while every other name - is refused. Treat these endpoints as unauthenticated: keep the PD client - Service on ClusterIP and do not expose it. All three outcomes, success, + `/v1/task/*`) authenticate on service name only on PD images up to 1.7.0. + Those images compare the Basic-auth username against a fixed internal set + (`hg`, `store`, `hubble`, `vermeer`) and do not validate the password at + all, so any password, including an empty one, is accepted for those names + while every other name is refused, and all three outcomes, success, refusal, and a missing credential, return HTTP 200 with the result in the - body, so the status code carries no signal and no health check should key on - it. The Disaster Recovery calls above therefore need `-u hg:` to run. + body. Treat these endpoints as unauthenticated on such images: keep the PD + client Service on ClusterIP and do not expose it, and do not key a health + check on the status code. PD images from 1.8.0 + ([#3189](https://github.com/apache/hugegraph/pull/3189)) check the password + against `auth.secret-key` and answer 401 on refusal; the chart supplies that + secret through `pd.auth` (see Chart Details), and the Disaster Recovery + calls above need it. - PD's `/v1/health` is liveness only: it answers 200 as soon as the REST listener is up and never consults raft. Measured on a 3-PD install with two PDs deleted, the survivor logged `Raft lost leader` within a second and kept diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 9bd81648c3..5c26da04cd 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -31,6 +31,11 @@ Watch the cluster come up: Store Pods stay in Init until a majority of PD peers answer store.waitPath (/v1/health by default, which proves the listener is up, not a raft quorum). +PD's management REST API (Disaster Recovery calls in the README) takes the +release's PD secret as the Basic-auth password: + + PD_SECRET="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.pd.authSecretName" . }} -o jsonpath='{.data.{{ include "hugegraph.pd.authSecretKey" . }}}' | base64 --decode)" + Verify the release: helm test {{ .Release.Name }} --namespace {{ .Release.Namespace }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index eeea380bbd..1fdb879225 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -170,6 +170,61 @@ same signing key. {{- end -}} {{- end }} +{{/* +Resolve the PD REST auth Secret. User-provided pd.auth.existingSecret wins; +otherwise a stable chart-managed name shared by PD, the Server storage wait +and Hubble. +*/}} +{{- define "hugegraph.pd.authSecretName" -}} +{{- $auth := get .Values.pd "auth" | default dict -}} +{{- $existing := get $auth "existingSecret" | default "" -}} +{{- if $existing -}} +{{- $existing -}} +{{- else -}} +{{- printf "%s-pd-auth" (.Release.Name | trunc 55 | trimSuffix "-") -}} +{{- end -}} +{{- end }} + +{{- define "hugegraph.pd.authSecretKey" -}} +{{- $auth := get .Values.pd "auth" | default dict -}} +{{- get $auth "key" | default "secret-key" -}} +{{- end }} + +{{/* +Return the chart-managed PD REST secret (base64). Inline pd.auth.value wins +on first write; otherwise lookup keeps every PD, Server and Hubble Pod, and +every upgrade, on the same secret. +*/}} +{{- define "hugegraph.pd.authSecretValue" -}} +{{- $auth := get .Values.pd "auth" | default dict -}} +{{- $value := get $auth "value" | default "" -}} +{{- if $value -}} +{{- $value | b64enc -}} +{{- else -}} +{{- $name := include "hugegraph.pd.authSecretName" . -}} +{{- $key := include "hugegraph.pd.authSecretKey" . -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace $name -}} +{{- if and $secret (hasKey $secret "data") (hasKey (get $secret "data") $key) -}} +{{- get (get $secret "data") $key -}} +{{- else -}} +{{- randAlphaNum 32 | b64enc -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +Checksum for the PD, Server and Hubble pod templates so rotating the PD REST +Secret rolls the Pods that read it. Same contract as hugegraph.server.authChecksum: +names, key and metadata.resourceVersion only, never Secret data; lookup-based, +so template-only renders emit a constant. +*/}} +{{- define "hugegraph.pd.authChecksum" -}} +{{- $parts := list (include "hugegraph.pd.authSecretName" .) (include "hugegraph.pd.authSecretKey" .) -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.pd.authSecretName" .) -}} +{{- if $secret -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $secret) -}}{{- end -}} +{{- join "|" $parts | sha256sum -}} +{{- end }} + {{/* PD Raft peers list: pod-0.svc.ns.svc:8610,... Uses short headless DNS (cluster.local optional) resolvable inside the namespace. @@ -556,10 +611,10 @@ JAVA_OPTIONS arrives preset in the environment (verified in start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). */}} {{- $reservedEnv := dict - "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") + "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "HG_PD_AUTH_SECRET_KEY" "JAVA_OPTS" "JAVA_OPTIONS") "store" (list "HG_STORE_PD_ADDRESS" "HG_STORE_GRPC_HOST" "HG_STORE_GRPC_PORT" "HG_STORE_REST_PORT" "HG_STORE_RAFT_ADDRESS" "HG_STORE_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") - "server" (list "POD_IP" "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PASSWORD" "HG_SERVER_AUTH_TOKEN_SECRET" "JAVA_OPTS" "JAVA_OPTIONS") - "hubble" (list "HG_HUBBLE_PD_PEERS" "HG_HUBBLE_PD_SERVER" "HG_HUBBLE_STORE_TARGETS" "HG_HUBBLE_SERVER_URL" "SPRING_DATASOURCE_URL") -}} + "server" (list "POD_IP" "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PD_AUTH_PASSWORD" "PASSWORD" "HG_SERVER_AUTH_TOKEN_SECRET" "JAVA_OPTS" "JAVA_OPTIONS") + "hubble" (list "HG_HUBBLE_PD_PEERS" "HG_HUBBLE_PD_SERVER" "HG_HUBBLE_PD_PASSWORD" "HG_HUBBLE_STORE_TARGETS" "HG_HUBBLE_SERVER_URL" "SPRING_DATASOURCE_URL") -}} {{- range $component, $reserved := $reservedEnv -}} {{- $componentValues := get $.Values $component | default dict -}} {{- range $entry := get $componentValues "extraEnv" | default list -}} @@ -586,6 +641,10 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- fail "hubble.ingress.enabled without tls publishes the plain-HTTP, unauthenticated Hubble UI; configure hubble.ingress.tls, or set hubble.ingress.allowPlainHttp=true to accept that on a trusted network" -}} {{- end -}} {{- end -}} +{{- $pdAuth := get .Values.pd "auth" | default dict -}} +{{- if and (not (get $pdAuth "existingSecret" | default "")) (not (get $pdAuth "value" | default "")) (not (get $pdAuth "autoGenerate" | default false)) -}} +{{- fail "pd.auth requires existingSecret, value, or autoGenerate=true: PD images from 1.8.0 refuse to start without a REST secret" -}} +{{- end -}} {{- $auth := get .Values.server "auth" | default dict -}} {{- $admin := get $auth "admin" | default dict -}} {{- $token := get $auth "token" | default dict -}} diff --git a/helm/hugegraph/templates/hubble-deployment.yaml b/helm/hugegraph/templates/hubble-deployment.yaml index b8645cb56f..df462ae1ab 100644 --- a/helm/hugegraph/templates/hubble-deployment.yaml +++ b/helm/hugegraph/templates/hubble-deployment.yaml @@ -47,10 +47,12 @@ spec: {{- include "hugegraph.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: hubble {{- with .Values.hubble.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} - {{- with .Values.hubble.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} - {{- end }} + {{- if $pdMode }} + # Rolls Hubble when the PD REST Secret written into its properties changes. + checksum/pd-auth: {{ include "hugegraph.pd.authChecksum" . | quote }} + {{- end }} + {{- with .Values.hubble.podAnnotations }}{{ toYaml . | nindent 8 }}{{- end }} spec: automountServiceAccountToken: {{ get (get .Values.hubble "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.hubble "name" (include "hugegraph.hubble.name" .) ) }} @@ -113,6 +115,8 @@ spec: {{- if $pdMode }} FOUND_PD_PEERS=false FOUND_PD_SERVER=false + FOUND_PD_USERNAME=false + FOUND_PD_PASSWORD=false FOUND_STORE_TARGETS=false {{- else }} FOUND_DIRECT_URL=false @@ -137,6 +141,14 @@ spec: printf 'pd.server=%s\n' "${HG_HUBBLE_PD_SERVER}" >>"${TMP}" FOUND_PD_SERVER=true ;; + operations.pd.username=*) + printf 'operations.pd.username=hubble\n' >>"${TMP}" + FOUND_PD_USERNAME=true + ;; + operations.pd.password=*) + printf 'operations.pd.password=%s\n' "${HG_HUBBLE_PD_PASSWORD}" >>"${TMP}" + FOUND_PD_PASSWORD=true + ;; operations.store.allowed_targets=*) printf 'operations.store.allowed_targets=%s\n' \ "${HG_HUBBLE_STORE_TARGETS}" >>"${TMP}" @@ -177,6 +189,13 @@ spec: if [[ "${FOUND_PD_SERVER}" == false ]]; then printf 'pd.server=%s\n' "${HG_HUBBLE_PD_SERVER}" >>"${TMP}" fi + # PD REST credential (PD images from 1.8.0 check the password). + if [[ "${FOUND_PD_USERNAME}" == false ]]; then + printf 'operations.pd.username=hubble\n' >>"${TMP}" + fi + if [[ "${FOUND_PD_PASSWORD}" == false ]]; then + printf 'operations.pd.password=%s\n' "${HG_HUBBLE_PD_PASSWORD}" >>"${TMP}" + fi if [[ "${FOUND_STORE_TARGETS}" == false ]]; then printf 'operations.store.allowed_targets=%s\n' \ "${HG_HUBBLE_STORE_TARGETS}" >>"${TMP}" @@ -213,6 +232,11 @@ spec: value: {{ include "hugegraph.pd.grpcPeersList" . | quote }} - name: HG_HUBBLE_PD_SERVER value: {{ include "hugegraph.pd.restClientEndpoint" . | quote }} + - name: HG_HUBBLE_PD_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "hugegraph.pd.authSecretName" . | quote }} + key: {{ include "hugegraph.pd.authSecretKey" . | quote }} - name: HG_HUBBLE_STORE_TARGETS value: {{ include "hugegraph.store.restOriginsList" . | quote }} {{- else }} diff --git a/helm/hugegraph/templates/pd-auth-secret.yaml b/helm/hugegraph/templates/pd-auth-secret.yaml new file mode 100644 index 0000000000..700d17529f --- /dev/null +++ b/helm/hugegraph/templates/pd-auth-secret.yaml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- $auth := get .Values.pd "auth" | default dict -}} +{{- if and (not (get $auth "existingSecret" | default "")) (or (get $auth "value" | default "") (get $auth "autoGenerate" | default false)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "hugegraph.pd.authSecretName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{ include "hugegraph.pd.authSecretKey" . }}: {{ include "hugegraph.pd.authSecretValue" . | quote }} +{{- end }} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index b7575eebe4..8934a88f75 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -44,10 +44,11 @@ spec: {{- include "hugegraph.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: pd {{- with .Values.pd.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} - {{- with .Values.pd.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} - {{- end }} + # Rolls PD pods when the resolved REST auth Secret changes, so rotating + # an existingSecret takes effect without a manual restart. + checksum/pd-auth: {{ include "hugegraph.pd.authChecksum" . | quote }} + {{- with .Values.pd.podAnnotations }}{{ toYaml . | nindent 8 }}{{- end }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -123,6 +124,11 @@ spec: value: {{ .Values.store.replicas | quote }} - name: HG_PD_DATA_PATH value: {{ .Values.pd.dataPath | quote }} + - name: HG_PD_AUTH_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "hugegraph.pd.authSecretName" . | quote }} + key: {{ include "hugegraph.pd.authSecretKey" . | quote }} {{- with .Values.pd.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} {{- with include "hugegraph.javaOptsEnv" (include "hugegraph.pd.effectiveJavaOpts" .) }}{{ . | trim | nindent 12 }}{{- end }} volumeMounts: diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 2c06d8be23..19751e2191 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -54,15 +54,15 @@ spec: {{- include "hugegraph.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: server {{- with .Values.server.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} - {{- if or .Values.server.auth.enabled .Values.server.podAnnotations }} annotations: {{- if .Values.server.auth.enabled }} # Rolls Server pods when the resolved auth Secrets change, so rotating # an existingSecret takes effect without a manual restart. checksum/auth: {{ include "hugegraph.server.authChecksum" . | quote }} {{- end }} + # Rolls Server pods when the PD REST Secret its storage wait uses changes. + checksum/pd-auth: {{ include "hugegraph.pd.authChecksum" . | quote }} {{- with .Values.server.podAnnotations }}{{ toYaml . | nindent 8 }}{{- end }} - {{- end }} spec: automountServiceAccountToken: {{ get (get .Values.server "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.server "name" (include "hugegraph.server.name" .) ) }} @@ -268,6 +268,13 @@ spec: value: {{ include "hugegraph.pd.grpcPeersList" . | quote }} - name: HG_SERVER_PD_REST_ENDPOINT value: {{ include "hugegraph.pd.restPeersList" . | quote }} + # wait-storage.sh authenticates its PD readiness checks as the + # store service user with this password (PD images from 1.8.0). + - name: PD_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "hugegraph.pd.authSecretName" . | quote }} + key: {{ include "hugegraph.pd.authSecretKey" . | quote }} - name: STORE_REST value: {{ include "hugegraph.store.restPrimary" . | quote }} - name: HG_SERVER_INIT_STORE_ENABLED diff --git a/helm/hugegraph/tests/pd_auth_secret_test.yaml b/helm/hugegraph/tests/pd_auth_secret_test.yaml new file mode 100644 index 0000000000..325679cdfa --- /dev/null +++ b/helm/hugegraph/tests/pd_auth_secret_test.yaml @@ -0,0 +1,139 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +suite: PD REST auth Secret wiring +tests: + - it: creates a kept release-pd-auth Secret by default + template: pd-auth-secret.yaml + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: RELEASE-NAME-pd-auth + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - isNotEmpty: + path: data["secret-key"] + + - it: renders no Secret when an operator-supplied Secret is named + template: pd-auth-secret.yaml + set: + pd.auth.existingSecret: my-pd-secret + asserts: + - hasDocuments: + count: 0 + + - it: hands PD the secret from the Secret, never a literal value + template: pd-statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_AUTH_SECRET_KEY + valueFrom: + secretKeyRef: + name: RELEASE-NAME-pd-auth + key: secret-key + - isNotEmpty: + path: spec.template.metadata.annotations["checksum/pd-auth"] + + - it: gives the Server storage wait the same secret as PD_AUTH_PASSWORD + template: server-deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PD_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: RELEASE-NAME-pd-auth + key: secret-key + - isNotEmpty: + path: spec.template.metadata.annotations["checksum/pd-auth"] + + - it: gives Hubble the secret and writes it into the properties file + template: hubble-deployment.yaml + set: + hubble.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_HUBBLE_PD_PASSWORD + valueFrom: + secretKeyRef: + name: RELEASE-NAME-pd-auth + key: secret-key + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "operations\\.pd\\.username=hubble" + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "operations\\.pd\\.password=%s" + - isNotEmpty: + path: spec.template.metadata.annotations["checksum/pd-auth"] + + - it: points PD at the operator-supplied Secret and key + template: pd-statefulset.yaml + set: + pd.auth.existingSecret: my-pd-secret + pd.auth.key: pd-pass + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_PD_AUTH_SECRET_KEY + valueFrom: + secretKeyRef: + name: my-pd-secret + key: pd-pass + + - it: points the Server storage wait at the operator-supplied Secret and key + template: server-deployment.yaml + set: + pd.auth.existingSecret: my-pd-secret + pd.auth.key: pd-pass + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PD_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: my-pd-secret + key: pd-pass + + - it: rejects a configuration with no secret source + template: pd-statefulset.yaml + set: + pd.auth.autoGenerate: false + asserts: + # The values schema refuses this before the template guard can; the + # guard still exists for values that bypass schema validation. + - failedTemplate: + errorPattern: "pd/auth|pd.auth requires" + + - it: rejects operator overrides of the chart-managed PD_AUTH_PASSWORD + template: server-deployment.yaml + set: + server.extraEnv: + - name: PD_AUTH_PASSWORD + value: x + asserts: + - failedTemplate: + errorPattern: "server.extraEnv must not set the chart-managed variable PD_AUTH_PASSWORD" diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index ffc1935054..d99b47f607 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -333,6 +333,63 @@ "type": "string", "minLength": 1 }, + "auth": { + "type": "object", + "additionalProperties": false, + "required": [ + "value", + "existingSecret", + "key", + "autoGenerate" + ], + "description": "PD REST Basic-auth secret (auth.secret-key). Required by PD images from 1.8.0; shared with the Server storage wait and Hubble.", + "properties": { + "value": { + "type": "string", + "pattern": "^[^\\r\\n\\\\]*$", + "description": "Plaintext secret. Empty defers to existingSecret or autoGenerate. No newlines, carriage returns, or backslashes: the value lands in Hubble's Java properties file." + }, + "existingSecret": { + "type": "string" + }, + "key": { + "type": "string", + "minLength": 1 + }, + "autoGenerate": { + "type": "boolean" + } + }, + "anyOf": [ + { + "properties": { + "existingSecret": { + "minLength": 1 + } + } + }, + { + "properties": { + "value": { + "minLength": 1 + }, + "existingSecret": { + "const": "" + } + } + }, + { + "properties": { + "autoGenerate": { + "const": true + }, + "existingSecret": { + "const": "" + } + } + } + ] + }, "storage": { "$ref": "#/definitions/storage" }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index d20be1e7dd..e5ec455a3b 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -66,6 +66,25 @@ pd: rest: 8620 raft: 8610 dataPath: /hugegraph-pd/pd_data + # PD REST Basic-auth credential. PD images from 1.8.0 + # (apache/hugegraph#3189) compare the password of every /v1 request + # against auth.secret-key and refuse to start without one + # (HG_PD_AUTH_SECRET_KEY). The chart hands the same value to the Server + # storage wait (PD_AUTH_PASSWORD, user store) and to Hubble + # (operations.pd.password, user hubble). Older images accept the variable + # and ignore the password, so the wiring is harmless there. The Secret is + # kept on uninstall like the Server auth Secrets. The value must not + # contain newlines, carriage returns, or backslashes: it is written into + # Hubble's Java properties file. Priority: existingSecret > value > + # autoGenerate. + auth: + # Optional plaintext secret (prefer existingSecret in shared clusters). + value: "" + # Pre-created Secret name. Must contain the key below; chart does not manage it. + existingSecret: "" + key: secret-key + # When existingSecret and value are empty, create a kept release-pd-auth Secret. + autoGenerate: true storage: size: 10Gi storageClassName: "" From 2bff77a683532807180d80dd9474c64c78d355a8 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 10 Sep 2026 03:44:59 +0530 Subject: [PATCH 30/61] feat(helm): default the PD paths to /v1/ready, pin the startup budget apache/hugegraph#3185, #3187 and #3189 are merged upstream and due in 1.8.0, so the chart no longer has to wait for them behind a TODO. - pd.readinessPath and store.waitPath default to /v1/ready, which answers 503 until a raft leader exists. PD readiness now means quorum membership and the Store's PD wait counts quorum members rather than live listeners. Startup and liveness stay on /v1/health so a PD that merely lost its leader is not restarted. On a PD image that predates #3185 both values have to be set back to /v1/health. - The Server gets HG_SERVER_STARTUP_TIMEOUT_S from the startup probe budget (effective failureThreshold * periodSeconds, 450s by default, capped at the entrypoint's 86400s). The image default of 120s is shorter than the storage wait alone, so the start command used to kill a Server that was still coming up. The variable joins the reserved list, because a server.extraEnv duplicate would silently decouple the process timeout from the probe. - Drops the two TODO comments the paths carried, and rewrites the README Limitations bullets on PD REST auth and /v1/health vs /v1/ready so the current behaviour leads and the older behaviour is the caveat. lint x3 presets clean, 63 unit tests pass (5 new), renders 16/14/16 with one /v1/ready path and HG_SERVER_STARTUP_TIMEOUT_S=450. Templates, values.schema.json and tests are byte-identical with the helm-dev testing branch. --- helm/hugegraph/README.md | 72 ++++++++++-------- helm/hugegraph/templates/_helpers.tpl | 18 ++++- .../templates/server-deployment.yaml | 5 ++ .../tests/pd_readiness_path_test.yaml | 28 +++---- .../tests/server_startup_timeout_test.yaml | 76 +++++++++++++++++++ helm/hugegraph/values.schema.json | 4 +- helm/hugegraph/values.yaml | 37 +++++---- 7 files changed, 172 insertions(+), 68 deletions(-) create mode 100644 helm/hugegraph/tests/server_startup_timeout_test.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 270ef1a906..f926f5a8cb 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -45,9 +45,10 @@ so operators do not have to: required for distributed HStore and does not make a local RocksDB backend shared across replicas. - **Store waits for PD** in an init container before starting: a majority of - the PD peers must answer `store.waitPath`. The default `/v1/health` proves - each PD's listener is up, not that a raft quorum exists; see Limitations for - the `/v1/ready` switch. + the PD peers must answer `store.waitPath`. The default `/v1/ready` stays 503 + until a raft leader exists, so that majority is a quorum and not merely a set + of live listeners. PD images that predate the endpoint need the value set + back to `/v1/health`; see Limitations. - **One PD REST secret, three readers.** PD images from 1.8.0 ([#3189](https://github.com/apache/hugegraph/pull/3189)) check the Basic-auth password of every management call against `auth.secret-key` and refuse to @@ -57,10 +58,16 @@ so operators do not have to: `operations.pd.password`; a `checksum/pd-auth` annotation rolls all three when the Secret changes. Older images ignore the password, so the wiring is harmless on them. -- **The Server startup probe allows at least 450 seconds.** The image may spend - 300 seconds waiting for storage and a further 120 seconds in the start - command. A lower configured `failureThreshold` is raised to this floor rather - than being rejected. +- **The Server startup probe allows at least 450 seconds, and the image gets + the same budget.** The container may spend 300 seconds waiting for storage + and the rest in the start command, so the chart sets + `HG_SERVER_STARTUP_TIMEOUT_S` to the startup probe's own budget + (`failureThreshold` * `periodSeconds`, 450 seconds by default) rather than + leaving the image's 120-second default, which would self-kill a Server that + was still starting. A lower configured `failureThreshold` is raised to the + 450-second floor rather than being rejected, and raising the probe budget + raises the timeout with it. The variable is chart-managed, so + `server.extraEnv` may not set it; change the probe instead. - **The wrapper writes `auth.admin_pa` from the auth Secret.** With `init_store.enabled=false` the admin credential is created on the PD startup path from `auth.admin_pa`, not from the Docker `PASSWORD` stdin path. When @@ -904,34 +911,35 @@ independently of the release name. exposed to those failure modes; use images built from a source tree that includes the switch. - The PD management REST endpoints (`/v1/members`, `/v1/stores`, - `/v1/task/*`) authenticate on service name only on PD images up to 1.7.0. - Those images compare the Basic-auth username against a fixed internal set - (`hg`, `store`, `hubble`, `vermeer`) and do not validate the password at - all, so any password, including an empty one, is accepted for those names - while every other name is refused, and all three outcomes, success, - refusal, and a missing credential, return HTTP 200 with the result in the - body. Treat these endpoints as unauthenticated on such images: keep the PD - client Service on ClusterIP and do not expose it, and do not key a health - check on the status code. PD images from 1.8.0 - ([#3189](https://github.com/apache/hugegraph/pull/3189)) check the password - against `auth.secret-key` and answer 401 on refusal; the chart supplies that - secret through `pd.auth` (see Chart Details), and the Disaster Recovery - calls above need it. + `/v1/task/*`) check the Basic-auth password against `auth.secret-key` and + answer 401 on refusal since + [#3189](https://github.com/apache/hugegraph/pull/3189), which is merged + upstream and due in 1.8.0. The chart supplies that secret through `pd.auth` + (see Chart Details), and the Disaster Recovery calls above need it. The + limitation is the older behaviour: PD images before that fix authenticate on + service name only. They compare the username against a fixed internal set + (`hg`, `store`, `hubble`, `vermeer`) and never look at the password, so any + password, including an empty one, is accepted for those names while every + other name is refused, and all three outcomes (success, refusal, and a + missing credential) return HTTP 200 with the result in the body. Treat these + endpoints as unauthenticated on such an image: keep the PD client Service on + ClusterIP and do not expose it, and do not key a health check on the status + code. - PD's `/v1/health` is liveness only: it answers 200 as soon as the REST listener is up and never consults raft. Measured on a 3-PD install with two PDs deleted, the survivor logged `Raft lost leader` within a second and kept - answering 200 while quorum-dependent calls failed. Every PD and Store probe - in this chart, and the Store init container's PD wait, key on that endpoint, - so a listening-but-leaderless PD passes readiness and the wait counts - listeners rather than quorum members. Tracked upstream as - [#3183](https://github.com/apache/hugegraph/issues/3183); the fix, - [#3185](https://github.com/apache/hugegraph/pull/3185), adds an - unauthenticated `/v1/ready` that answers 503 without a raft leader, plus - raft gauges, from 1.8.0. On such an image set `pd.readinessPath=/v1/ready` - and `store.waitPath=/v1/ready`; keep startup and liveness on `/v1/health` - so a PD that merely lost its leader is not restarted. Do not set either - path on an older image: it does not exist there, the PD never turns Ready - and Stores never leave Init. + answering 200 while quorum-dependent calls failed + ([#3183](https://github.com/apache/hugegraph/issues/3183)). The fix, + [#3185](https://github.com/apache/hugegraph/pull/3185), is merged upstream + and due in 1.8.0: an unauthenticated `/v1/ready` that answers 503 without a + raft leader, plus raft gauges. The chart defaults `pd.readinessPath` and + `store.waitPath` to `/v1/ready` accordingly, and keeps PD startup and + liveness on `/v1/health` so a PD that merely lost its leader is not + restarted. The limitation is what happens on a PD image that predates the + fix: `/v1/ready` does not exist there, so the PD never turns Ready and + Stores never leave Init. Set both values back to `/v1/health` on such an + image, and accept that readiness then passes for a leaderless PD and the + Store wait counts listeners rather than quorum members. - Server discovery is a lease. Each Server re-registers its Pod IP with PD every 15 seconds and PD drops an entry after three missed heartbeats, so a replaced or evicted Server can stay in PD's list for up to 45 seconds after diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 1fdb879225..5f43ef987f 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -471,6 +471,22 @@ values remain accepted, but their rendered threshold is raised to this floor. {{- max $configured $minimum -}} {{- end }} +{{/* +Seconds the chart gives the Server image to finish starting, passed as +HG_SERVER_STARTUP_TIMEOUT_S. The image defaults that to 120 seconds, which is +shorter than the storage wait alone, so a Server still coming up kills itself +before Kubernetes has given up on it. The value therefore tracks the startup +probe: the effective failureThreshold above, already floored at 450 seconds, +times periodSeconds. Raising the probe budget raises this with it. The +entrypoint rejects anything over 86400, so the product is capped there rather +than rendered into a Pod that refuses to start. +*/}} +{{- define "hugegraph.server.startupTimeoutSeconds" -}} +{{- $period := int .Values.server.probes.startup.periodSeconds -}} +{{- $threshold := include "hugegraph.server.startupFailureThreshold" . | int -}} +{{- min 86400 (mul $threshold $period) -}} +{{- end }} + {{/* Optional probe tunables, emitted only when explicitly set. Kubernetes defaults timeoutSeconds to 1 second, which a garbage-collection pause can exceed on a @@ -613,7 +629,7 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- $reservedEnv := dict "pd" (list "HG_PD_GRPC_HOST" "HG_PD_GRPC_PORT" "HG_PD_REST_PORT" "HG_PD_RAFT_ADDRESS" "HG_PD_RAFT_PEERS_LIST" "HG_PD_INITIAL_STORE_LIST" "HG_PD_INITIAL_STORE_COUNT" "HG_PD_DATA_PATH" "HG_PD_AUTH_SECRET_KEY" "JAVA_OPTS" "JAVA_OPTIONS") "store" (list "HG_STORE_PD_ADDRESS" "HG_STORE_GRPC_HOST" "HG_STORE_GRPC_PORT" "HG_STORE_REST_PORT" "HG_STORE_RAFT_ADDRESS" "HG_STORE_DATA_PATH" "JAVA_OPTS" "JAVA_OPTIONS") - "server" (list "POD_IP" "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "PD_AUTH_PASSWORD" "PASSWORD" "HG_SERVER_AUTH_TOKEN_SECRET" "JAVA_OPTS" "JAVA_OPTIONS") + "server" (list "POD_IP" "HG_SERVER_BACKEND" "HG_SERVER_PD_PEERS" "HG_SERVER_PD_REST_ENDPOINT" "STORE_REST" "HG_SERVER_INIT_STORE_ENABLED" "HG_SERVER_URLS_TO_PD" "HG_SERVER_STARTUP_TIMEOUT_S" "PD_AUTH_PASSWORD" "PASSWORD" "HG_SERVER_AUTH_TOKEN_SECRET" "JAVA_OPTS" "JAVA_OPTIONS") "hubble" (list "HG_HUBBLE_PD_PEERS" "HG_HUBBLE_PD_SERVER" "HG_HUBBLE_PD_PASSWORD" "HG_HUBBLE_STORE_TARGETS" "HG_HUBBLE_SERVER_URL" "SPRING_DATASOURCE_URL") -}} {{- range $component, $reserved := $reservedEnv -}} {{- $componentValues := get $.Values $component | default dict -}} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 19751e2191..7bc1ef23c3 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -279,6 +279,11 @@ spec: value: {{ include "hugegraph.store.restPrimary" . | quote }} - name: HG_SERVER_INIT_STORE_ENABLED value: {{ .Values.server.initStoreEnabled | quote }} + # The image would otherwise give the start command 120 seconds, + # less than the storage wait, and kill a Server that is still + # coming up. Track the startup probe's own budget instead. + - name: HG_SERVER_STARTUP_TIMEOUT_S + value: {{ include "hugegraph.server.startupTimeoutSeconds" . | quote }} {{- if $pdMeta }} - name: HG_SERVER_URLS_TO_PD value: {{ include "hugegraph.server.urlsToPd" . | quote }} diff --git a/helm/hugegraph/tests/pd_readiness_path_test.yaml b/helm/hugegraph/tests/pd_readiness_path_test.yaml index 6042dec259..3cd9ac9be3 100644 --- a/helm/hugegraph/tests/pd_readiness_path_test.yaml +++ b/helm/hugegraph/tests/pd_readiness_path_test.yaml @@ -17,27 +17,27 @@ suite: PD readiness path and Store wait path tests: - - it: keeps every PD probe on /v1/health by default + - it: reads PD readiness from /v1/ready by default, startup and liveness from /v1/health template: pd-statefulset.yaml asserts: - - equal: - path: spec.template.spec.containers[0].startupProbe.httpGet.path - value: /v1/health - equal: path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /v1/ready + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path value: /v1/health - equal: path: spec.template.spec.containers[0].livenessProbe.httpGet.path value: /v1/health - - it: moves only the PD readiness probe when pd.readinessPath is set + - it: moves only the PD readiness probe when pd.readinessPath is set back for an older image template: pd-statefulset.yaml set: - pd.readinessPath: /v1/ready + pd.readinessPath: /v1/health asserts: - equal: path: spec.template.spec.containers[0].readinessProbe.httpGet.path - value: /v1/ready + value: /v1/health - equal: path: spec.template.spec.containers[0].startupProbe.httpGet.path value: /v1/health @@ -45,29 +45,29 @@ tests: path: spec.template.spec.containers[0].livenessProbe.httpGet.path value: /v1/health - - it: polls /v1/health on each PD peer in the Store wait by default + - it: polls /v1/ready on each PD peer in the Store wait by default template: store-statefulset.yaml asserts: - matchRegex: path: spec.template.spec.initContainers[0].command[2] - pattern: 'WAIT_PATH="/v1/health"' + pattern: 'WAIT_PATH="/v1/ready"' - matchRegex: path: spec.template.spec.initContainers[0].command[2] pattern: 'curl -fsS "http://\${peer}\${WAIT_PATH}"' - - it: polls store.waitPath on each PD peer when set + - it: polls store.waitPath on each PD peer when set back for an older image template: store-statefulset.yaml set: - store.waitPath: /v1/ready + store.waitPath: /v1/health asserts: - matchRegex: path: spec.template.spec.initContainers[0].command[2] - pattern: 'WAIT_PATH="/v1/ready"' + pattern: 'WAIT_PATH="/v1/health"' - notMatchRegex: path: spec.template.spec.initContainers[0].command[2] - pattern: 'v1/health' + pattern: 'v1/ready' - - it: leaves the Store's own probes on /v1/health when store.waitPath is set + - it: leaves the Store's own probes on /v1/health whatever store.waitPath is template: store-statefulset.yaml set: store.waitPath: /v1/ready diff --git a/helm/hugegraph/tests/server_startup_timeout_test.yaml b/helm/hugegraph/tests/server_startup_timeout_test.yaml new file mode 100644 index 0000000000..d47d5567e1 --- /dev/null +++ b/helm/hugegraph/tests/server_startup_timeout_test.yaml @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Server startup timeout tracks the startup probe budget +templates: + - server-deployment.yaml +tests: + - it: passes the default 450 second budget to the image + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_STARTUP_TIMEOUT_S + value: "450" + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 90 + - equal: + path: spec.template.spec.containers[0].startupProbe.periodSeconds + value: 5 + + - it: follows a raised startup probe budget + set: + server.probes.startup.failureThreshold: 200 + server.probes.startup.periodSeconds: 10 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_STARTUP_TIMEOUT_S + value: "2000" + + - it: follows the 450 second floor when a lower failureThreshold is configured + set: + server.probes.startup.failureThreshold: 1 + server.probes.startup.periodSeconds: 5 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_STARTUP_TIMEOUT_S + value: "450" + + - it: caps at the entrypoint's 86400 second maximum + set: + server.probes.startup.failureThreshold: 100000 + server.probes.startup.periodSeconds: 10 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_STARTUP_TIMEOUT_S + value: "86400" + + - it: refuses an extraEnv override that would silently diverge from the probe + set: + server.extraEnv: + - name: HG_SERVER_STARTUP_TIMEOUT_S + value: "60" + asserts: + - failedTemplate: + errorMessage: server.extraEnv must not set the chart-managed variable HG_SERVER_STARTUP_TIMEOUT_S diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index d99b47f607..b960a4dd76 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -497,7 +497,7 @@ "readinessPath": { "type": "string", "pattern": "^/", - "description": "HTTP path for the PD readinessProbe; /v1/health (default) or /v1/ready on PD images that have it" + "description": "HTTP path for the PD readinessProbe; /v1/ready (default) on PD images that serve it, /v1/health on older ones" } } }, @@ -626,7 +626,7 @@ "waitPath": { "type": "string", "pattern": "^/", - "description": "HTTP path the Store init container polls on each PD peer; /v1/health (default) or /v1/ready on PD images that have it" + "description": "HTTP path the Store init container polls on each PD peer; /v1/ready (default) on PD images that serve it, /v1/health on older ones" } } }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index e5ec455a3b..9e3757204e 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -150,17 +150,13 @@ pd: whenDeleted: Retain whenScaled: Retain # Startup can take a while during Raft bootstrap - # HTTP path the PD readinessProbe hits. /v1/health answers 200 as soon as - # the REST listener is up and never consults raft, so a PD outside the - # quorum still passes readiness. PD images from 1.8.0 on add /v1/ready - # (apache/hugegraph#3185), which answers 503 without a raft leader; switch - # this to /v1/ready only on such an image, an older image has no such - # path and the PD never turns Ready. Startup and liveness stay on - # /v1/health regardless, so a PD that merely lost its leader is not - # restarted. - # TODO(apache/hugegraph#3185): change the default to /v1/ready in the same - # commit that pins the image tag to 1.8.0 or newer. - readinessPath: /v1/health + # HTTP path the PD readinessProbe hits. /v1/ready (apache/hugegraph#3185, + # merged upstream, due in 1.8.0) answers 503 while there is no raft leader, + # so a PD outside the quorum stops passing readiness. Images that predate + # the fix do not serve that path and never turn Ready on it: set this back + # to /v1/health on one of those. Startup and liveness stay on /v1/health + # regardless, so a PD that merely lost its leader is not restarted. + readinessPath: /v1/ready probes: startup: failureThreshold: 30 @@ -232,13 +228,12 @@ store: minAvailable: 2 waitImage: curlimages/curl:8.5.0 # HTTP path the init container polls on every PD peer; a majority must - # answer 2xx before the Store starts. /v1/health proves the listener is - # up, not a raft quorum. On PD images that carry /v1/ready (1.8.0 on, - # apache/hugegraph#3185) set /v1/ready so the count means quorum members; - # on older images that path does not exist and Stores never leave Init. - # TODO(apache/hugegraph#3185): change the default to /v1/ready in the same - # commit that pins the image tag to 1.8.0 or newer. - waitPath: /v1/health + # answer 2xx before the Store starts. /v1/ready (apache/hugegraph#3185, + # merged upstream, due in 1.8.0) makes that majority a raft quorum instead + # of a set of live listeners. Images that predate the fix do not serve the + # path and their Stores never leave Init: set this back to /v1/health on + # one of those. + waitPath: /v1/ready # Bound the PD wait so a cluster whose PDs never come up fails visibly # instead of sitting in Init:0/1 forever. waitTimeoutSeconds: 900 @@ -388,7 +383,11 @@ server: targetCPUUtilizationPercentage: 70 probes: startup: - # 450s covers the 300s storage wait and Server process startup. + # 450s covers the 300s storage wait and Server process startup. The + # chart passes the same budget (failureThreshold * periodSeconds) to + # the image as HG_SERVER_STARTUP_TIMEOUT_S, so the start command and + # the probe give up together instead of the command self-killing at + # the image default of 120s. failureThreshold: 90 periodSeconds: 5 timeoutSeconds: 5 From b94ff5d1e06893f74a243bb4b7575ef22110a84e Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 10 Sep 2026 09:14:59 +0530 Subject: [PATCH 31/61] docs(helm): fix the readiness path defaults in NOTES and README Chart 0.1.6 moved pd.readinessPath and store.waitPath to /v1/ready and updated most of the surrounding prose, but three places were left describing the old default: templates/NOTES.txt, which is printed to the operator on every install and every upgrade, and the Configuration rows for both values, where the Default column still read /v1/health. The two rows also gave the wrong reason for setting /v1/health on an older image. They said /v1/ready "exists only on PD images from 1.8.0", which reads as though an older image would simply reject it. It does not. On a PD image predating apache/hugegraph#3189 the auth interceptor writes its error body without setting a status, so every unmapped path under /v1/ answers 200, and /v1/ready is such a path. The readiness probe then passes unconditionally and the signal means nothing. Measured on a PD image built from bed2e457, the parent of the #3185 merge, so it carries #3187 but neither #3185 nor #3189: all three PD pods reported Ready, /v1/ready returned 200 with an Unauthorized body, and an arbitrary nonexistent path returned the same 200. On merged master both return 401. No template behaviour changes. Renders stay 16/14/16 and the 63 unit tests pass. --- helm/hugegraph/README.md | 4 ++-- helm/hugegraph/templates/NOTES.txt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index f926f5a8cb..d6abe32075 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -303,7 +303,7 @@ default values. | `pd.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | -| `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/health` is liveness only; set `/v1/ready` on PD images from 1.8.0 that carry it, never on older images | `/v1/health` | +| `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader. Set `/v1/health` on PD images that predate apache/hugegraph#3189: there every unmapped `/v1/` path answers 200, so `/v1/ready` passes unconditionally and readiness means nothing | `/v1/ready` | | `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. No newlines, carriage returns, or backslashes | `""` | | `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it | `""` | | `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | @@ -333,7 +333,7 @@ default values. | `store.resources` | Store container resources. Set these for production | `{}` | | `store.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | | `store.securityContext` | Container-level securityContext; also applied to the PD wait init container. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | -| `store.waitPath` | Path the init container polls on each PD peer; a majority must answer 2xx. `/v1/health` counts listeners; `/v1/ready` counts quorum members but exists only on PD images from 1.8.0 | `/v1/health` | +| `store.waitPath` | Path the init container polls on each PD peer; a majority must answer 2xx. `/v1/ready` counts quorum members. Set `/v1/health`, which counts listeners only, on PD images that predate apache/hugegraph#3189, for the reason given under `pd.readinessPath` | `/v1/ready` | | `store.waitTimeoutSeconds` | Bound on the PD wait before the init container fails | `900` | | `store.antiAffinity` | One of `required`, `preferred`, `disabled`. `preferred` schedules on clusters with fewer nodes than replicas; production should use `required` so one node failure cannot co-locate shard replicas | `preferred` | | `store.nodeSelector` | Node selector for store Pods | `{}` | diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 5c26da04cd..eb153ff86a 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -29,7 +29,8 @@ Watch the cluster come up: kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -w Store Pods stay in Init until a majority of PD peers answer store.waitPath -(/v1/health by default, which proves the listener is up, not a raft quorum). +(/v1/ready by default, which stays 503 until a raft leader exists, so the +majority counted is a quorum and not merely a set of live listeners). PD's management REST API (Disaster Recovery calls in the README) takes the release's PD secret as the Basic-auth password: From 239e13ae0f2c9d170bba61a65433985b11a74349 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 12 Sep 2026 19:16:11 +0530 Subject: [PATCH 32/61] docs(helm): drop the pre-release image caveats from the chart docs The chart targets images that carry the PD REST password check, the /v1/ready endpoint, and the raft allowlist switch; on anything older the install does not get far enough for the fallback guidance to matter. Remove the old-image caveats and the release-version promises from the README, values comments, schema description, and two template comments, and state the current behavior as plain fact. Image tags and appVersion still track latest until a release tag exists to pin. --- helm/hugegraph/README.md | 100 +++++------------- .../templates/hubble-deployment.yaml | 2 +- .../templates/server-deployment.yaml | 2 +- helm/hugegraph/values.schema.json | 2 +- helm/hugegraph/values.yaml | 51 ++++----- 5 files changed, 52 insertions(+), 105 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index d6abe32075..35488080a5 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -47,17 +47,14 @@ so operators do not have to: - **Store waits for PD** in an init container before starting: a majority of the PD peers must answer `store.waitPath`. The default `/v1/ready` stays 503 until a raft leader exists, so that majority is a quorum and not merely a set - of live listeners. PD images that predate the endpoint need the value set - back to `/v1/health`; see Limitations. -- **One PD REST secret, three readers.** PD images from 1.8.0 - ([#3189](https://github.com/apache/hugegraph/pull/3189)) check the Basic-auth - password of every management call against `auth.secret-key` and refuse to - start without one. The chart keeps that value in a release-pd-auth Secret + of live listeners. +- **One PD REST secret, three readers.** PD checks the Basic-auth password of + every management call against `auth.secret-key` and refuses to start + without one. The chart keeps that value in a release-pd-auth Secret (or `pd.auth.existingSecret`) and hands it to PD as `HG_PD_AUTH_SECRET_KEY`, to the Server storage wait as `PD_AUTH_PASSWORD`, and to Hubble as `operations.pd.password`; a `checksum/pd-auth` annotation rolls all three - when the Secret changes. Older images ignore the password, so the wiring is - harmless on them. + when the Secret changes. - **The Server startup probe allows at least 450 seconds, and the image gets the same budget.** The container may spend 300 seconds waiting for storage and the rest in the start command, so the chart sets @@ -148,12 +145,9 @@ A fresh install seeds PD with a partition shard count of 3 when default of 1. The seed applies at first bootstrap only; see Partition Sharding below. -This chart is at an early 0.1.x version. While the contribution is a draft, its -component image tags and `appVersion` track `latest` with pull policy `Always`. -Before stable publication, pin all four component tags (PD, Store, Server and -Hubble) and `appVersion` to the -next HugeGraph release and switch the component pull policies to -`IfNotPresent`. +The component image tags and `appVersion` track `latest` until the next +HugeGraph release tag is published. For production, pin the image tags (or +digests) and switch the component pull policies to `IfNotPresent`. Verify the release: @@ -184,12 +178,7 @@ are worth knowing about in advance: - **PD** restarts one pod at a time whenever its Pod template changes, which includes adopting the `-Draft.ip-whitelist.enabled=false` setting described - under Limitations. With a PD image that carries the upstream whitelist - switch this roll is uneventful. On an older image the whitelist stays - active, and a restarted PD returning on a new Pod IP may be rejected by - peers holding stale allowlists; if PDs log `Blocked connection` after a - roll, delete all PD pods at once so they cold-start together and - re-resolve. For a maintenance-window upgrade, set + under Limitations. For a maintenance-window upgrade, set `pd.updateStrategy.type=OnDelete` and restart the pods yourself. - **Server** rolls once on the first `helm upgrade` after a fresh install, when the `checksum/auth` annotation first observes the install-created @@ -271,7 +260,7 @@ default values. |---|---|---| | `pd.replicas` | PD StatefulSet replicas. Maximum `99` | `3` | | `pd.image.repository` | PD image repository | `hugegraph/pd` | -| `pd.image.tag` | PD image tag. Tracks the development image until the next release is pinned | `latest` | +| `pd.image.tag` | PD image tag; pin it (or a digest) for production | `latest` | | `pd.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `pd.image.pullPolicy` | PD image pull policy | `Always` | | `pd.javaOpts` | Extra JVM flags, rendered after the chart-derived `-D` properties below so an explicit duplicate here wins. The image's automatic heap sizing is preserved unless heap flags are set | `""` | @@ -303,7 +292,7 @@ default values. | `pd.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | -| `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader. Set `/v1/health` on PD images that predate apache/hugegraph#3189: there every unmapped `/v1/` path answers 200, so `/v1/ready` passes unconditionally and readiness means nothing | `/v1/ready` | +| `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader | `/v1/ready` | | `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. No newlines, carriage returns, or backslashes | `""` | | `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it | `""` | | `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | @@ -320,7 +309,7 @@ default values. |---|---|---| | `store.replicas` | Store StatefulSet replicas. Maximum `99` | `3` | | `store.image.repository` | Store image repository | `hugegraph/store` | -| `store.image.tag` | Store image tag. Tracks the development image until the next release is pinned | `latest` | +| `store.image.tag` | Store image tag; pin it (or a digest) for production | `latest` | | `store.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `store.image.pullPolicy` | Store image pull policy | `Always` | | `store.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | @@ -333,7 +322,7 @@ default values. | `store.resources` | Store container resources. Set these for production | `{}` | | `store.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | | `store.securityContext` | Container-level securityContext; also applied to the PD wait init container. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | -| `store.waitPath` | Path the init container polls on each PD peer; a majority must answer 2xx. `/v1/ready` counts quorum members. Set `/v1/health`, which counts listeners only, on PD images that predate apache/hugegraph#3189, for the reason given under `pd.readinessPath` | `/v1/ready` | +| `store.waitPath` | Path the init container polls on each PD peer; a majority must answer 2xx. `/v1/ready` counts quorum members, not merely live listeners | `/v1/ready` | | `store.waitTimeoutSeconds` | Bound on the PD wait before the init container fails | `900` | | `store.antiAffinity` | One of `required`, `preferred`, `disabled`. `preferred` schedules on clusters with fewer nodes than replicas; production should use `required` so one node failure cannot co-locate shard replicas | `preferred` | | `store.nodeSelector` | Node selector for store Pods | `{}` | @@ -361,7 +350,7 @@ default values. |---|---|---| | `server.replicas` | Server Deployment replicas. Ignored when `server.hpa.enabled` | `3` | | `server.image.repository` | Server image repository | `hugegraph/server` | -| `server.image.tag` | Server image tag. Tracks the development image until the next release is pinned | `latest` | +| `server.image.tag` | Server image tag; pin it (or a digest) for production | `latest` | | `server.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `server.image.pullPolicy` | Server image pull policy | `Always` | | `server.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | @@ -573,7 +562,7 @@ trusted network. | `hubble.mode` | `pd` discovers the cluster through PD and enables the operations view; `direct` talks to the Server client Service only | `pd` | | `hubble.allowWithoutServerAuth` | Renders Hubble without `server.auth`, for future images whose login does not require cluster authentication | `false` | | `hubble.image.repository` | Hubble image repository | `hugegraph/hubble` | -| `hubble.image.tag` | Hubble image tag. Tracks the development image until the next release is pinned | `latest` | +| `hubble.image.tag` | Hubble image tag; pin it (or a digest) for production | `latest` | | `hubble.image.digest` | Optional immutable digest such as `sha256:...`; when set it takes priority over the tag | `""` | | `hubble.image.pullPolicy` | Hubble image pull policy | `Always` | | `hubble.port` | Hubble HTTP port, container port, and Service port | `8088` | @@ -767,12 +756,9 @@ curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balanceLeaders # spr curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data ``` -The credential is required. PD images from 1.8.0 answer 401 without it; older -images answer HTTP 200 with an `Unauthorized` body and the task does not run, -so a recovery attempt looks successful while doing nothing. On those older -images the password is not checked, so `-u hg:` alone also works. The Secret -name follows the release (`-pd-auth`) unless `pd.auth.existingSecret` -is set. See Limitations. +The credential is required; PD answers 401 without it. The Secret name +follows the release (`-pd-auth`) unless `pd.auth.existingSecret` +is set. Run `patrolPartitions` after replacing a Store that is not coming back, `balancePartitions` once the cluster is stable again, and `balanceLeaders` @@ -836,12 +822,10 @@ curl -s --user "admin:${PASSWORD}" http://127.0.0.1:8080/graphs ### Queries Fail with "Could not rebind" Right After Creating a Graph -**Update (2026-08-12):** [#3138](https://github.com/apache/hugegraph/pull/3138) -merged on `master` and closes Phase 1 of -[#3137](https://github.com/apache/hugegraph/issues/3137). The Server that -handles `CreateGraph` now waits for its own Gremlin binding before returning -HTTP 200, so create-then-query on the **same** Server (or sticky routing to -that Pod) is reliable. +The Server that handles `CreateGraph` waits for its own Gremlin binding +before returning HTTP 200 +([#3138](https://github.com/apache/hugegraph/pull/3138)), so create-then-query +on the **same** Server (or sticky routing to that Pod) is reliable. Other Server replicas still converge independently through a PD metadata watch plus a local graph open. Until they finish, a Gremlin query routed @@ -906,40 +890,12 @@ independently of the release name. peer authentication to Kubernetes-level controls. Setting `pd.raftIpWhitelistEnabled=true` restores the image default along with its one-shot resolution semantics (bring-up races and pod-IP-change - rejections included) at the operator's own risk. PD images that predate - the switch ignore the flag and keep the whitelist active, so they remain - exposed to those failure modes; use images built from a source tree that - includes the switch. -- The PD management REST endpoints (`/v1/members`, `/v1/stores`, - `/v1/task/*`) check the Basic-auth password against `auth.secret-key` and - answer 401 on refusal since - [#3189](https://github.com/apache/hugegraph/pull/3189), which is merged - upstream and due in 1.8.0. The chart supplies that secret through `pd.auth` - (see Chart Details), and the Disaster Recovery calls above need it. The - limitation is the older behaviour: PD images before that fix authenticate on - service name only. They compare the username against a fixed internal set - (`hg`, `store`, `hubble`, `vermeer`) and never look at the password, so any - password, including an empty one, is accepted for those names while every - other name is refused, and all three outcomes (success, refusal, and a - missing credential) return HTTP 200 with the result in the body. Treat these - endpoints as unauthenticated on such an image: keep the PD client Service on - ClusterIP and do not expose it, and do not key a health check on the status - code. -- PD's `/v1/health` is liveness only: it answers 200 as soon as the REST - listener is up and never consults raft. Measured on a 3-PD install with two - PDs deleted, the survivor logged `Raft lost leader` within a second and kept - answering 200 while quorum-dependent calls failed - ([#3183](https://github.com/apache/hugegraph/issues/3183)). The fix, - [#3185](https://github.com/apache/hugegraph/pull/3185), is merged upstream - and due in 1.8.0: an unauthenticated `/v1/ready` that answers 503 without a - raft leader, plus raft gauges. The chart defaults `pd.readinessPath` and - `store.waitPath` to `/v1/ready` accordingly, and keeps PD startup and - liveness on `/v1/health` so a PD that merely lost its leader is not - restarted. The limitation is what happens on a PD image that predates the - fix: `/v1/ready` does not exist there, so the PD never turns Ready and - Stores never leave Init. Set both values back to `/v1/health` on such an - image, and accept that readiness then passes for a leaderless PD and the - Store wait counts listeners rather than quorum members. + rejections included) at the operator's own risk. +- PD's `/v1/health` answers 200 as soon as the REST listener is up and never + consults raft, so it cannot see a lost quorum. The chart therefore uses it + only for PD startup and liveness (a PD that merely lost its leader is not + restarted) and puts readiness and the Store wait on `/v1/ready`, which + answers 503 without a raft leader. - Server discovery is a lease. Each Server re-registers its Pod IP with PD every 15 seconds and PD drops an entry after three missed heartbeats, so a replaced or evicted Server can stay in PD's list for up to 45 seconds after diff --git a/helm/hugegraph/templates/hubble-deployment.yaml b/helm/hugegraph/templates/hubble-deployment.yaml index df462ae1ab..67f5d6d405 100644 --- a/helm/hugegraph/templates/hubble-deployment.yaml +++ b/helm/hugegraph/templates/hubble-deployment.yaml @@ -189,7 +189,7 @@ spec: if [[ "${FOUND_PD_SERVER}" == false ]]; then printf 'pd.server=%s\n' "${HG_HUBBLE_PD_SERVER}" >>"${TMP}" fi - # PD REST credential (PD images from 1.8.0 check the password). + # PD REST credential for the operations API. if [[ "${FOUND_PD_USERNAME}" == false ]]; then printf 'operations.pd.username=hubble\n' >>"${TMP}" fi diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 7bc1ef23c3..c075cf0291 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -269,7 +269,7 @@ spec: - name: HG_SERVER_PD_REST_ENDPOINT value: {{ include "hugegraph.pd.restPeersList" . | quote }} # wait-storage.sh authenticates its PD readiness checks as the - # store service user with this password (PD images from 1.8.0). + # store service user with this password. - name: PD_AUTH_PASSWORD valueFrom: secretKeyRef: diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index b960a4dd76..3c7253060d 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -342,7 +342,7 @@ "key", "autoGenerate" ], - "description": "PD REST Basic-auth secret (auth.secret-key). Required by PD images from 1.8.0; shared with the Server storage wait and Hubble.", + "description": "PD REST Basic-auth secret (auth.secret-key), shared with the Server storage wait and Hubble.", "properties": { "value": { "type": "string", diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 9e3757204e..98609e7065 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -27,8 +27,8 @@ pd: replicas: 3 image: repository: hugegraph/pd - # The draft tracks latest until the next HugeGraph release tag is available. - # Pin the release tag and switch to IfNotPresent before stable publication. + # Tracks latest until the next HugeGraph release tag is published. Pin the + # tag (or digest) and switch to IfNotPresent for production. tag: latest # Optional immutable digest, for example sha256:abc... When set it wins over # tag and the image is pulled by digest, which is what a release gate should @@ -66,17 +66,14 @@ pd: rest: 8620 raft: 8610 dataPath: /hugegraph-pd/pd_data - # PD REST Basic-auth credential. PD images from 1.8.0 - # (apache/hugegraph#3189) compare the password of every /v1 request - # against auth.secret-key and refuse to start without one + # PD REST Basic-auth credential. PD compares the password of every /v1 + # request against auth.secret-key and refuses to start without one # (HG_PD_AUTH_SECRET_KEY). The chart hands the same value to the Server # storage wait (PD_AUTH_PASSWORD, user store) and to Hubble - # (operations.pd.password, user hubble). Older images accept the variable - # and ignore the password, so the wiring is harmless there. The Secret is - # kept on uninstall like the Server auth Secrets. The value must not - # contain newlines, carriage returns, or backslashes: it is written into - # Hubble's Java properties file. Priority: existingSecret > value > - # autoGenerate. + # (operations.pd.password, user hubble). The Secret is kept on uninstall + # like the Server auth Secrets. The value must not contain newlines, + # carriage returns, or backslashes: it is written into Hubble's Java + # properties file. Priority: existingSecret > value > autoGenerate. auth: # Optional plaintext secret (prefer existingSecret in shared clusters). value: "" @@ -136,8 +133,7 @@ pd: # Kubernetes blocks peers whose pod IPs were not yet published or change # later. Off by default in-cluster (upstream switch # raft.ip-whitelist.enabled); k8s network policy/auth owns that layer. - # Ignored by images that predate the switch. Set true to restore the - # image default. + # Set true to restore the image default. raftIpWhitelistEnabled: false # Explicit rollout strategy instead of the implicit StatefulSet default. updateStrategy: @@ -150,12 +146,10 @@ pd: whenDeleted: Retain whenScaled: Retain # Startup can take a while during Raft bootstrap - # HTTP path the PD readinessProbe hits. /v1/ready (apache/hugegraph#3185, - # merged upstream, due in 1.8.0) answers 503 while there is no raft leader, - # so a PD outside the quorum stops passing readiness. Images that predate - # the fix do not serve that path and never turn Ready on it: set this back - # to /v1/health on one of those. Startup and liveness stay on /v1/health - # regardless, so a PD that merely lost its leader is not restarted. + # HTTP path the PD readinessProbe hits. /v1/ready answers 503 while there + # is no raft leader, so a PD outside the quorum stops passing readiness. + # Startup and liveness stay on /v1/health so a PD that merely lost its + # leader is not restarted. readinessPath: /v1/ready probes: startup: @@ -175,8 +169,8 @@ store: replicas: 3 image: repository: hugegraph/store - # The draft tracks latest until the next HugeGraph release tag is available. - # Pin the release tag and switch to IfNotPresent before stable publication. + # Tracks latest until the next HugeGraph release tag is published. Pin the + # tag (or digest) and switch to IfNotPresent for production. tag: latest # Optional immutable digest, for example sha256:abc... When set it wins over # tag and the image is pulled by digest, which is what a release gate should @@ -228,11 +222,8 @@ store: minAvailable: 2 waitImage: curlimages/curl:8.5.0 # HTTP path the init container polls on every PD peer; a majority must - # answer 2xx before the Store starts. /v1/ready (apache/hugegraph#3185, - # merged upstream, due in 1.8.0) makes that majority a raft quorum instead - # of a set of live listeners. Images that predate the fix do not serve the - # path and their Stores never leave Init: set this back to /v1/health on - # one of those. + # answer 2xx before the Store starts. /v1/ready makes that majority a + # raft quorum instead of a set of live listeners. waitPath: /v1/ready # Bound the PD wait so a cluster whose PDs never come up fails visibly # instead of sitting in Init:0/1 forever. @@ -265,8 +256,8 @@ server: replicas: 3 image: repository: hugegraph/server - # The draft tracks latest until the next HugeGraph release tag is available. - # Pin the release tag and switch to IfNotPresent before stable publication. + # Tracks latest until the next HugeGraph release tag is published. Pin the + # tag (or digest) and switch to IfNotPresent for production. tag: latest # Optional immutable digest, for example sha256:abc... When set it wins over # tag and the image is pulled by digest, which is what a release gate should @@ -413,8 +404,8 @@ hubble: allowWithoutServerAuth: false image: repository: hugegraph/hubble - # The draft tracks latest until the next HugeGraph release tag is available. - # Pin the release tag and switch to IfNotPresent before stable publication. + # Tracks latest until the next HugeGraph release tag is published. Pin the + # tag (or digest) and switch to IfNotPresent for production. tag: latest # Optional immutable digest, for example sha256:abc... When set it wins over # tag and the image is pulled by digest, which is what a release gate should From 895019159439a7ea0d3a35c20e656ac53c554004 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 12 Sep 2026 19:23:20 +0530 Subject: [PATCH 33/61] fix(helm): render legacy values without pd.auth, bound the wait curls A values set stored before pd.auth existed (replayed by --reuse-values, guarded in CI by the pre-hardening fixture) failed the render: the guard and the Secret template read a missing autoGenerate as false. Treat an absent pd.auth block as the chart default, autoGenerate, in both places; an explicit autoGenerate=false with no value or existingSecret still fails. This turns the lint-and-render job green again; it has been red since the pd.auth guard landed. Also bound each probe in the Store PD-wait loop with --connect-timeout 2 and --max-time 5 so one blackholed PD cannot stall an iteration for the curl default and push the wait far past store.waitTimeoutSeconds, which is only checked between sweeps. The Server wait already bounds its curls. --- helm/hugegraph/templates/_helpers.tpl | 9 +++++++-- helm/hugegraph/templates/pd-auth-secret.yaml | 3 ++- helm/hugegraph/templates/store-statefulset.yaml | 2 +- helm/hugegraph/tests/pd_readiness_path_test.yaml | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 5f43ef987f..57f1c5fff2 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -658,8 +658,13 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- end -}} {{- end -}} {{- $pdAuth := get .Values.pd "auth" | default dict -}} -{{- if and (not (get $pdAuth "existingSecret" | default "")) (not (get $pdAuth "value" | default "")) (not (get $pdAuth "autoGenerate" | default false)) -}} -{{- fail "pd.auth requires existingSecret, value, or autoGenerate=true: PD images from 1.8.0 refuse to start without a REST secret" -}} +{{/* A values set with no pd.auth block at all (a release stored before the + field existed, replayed by --reuse-values) gets the chart default, + autoGenerate; only an explicit autoGenerate=false with nothing else set + is an error. */}} +{{- $pdAutoGen := ternary (get $pdAuth "autoGenerate") true (hasKey $pdAuth "autoGenerate") -}} +{{- if and (not (get $pdAuth "existingSecret" | default "")) (not (get $pdAuth "value" | default "")) (not $pdAutoGen) -}} +{{- fail "pd.auth requires existingSecret, value, or autoGenerate=true: PD refuses to start without a REST secret" -}} {{- end -}} {{- $auth := get .Values.server "auth" | default dict -}} {{- $admin := get $auth "admin" | default dict -}} diff --git a/helm/hugegraph/templates/pd-auth-secret.yaml b/helm/hugegraph/templates/pd-auth-secret.yaml index 700d17529f..638e8b24e5 100644 --- a/helm/hugegraph/templates/pd-auth-secret.yaml +++ b/helm/hugegraph/templates/pd-auth-secret.yaml @@ -16,7 +16,8 @@ # {{- $auth := get .Values.pd "auth" | default dict -}} -{{- if and (not (get $auth "existingSecret" | default "")) (or (get $auth "value" | default "") (get $auth "autoGenerate" | default false)) }} +{{- $autoGen := ternary (get $auth "autoGenerate") true (hasKey $auth "autoGenerate") -}} +{{- if and (not (get $auth "existingSecret" | default "")) (or (get $auth "value" | default "") $autoGen) }} apiVersion: v1 kind: Secret metadata: diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index 10a0934079..6b9035add4 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -104,7 +104,7 @@ spec: until [ "$( ok=0 for peer in ${HEALTH_PEERS}; do - if curl -fsS "http://${peer}${WAIT_PATH}" >/dev/null 2>&1; then + if curl -fsS --connect-timeout 2 --max-time 5 "http://${peer}${WAIT_PATH}" >/dev/null 2>&1; then ok=$((ok+1)) fi done diff --git a/helm/hugegraph/tests/pd_readiness_path_test.yaml b/helm/hugegraph/tests/pd_readiness_path_test.yaml index 3cd9ac9be3..3025984985 100644 --- a/helm/hugegraph/tests/pd_readiness_path_test.yaml +++ b/helm/hugegraph/tests/pd_readiness_path_test.yaml @@ -53,7 +53,7 @@ tests: pattern: 'WAIT_PATH="/v1/ready"' - matchRegex: path: spec.template.spec.initContainers[0].command[2] - pattern: 'curl -fsS "http://\${peer}\${WAIT_PATH}"' + pattern: 'curl -fsS --connect-timeout 2 --max-time 5 "http://\${peer}\${WAIT_PATH}"' - it: polls store.waitPath on each PD peer when set back for an older image template: store-statefulset.yaml From 917ac2cd24c95058922d09f4af9798b498b017a1 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 17 Sep 2026 03:03:15 +0530 Subject: [PATCH 34/61] docs(helm): port the onboarding sections, close three review notes Bring over the onboarding pieces the testing branch already had: a pre-install cluster check, --wait --timeout on the primary install command, the Kind/minikube local-build section (with the pullPolicy override called out as required, since this chart defaults Always), a Hubble row in the component table, and the resource naming pattern. Close three notes from the 2026-09-16 independent test run on #3132: document the liveness bound on a stalled component, document how to stage a rollout given the schema's one-replica floor, and enable the Server PodDisruptionBudget in values-cluster.yaml so a node drain cannot evict every Server at once (the default stays off, as recorded). Also move the server.ingress.allowPlainHttp rejection into the schema (a server-specific ingress definition without the key), so the refusal happens at validation with a clear message instead of mid-render; the template guard stays as a backstop. --- helm/hugegraph/README.md | 62 +++++++++++++++++++++++++- helm/hugegraph/values-cluster.yaml | 5 +++ helm/hugegraph/values.schema.json | 70 +++++++++++++++++++++++++++++- 3 files changed, 134 insertions(+), 3 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 35488080a5..0790dc1e96 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -26,6 +26,7 @@ under Upgrading, requires Helm 3.14 or later. | PD | StatefulSet + PVC | Placement driver; Raft group tracking Stores and partitions | | Store | StatefulSet + PVC | Graph data storage (HStore) | | Server | Deployment | Gremlin and REST query layer | +| Hubble | Deployment + optional PVC | Web UI, off by default; enable with `hubble.enabled` | A distributed HugeGraph cluster has a startup contract that this chart encodes so operators do not have to: @@ -84,8 +85,20 @@ so operators do not have to: ## Installing the Chart +Before installing, confirm `kubectl` points at the intended cluster and that +it can provision volumes. The default topology needs 3 PD and 3 Store PVCs, +and PVCs stuck `Pending` for want of a StorageClass are the most common +first-run failure: + +```bash +kubectl config current-context +kubectl get nodes +kubectl get storageclass +``` + ```bash -helm install hugegraph ./helm/hugegraph --namespace hugegraph --create-namespace +helm install hugegraph ./helm/hugegraph --namespace hugegraph \ + --create-namespace --wait --timeout 15m ``` This deploys 3 PD + 3 Store + 3 Server, preserves the image's automatic JVM @@ -95,6 +108,9 @@ production use. The command examples in this document assume the release is named `hugegraph`. With a different release name, substitute the release-prefixed resource names (`kubectl get svc,secret -n ` lists them). +Workloads and Services are named `-hugegraph-*`, while the kept +Secrets are `-admin`, `-auth-token`, and +`-pd-auth`. **Authentication is enabled by default.** The chart creates a kept Secret named `-admin` (for example `hugegraph-admin`) with a random @@ -167,6 +183,37 @@ helm test hugegraph --namespace hugegraph guarantee. Recalculate capacity for the graph size, traffic, failure budget, node topology, and storage class before production use. +### Local Kubernetes (Kind / minikube) + +Only needed when there is no cluster yet, or to test locally built images. +Build the three images, load them into the cluster, and override their tags +and pull policies. The override is required, not optional: this chart +defaults `pullPolicy: Always`, so without `Never` the kubelet tries to pull +your local tag from Docker Hub and fails even though the image is loaded. + +```bash +kind create cluster --name hg + +docker build -f hugegraph-pd/Dockerfile -t hugegraph/pd:local . +docker build -f hugegraph-store/Dockerfile -t hugegraph/store:local . +docker build -f hugegraph-server/Dockerfile-hstore -t hugegraph/server:local . + +kind load docker-image hugegraph/pd:local hugegraph/store:local \ + hugegraph/server:local --name hg +# minikube: minikube image load + +helm upgrade --install hugegraph ./helm/hugegraph \ + --namespace hugegraph --create-namespace \ + -f helm/hugegraph/values-single.yaml \ + --set pd.image.tag=local --set pd.image.pullPolicy=Never \ + --set store.image.tag=local --set store.image.pullPolicy=Never \ + --set server.image.tag=local --set server.image.pullPolicy=Never +``` + +Server uses `Dockerfile-hstore` so the image's default backend is HStore. +Skipping the load step fails the Pods with `ErrImageNeverPull`; do not retag +Docker Hub images as `local`. + ## Upgrading the Chart ```bash @@ -359,7 +406,7 @@ default values. | `server.resources` | Server resources. `requests.cpu` is required when HPA is enabled | `{}` | | `server.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | | `server.securityContext` | Container-level securityContext. Hardened by default; `runAsNonRoot` is not set because the published images run as root | `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault` | -| `server.pdb.enabled` | Create a PodDisruptionBudget for Server. Off by default: Server holds no quorum | `false` | +| `server.pdb.enabled` | Create a PodDisruptionBudget for Server. Off by default: Server holds no quorum. `values-cluster.yaml` enables it so a node drain cannot evict every Server at once | `false` | | `server.pdb.minAvailable` | Must be less than `server.hpa.minReplicas` when HPA is enabled, otherwise less than `server.replicas` | `2` | | `server.antiAffinity` | One of `required`, `preferred`, `disabled`. Defaults to `preferred` rather than `required` because HPA may scale Server past the node count; set `required` when replicas always stay below it | `preferred` | | `server.nodeSelector` | Node selector for server Pods | `{}` | @@ -664,6 +711,11 @@ curl --user "admin:${PASSWORD}" http://127.0.0.1:8080/graphs All ports are configurable through `values.yaml`. Changing `server.port` updates the listener, container port, and Service together. +A stalled component (process alive but frozen) is ended by its liveness +probe, so the default 20 s period and 3-failure threshold bound the blast +radius of a stalled Store at roughly one minute; raft moves its partition +leaders within seconds of the restart. + --- ### Scheduling @@ -783,6 +835,12 @@ Server scales through `server.replicas`, or by enabling `server.hpa`. With HPA enabled the Deployment omits `spec.replicas`, so a Helm upgrade does not overwrite the autoscaler's live replica count. +`values.schema.json` requires at least one replica per component, so a +staged rollout (PD and Server first, Stores later) cannot be written in a +values file. Install the full topology and stage it with +`kubectl scale statefulset -hugegraph-store --replicas=0`, scaling +back up when ready; the Servers wait, not-ready, until Stores register. + ## Troubleshooting ### Store Pods Stuck in `Init:0/1` diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index 0518468666..dee8edd1c1 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -86,6 +86,11 @@ server: memory: 2Gi hpa: enabled: false + # Keep at least two Servers through voluntary evictions such as node + # drains; the default leaves the PDB off because Server holds no quorum. + pdb: + enabled: true + minAvailable: 2 # Auth is on by default (chart-managed admin Secret). Pin it here so a # production overlay cannot accidentally drop authentication. auth: diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 3c7253060d..f8874390bd 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -895,7 +895,7 @@ ] }, "ingress": { - "$ref": "#/definitions/ingress" + "$ref": "#/definitions/serverIngress" }, "hpa": { "type": "object", @@ -1098,6 +1098,74 @@ } } }, + "serverIngress": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "className", + "hosts", + "tls" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "className": { + "type": "string" + }, + "hosts": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "host", + "paths" + ], + "properties": { + "host": { + "type": "string", + "minLength": 1 + }, + "paths": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "pathType" + ], + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string", + "enum": [ + "Exact", + "Prefix", + "ImplementationSpecific" + ] + } + } + } + } + } + } + }, + "tls": { + "type": "array", + "items": { + "type": "object" + } + }, + "annotations": { + "type": "object" + } + } + }, "hubble": { "type": "object", "additionalProperties": false, From 5ff67ab3deacc2cc0272efe230dbcc4e1e9a6175 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 17 Sep 2026 03:13:09 +0530 Subject: [PATCH 35/61] feat(helm): default the PD raft RPC timeout to 3000 ms A PD leader that vanishes without closing its sockets (a freeze or a network partition, unlike a crash) is waited on for the full raft.rpc-timeout per attempt, so the image default of 10000 ms leaves the cluster without a PD leader for about a minute. Measured twice on the chart: 67 s with a blackholed leader (2026-09-10 campaign) and 56 s with a SIGSTOP-frozen leader, against 8.5 s with the timeout at 3000 ms (independent k3s run on #3132, 2026-09-16). Data-plane writes were unaffected in both runs; everything that needs PD waited. Render -Draft.rpc-timeout from a new pd.raftRpcTimeoutMs value, defaulting to 3000, placed with the other derived -D properties ahead of pd.javaOpts so an explicit flag there still wins. Empty preserves the image default, and a values set stored before the key existed renders without the flag. Two unit tests cover the override and the opt-out; the CI verbatim JAVA_OPTS greps carry the new flag. --- .github/workflows/helm-chart-ci.yml | 6 +++--- helm/hugegraph/README.md | 1 + helm/hugegraph/templates/_helpers.tpl | 9 +++++++++ helm/hugegraph/tests/pd_javaopts_test.yaml | 18 ++++++++++++++++-- helm/hugegraph/values.schema.json | 3 +++ helm/hugegraph/values.yaml | 6 ++++++ 6 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 63826fd936..0564f675bb 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -77,13 +77,13 @@ jobs: --set pd.partition.storeMaxShardCount=12 > /dev/null helm template ci helm/hugegraph \ --set-string pd.partition.defaultShardCount=3 > /dev/null - # The derived shard count and the raft whitelist switch must land in + # The derived shard count and the raft whitelist and RPC-timeout flags must land in # the PD JAVA_OPTS verbatim: shard count 3 on the default topology, # 1 on the single-node preset, whitelist disabled in both. helm template ci helm/hugegraph \ - | grep -qF 'value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false"' + | grep -qF 'value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false -Draft.rpc-timeout=3000"' helm template ci helm/hugegraph -f helm/hugegraph/values-single.yaml \ - | grep -qF 'value: "-Dpartition.default-shard-count=1 -Draft.ip-whitelist.enabled=false"' + | grep -qF 'value: "-Dpartition.default-shard-count=1 -Draft.ip-whitelist.enabled=false -Draft.rpc-timeout=3000"' # The stock distributed install must put every Server replica on # the shared PD graph catalog, even without Hubble. Auth is on by # default, so the chart-managed admin Secret must render too. diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 0790dc1e96..5941ce027a 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -312,6 +312,7 @@ default values. | `pd.image.pullPolicy` | PD image pull policy | `Always` | | `pd.javaOpts` | Extra JVM flags, rendered after the chart-derived `-D` properties below so an explicit duplicate here wins. The image's automatic heap sizing is preserved unless heap flags are set | `""` | | `pd.raftIpWhitelistEnabled` | Enable PD's raft peer IP whitelist. Off in-cluster because PD resolves peers once at boot; requires a PD image carrying the upstream switch | `false` | +| `pd.raftRpcTimeoutMs` | Raft RPC timeout (`-Draft.rpc-timeout`). Bounds the wait on a vanished leader, so it bounds leader elections: the image default of 10000 was measured leaderless for about a minute, 3000 elects in seconds. Empty preserves the image default | `3000` | | `pd.partition.defaultShardCount` | Shard replicas per partition, seeded into PD's persisted config at first bootstrap only; inert on an initialized cluster (see Partition Sharding). Empty derives 3 when `store.replicas` is at least 3, else 1. An explicit value must be odd and must not exceed `store.replicas` | `""` | | `pd.partition.storeMaxShardCount` | Maximum shards per Store, seeded at first bootstrap only. Also fixes the initial partition count, `store.replicas x storeMaxShardCount / shardCount` (see Partition Sharding). Empty preserves the image default of `12` | `""` | | `pd.ports.grpc` | PD gRPC port | `8686` | diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 57f1c5fff2..3cb656f7c6 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -436,6 +436,11 @@ pod IPs were unpublished at that moment or change later, so the switch is off in-cluster per the upstream design and k8s auth owns that layer. Images without the property ignore the flag. Set pd.raftIpWhitelistEnabled=true to restore the image default. + +raft.rpc-timeout is a plain runtime property, applied on every start rather +than seeded at bootstrap. It bounds how long the surviving PDs wait on a +peer that stopped answering without closing its sockets, which is what a +leader election waits on; empty preserves the image default. */}} {{- define "hugegraph.pd.effectiveJavaOpts" -}} {{- $pd := .Values.pd -}} @@ -452,6 +457,10 @@ restore the image default. {{- end -}} {{- $ipWhitelist := ternary "true" "false" (eq (get $pd "raftIpWhitelistEnabled" | toString) "true") -}} {{- $flags = append $flags (printf "-Draft.ip-whitelist.enabled=%s" $ipWhitelist) -}} +{{- $rpcTimeout := include "hugegraph.optionalScalar" (get $pd "raftRpcTimeoutMs") -}} +{{- if ne $rpcTimeout "" -}} +{{- $flags = append $flags (printf "-Draft.rpc-timeout=%s" $rpcTimeout) -}} +{{- end -}} {{- $userOpts := trim (get $pd "javaOpts" | default "") -}} {{- if ne $userOpts "" -}} {{- $flags = append $flags $userOpts -}} diff --git a/helm/hugegraph/tests/pd_javaopts_test.yaml b/helm/hugegraph/tests/pd_javaopts_test.yaml index 396d293094..fa0d506fc4 100644 --- a/helm/hugegraph/tests/pd_javaopts_test.yaml +++ b/helm/hugegraph/tests/pd_javaopts_test.yaml @@ -25,7 +25,7 @@ tests: path: spec.template.spec.containers[0].env content: name: JAVA_OPTS - value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false" + value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false -Draft.rpc-timeout=3000" - it: re-enables the whitelist when the operator opts in set: @@ -49,4 +49,18 @@ tests: asserts: - matchRegex: path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value - pattern: "-Draft\\.ip-whitelist\\.enabled=false -Xmx2g$" + pattern: "-Draft\\.rpc-timeout=3000 -Xmx2g$" + - it: renders a custom raft RPC timeout + set: + pd.raftRpcTimeoutMs: 5000 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value + pattern: "-Draft\\.rpc-timeout=5000" + - it: omits the raft RPC timeout when set empty, preserving the image default + set: + pd.raftRpcTimeoutMs: "" + asserts: + - notMatchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value + pattern: "rpc-timeout" diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index f8874390bd..e1bebdba1c 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -308,6 +308,9 @@ "raftIpWhitelistEnabled": { "type": "boolean" }, + "raftRpcTimeoutMs": { + "$ref": "#/definitions/optionalPositiveInteger" + }, "updateStrategy": { "$ref": "#/definitions/updateStrategy" }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 98609e7065..f6e128a2dd 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -135,6 +135,12 @@ pd: # raft.ip-whitelist.enabled); k8s network policy/auth owns that layer. # Set true to restore the image default. raftIpWhitelistEnabled: false + # Raft RPC timeout in milliseconds, rendered as -Draft.rpc-timeout. A + # vanished (not crashed) leader is waited on for this long per attempt, so + # the image default of 10000 leaves the cluster leaderless for about a + # minute where 3000 elects a new leader in seconds. Empty preserves the + # image default. + raftRpcTimeoutMs: 3000 # Explicit rollout strategy instead of the implicit StatefulSet default. updateStrategy: type: RollingUpdate From 94b524f78e45bcfd0b44c337a52bb7413f389e0e Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 17 Sep 2026 23:04:06 +0800 Subject: [PATCH 36/61] fix(helm): preserve zero termination grace - render explicit zero grace periods for all workloads - preserve omitted fields for legacy values - cover zero overrides across four workload templates --- .../templates/hubble-deployment.yaml | 4 +-- helm/hugegraph/templates/pd-statefulset.yaml | 4 +-- .../templates/server-deployment.yaml | 4 +-- .../templates/store-statefulset.yaml | 4 +-- .../tests/termination_grace_period_test.yaml | 35 +++++++++++++++++++ 5 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 helm/hugegraph/tests/termination_grace_period_test.yaml diff --git a/helm/hugegraph/templates/hubble-deployment.yaml b/helm/hugegraph/templates/hubble-deployment.yaml index 67f5d6d405..3f13339342 100644 --- a/helm/hugegraph/templates/hubble-deployment.yaml +++ b/helm/hugegraph/templates/hubble-deployment.yaml @@ -56,8 +56,8 @@ spec: spec: automountServiceAccountToken: {{ get (get .Values.hubble "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.hubble "name" (include "hugegraph.hubble.name" .) ) }} - {{- with .Values.hubble.terminationGracePeriodSeconds }} - terminationGracePeriodSeconds: {{ . }} + {{- if hasKey .Values.hubble "terminationGracePeriodSeconds" }} + terminationGracePeriodSeconds: {{ .Values.hubble.terminationGracePeriodSeconds }} {{- end }} {{- with .Values.hubble.priorityClassName }} priorityClassName: {{ . | quote }} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 8934a88f75..e9768ea129 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -56,8 +56,8 @@ spec: {{- end }} automountServiceAccountToken: {{ get (get .Values.pd "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.pd "name" (include "hugegraph.pd.name" .) ) }} - {{- with .Values.pd.terminationGracePeriodSeconds }} - terminationGracePeriodSeconds: {{ . }} + {{- if hasKey .Values.pd "terminationGracePeriodSeconds" }} + terminationGracePeriodSeconds: {{ .Values.pd.terminationGracePeriodSeconds }} {{- end }} {{- with .Values.pd.priorityClassName }} priorityClassName: {{ . | quote }} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index c075cf0291..44e4f63cea 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -66,8 +66,8 @@ spec: spec: automountServiceAccountToken: {{ get (get .Values.server "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.server "name" (include "hugegraph.server.name" .) ) }} - {{- with .Values.server.terminationGracePeriodSeconds }} - terminationGracePeriodSeconds: {{ . }} + {{- if hasKey .Values.server "terminationGracePeriodSeconds" }} + terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds }} {{- end }} {{- with .Values.server.priorityClassName }} priorityClassName: {{ . | quote }} diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index 6b9035add4..ab97602d2c 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -55,8 +55,8 @@ spec: {{- end }} automountServiceAccountToken: {{ get (get .Values.store "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.store "name" (include "hugegraph.store.name" .) ) }} - {{- with .Values.store.terminationGracePeriodSeconds }} - terminationGracePeriodSeconds: {{ . }} + {{- if hasKey .Values.store "terminationGracePeriodSeconds" }} + terminationGracePeriodSeconds: {{ .Values.store.terminationGracePeriodSeconds }} {{- end }} {{- with .Values.store.priorityClassName }} priorityClassName: {{ . | quote }} diff --git a/helm/hugegraph/tests/termination_grace_period_test.yaml b/helm/hugegraph/tests/termination_grace_period_test.yaml new file mode 100644 index 0000000000..0c65493b95 --- /dev/null +++ b/helm/hugegraph/tests/termination_grace_period_test.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Workload termination grace period overrides +templates: + - pd-statefulset.yaml + - store-statefulset.yaml + - server-deployment.yaml + - hubble-deployment.yaml +tests: + - it: preserves explicit zero grace periods + set: + hubble.enabled: true + pd.terminationGracePeriodSeconds: 0 + store.terminationGracePeriodSeconds: 0 + server.terminationGracePeriodSeconds: 0 + hubble.terminationGracePeriodSeconds: 0 + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 0 From 020b2dfb6513fdf2863fde35118658b9e297f78a Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 17 Sep 2026 23:07:30 +0800 Subject: [PATCH 37/61] fix(helm): validate probe success thresholds - restrict startup and liveness success thresholds to one - preserve readiness thresholds above one - cover invalid probe overrides for all components --- .github/workflows/helm-chart-ci.yml | 8 ++++++++ helm/hugegraph/values.schema.json | 10 ++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 0564f675bb..1d0dc64cd2 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -108,6 +108,14 @@ jobs: exit 1 fi } + # Kubernetes permits successThreshold > 1 only for readiness. + for component in pd store server hubble; do + for probe in startup liveness; do + must_fail --set "${component}.probes.${probe}.successThreshold=2" + done + helm template ci helm/hugegraph \ + --set "${component}.probes.readiness.successThreshold=2" > /dev/null + done must_fail --set pd.replicas=100 must_fail --set pd.pdb.minAvailable=3 must_fail --set server.hpa.enabled=true diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index e1bebdba1c..eb75155335 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -243,13 +243,19 @@ ], "properties": { "startup": { - "$ref": "#/definitions/probe" + "allOf": [ + { "$ref": "#/definitions/probe" }, + { "properties": { "successThreshold": { "maximum": 1 } } } + ] }, "readiness": { "$ref": "#/definitions/probe" }, "liveness": { - "$ref": "#/definitions/probe" + "allOf": [ + { "$ref": "#/definitions/probe" }, + { "properties": { "successThreshold": { "maximum": 1 } } } + ] } } }, From 9b54257bacceea3ededfa16e953abb0c6bd3553f Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 19:13:57 +0530 Subject: [PATCH 38/61] ci(helm): run the unit suites, verify the downloaded binaries The workflow lints, renders and validates but never invokes the helm-unittest suites under helm/hugegraph/tests/, so a change can break every one of them while CI stays green. Install the plugin pinned at v1.1.2 and run it after lint. Also check the helm and kubeconform tarballs against their published sha256 sums instead of piping an unverified download into the job. --- .github/workflows/helm-chart-ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 1d0dc64cd2..a9471597bf 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -39,7 +39,9 @@ jobs: # fails the workflow at startup; install the pinned release directly. - name: install helm run: | - curl -fsSL https://get.helm.sh/helm-v3.16.2-linux-amd64.tar.gz | tar -xz -C /tmp + curl -fsSLo /tmp/helm.tgz https://get.helm.sh/helm-v3.16.2-linux-amd64.tar.gz + echo "9318379b847e333460d33d291d4c088156299a26cd93d570a7f5d0c36e50b5bb /tmp/helm.tgz" | sha256sum -c - + tar -xzf /tmp/helm.tgz -C /tmp sudo install -m 0755 /tmp/linux-amd64/helm /usr/local/bin/helm helm version @@ -49,6 +51,11 @@ jobs: helm lint helm/hugegraph -f helm/hugegraph/values-single.yaml helm lint helm/hugegraph -f helm/hugegraph/values-cluster.yaml + - name: helm unittest + run: | + helm plugin install https://github.com/helm-unittest/helm-unittest.git --version v1.1.2 + helm unittest helm/hugegraph + - name: helm template run: | for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do @@ -172,6 +179,7 @@ jobs: run: | set -o pipefail curl -sSLo /tmp/kc.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz + echo "95f14e87aa28c09d5941f11bd024c1d02fdc0303ccaa23f61cef67bc92619d73 /tmp/kc.tar.gz" | sha256sum -c - tar -xzf /tmp/kc.tar.gz -C /tmp for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do # shellcheck disable=SC2086 # $preset intentionally splits into flags From 344fa81903bb9c7646cbeb435f5f9c8a89a41cb2 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 19:36:27 +0530 Subject: [PATCH 39/61] fix(helm): bound Store evictions, reserve selector labels Review fixes from hugegraph/hugegraph#221: - Require store.pdb.minAvailable >= store.replicas - 1. Shards keep their copies on a subset of the Stores, so a budget that permits two concurrent evictions (the old default with four or five Stores) can remove a shard majority even while the PDB is satisfied. The default 3-Store topology is unchanged. - Reject podLabels that set app.kubernetes.io/name, instance or component on any of the four workloads: they render after the chart-managed labels and would break the selectors, Services and PDBs that match on them. - Accept an empty hubble.image.tag when a digest is set; the digest is a complete image identity. --- helm/hugegraph/README.md | 4 ++-- helm/hugegraph/templates/_helpers.tpl | 15 +++++++++++++-- helm/hugegraph/tests/validate_values_test.yaml | 17 ++++++++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 5941ce027a..30606421f9 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -331,7 +331,7 @@ default values. | `pd.topologySpreadConstraints` | Topology spread constraints for pd Pods | `[]` | | `pd.priorityClassName` | PriorityClass for pd Pods | `""` | | `pd.podAnnotations` | Extra annotations on pd Pods | `{}` | -| `pd.podLabels` | Extra labels on pd Pods | `{}` | +| `pd.podLabels` | Extra labels on pd Pods. The `app.kubernetes.io/name`, `instance` and `component` keys are chart-managed and rejected | `{}` | | `pd.extraEnv` | Extra environment variables for the PD container | `[]` | | `pd.terminationGracePeriodSeconds` | Shutdown grace period | `300` | | `pd.serviceAccount.create` | Create a ServiceAccount for pd | `true` | @@ -387,7 +387,7 @@ default values. | `store.serviceAccount.annotations` | Annotations on the created ServiceAccount | `{}` | | `store.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | | `store.pdb.enabled` | Create a PodDisruptionBudget for Store | `true` | -| `store.pdb.minAvailable` | Must be strictly less than `store.replicas`. No PDB is rendered when `store.replicas` is 1 | `2` | +| `store.pdb.minAvailable` | Must be strictly less than `store.replicas` and at least `store.replicas - 1`, so voluntary evictions cannot remove two copies of one shard at once. No PDB is rendered when `store.replicas` is 1 | `2` | | `store.waitImage` | Image for the PD wait init container | `curlimages/curl:8.5.0` | | `store.waitResources` | Resources for the init container | `{}` | | `store.probes.*` | Same probe keys as PD | see `values.yaml` | diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 3cb656f7c6..e4fdcd5a4f 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -544,6 +544,14 @@ The minimum Server replica count that a PDB must remain valid against. Cross-field validation that JSON Schema draft-07 cannot express. */}} {{- define "hugegraph.validateValues" -}} +{{- range $comp := list "pd" "store" "server" "hubble" -}} +{{- $compLabels := get (get $.Values $comp | default dict) "podLabels" | default dict -}} +{{- range $reserved := list "app.kubernetes.io/name" "app.kubernetes.io/instance" "app.kubernetes.io/component" -}} +{{- if hasKey $compLabels $reserved -}} +{{- fail (printf "%s.podLabels must not set %s: the chart manages it and the workload selectors, Services and PDBs match on it" $comp $reserved) -}} +{{- end -}} +{{- end -}} +{{- end -}} {{- $networkPolicy := get .Values "networkPolicy" | default dict -}} {{- if (get $networkPolicy "enabled" | default false) -}} {{- fail "networkPolicy.enabled=true is unsupported because this chart does not implement NetworkPolicy resources" -}} @@ -576,6 +584,9 @@ and must not be failed for a value that has no effect. {{- if and .Values.store.pdb.enabled (gt (int .Values.store.replicas) 1) (ge (int .Values.store.pdb.minAvailable) (int .Values.store.replicas)) -}} {{- fail "store.pdb.minAvailable must be less than store.replicas, otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} +{{- if and .Values.store.pdb.enabled (gt (int .Values.store.replicas) 1) (lt (int .Values.store.pdb.minAvailable) (sub (int .Values.store.replicas) 1)) -}} +{{- fail "store.pdb.minAvailable must be at least store.replicas - 1: each shard keeps its copies on a subset of the Stores, so permitting more than one concurrent voluntary eviction can remove a shard majority regardless of the Store count" -}} +{{- end -}} {{/* PD -D system properties must be empty or a positive integer. The schema enforces the types; these checks add a named failure for zero, negative, and @@ -658,8 +669,8 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- fail "hubble.service.nodePort requires hubble.service.type to be NodePort or LoadBalancer" -}} {{- end -}} {{- $hubbleImage := get $hubble "image" | default dict -}} -{{- if eq (trim (get $hubbleImage "tag" | default "")) "" -}} -{{- fail "hubble.image.tag must not be empty: the chart appVersion tracks the Server release, not Hubble, so there is no meaningful fallback" -}} +{{- if and (eq (trim (get $hubbleImage "tag" | default "")) "") (eq (trim (get $hubbleImage "digest" | default "")) "") -}} +{{- fail "hubble.image needs a tag or a digest: the chart appVersion tracks the Server release, not Hubble, so there is no meaningful fallback" -}} {{- end -}} {{- $hubbleIngress := get $hubble "ingress" | default dict -}} {{- if and (get $hubbleIngress "enabled" | default false) (empty (get $hubbleIngress "tls")) (not (get $hubbleIngress "allowPlainHttp" | default false)) -}} diff --git a/helm/hugegraph/tests/validate_values_test.yaml b/helm/hugegraph/tests/validate_values_test.yaml index f0700982fc..7fc861b099 100644 --- a/helm/hugegraph/tests/validate_values_test.yaml +++ b/helm/hugegraph/tests/validate_values_test.yaml @@ -85,4 +85,19 @@ tests: hubble.enabled: true asserts: - failedTemplate: - errorPattern: "hubble.image.tag must not be empty" + errorPattern: "hubble.image needs a tag or a digest" + + - it: rejects a Store PDB that permits two concurrent evictions + set: + store.replicas: 4 + asserts: + - failedTemplate: + errorPattern: "store.pdb.minAvailable must be at least store.replicas - 1" + + - it: rejects podLabels that overwrite a selector label + set: + store.podLabels: + app.kubernetes.io/component: hacked + asserts: + - failedTemplate: + errorPattern: "store.podLabels must not set app.kubernetes.io/component" From 0faed0a0c24354ece15f774377eb58d664ae53db Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 19:36:27 +0530 Subject: [PATCH 40/61] fix(helm): roll readers on inline credential change, validate shapes Review fixes from hugegraph/hugegraph#221: - The checksum annotations were lookup-only, so an upgrade that changes an inline pd.auth.value, admin password or JWT value hashed the OLD Secret's resourceVersion and did not roll the reading pods in that same upgrade. Fold a hash of the desired inline value into hugegraph.pd.authChecksum and hugegraph.server.authChecksum, only when that inline value is live (no existingSecret overriding it); the autoGenerate and existingSecret paths are unchanged. - Schema now rejects inline admin passwords the Server wrapper refuses at startup (leading whitespace, backslash, CR/LF), failing at install instead of CrashLooping after it. - pd.auth.value is restricted to printable ASCII with no leading whitespace and no backslashes: Hubble reads its properties file as ISO-8859-1, strips leading whitespace, and its Java properties parser unescapes backslashes, so such values silently stop matching what PD holds. --- helm/hugegraph/README.md | 2 +- helm/hugegraph/templates/_helpers.tpl | 10 ++++++++++ helm/hugegraph/values.schema.json | 8 +++++--- helm/hugegraph/values.yaml | 7 ++++--- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 30606421f9..bac2ada260 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -341,7 +341,7 @@ default values. | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | | `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader | `/v1/ready` | -| `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. No newlines, carriage returns, or backslashes | `""` | +| `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. Printable ASCII with no leading whitespace | `""` | | `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it | `""` | | `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | | `pd.auth.autoGenerate` | Create and keep a random release-pd-auth Secret when `value` and `existingSecret` are empty | `true` | diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index e4fdcd5a4f..a556cb179c 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -220,6 +220,9 @@ so template-only renders emit a constant. */}} {{- define "hugegraph.pd.authChecksum" -}} {{- $parts := list (include "hugegraph.pd.authSecretName" .) (include "hugegraph.pd.authSecretKey" .) -}} +{{- $pdAuthCfg := get .Values.pd "auth" | default dict -}} +{{- $inline := get $pdAuthCfg "value" | default "" -}} +{{- if and $inline (not (get $pdAuthCfg "existingSecret" | default "")) -}}{{- $parts = append $parts (sha256sum $inline) -}}{{- end -}} {{- $secret := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.pd.authSecretName" .) -}} {{- if $secret -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $secret) -}}{{- end -}} {{- join "|" $parts | sha256sum -}} @@ -283,6 +286,13 @@ existingSecret applies on the next `helm upgrade`. */}} {{- define "hugegraph.server.authChecksum" -}} {{- $parts := list (include "hugegraph.server.authSecretName" .) (include "hugegraph.server.authSecretKey" .) (include "hugegraph.server.authTokenSecretName" .) (include "hugegraph.server.authTokenSecretKey" .) -}} +{{- $srvAuth := get .Values.server "auth" | default dict -}} +{{- $adminCfg := get $srvAuth "admin" | default dict -}} +{{- $inlineAdmin := get $adminCfg "password" | default "" -}} +{{- if and $inlineAdmin (not (get $adminCfg "existingSecret" | default "")) -}}{{- $parts = append $parts (sha256sum $inlineAdmin) -}}{{- end -}} +{{- $tokenCfg := get $srvAuth "token" | default dict -}} +{{- $inlineToken := get $tokenCfg "value" | default "" -}} +{{- if and $inlineToken (not (get $tokenCfg "existingSecret" | default "")) -}}{{- $parts = append $parts (sha256sum $inlineToken) -}}{{- end -}} {{- $admin := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authSecretName" .) -}} {{- if $admin -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $admin) -}}{{- end -}} {{- $token := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authTokenSecretName" .) -}} diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index eb75155335..45ffa2b976 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -355,8 +355,8 @@ "properties": { "value": { "type": "string", - "pattern": "^[^\\r\\n\\\\]*$", - "description": "Plaintext secret. Empty defers to existingSecret or autoGenerate. No newlines, carriage returns, or backslashes: the value lands in Hubble's Java properties file." + "pattern": "^([\\x21-\\x5b\\x5d-\\x7e][\\x20-\\x5b\\x5d-\\x7e]*)?$", + "description": "Plaintext secret. Empty defers to existingSecret or autoGenerate. Printable ASCII with no leading whitespace and no backslashes: the value lands in Hubble's Java properties file, which strips leading whitespace and reads the file as ISO-8859-1." }, "existingSecret": { "type": "string" @@ -743,7 +743,9 @@ ], "properties": { "password": { - "type": "string" + "type": "string", + "pattern": "^([^ \\t\\r\\n\\\\][^\\r\\n\\\\]*)?$", + "description": "Inline admin password. Empty defers to existingSecret or autoGenerate. The Server wrapper rejects newlines, carriage returns, backslashes and leading whitespace, so the schema rejects them before install." }, "existingSecret": { "type": "string" diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index f6e128a2dd..9ec52fbb25 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -71,9 +71,10 @@ pd: # (HG_PD_AUTH_SECRET_KEY). The chart hands the same value to the Server # storage wait (PD_AUTH_PASSWORD, user store) and to Hubble # (operations.pd.password, user hubble). The Secret is kept on uninstall - # like the Server auth Secrets. The value must not contain newlines, - # carriage returns, or backslashes: it is written into Hubble's Java - # properties file. Priority: existingSecret > value > autoGenerate. + # like the Server auth Secrets. The value must be printable ASCII with + # no leading whitespace: it is written into Hubble's Java properties + # file, which strips leading whitespace and reads the file as + # ISO-8859-1. Priority: existingSecret > value > autoGenerate. auth: # Optional plaintext secret (prefer existingSecret in shared clusters). value: "" From 9217b531a2803530754d2707ef885abdc49df526 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 19:36:28 +0530 Subject: [PATCH 41/61] fix(helm): reject replica shrinks that lose persisted membership Review fixes from hugegraph/hugegraph#221: Raft and shard membership are persisted, and deleting Pods does not reconfigure them: a 3-to-1 PD shrink permanently loses quorum, and removing a Store strands the shard copies it holds. validateValues now reads the live StatefulSet through lookup and fails an upgrade whose replica count is below it. Template-only renders have no live object and skip the guard, and an operator who finished the documented manual procedure has already scaled the live StatefulSet, so the upgrade passes once live and desired match. README: the manual scale-down procedure under Scaling, a note that kubectl scale does not survive the next helm upgrade, and an Upgrading bullet recommending store.updateStrategy=OnDelete for production image rolls, because Store readiness reports the listener while shard recovery is still catching up. --- helm/hugegraph/README.md | 23 +++++++++++++++++++++++ helm/hugegraph/templates/_helpers.tpl | 21 +++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index bac2ada260..8f4da1ef70 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -227,6 +227,12 @@ are worth knowing about in advance: includes adopting the `-Draft.ip-whitelist.enabled=false` setting described under Limitations. For a maintenance-window upgrade, set `pd.updateStrategy.type=OnDelete` and restart the pods yourself. +- **Store** rolling updates advance on `/v1/health`, which reports the + listener, not shard recovery: the controller can replace the next Store + while the previous one is still rejoining its shard groups. For a + production image roll, set `store.updateStrategy.type=OnDelete` and delete + Store Pods one at a time, waiting for the replaced Store to show `Up` in + PD (see Cluster Health) before the next. - **Server** rolls once on the first `helm upgrade` after a fresh install, when the `checksum/auth` annotation first observes the install-created Secrets. Template-only pipelines (`helm template`, GitOps renderers) never @@ -841,6 +847,23 @@ staged rollout (PD and Server first, Stores later) cannot be written in a values file. Install the full topology and stage it with `kubectl scale statefulset -hugegraph-store --replicas=0`, scaling back up when ready; the Servers wait, not-ready, until Stores register. +`kubectl scale` changes only the live StatefulSet: the next `helm upgrade` +renders `store.replicas` from values again and restores the full topology. + +Scaling **down** PD or Store is not a values change. Raft and shard +membership are persisted, and deleting Pods does not reconfigure them: a +3-to-1 PD shrink permanently loses quorum, and removing a Store strands the +shard copies it holds. The chart therefore rejects an upgrade whose replica +count is below the live StatefulSet. The manual procedure: for Store, drain +the leaving Stores first (trigger `patrolPartitions` and +`balancePartitions`, then verify in Cluster Health that no shard lists +them); for PD, the persisted raft membership must be reduced through PD +itself before Pods are removed. Then scale the live StatefulSet with +`kubectl -n scale statefulset --replicas=` and run +`helm upgrade` with the matching value. The same applies after a manual +scale up: upgrade with the matching value, because the guard reads any +value below the live StatefulSet as a shrink. The guard needs the live +object, so a client-side `--dry-run` does not show it. ## Troubleshooting diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index a556cb179c..82335f849b 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -581,6 +581,27 @@ Cross-field validation that JSON Schema draft-07 cannot express. {{- end -}} {{- end -}} {{/* +Raft and shard membership are persisted; deleting Pods does not reconfigure +them, so an in-place replica shrink permanently loses PD quorum or Store +shard majorities. The guard reads the live StatefulSet, so it fires only on +a real upgrade against a cluster; template-only renders have no live object +and skip it. An operator who has completed the documented manual scale-down +procedure has already scaled the live StatefulSet, so desired equals live +and the upgrade passes. +*/}} +{{- range $comp := list "pd" "store" -}} +{{- $stsName := "" -}} +{{- if eq $comp "pd" -}}{{- $stsName = include "hugegraph.pd.name" $ -}}{{- else -}}{{- $stsName = include "hugegraph.store.name" $ -}}{{- end -}} +{{- $live := lookup "apps/v1" "StatefulSet" $.Release.Namespace $stsName -}} +{{- if $live -}} +{{- $liveReplicas := int (dig "spec" "replicas" 0 $live) -}} +{{- $desired := int (get (get $.Values $comp) "replicas") -}} +{{- if and (gt $liveReplicas 0) (lt $desired $liveReplicas) -}} +{{- fail (printf "%s.replicas cannot shrink from %d to %d through a helm upgrade: raft and shard membership are persisted, and removing Pods does not reconfigure them. Follow the manual scale-down procedure in the README (Scaling), which ends by scaling the live StatefulSet; the upgrade passes once the live replicas match the value" $comp $liveReplicas $desired) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{/* Only validate minAvailable where a PDB is actually rendered. The pd/store PDB templates require replicas > 1, so a single-replica release never creates one and must not be failed for a value that has no effect. From e62cac3d277a7e87a1b683df1d3134402a75b2e5 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 19:37:38 +0530 Subject: [PATCH 42/61] fix(helm): keep configurable Secret keys literal, align doc commands Review fixes from hugegraph/hugegraph#221: - Quote the configurable Secret data keys in the three Secret templates, so a key like "on" or "01" stays the literal string the workloads reference instead of being reinterpreted by YAML. - A regression test renders a Secret with the key "on" and asserts the literal key survives; a CI render greps the quoted key verbatim, since the unit framework's YAML 1.2 parser cannot see the 1.1 coercion. - Escape dots in the NOTES.txt jsonpath commands, so a dotted key such as admin.password reads the flat Secret entry instead of querying a nested field and printing an empty credential. - Default the pd readiness path and the Store wait path to /v1/ready in the templates too, matching values.yaml, NOTES and the schema for a release whose stored values predate the keys. - Scope the BestEffort NOTES warning to the case where no component sets resources; a milder note already covers the partial case. - README: enable Hubble with --reuse-values so the upgrade keeps the release's existing overrides; add -n to the troubleshooting commands; use four-space continuations in the new command blocks. --- .github/workflows/helm-chart-ci.yml | 4 +++ helm/hugegraph/README.md | 30 ++++++++++--------- helm/hugegraph/templates/NOTES.txt | 6 ++-- helm/hugegraph/templates/pd-auth-secret.yaml | 2 +- helm/hugegraph/templates/pd-statefulset.yaml | 2 +- .../templates/server-auth-token-secret.yaml | 2 +- helm/hugegraph/templates/server-secret.yaml | 2 +- .../templates/store-statefulset.yaml | 2 +- helm/hugegraph/tests/pd_auth_secret_test.yaml | 8 +++++ 9 files changed, 36 insertions(+), 22 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index a9471597bf..89a8c4c271 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -103,6 +103,10 @@ jobs: grep -qE '^kind: Secret$' <<<"$DEFAULT" grep -qF 'name: ci-admin' <<<"$DEFAULT" grep -qF 'name: ci-auth-token' <<<"$DEFAULT" + # A YAML-coercible Secret key must render quoted, or the API + # server stores the key as a boolean and the pods cannot find it. + helm template ci helm/hugegraph --set-string server.auth.admin.key=on \ + | grep -qF '"on":' - name: reject invalid values run: | diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 8f4da1ef70..afde887e3d 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -98,7 +98,7 @@ kubectl get storageclass ```bash helm install hugegraph ./helm/hugegraph --namespace hugegraph \ - --create-namespace --wait --timeout 15m + --create-namespace --wait --timeout 15m ``` This deploys 3 PD + 3 Store + 3 Server, preserves the image's automatic JVM @@ -141,11 +141,13 @@ curl --user "admin:${PASSWORD}" http://127.0.0.1:8080/versions **Hubble is not installed by default.** Enable the optional UI after install: ```bash -helm upgrade --install hugegraph ./helm/hugegraph --namespace hugegraph \ - --set hubble.enabled=true +helm upgrade hugegraph ./helm/hugegraph --namespace hugegraph \ + --reuse-values --set hubble.enabled=true ``` -Auth is already on, so that single flag is enough. Login uses the same admin +`--reuse-values` keeps the release's existing overrides (presets, images, +resources, Secrets); without it the upgrade rebuilds the release from chart +defaults. Auth is already on, so that single flag is enough. Login uses the same admin credential from the chart-managed (or BYO) Secret. The default anti-affinity for `pd`, `store`, and `server` is `preferred` @@ -199,15 +201,15 @@ docker build -f hugegraph-store/Dockerfile -t hugegraph/store:local . docker build -f hugegraph-server/Dockerfile-hstore -t hugegraph/server:local . kind load docker-image hugegraph/pd:local hugegraph/store:local \ - hugegraph/server:local --name hg + hugegraph/server:local --name hg # minikube: minikube image load helm upgrade --install hugegraph ./helm/hugegraph \ - --namespace hugegraph --create-namespace \ - -f helm/hugegraph/values-single.yaml \ - --set pd.image.tag=local --set pd.image.pullPolicy=Never \ - --set store.image.tag=local --set store.image.pullPolicy=Never \ - --set server.image.tag=local --set server.image.pullPolicy=Never + --namespace hugegraph --create-namespace \ + -f helm/hugegraph/values-single.yaml \ + --set pd.image.tag=local --set pd.image.pullPolicy=Never \ + --set store.image.tag=local --set store.image.pullPolicy=Never \ + --set server.image.tag=local --set server.image.pullPolicy=Never ``` Server uses `Dockerfile-hstore` so the image's default backend is HStore. @@ -873,8 +875,8 @@ The Store init container waits for a majority of PD peers to answer `store.waitPath`. Check PD first: ```bash -kubectl get pods -l app.kubernetes.io/component=pd -kubectl logs -c wait-for-pd +kubectl -n get pods -l app.kubernetes.io/component=pd +kubectl -n logs -c wait-for-pd ``` The wait is bounded by `store.waitTimeoutSeconds` (default 900). On timeout the @@ -887,7 +889,7 @@ No default StorageClass, or the provisioner is unhealthy: ```bash kubectl get sc -kubectl get pvc -l app.kubernetes.io/instance= +kubectl -n get pvc -l app.kubernetes.io/instance= kubectl -n get pods ``` @@ -935,7 +937,7 @@ use; see `values-cluster.yaml`. ```bash kubectl get pods -o wide -kubectl describe pod | grep -A5 "Last State" +kubectl -n describe pod | grep -A5 "Last State" ``` ### Release Name Too Long diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index eb153ff86a..53bf833df8 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -35,7 +35,7 @@ majority counted is a quorum and not merely a set of live listeners). PD's management REST API (Disaster Recovery calls in the README) takes the release's PD secret as the Basic-auth password: - PD_SECRET="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.pd.authSecretName" . }} -o jsonpath='{.data.{{ include "hugegraph.pd.authSecretKey" . }}}' | base64 --decode)" + PD_SECRET="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.pd.authSecretName" . }} -o jsonpath='{.data.{{ include "hugegraph.pd.authSecretKey" . | replace "." "\\." }}}' | base64 --decode)" Verify the release: @@ -45,7 +45,7 @@ Reach the Server API: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.server.name" . }} {{ .Values.server.port }}:{{ .Values.server.port }} {{- if .Values.server.auth.enabled }} - PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.server.authSecretName" . }} -o jsonpath='{.data.{{ include "hugegraph.server.authSecretKey" . }}}' | base64 --decode)" + PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ include "hugegraph.server.authSecretName" . }} -o jsonpath='{.data.{{ include "hugegraph.server.authSecretKey" . | replace "." "\\." }}}' | base64 --decode)" curl --user "admin:${PASSWORD}" http://127.0.0.1:{{ .Values.server.port }}/versions {{- else }} curl http://127.0.0.1:{{ .Values.server.port }}/versions @@ -104,7 +104,7 @@ One or more components have no resource requests or limits. Set them before production use; see values-cluster.yaml. {{- end }} -{{- if not .Values.pd.resources }} +{{- if and (empty .Values.pd.resources) (empty .Values.store.resources) (empty .Values.server.resources) }} WARNING: no resources are set, so every pod is BestEffort and each JVM sizes its heap against total node memory. On a multi-node cluster this oversubscribes the nodes and pods may abort. Use values-cluster.yaml or set resources explicitly. diff --git a/helm/hugegraph/templates/pd-auth-secret.yaml b/helm/hugegraph/templates/pd-auth-secret.yaml index 638e8b24e5..66a484bdf5 100644 --- a/helm/hugegraph/templates/pd-auth-secret.yaml +++ b/helm/hugegraph/templates/pd-auth-secret.yaml @@ -28,5 +28,5 @@ metadata: helm.sh/resource-policy: keep type: Opaque data: - {{ include "hugegraph.pd.authSecretKey" . }}: {{ include "hugegraph.pd.authSecretValue" . | quote }} + {{ include "hugegraph.pd.authSecretKey" . | quote }}: {{ include "hugegraph.pd.authSecretValue" . | quote }} {{- end }} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index e9768ea129..62b78ee7a5 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -143,7 +143,7 @@ spec: {{- with include "hugegraph.probeTuning" .Values.pd.probes.startup }}{{ . | trim | nindent 12 }}{{- end }} readinessProbe: httpGet: - path: {{ .Values.pd.readinessPath | default "/v1/health" }} + path: {{ .Values.pd.readinessPath | default "/v1/ready" }} port: rest periodSeconds: {{ .Values.pd.probes.readiness.periodSeconds }} failureThreshold: {{ .Values.pd.probes.readiness.failureThreshold }} diff --git a/helm/hugegraph/templates/server-auth-token-secret.yaml b/helm/hugegraph/templates/server-auth-token-secret.yaml index f416e313de..717d4fc23b 100644 --- a/helm/hugegraph/templates/server-auth-token-secret.yaml +++ b/helm/hugegraph/templates/server-auth-token-secret.yaml @@ -28,5 +28,5 @@ metadata: helm.sh/resource-policy: keep type: Opaque data: - {{ include "hugegraph.server.authTokenSecretKey" . }}: {{ include "hugegraph.server.authTokenSecretValue" . | quote }} + {{ include "hugegraph.server.authTokenSecretKey" . | quote }}: {{ include "hugegraph.server.authTokenSecretValue" . | quote }} {{- end }} diff --git a/helm/hugegraph/templates/server-secret.yaml b/helm/hugegraph/templates/server-secret.yaml index c25f9a72b7..4d132224a6 100644 --- a/helm/hugegraph/templates/server-secret.yaml +++ b/helm/hugegraph/templates/server-secret.yaml @@ -28,5 +28,5 @@ metadata: helm.sh/resource-policy: keep type: Opaque data: - {{ include "hugegraph.server.authSecretKey" . }}: {{ include "hugegraph.server.authSecretPassword" . | quote }} + {{ include "hugegraph.server.authSecretKey" . | quote }}: {{ include "hugegraph.server.authSecretPassword" . | quote }} {{- end }} diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index ab97602d2c..69dc03d746 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -99,7 +99,7 @@ spec: HEALTH_PEERS=$(echo "{{ include "hugegraph.pd.restPeersList" . }}" | tr ',' ' ') TIMEOUT={{ .Values.store.waitTimeoutSeconds | default 900 }} DEADLINE=$(( $(date +%s) + TIMEOUT )) - WAIT_PATH={{ .Values.store.waitPath | default "/v1/health" | quote }} + WAIT_PATH={{ .Values.store.waitPath | default "/v1/ready" | quote }} echo "Waiting for ${REQUIRED} PD peers to answer ${WAIT_PATH} among: ${HEALTH_PEERS}" until [ "$( ok=0 diff --git a/helm/hugegraph/tests/pd_auth_secret_test.yaml b/helm/hugegraph/tests/pd_auth_secret_test.yaml index 325679cdfa..ae0ee1bf34 100644 --- a/helm/hugegraph/tests/pd_auth_secret_test.yaml +++ b/helm/hugegraph/tests/pd_auth_secret_test.yaml @@ -137,3 +137,11 @@ tests: asserts: - failedTemplate: errorPattern: "server.extraEnv must not set the chart-managed variable PD_AUTH_PASSWORD" + + - it: keeps a YAML-coercible Secret key literal + template: server-secret.yaml + set: + server.auth.admin.key: "on" + asserts: + - exists: + path: data.on From cbc3739559e3e52b86f2cdf752d5599342934c23 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 19:38:04 +0530 Subject: [PATCH 43/61] fix(helm): gate insecure exposure, align the start budget, bracket IPv6 Review fixes from hugegraph/hugegraph#221: - A non-ClusterIP pd.service now requires pd.service.allowInsecureExposure=true: the exposed gRPC port carries no authentication, raft membership RPCs included, so exposure must be an explicit decision taken after restricting reachability elsewhere. - A Server Ingress without tls is refused unless server.ingress.allowPlainHttp=true; auth is on by default, so a plain HTTP route carries Basic-auth credentials and JWTs. Replaces the old rule that rejected the key outright, and mirrors the Hubble opt-in. - hubble.securityContext.readOnlyRootFilesystem=true is refused: the Hubble wrapper writes its properties file inside the image at startup and the chart mounts no writable volume there. - HG_SERVER_STARTUP_TIMEOUT_S is now the startup probe budget minus the image's 300-second storage wait (150 s by default; the 450 s probe floor keeps the result at 150 or more, with the image's 120 s minimum kept as a guard), so the start command and kubelet give up together instead of the image outliving the probe. - The default PD announcement brackets an IPv6 POD_IP, which a URL requires; an explicit server.advertiseUrl stays untouched. - README documents the exposure acknowledgements, refreshes the validation-rules list for the new gates, adds an Upgrading note for releases that already expose PD or serve a TLS-less Ingress, and uses HTTPS in the outside-Hubble direct URL example, since login sends credentials over it. --- .github/workflows/helm-chart-ci.yml | 7 ++- helm/hugegraph/README.md | 51 ++++++++++++++----- helm/hugegraph/templates/_helpers.tpl | 21 +++++--- .../templates/server-deployment.yaml | 6 +++ .../tests/server_startup_timeout_test.yaml | 10 ++-- .../hugegraph/tests/validate_values_test.yaml | 22 ++++++++ helm/hugegraph/values.schema.json | 8 +++ helm/hugegraph/values.yaml | 12 +++-- 8 files changed, 106 insertions(+), 31 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 89a8c4c271..2a234ae831 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -149,8 +149,11 @@ jobs: must_fail "${A[@]}" --set hubble.mode=bogus must_fail "${A[@]}" --set hubble.image.tag="" must_fail "${A[@]}" --set hubble.ingress.enabled=true - must_fail --set server.ingress.enabled=true \ - --set server.ingress.allowPlainHttp=true + # A TLS-less Server Ingress needs the explicit plain-HTTP opt-in + must_fail --set server.ingress.enabled=true + helm template ci helm/hugegraph \ + --set server.ingress.enabled=true \ + --set server.ingress.allowPlainHttp=true > /dev/null # PD PDB must keep the Raft majority: floor(replicas/2)+1 must_fail --set pd.replicas=5 --set pd.pdb.minAvailable=2 must_fail --set pd.replicas=4 --set pd.pdb.minAvailable=2 diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index afde887e3d..c87494d78f 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -222,8 +222,15 @@ Docker Hub images as `local`. helm upgrade hugegraph ./helm/hugegraph --namespace hugegraph --reuse-values ``` -Any upgrade that changes a Pod template rolls that workload once. Two cases -are worth knowing about in advance: +Any upgrade that changes a Pod template rolls that workload once. + +A release created before the exposure gates existed can hit them on its +next upgrade, `--reuse-values` included: a non-ClusterIP `pd.service.type` +now needs `pd.service.allowInsecureExposure=true`, and a TLS-less Server +Ingress needs `server.ingress.allowPlainHttp=true`. The render error names +the value to set. + +Two cases are worth knowing about in advance: - **PD** restarts one pod at a time whenever its Pod template changes, which includes adopting the `-Draft.ip-whitelist.enabled=false` setting described @@ -349,7 +356,7 @@ default values. | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | | `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader | `/v1/ready` | -| `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. Printable ASCII with no leading whitespace | `""` | +| `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. Printable ASCII, no leading whitespace, no backslashes | `""` | | `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it | `""` | | `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | | `pd.auth.autoGenerate` | Create and keep a random release-pd-auth Secret when `value` and `existingSecret` are empty | `true` | @@ -452,7 +459,8 @@ default values. | `server.service.type` | Server Service type | `ClusterIP` | | `server.service.annotations` | Server Service annotations | `{}` | | `server.ingress.hosts` | Ingress hosts and paths | see `values.yaml` | -| `server.ingress.tls` | Ingress TLS configuration | `[]` | +| `server.ingress.tls` | Ingress TLS configuration. Empty is refused unless `allowPlainHttp` opts in: the Server carries Basic-auth credentials and JWTs | `[]` | +| `server.ingress.allowPlainHttp` | Explicit opt-in to a TLS-less Server Ingress on a trusted network | unset | | `server.hpa.enabled` | Create a HorizontalPodAutoscaler | `false` | | `server.hpa.minReplicas` | HPA minimum replicas | `3` | | `server.hpa.maxReplicas` | HPA maximum replicas | `10` | @@ -502,11 +510,14 @@ need graph / schema / data / Gremlin (not PD discovery). `server.direct_url` set to that reachable Server URL (match Server auth). 4. Open the standalone Hubble port in a browser (or SSH tunnel to it). +Use HTTPS (or a trusted channel such as a local port-forward) for +`server.direct_url`: login sends the Server credentials over that URL. + Example property fragment for the standalone process: ```properties pd.enabled=false -server.direct_url=http://: +server.direct_url=https://: ``` Mount the file at `/hubble/conf/hugegraph-hubble.properties` inside the @@ -528,7 +539,9 @@ is honored whenever it is set. 2. Expose Server and set `server.advertiseUrl` to the absolute `http(s)://` URL outside Hubble will use after discovery. The chart registers it via `server.urls_to_pd` instead of the in-cluster Service URL. -3. Expose PD (`pd.service.type` NodePort/LoadBalancer) so Hubble can dial PD +3. Expose PD (`pd.service.type` NodePort/LoadBalancer, which needs + `pd.service.allowInsecureExposure=true`; PD gRPC has no authentication, + so restrict who can reach it first) so Hubble can dial PD REST and gRPC. 4. Run standalone Hubble with `pd.enabled=true` and `pd.peers` / `pd.server` pointed at those external PD addresses. Mount config at @@ -553,7 +566,8 @@ Local quick test (cluster and Hubble on the same machine): port-forward Server | Parameter | Description | Default | |---|---|---| | `server.advertiseUrl` | Absolute Server URL registered with PD for discovery clients. Empty registers each Server Pod IP for in-cluster discovery | `""` | -| `pd.service.type` | PD client Service type (`ClusterIP`, `NodePort`, `LoadBalancer`) | `ClusterIP` | +| `pd.service.type` | PD client Service type (`ClusterIP`, `NodePort`, `LoadBalancer`). A non-ClusterIP type requires `pd.service.allowInsecureExposure` | `ClusterIP` | +| `pd.service.allowInsecureExposure` | Acknowledgement that a non-ClusterIP PD Service exposes the unauthenticated gRPC port; restrict reachability by other means first | `false` | | `pd.service.annotations` | Annotations on the PD client Service | `{}` | | `pd.service.restNodePort` | Optional fixed NodePort for PD REST; requires NodePort/LoadBalancer | unset | | `pd.service.grpcNodePort` | Optional fixed NodePort for PD gRPC; requires NodePort/LoadBalancer | unset | @@ -631,7 +645,7 @@ trusted network. | `hubble.service.type` | Hubble Service type | `ClusterIP` | | `hubble.service.annotations` | Hubble Service annotations | `{}` | | `hubble.service.nodePort` | Requires a `NodePort` or `LoadBalancer` Service type | unset | -| `hubble.ingress.*` | Same Ingress keys as `server.ingress.*`, plus `allowPlainHttp`, which applies to the Hubble Ingress only | `enabled: false` | +| `hubble.ingress.*` | Same Ingress keys as `server.ingress.*`, including `allowPlainHttp` | `enabled: false` | | `hubble.serviceAccount.*` | Same ServiceAccount keys as the other components | `create: true` | | `hubble.nodeSelector` / `tolerations` / `affinity` / `topologySpreadConstraints` | Scheduling controls | unset | | `hubble.priorityClassName` | PriorityClass for the hubble Pod | `""` | @@ -685,13 +699,22 @@ before anything reaches the cluster: - `hubble.port` must be a valid port, `hubble.persistence.size` must be non-empty, and `hubble.service.nodePort` requires a `NodePort` or `LoadBalancer` Service type. -- `hubble.image.tag` must be non-empty (the chart `appVersion` tracks the - Server release, not Hubble), and a Hubble Ingress without `tls` is rejected - unless `hubble.ingress.allowPlainHttp=true`. +- `hubble.image` needs a tag or a digest (the chart `appVersion` tracks the + Server release, not Hubble), and an Ingress without `tls` is rejected for + Server and Hubble alike unless the matching `ingress.allowPlainHttp=true` + opts in. - `hubble.enabled` without `server.auth.enabled` is rejected unless - `hubble.allowWithoutServerAuth=true`, and setting - `server.ingress.allowPlainHttp` is rejected because the plain-HTTP opt-in - applies to the Hubble Ingress only. + `hubble.allowWithoutServerAuth=true`, and + `hubble.securityContext.readOnlyRootFilesystem=true` is rejected because + the Hubble wrapper writes its properties file inside the image at startup. +- `store.pdb.minAvailable` must be at least `store.replicas - 1`, so + voluntary evictions cannot remove two copies of one shard at once. +- `podLabels` may not override the chart-managed `app.kubernetes.io/name`, + `instance` or `component` keys on any workload. +- A non-ClusterIP `pd.service.type` requires + `pd.service.allowInsecureExposure=true`. +- An upgrade may not shrink `pd.replicas` or `store.replicas` below the + live StatefulSet; see Scaling for the manual procedure. ## Deep Dive diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 82335f849b..301856897e 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -495,15 +495,16 @@ Seconds the chart gives the Server image to finish starting, passed as HG_SERVER_STARTUP_TIMEOUT_S. The image defaults that to 120 seconds, which is shorter than the storage wait alone, so a Server still coming up kills itself before Kubernetes has given up on it. The value therefore tracks the startup -probe: the effective failureThreshold above, already floored at 450 seconds, -times periodSeconds. Raising the probe budget raises this with it. The -entrypoint rejects anything over 86400, so the product is capped there rather -than rendered into a Pod that refuses to start. +probe: the effective budget above (floored at 450 seconds) minus the +300-second storage wait the entrypoint runs first, so the start command and +kubelet give up together instead of the image outliving the probe. Floored +at the image's own 120-second default, and capped at the entrypoint's 86400 +maximum rather than rendered into a Pod that refuses to start. */}} {{- define "hugegraph.server.startupTimeoutSeconds" -}} {{- $period := int .Values.server.probes.startup.periodSeconds -}} {{- $threshold := include "hugegraph.server.startupFailureThreshold" . | int -}} -{{- min 86400 (mul $threshold $period) -}} +{{- min 86400 (max 120 (sub (mul $threshold $period) 300)) -}} {{- end }} {{/* @@ -658,14 +659,17 @@ keys for releases stored before the values existed. {{- if and (or (get $pdSvc "restNodePort") (get $pdSvc "grpcNodePort")) (not (has $pdSvcType (list "NodePort" "LoadBalancer"))) -}} {{- fail "pd.service.restNodePort and pd.service.grpcNodePort require pd.service.type to be NodePort or LoadBalancer" -}} {{- end -}} +{{- if and (ne $pdSvcType "ClusterIP") (not (get $pdSvc "allowInsecureExposure" | default false)) -}} +{{- fail "pd.service.type NodePort or LoadBalancer exposes PD's unauthenticated gRPC port outside the cluster, raft membership RPCs included; keep ClusterIP, or set pd.service.allowInsecureExposure=true once reachability is restricted by other means (NetworkPolicy, load balancer allowlist, firewall)" -}} +{{- end -}} {{- $serverPdb := get .Values.server "pdb" | default dict -}} {{- $serverReplicaFloor := include "hugegraph.server.replicaFloor" . | int -}} {{- if and (get $serverPdb "enabled" | default false) (gt $serverReplicaFloor 1) (ge (int (get $serverPdb "minAvailable" | default 1)) $serverReplicaFloor) -}} {{- fail "server.pdb.minAvailable must be less than the active Server replica floor (server.hpa.minReplicas when HPA is enabled, otherwise server.replicas), otherwise the PDB permanently blocks voluntary disruptions such as node drains" -}} {{- end -}} {{- $serverIngress := get .Values.server "ingress" | default dict -}} -{{- if hasKey $serverIngress "allowPlainHttp" -}} -{{- fail "server.ingress.allowPlainHttp has no effect; the plain-HTTP opt-in applies to hubble.ingress only" -}} +{{- if and (get $serverIngress "enabled" | default false) (empty (get $serverIngress "tls")) (not (get $serverIngress "allowPlainHttp" | default false)) -}} +{{- fail "server.ingress.enabled without tls publishes Basic-auth credentials and JWTs over plain HTTP; configure server.ingress.tls, or set server.ingress.allowPlainHttp=true to accept that on a trusted network" -}} {{- end -}} {{/* extraEnv entries render after the chart-owned variables and Kubernetes lets @@ -703,6 +707,9 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- if and (eq (trim (get $hubbleImage "tag" | default "")) "") (eq (trim (get $hubbleImage "digest" | default "")) "") -}} {{- fail "hubble.image needs a tag or a digest: the chart appVersion tracks the Server release, not Hubble, so there is no meaningful fallback" -}} {{- end -}} +{{- if get (get $hubble "securityContext" | default dict) "readOnlyRootFilesystem" | default false -}} +{{- fail "hubble.securityContext.readOnlyRootFilesystem=true breaks Hubble: its wrapper writes conf/hugegraph-hubble.properties inside the image at startup and the chart mounts no writable volume there" -}} +{{- end -}} {{- $hubbleIngress := get $hubble "ingress" | default dict -}} {{- if and (get $hubbleIngress "enabled" | default false) (empty (get $hubbleIngress "tls")) (not (get $hubbleIngress "allowPlainHttp" | default false)) -}} {{- fail "hubble.ingress.enabled without tls publishes the plain-HTTP, unauthenticated Hubble UI; configure hubble.ingress.tls, or set hubble.ingress.allowPlainHttp=true to accept that on a trusted network" -}} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 44e4f63cea..e7bccebde7 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -142,6 +142,12 @@ spec: FOUND_PD_PEERS=false FOUND_URLS_TO_PD=false FOUND_DEPLOY_IN_K8S=false + # An IPv6 POD_IP must be bracketed inside the announcement URL. + # Only the chart's own default is rewritten; an explicit + # server.advertiseUrl is the operator's to format. + if [[ "${POD_IP:-}" == *:* && "${HG_SERVER_URLS_TO_PD:-}" == "http://${POD_IP}:"* ]]; then + HG_SERVER_URLS_TO_PD="http://[${POD_IP}]:${HG_SERVER_URLS_TO_PD##*:}" + fi {{- end }} {{- if .Values.server.auth.enabled }} FOUND_AUTH_ADMIN_PA=false diff --git a/helm/hugegraph/tests/server_startup_timeout_test.yaml b/helm/hugegraph/tests/server_startup_timeout_test.yaml index d47d5567e1..9ba6f68cf9 100644 --- a/helm/hugegraph/tests/server_startup_timeout_test.yaml +++ b/helm/hugegraph/tests/server_startup_timeout_test.yaml @@ -19,13 +19,13 @@ suite: Server startup timeout tracks the startup probe budget templates: - server-deployment.yaml tests: - - it: passes the default 450 second budget to the image + - it: passes the derived 150 second start budget to the image asserts: - contains: path: spec.template.spec.containers[0].env content: name: HG_SERVER_STARTUP_TIMEOUT_S - value: "450" + value: "150" - equal: path: spec.template.spec.containers[0].startupProbe.failureThreshold value: 90 @@ -42,9 +42,9 @@ tests: path: spec.template.spec.containers[0].env content: name: HG_SERVER_STARTUP_TIMEOUT_S - value: "2000" + value: "1700" - - it: follows the 450 second floor when a lower failureThreshold is configured + - it: follows the 450 second probe floor when a lower failureThreshold is configured set: server.probes.startup.failureThreshold: 1 server.probes.startup.periodSeconds: 5 @@ -53,7 +53,7 @@ tests: path: spec.template.spec.containers[0].env content: name: HG_SERVER_STARTUP_TIMEOUT_S - value: "450" + value: "150" - it: caps at the entrypoint's 86400 second maximum set: diff --git a/helm/hugegraph/tests/validate_values_test.yaml b/helm/hugegraph/tests/validate_values_test.yaml index 7fc861b099..dbe36bd0c8 100644 --- a/helm/hugegraph/tests/validate_values_test.yaml +++ b/helm/hugegraph/tests/validate_values_test.yaml @@ -101,3 +101,25 @@ tests: asserts: - failedTemplate: errorPattern: "store.podLabels must not set app.kubernetes.io/component" + + - it: rejects a TLS-less Server Ingress without the plain-HTTP opt-in + set: + server.ingress.enabled: true + asserts: + - failedTemplate: + errorPattern: "server.ingress.enabled without tls" + + - it: rejects a non-ClusterIP PD Service without the exposure acknowledgement + set: + pd.service.type: NodePort + asserts: + - failedTemplate: + errorPattern: "pd.service.type NodePort or LoadBalancer exposes" + + - it: rejects a read-only root filesystem on Hubble + set: + hubble.enabled: true + hubble.securityContext.readOnlyRootFilesystem: true + asserts: + - failedTemplate: + errorPattern: "readOnlyRootFilesystem=true breaks Hubble" diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 45ffa2b976..4c45dfe8b7 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -485,6 +485,10 @@ "annotations": { "type": "object" }, + "allowInsecureExposure": { + "type": "boolean", + "description": "Required acknowledgement for a non-ClusterIP PD Service: the exposed gRPC port has no authentication." + }, "restNodePort": { "type": [ "integer", @@ -1174,6 +1178,10 @@ }, "annotations": { "type": "object" + }, + "allowPlainHttp": { + "type": "boolean", + "description": "Explicit opt-in to a TLS-less Server Ingress; without it the render fails when tls is empty." } } }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 9ec52fbb25..0dcb0f167a 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -72,9 +72,10 @@ pd: # storage wait (PD_AUTH_PASSWORD, user store) and to Hubble # (operations.pd.password, user hubble). The Secret is kept on uninstall # like the Server auth Secrets. The value must be printable ASCII with - # no leading whitespace: it is written into Hubble's Java properties - # file, which strips leading whitespace and reads the file as - # ISO-8859-1. Priority: existingSecret > value > autoGenerate. + # no leading whitespace and no backslashes: it is written into Hubble's + # Java properties file, which strips leading whitespace, unescapes + # backslashes, and reads the file as ISO-8859-1. Priority: + # existingSecret > value > autoGenerate. auth: # Optional plaintext secret (prefer existingSecret in shared clusters). value: "" @@ -124,6 +125,11 @@ pd: service: type: ClusterIP annotations: {} + # A non-ClusterIP type exposes PD's gRPC port, which has no + # authentication (raft membership RPCs included). The render refuses it + # unless this acknowledgement is set and reachability is restricted by + # other means (NetworkPolicy, load balancer allowlist, firewall). + allowInsecureExposure: false # Optional fixed NodePorts; require service.type NodePort or LoadBalancer. restNodePort: grpcNodePort: From 2a7981f60373d775a098283be7dac17202517a1f Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 19:38:04 +0530 Subject: [PATCH 44/61] docs(helm): target the PD leader in recovery, retire a replaced Store ID Review fixes from hugegraph/hugegraph#221: The recovery task endpoints execute locally on the PD that receives them and answer a follower with empty success, so the documented procedure could report three successes while recovering nothing. The runbook now identifies the leader through /v1/members and port-forwards that Pod before triggering tasks. A Store replaced with an empty PVC registers under a new Store ID while the old one stays Offline with its shard memberships; the patrol repairs only Tombstone members, so it never touches the Offline entry. The runbook adds the retirement step: mark the old ID Tombstone with POST /v1/store/ {"storeState":"Tombstone"}, which hands its shards to the patrol, then patrol and verify the shard groups. DELETE only erases the record and strands the memberships, so it is documented as cleanup after the patrol, never the retirement itself. --- helm/hugegraph/README.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index c87494d78f..bc65db7aea 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -828,13 +828,18 @@ explicitly. PD's configuration binds `pd.patrol-interval` and `store.max-down-time` keys, but no code path on current builds reads either, which is why this chart does not expose them. -Recovery and rebalancing are operator-triggered. PD exposes REST triggers, -reachable through the PD client Service: +Recovery and rebalancing are operator-triggered, and the task endpoints +execute **locally on the PD that receives them**: a follower answers with +an empty success and does no recovery work. Port-forwarding the client +Service selects an arbitrary PD, so identify the leader first and +port-forward that Pod: ```bash kubectl port-forward -n hugegraph svc/hugegraph-pd-client 8620:8620 PD_SECRET="$(kubectl -n hugegraph get secret hugegraph-pd-auth \ -o jsonpath='{.data.secret-key}' | base64 --decode)" +curl -su "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/members # read .data.pdLeader.raftUrl; its host names the leader Pod +kubectl port-forward -n hugegraph pod/ 8620:8620 # replace the Service forward with the leader curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/patrolPartitions # reconcile shard groups, process tombstoned Stores curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balanceLeaders # spread Raft leaders curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data @@ -848,6 +853,20 @@ Run `patrolPartitions` after replacing a Store that is not coming back, `balancePartitions` once the cluster is stable again, and `balanceLeaders` after restarts that skewed leader placement. +A Store replaced with an empty PVC registers under a **new Store ID**, even +though its Pod name and DNS address are unchanged, and the old ID stays +`Offline` in PD with its shard memberships intact; the patrol repairs only +`Tombstone` members, so it never touches the `Offline` entry. After such a +replacement, retire the old ID explicitly on the leader: find the +`Offline` entry in `/v1/stores` whose address matches the replaced Pod, +mark it `Tombstone` with `curl -u "hg:${PD_SECRET}" -X POST -H +'Content-Type: application/json' -d '{"storeState":"Tombstone"}' +http://127.0.0.1:8620/v1/store/` (this hands its shards to the +patrol), then run `patrolPartitions` and verify every shard group lists +only `Up` Stores. `DELETE /v1/store/` only erases the record and +strands the shard memberships; use it, if at all, as cleanup after the +patrol has finished. + Periodic balancing and shard-sync progress metrics do not exist upstream yet and are out of scope for this chart. Periodic leader balancing is tracked in From 0b15d889abda874728749f98c0b2c58269f38ecc Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 20:24:00 +0530 Subject: [PATCH 45/61] ci(helm): pin helm-unittest to v1.0.0, the newest Helm 3.16.2 can load The v1.1.x plugin manifests carry a platformHooks field that Helm 3.16.2 rejects at plugin load, so the new unittest step failed in seconds on both PRs; the local gate missed it because this machine runs Helm 4. Verified against the CI's exact 3.16.2 binary with an isolated plugin home: v1.0.0 installs and runs the suites. Under v1.0.0 a schema abort is a suite error rather than a matchable failed render, which broke exactly one case, the pd.auth no-secret-source rejection. That input moves to the workflow's must-fail render guards, where the same class of schema rejection is already asserted; nothing is asserted less than before. 71 cases pass under 3.16.2 with v1.0.0 and under Helm 4 alike. --- .github/workflows/helm-chart-ci.yml | 8 +++++++- helm/hugegraph/tests/pd_auth_secret_test.yaml | 10 ---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 2a234ae831..4c9a25465e 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -53,7 +53,9 @@ jobs: - name: helm unittest run: | - helm plugin install https://github.com/helm-unittest/helm-unittest.git --version v1.1.2 + # v1.0.0 is the newest helm-unittest whose plugin.yaml Helm 3.16.2 can + # parse; v1.1.x adds a platformHooks field that 3.16.2 rejects. + helm plugin install https://github.com/helm-unittest/helm-unittest.git --version v1.0.0 helm unittest helm/hugegraph - name: helm template @@ -137,6 +139,10 @@ jobs: --set server.pdb.enabled=true \ --set server.pdb.minAvailable=2 must_fail --set server.auth.enabled=true --set server.auth.admin.autoGenerate=false --set server.auth.token.autoGenerate=false + # pd.auth with every source disabled fails the schema anyOf; the + # unit framework at v1.0.0 cannot match schema aborts, so the case + # lives here. + must_fail --set pd.auth.autoGenerate=false # Auth defaults to on, so Hubble alone is valid; refuse Hubble only # when authentication is explicitly disabled. must_fail --set hubble.enabled=true --set server.auth.enabled=false diff --git a/helm/hugegraph/tests/pd_auth_secret_test.yaml b/helm/hugegraph/tests/pd_auth_secret_test.yaml index ae0ee1bf34..89ed20302c 100644 --- a/helm/hugegraph/tests/pd_auth_secret_test.yaml +++ b/helm/hugegraph/tests/pd_auth_secret_test.yaml @@ -118,16 +118,6 @@ tests: name: my-pd-secret key: pd-pass - - it: rejects a configuration with no secret source - template: pd-statefulset.yaml - set: - pd.auth.autoGenerate: false - asserts: - # The values schema refuses this before the template guard can; the - # guard still exists for values that bypass schema validation. - - failedTemplate: - errorPattern: "pd/auth|pd.auth requires" - - it: rejects operator overrides of the chart-managed PD_AUTH_PASSWORD template: server-deployment.yaml set: From b2344a81eff08bdc8a47b6a716518436c1502e88 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 20:26:04 +0530 Subject: [PATCH 46/61] fix(helm): pin the plugin by commit, unblock the recovery forwards Follow-ups from CodeRabbit's 2026-09-18 pass on hugegraph/hugegraph#221: - Install helm-unittest by the v1.0.0 tag's commit SHA, so a moved tag cannot change what executes on the runner. - The recovery runbook ran two port-forwards and three curls as one foreground sequence, which blocks at the first forward. Note the second terminal, stop the Service forward before the leader forward, and move the inline comments to their own lines within 100 columns. - Exclude a leading form feed from inline admin passwords: the config reader trims it when the admin is first created, so the Secret and the effective password would silently differ. --- .github/workflows/helm-chart-ci.yml | 6 ++++-- helm/hugegraph/README.md | 18 +++++++++++++----- helm/hugegraph/values.schema.json | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 4c9a25465e..06adf85907 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -54,8 +54,10 @@ jobs: - name: helm unittest run: | # v1.0.0 is the newest helm-unittest whose plugin.yaml Helm 3.16.2 can - # parse; v1.1.x adds a platformHooks field that 3.16.2 rejects. - helm plugin install https://github.com/helm-unittest/helm-unittest.git --version v1.0.0 + # parse (v1.1.x adds a platformHooks field that 3.16.2 rejects). + # Pinned by the tag's commit so a moved tag cannot change what runs. + helm plugin install https://github.com/helm-unittest/helm-unittest.git \ + --version ddd5feaa9465e7a3591792524ce7f441d4c5157e helm unittest helm/hugegraph - name: helm template diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index bc65db7aea..fc994d9e01 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -834,15 +834,23 @@ an empty success and does no recovery work. Port-forwarding the client Service selects an arbitrary PD, so identify the leader first and port-forward that Pod: +`kubectl port-forward` runs in the foreground, so use a second terminal +(or background the forward) for the curls, and stop the Service forward +before starting the leader one: + ```bash kubectl port-forward -n hugegraph svc/hugegraph-pd-client 8620:8620 PD_SECRET="$(kubectl -n hugegraph get secret hugegraph-pd-auth \ -o jsonpath='{.data.secret-key}' | base64 --decode)" -curl -su "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/members # read .data.pdLeader.raftUrl; its host names the leader Pod -kubectl port-forward -n hugegraph pod/ 8620:8620 # replace the Service forward with the leader -curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/patrolPartitions # reconcile shard groups, process tombstoned Stores -curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balanceLeaders # spread Raft leaders -curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions # spread partition data +# Read .data.pdLeader.raftUrl; its host names the leader Pod. +curl -su "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/members +# Stop the Service forward, then forward the leader Pod instead. +kubectl port-forward -n hugegraph pod/ 8620:8620 +# Reconcile shard groups and process tombstoned Stores. +curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/patrolPartitions +# Spread Raft leaders, then partition data. +curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balanceLeaders +curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions ``` The credential is required; PD answers 401 without it. The Secret name diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 4c45dfe8b7..211e88f062 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -748,7 +748,7 @@ "properties": { "password": { "type": "string", - "pattern": "^([^ \\t\\r\\n\\\\][^\\r\\n\\\\]*)?$", + "pattern": "^([^ \\t\\f\\r\\n\\\\][^\\r\\n\\\\]*)?$", "description": "Inline admin password. Empty defers to existingSecret or autoGenerate. The Server wrapper rejects newlines, carriage returns, backslashes and leading whitespace, so the schema rejects them before install." }, "existingSecret": { From 52d03a31b66ca635377504c72d3d1039e83c73b9 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 20:27:32 +0530 Subject: [PATCH 47/61] fix(helm): pull secrets for the test hook, document the PVC resize path Review fixes from hugegraph/hugegraph#221: The helm test Pod never received .Values.imagePullSecrets, so on an authenticated private mirror the four workloads pull with credentials while helm test enters ImagePullBackOff. The hook now propagates the setting like the workloads do, with a unit case. PD and Store storage sizes live in the StatefulSet volumeClaimTemplates, which Kubernetes forbids changing, so an upgrade with a new size was rejected in full with nothing telling the operator the supported path. The parameter tables mark both sizes install-time and Upgrading gains the resize procedure: patch each PVC, wait, recreate the StatefulSet with --cascade=orphan, upgrade with the matching value. --- helm/hugegraph/README.md | 12 ++++++++++-- helm/hugegraph/templates/tests/test-connection.yaml | 4 ++++ helm/hugegraph/tests/test_hook_resources_test.yaml | 10 ++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index fc994d9e01..ef7a00ea3f 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -230,6 +230,14 @@ now needs `pd.service.allowInsecureExposure=true`, and a TLS-less Server Ingress needs `server.ingress.allowPlainHttp=true`. The render error names the value to set. +PD and Store storage sizes live in the StatefulSet `volumeClaimTemplates`, +which Kubernetes forbids changing, so an upgrade with a new size is +rejected in full. To grow storage on a StorageClass that supports volume +expansion: patch each PVC's `spec.resources.requests.storage`, wait for +the resize to finish, recreate the StatefulSet object without touching +Pods (`kubectl delete statefulset --cascade=orphan`), then upgrade +with the matching value. + Two cases are worth knowing about in advance: - **PD** restarts one pod at a time whenever its Pod template changes, which @@ -334,7 +342,7 @@ default values. | `pd.ports.rest` | PD REST port, also used by probes | `8620` | | `pd.ports.raft` | PD Raft port | `8610` | | `pd.dataPath` | PD data directory inside the container | `/hugegraph-pd/pd_data` | -| `pd.storage.size` | PD PersistentVolumeClaim size | `10Gi` | +| `pd.storage.size` | PD PersistentVolumeClaim size. Applies at install; see Upgrading for the resize procedure | `10Gi` | | `pd.storage.storageClassName` | Empty uses the cluster default StorageClass | `""` | | `pd.resources` | PD container resources. Set these for production | `{}` | | `pd.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | @@ -380,7 +388,7 @@ default values. | `store.ports.raft` | Store Raft port | `8510` | | `store.ports.rest` | Store REST port | `8520` | | `store.dataPath` | Store data directory | `/hugegraph-store/storage` | -| `store.storage.size` | Store PersistentVolumeClaim size | `50Gi` | +| `store.storage.size` | Store PersistentVolumeClaim size. Applies at install; see Upgrading for the resize procedure | `50Gi` | | `store.storage.storageClassName` | Empty uses the cluster default StorageClass | `""` | | `store.resources` | Store container resources. Set these for production | `{}` | | `store.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | diff --git a/helm/hugegraph/templates/tests/test-connection.yaml b/helm/hugegraph/templates/tests/test-connection.yaml index 1176538a91..4a15184a78 100644 --- a/helm/hugegraph/templates/tests/test-connection.yaml +++ b/helm/hugegraph/templates/tests/test-connection.yaml @@ -28,6 +28,10 @@ metadata: "helm.sh/hook": test "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} automountServiceAccountToken: false restartPolicy: Never securityContext: diff --git a/helm/hugegraph/tests/test_hook_resources_test.yaml b/helm/hugegraph/tests/test_hook_resources_test.yaml index f57334d8a0..e96af6ba22 100644 --- a/helm/hugegraph/tests/test_hook_resources_test.yaml +++ b/helm/hugegraph/tests/test_hook_resources_test.yaml @@ -43,3 +43,13 @@ tests: - equal: path: spec.containers[0].resources.limits.cpu value: 500m + + - it: passes imagePullSecrets to the test hook Pod + set: + imagePullSecrets: + - name: registry-creds + asserts: + - contains: + path: spec.imagePullSecrets + content: + name: registry-creds From 05f3d9e0b940aebece2569378bd230b328696957 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 18 Sep 2026 20:54:19 +0530 Subject: [PATCH 48/61] docs(helm): constrain BYO PD Secrets, re-verify leadership after recovery Two gaps a review of the replies surfaced before posting them: The pd.auth.existingSecret row now states the same value constraint as the inline path (printable ASCII, no leading whitespace, no backslashes); an external Secret bypasses the schema, so the rule has to live in the contract text. The recovery runbook now re-reads /v1/members after the task sequence: if leadership moved mid-sequence, the later tasks ran on a follower and did nothing, so they are rerun on the new leader. Also strips trailing whitespace the workflow picked up in e62cac3d2. --- .github/workflows/helm-chart-ci.yml | 2 +- helm/hugegraph/README.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 06adf85907..5ecc700c8f 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -110,7 +110,7 @@ jobs: # A YAML-coercible Secret key must render quoted, or the API # server stores the key as a boolean and the pods cannot find it. helm template ci helm/hugegraph --set-string server.auth.admin.key=on \ - | grep -qF '"on":' + | grep -qF '"on":' - name: reject invalid values run: | diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index ef7a00ea3f..052c073efb 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -365,7 +365,7 @@ default values. | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | | `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader | `/v1/ready` | | `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. Printable ASCII, no leading whitespace, no backslashes | `""` | -| `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it | `""` | +| `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it. Its value must meet the same constraint as `pd.auth.value`: printable ASCII, no leading whitespace, no backslashes | `""` | | `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | | `pd.auth.autoGenerate` | Create and keep a random release-pd-auth Secret when `value` and `existingSecret` are empty | `true` | | `pd.probes.*.periodSeconds` | Probe interval | see `values.yaml` | @@ -861,6 +861,10 @@ curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balanceLeaders curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions ``` +Read `/v1/members` again after the tasks: if leadership moved mid-sequence, +the later tasks ran on a follower and did nothing, so rerun them on the new +leader. + The credential is required; PD answers 401 without it. The Secret name follows the release (`-pd-auth`) unless `pd.auth.existingSecret` is set. From 62aa010df7f880e2216a28555201c566ce62d942 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sat, 19 Sep 2026 10:41:40 +0530 Subject: [PATCH 49/61] feat(helm): make the Server readiness path configurable Add server.readinessPath (default /versions, so nothing changes on current images), mirroring pd.readinessPath: values.yaml, the schema (pattern ^/) and the Server readinessProbe in server-deployment.yaml, plus a README parameter row and a two-case unit suite (74 total). On a Server image that serves GET /readiness (apache/hugegraph#3212, proposed in apache/hugegraph#3221), setting the value to /readiness makes a Server that cannot serve graph traffic answer 503 and drop out of the Service instead of returning 500 to every graph request. Startup and liveness stay on /versions so a Server that merely lost its storage is not restarted. Implements the change proposed and measured by @SebastianGruza in hugegraph/hugegraph#229 (six fault scenarios at 1 Hz sampling, zero readiness transitions across Store and PD rolling restarts). --- helm/hugegraph/README.md | 1 + .../templates/server-deployment.yaml | 2 +- .../tests/server_readiness_path_test.yaml | 46 +++++++++++++++++++ helm/hugegraph/values.schema.json | 5 ++ helm/hugegraph/values.yaml | 6 +++ 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 helm/hugegraph/tests/server_readiness_path_test.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 052c073efb..cc5874b7e8 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -426,6 +426,7 @@ default values. | `server.image.pullPolicy` | Server image pull policy | `Always` | | `server.javaOpts` | Empty preserves the image's automatic JVM sizing | `""` | | `server.port` | Server REST port, container port, and Service port | `8080` | +| `server.readinessPath` | Path the Server readinessProbe hits. Set `/readiness` once the Server image serves it (apache/hugegraph#3212); it answers 503 while the Server cannot serve graph traffic. Startup and liveness stay on `/versions` | `/versions` | | `server.backend` | Storage backend | `hstore` | | `server.resources` | Server resources. `requests.cpu` is required when HPA is enabled | `{}` | | `server.podSecurityContext` | Pod-level securityContext, rendered only when set | `{}` | diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index e7bccebde7..93709f0c9b 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -317,7 +317,7 @@ spec: {{- with include "hugegraph.probeTuning" .Values.server.probes.startup }}{{ . | trim | nindent 12 }}{{- end }} readinessProbe: httpGet: - path: /versions + path: {{ .Values.server.readinessPath | default "/versions" }} port: http periodSeconds: {{ .Values.server.probes.readiness.periodSeconds }} failureThreshold: {{ .Values.server.probes.readiness.failureThreshold }} diff --git a/helm/hugegraph/tests/server_readiness_path_test.yaml b/helm/hugegraph/tests/server_readiness_path_test.yaml new file mode 100644 index 0000000000..aa4eeecfdf --- /dev/null +++ b/helm/hugegraph/tests/server_readiness_path_test.yaml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Server readiness path +tests: + - it: reads all three Server probes from /versions by default + template: server-deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /versions + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /versions + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /versions + + - it: moves only the Server readiness probe when server.readinessPath targets /readiness + template: server-deployment.yaml + set: + server.readinessPath: /readiness + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /readiness + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /versions + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /versions diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 211e88f062..dec27dbb78 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -940,6 +940,11 @@ } } }, + "readinessPath": { + "type": "string", + "pattern": "^/", + "description": "HTTP path for the Server readinessProbe; /readiness (apache/hugegraph#3212) on Server images that serve it, /versions (default) on older ones" + }, "probes": { "$ref": "#/definitions/probes" }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 0dcb0f167a..c84379a37f 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -280,6 +280,12 @@ server: # Empty preserves the image entrypoint's automatic JVM sizing. javaOpts: "" port: 8080 + # HTTP path the Server readinessProbe hits. /readiness (apache/hugegraph#3212) + # answers 503 while the Server cannot serve graph traffic, so such a Server + # drops out of the Service instead of answering 500 to every graph request. + # Keep /versions on Server images that do not serve it. Startup and liveness + # stay on /versions so a Server that merely lost its storage is not restarted. + readinessPath: /versions backend: hstore resources: {} # Server is stateless and may scale past the node count via HPA, so the From 5c151163d3abb9c8dc8c12f1e7d2364b15dcd70f Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 01:01:51 +0530 Subject: [PATCH 50/61] fix(helm): raise the cluster-preset Store memory ceiling The Store commits well beyond its heap. conf/application.yml includes the pd Spring profile, and conf/application-pd.yml sets rocksdb.total_memory_size to 32000000000; RaftRocksdbOptions splits that into a RocksDB write cache and block cache, so both are bounded by 32 GB rather than by the container. The jraft log storage registers a further 1 GiB LRU cache once per process. Against -Xmx1024m -XX:MaxDirectMemorySize=512m the old 4Gi limit sat below the steady state: a k3s run OOM-killed all three Stores after about 1 GB of data and they stayed in CrashLoopBackOff, while 8Gi held at 4.42 GiB anonymous RSS. Request 5Gi, limit 8Gi, with the accounting recorded in the preset and in Limitations. The limit bounds the damage, not RocksDB. The Store entrypoint rebuilds SPRING_APPLICATION_JSON from its own variables and the chart mounts no config file, so rocksdb.total_memory_size cannot be set from the chart today; that is noted as an image-side gap. Reported with measurements by @SebastianGruza on hugegraph/hugegraph#221. --- helm/hugegraph/README.md | 15 +++++++ .../tests/cluster_preset_resources_test.yaml | 40 +++++++++++++++++++ helm/hugegraph/values-cluster.yaml | 18 ++++++++- 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 helm/hugegraph/tests/cluster_preset_resources_test.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index cc5874b7e8..1f0caf03f1 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -1030,6 +1030,21 @@ independently of the release name. gave PD `-Xmx3299m` and, with three to four pods per node, never converged. Use `values-cluster.yaml`, or set your own `resources`, for any multi-node deployment. +- The Store's memory ceiling is not its heap. The shipped + `conf/application.yml` includes the `pd` Spring profile, and + `conf/application-pd.yml` sets `rocksdb.total_memory_size` to + `32000000000`; `RaftRocksdbOptions` splits that number into a RocksDB + write cache and block cache, so those native caches are bounded by 32 GB + and not by the container. The jraft log storage also registers its own + 1 GiB LRU block cache once per process. With the cluster preset's + `-Xmx1024m -XX:MaxDirectMemorySize=512m`, a 4Gi limit sat below the + steady state and the kernel OOM-killed all three Stores after about 1 GB + of data; the preset now asks for 5Gi and limits at 8Gi, where a k3s run + measured 4.42 GiB anonymous RSS (2026-09-19). Scale both numbers with the + data size. The chart cannot lower the RocksDB budget itself: the Store + entrypoint rebuilds `SPRING_APPLICATION_JSON` from its own variables and + the chart mounts no config file, so `rocksdb.total_memory_size` can only + be changed in the image or through a custom config mount. - PD's raft IP whitelist resolves peer hostnames to IPs once at startup, which under Kubernetes can block peers whose pod IPs were unpublished at that moment or change later. The chart therefore disables the whitelist diff --git a/helm/hugegraph/tests/cluster_preset_resources_test.yaml b/helm/hugegraph/tests/cluster_preset_resources_test.yaml new file mode 100644 index 0000000000..44a515c056 --- /dev/null +++ b/helm/hugegraph/tests/cluster_preset_resources_test.yaml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Cluster preset Store memory +values: + - ../values-cluster.yaml +tests: + - it: gives the Store room for the caches the image commits outside its heap + template: store-statefulset.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 8Gi + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 5Gi + + - it: keeps the Store heap flags the memory budget is derived from + template: store-statefulset.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value + pattern: -Xmx1024m + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="JAVA_OPTS")].value + pattern: -XX:MaxDirectMemorySize=512m diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index dee8edd1c1..35c9d7a26d 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -48,13 +48,27 @@ store: -XX:MaxMetaspaceSize=256m -XX:MaxDirectMemorySize=512m -XX:+UseContainerSupport + # The Store commits far more than its heap. conf/application-pd.yml, which + # the shipped conf/application.yml includes as a Spring profile, sets + # rocksdb.total_memory_size to 32000000000, and RaftRocksdbOptions splits + # that value into a RocksDB write cache and block cache, so those native + # caches are bounded by 32 GB rather than by the container. On top of that + # the jraft log storage registers its own 1 GiB LRU block cache once per + # process, plus heap, direct memory, metaspace and about a thousand threads. + # A 4Gi limit was below the steady state and OOM-killed all three Stores + # after roughly 1 GB of data; 8Gi holds, with 4.42 GiB anonymous RSS + # measured on k3s (2026-09-19). The limit bounds the damage, it does not + # bound RocksDB: the chart cannot set rocksdb.total_memory_size today, + # because the Store entrypoint rebuilds SPRING_APPLICATION_JSON from its + # own variables and no conf file is mounted. Raise both numbers together + # with the data size, or lower the RocksDB budget in a custom image. resources: requests: cpu: "1" - memory: 2Gi + memory: 5Gi limits: cpu: "4" - memory: 4Gi + memory: 8Gi waitResources: requests: cpu: 25m From f7c2f312a0d057938f0c30aa6d57ab6401167414 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 01:03:46 +0530 Subject: [PATCH 51/61] fix(helm): pick the rollout checksum revision by credential source Both auth checksums appended the live Secret resourceVersion unconditionally, including when the credential was an active inline value. That rolled the Pods a second time on the no-change upgrade after a rotation: the rotation render still saw the old resourceVersion, and the next render saw the new one with the same desired value. The revision input is now chosen per credential. An active inline value contributes its digest alone, so a rotation rolls once. An external or chart-generated Secret keeps the live resourceVersion, which is the only signal it has. Server admin and token select independently. Tests pin the exact annotation for four source combinations, so the parts list cannot change unnoticed; the existingSecret cases set an inline value too and prove it stays out. A render carries no live Secret, so the lookup half stays for the lifecycle run. Answers the open thread on hugegraph/hugegraph#221. --- helm/hugegraph/templates/_helpers.tpl | 46 +++++++--- .../tests/auth_checksum_source_test.yaml | 89 +++++++++++++++++++ 2 files changed, 124 insertions(+), 11 deletions(-) create mode 100644 helm/hugegraph/tests/auth_checksum_source_test.yaml diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 301856897e..ef5a3d3a03 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -214,17 +214,30 @@ every upgrade, on the same secret. {{/* Checksum for the PD, Server and Hubble pod templates so rotating the PD REST -Secret rolls the Pods that read it. Same contract as hugegraph.server.authChecksum: -names, key and metadata.resourceVersion only, never Secret data; lookup-based, -so template-only renders emit a constant. +Secret rolls the Pods that read it. Never hashes Secret data: it hashes the +Secret name, its key, and one revision input chosen by where the credential +comes from. + +The revision input is per credential source, because the two sources move at +different times. An active inline value is known at render time, so its +digest is the revision and it changes exactly once, on the upgrade that +rotates it. Mixing the live resourceVersion into that case would roll the +Pods a second time on the next no-change upgrade, once the rotated Secret had +been applied and its resourceVersion moved. An external or chart-generated +Secret has no render-time value to hash, so the live resourceVersion is the +only signal that it changed; there the lookup is kept and template-only +renders emit a constant. */}} {{- define "hugegraph.pd.authChecksum" -}} {{- $parts := list (include "hugegraph.pd.authSecretName" .) (include "hugegraph.pd.authSecretKey" .) -}} {{- $pdAuthCfg := get .Values.pd "auth" | default dict -}} {{- $inline := get $pdAuthCfg "value" | default "" -}} -{{- if and $inline (not (get $pdAuthCfg "existingSecret" | default "")) -}}{{- $parts = append $parts (sha256sum $inline) -}}{{- end -}} +{{- if and $inline (not (get $pdAuthCfg "existingSecret" | default "")) -}} +{{- $parts = append $parts (sha256sum $inline) -}} +{{- else -}} {{- $secret := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.pd.authSecretName" .) -}} {{- if $secret -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $secret) -}}{{- end -}} +{{- end -}} {{- join "|" $parts | sha256sum -}} {{- end }} @@ -276,13 +289,20 @@ PD REST endpoints for Server storage-readiness checks. {{/* Checksum for the Server pod template so rotating the referenced auth Secrets -rolls Server pods. Hashes Secret names, keys, and metadata.resourceVersion - -never Secret data - so the annotation carries no credential-derived material. -Lookup-based and therefore best-effort: plain `helm template` (and -template-only GitOps renderers) see no live Secrets and emit a constant; the -first upgrade after a fresh install rolls Server once as the checksum picks -up the Secrets created by that install; out-of-band rotation of an -existingSecret applies on the next `helm upgrade`. +rolls Server pods. Hashes Secret names and keys, never Secret data, so the +annotation carries no credential-derived material. + +The admin and token credentials pick their revision input independently, by +source, for the reason given on hugegraph.pd.authChecksum: an active inline +value contributes its digest and nothing else, so a rotation rolls Server +once rather than again on the next no-change upgrade; an external or +chart-generated Secret contributes its live metadata.resourceVersion. + +The lookup half is best-effort: plain `helm template` (and template-only +GitOps renderers) see no live Secrets and emit a constant; the first upgrade +after a fresh install rolls Server once as the checksum picks up the Secrets +created by that install; out-of-band rotation of an existingSecret applies on +the next `helm upgrade`. */}} {{- define "hugegraph.server.authChecksum" -}} {{- $parts := list (include "hugegraph.server.authSecretName" .) (include "hugegraph.server.authSecretKey" .) (include "hugegraph.server.authTokenSecretName" .) (include "hugegraph.server.authTokenSecretKey" .) -}} @@ -293,10 +313,14 @@ existingSecret applies on the next `helm upgrade`. {{- $tokenCfg := get $srvAuth "token" | default dict -}} {{- $inlineToken := get $tokenCfg "value" | default "" -}} {{- if and $inlineToken (not (get $tokenCfg "existingSecret" | default "")) -}}{{- $parts = append $parts (sha256sum $inlineToken) -}}{{- end -}} +{{- if not (and $inlineAdmin (not (get $adminCfg "existingSecret" | default ""))) -}} {{- $admin := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authSecretName" .) -}} {{- if $admin -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $admin) -}}{{- end -}} +{{- end -}} +{{- if not (and $inlineToken (not (get $tokenCfg "existingSecret" | default ""))) -}} {{- $token := lookup "v1" "Secret" .Release.Namespace (include "hugegraph.server.authTokenSecretName" .) -}} {{- if $token -}}{{- $parts = append $parts (dig "metadata" "resourceVersion" "" $token) -}}{{- end -}} +{{- end -}} {{- join "|" $parts | sha256sum -}} {{- end }} diff --git a/helm/hugegraph/tests/auth_checksum_source_test.yaml b/helm/hugegraph/tests/auth_checksum_source_test.yaml new file mode 100644 index 0000000000..a2f4fea187 --- /dev/null +++ b/helm/hugegraph/tests/auth_checksum_source_test.yaml @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Each rollout checksum picks its revision input by credential source. These +# cases pin the exact digest, so they fail if the parts list changes at all, +# not merely if it stops being 64 hex characters. Each expected value is +# sha256 of the parts joined with "|", computed outside Helm: +# +# inline PD value sha256("t-pd-auth|secret-key|" + sha256(value)) +# PD existingSecret sha256("byo-pd-secret|secret-key") +# +# The existingSecret cases are the ones that prove the selection: the inline +# value is set there too, and it must not reach the annotation, because the +# Pods do not read it. +# +# A render has no live Secret, so these cases cover the inline half only. The +# lookup half, where mixing a live resourceVersion into an inline rotation +# caused a second rollout on the following no-change upgrade, is observable +# only against a cluster and belongs to the lifecycle run. +suite: Rollout checksum revision source +release: + name: t +tests: + - it: hashes the inline PD value with the Secret name and key, and nothing else + template: pd-statefulset.yaml + set: + pd.auth.value: pd-secret-steady + asserts: + - equal: + path: spec.template.metadata.annotations["checksum/pd-auth"] + value: c3e18a4d518ea495ef75e4fec03a68675b23ef0be93714e7a777fbb50a5957c1 + + - it: drops the inline PD digest once an existingSecret supplies the credential + template: pd-statefulset.yaml + set: + pd.auth.existingSecret: byo-pd-secret + pd.auth.value: ignored-by-the-pods + asserts: + - equal: + path: spec.template.metadata.annotations["checksum/pd-auth"] + value: 86fdc89040752c37324032343cd1d9a53ca23a46f724689dd61c2f5f829c0d33 + + - it: hashes both inline Server credentials with their Secret names and keys + template: server-deployment.yaml + set: + server.auth.admin.password: admin-after + server.auth.token.value: 0123456789abcdef0123456789abcdef + asserts: + - equal: + path: spec.template.metadata.annotations["checksum/auth"] + value: d6ce6518e8b740a46bbacf0b55e4094580048c87912c85f21448b701f972fd72 + + - it: selects the admin and token sources independently + template: server-deployment.yaml + set: + server.auth.admin.existingSecret: byo-admin + server.auth.admin.password: ignored-by-the-pods + server.auth.token.value: 0123456789abcdef0123456789abcdef + asserts: + - equal: + path: spec.template.metadata.annotations["checksum/auth"] + value: 7f13191bccb01ef3b2c6ffcce2fc6b6a73a6c6416583dcc4c593a5cfde1a24a0 + + - it: keeps credential plaintext out of the annotation + template: server-deployment.yaml + set: + server.auth.admin.password: admin-plaintext-value + server.auth.token.value: 0123456789abcdef0123456789abcdef + asserts: + - notMatchRegex: + path: spec.template.metadata.annotations["checksum/auth"] + pattern: admin-plaintext-value + - notMatchRegex: + path: spec.template.metadata.annotations["checksum/auth"] + pattern: 0123456789abcdef0123456789abcdef From 48062ad9657153a09068374e33b5dd92596ecd4d Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 01:04:14 +0530 Subject: [PATCH 52/61] docs(helm): give the Store roll a shard-level barrier The OnDelete procedure told the operator to wait for the replaced Store to show Up in PD. That is not a recovery barrier. StoreNodeService.register() persists StoreState.Up and only then notifies the Store, where HgStoreEngine.stateChanged starts restoreLocalPartitionEngine(); a failure there is logged and the state stays Up. The Store is Up before it has restored anything, so an operator following the old text could delete the next replica while the first was still rejoining. Replace it with the strongest check these images support: per shard group, the full shard count, exactly one leader, and the replaced Store back in the groups it holds, read from the PD leader through /v1/shardGroups. State the residue plainly. That is PD's membership record, not proof that the Store finished loading its partitions and caught up, and no endpoint reports restoration-complete, so the text asks for a margin, keeps the PDB floor as the backstop, and names the missing image-side signal. Answers the P1 recheck on hugegraph/hugegraph#221. --- helm/hugegraph/README.md | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 1f0caf03f1..2e7f9eae0c 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -248,8 +248,42 @@ Two cases are worth knowing about in advance: listener, not shard recovery: the controller can replace the next Store while the previous one is still rejoining its shard groups. For a production image roll, set `store.updateStrategy.type=OnDelete` and delete - Store Pods one at a time, waiting for the replaced Store to show `Up` in - PD (see Cluster Health) before the next. + Store Pods one at a time, checking shard membership between deletions. + + `Up` in PD is not that check. PD sets `StoreState.Up` and persists it in + `StoreNodeService.register()`, and only then does the notification reach + the Store, whose `HgStoreEngine.stateChanged` starts + `restoreLocalPartitionEngine()`; a failure there is logged and leaves the + state `Up`. A Store is therefore `Up` before it has restored anything, and + stays `Up` if restoring fails. + + The strongest check the current images support is shard membership and + leadership per group, read from the PD leader: + + ```bash + # PD leader, then its shard groups (see Disaster Recovery for the port-forward) + curl -s -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/shardGroups | jq ' + .shardGroups[] | {id, + shards: [.shards[] | {storeId, role}], + leaders: [.shards[] | select(.role=="Leader")] | length}' + ``` + + Delete the next Store only when every group reports the full shard count + from `pd.partition.shardCount`, exactly one `Leader`, and the replaced + Store's id back in the groups it holds. `/v1/shardLeaders` gives the same + leadership view grouped by Store raft address. + + Know what this does not prove. The shard list is PD's membership record, + not a statement that the Store finished loading those partitions locally + and caught up on the raft log. No endpoint in these images reports + restoration-complete, so a group can list a Store whose local engine is + still behind. Leave a margin after the membership check rather than + deleting the next Pod on the same second, keep `store.pdb.minAvailable` at + `replicas - 1` so an accidental second eviction is refused, and treat a + group that is short a shard or has no leader as a stop. Closing that gap + needs an image-side readiness signal for partition restoration, which is + the Store-side counterpart of the Server work in + [apache/hugegraph#3212](https://github.com/apache/hugegraph/issues/3212). - **Server** rolls once on the first `helm upgrade` after a fresh install, when the `checksum/auth` annotation first observes the install-created Secrets. Template-only pipelines (`helm template`, GitOps renderers) never From cf9049e0131da0912b0a9572c6b7f15e39ecf864 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 01:05:24 +0530 Subject: [PATCH 53/61] fix(helm): reject credentials with surrounding whitespace HugeConfig reads the Server properties file through Configurations.properties(), whose reader trims the value, so a password with a trailing space or tab was stored padded in the Secret and applied to the first-created admin account trimmed. Authenticating with the value the Secret holds then fails, with nothing in the install to suggest why. The admin-password pattern already refused leading whitespace but accepted a trailing space, tab or form feed; pd.auth.value accepted a trailing space. Both now require the last character to be non-whitespace, and a single character value stays valid. An inner space is still allowed, because only the ends are trimmed. Schema aborts cannot be matched by the unit framework at v1.0.0, so the CI job gains three must-fail cases and one positive control for the inner space. Answers the P2 recheck on hugegraph/hugegraph#221. --- .github/workflows/helm-chart-ci.yml | 11 +++++++++++ helm/hugegraph/README.md | 12 +++++++++--- helm/hugegraph/values.schema.json | 8 ++++---- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 5ecc700c8f..34e606698e 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -145,6 +145,17 @@ jobs: # unit framework at v1.0.0 cannot match schema aborts, so the case # lives here. must_fail --set pd.auth.autoGenerate=false + # Commons Configuration trims a properties value, so a credential + # with surrounding whitespace would be stored in the Secret and + # applied to the account without it. The schema rejects both ends; + # these are schema aborts, so they live here rather than in the + # unit suite. --set-string keeps the value a string. + must_fail --set-string "server.auth.admin.password=review-secret " + must_fail --set-string "$(printf 'server.auth.admin.password=review-secret\t')" + must_fail --set-string "pd.auth.value=review-secret " + # An inner space stays legal: only the trimmed ends are the problem. + helm template ci helm/hugegraph \ + --set-string "server.auth.admin.password=review secret" >/dev/null # Auth defaults to on, so Hubble alone is valid; refuse Hubble only # when authentication is explicitly disabled. must_fail --set hubble.enabled=true --set server.auth.enabled=false diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 2e7f9eae0c..185c9dde17 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -126,7 +126,13 @@ kubectl -n hugegraph create secret generic my-hugegraph-admin \ Then add `--set-string server.auth.admin.existingSecret=my-hugegraph-admin` to the install command. The Secret must contain a `password` key with no newlines, -carriage returns, backslashes, or leading whitespace. The JWT signing key uses +carriage returns, backslashes, or surrounding whitespace. The last one bites +quietly: the Server wrapper writes the value into a properties file, and +Commons Configuration trims it when the Server reads it back, so a padded +Secret would create the account under the trimmed password and then fail to +authenticate with the value the Secret holds. The schema rejects padding on +inline values; for a bring-your-own Secret the chart cannot see the value, so +check it yourself. The JWT signing key uses the same shape under `server.auth.token` (`value`, `existingSecret`, `autoGenerate`), and its value must be at least 32 bytes. Read the password and exercise the API: @@ -398,8 +404,8 @@ default values. | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | | `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader | `/v1/ready` | -| `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. Printable ASCII, no leading whitespace, no backslashes | `""` | -| `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it. Its value must meet the same constraint as `pd.auth.value`: printable ASCII, no leading whitespace, no backslashes | `""` | +| `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. Printable ASCII, no backslashes, no leading or trailing space (a properties read trims it) | `""` | +| `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it. Its value must meet the same constraint as `pd.auth.value`: printable ASCII, no backslashes, no leading or trailing space | `""` | | `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | | `pd.auth.autoGenerate` | Create and keep a random release-pd-auth Secret when `value` and `existingSecret` are empty | `true` | | `pd.probes.*.periodSeconds` | Probe interval | see `values.yaml` | diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index dec27dbb78..ece2dbc44e 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -355,8 +355,8 @@ "properties": { "value": { "type": "string", - "pattern": "^([\\x21-\\x5b\\x5d-\\x7e][\\x20-\\x5b\\x5d-\\x7e]*)?$", - "description": "Plaintext secret. Empty defers to existingSecret or autoGenerate. Printable ASCII with no leading whitespace and no backslashes: the value lands in Hubble's Java properties file, which strips leading whitespace and reads the file as ISO-8859-1." + "pattern": "^([\\x21-\\x5b\\x5d-\\x7e]([\\x20-\\x5b\\x5d-\\x7e]*[\\x21-\\x5b\\x5d-\\x7e])?)?$", + "description": "Plaintext secret. Empty defers to existingSecret or autoGenerate. Printable ASCII with no backslash, and no leading or trailing space: Commons Configuration trims a properties value, so padding would make the stored secret and the effective one differ." }, "existingSecret": { "type": "string" @@ -748,8 +748,8 @@ "properties": { "password": { "type": "string", - "pattern": "^([^ \\t\\f\\r\\n\\\\][^\\r\\n\\\\]*)?$", - "description": "Inline admin password. Empty defers to existingSecret or autoGenerate. The Server wrapper rejects newlines, carriage returns, backslashes and leading whitespace, so the schema rejects them before install." + "pattern": "^([^ \\t\\f\\r\\n\\\\]([^\\r\\n\\\\]*[^ \\t\\f\\r\\n\\\\])?)?$", + "description": "Inline admin password. Empty defers to existingSecret or autoGenerate. The Server wrapper rejects newlines, carriage returns and backslashes, so the schema rejects them before install. Leading and trailing whitespace is rejected too: Commons Configuration trims a properties value, so the Secret would hold the padded string while the account was created with the trimmed one." }, "existingSecret": { "type": "string" From fccfd25a86e1b80a9ab03ae2a80415b9e055f003 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 01:06:41 +0530 Subject: [PATCH 54/61] fix(helm): refuse PD replica growth, correct the Store drain Two halves of the same mistake: the chart guarded PD and Store shrinks but let a PD increase through, and the documented Store drain named two tasks that cannot retire a healthy Store. PD. The rendered peer list reaches raft only as NodeOptions.setInitialConf, which jraft applies when a node bootstraps without its own configuration. On an initialized group a 3-to-5 upgrade adds Pods and leaves the voting configuration at three. Membership moves through RaftEngine.changePeerList, which the PD client API reaches and no REST route exposes, so the chart refuses the upgrade and says where the operation actually lives instead of offering a sequence nobody has run here. Store. patrolPartitions reallocates groups whose shard count is wrong and hands off groups belonging to Stores already in Tombstone; balancePartitions spreads shards over the active Stores, the leaving one included. Neither retires it, so the old completion condition could never arrive. The procedure now transitions the leaving Store to Tombstone, the same path the recovery runbook uses, checks the remaining Stores still satisfy the persisted replication factor, maps ordinals to Store ids, and waits on /v1/shardGroups before the StatefulSet shrinks. The guard needs a live StatefulSet, so a fresh install at five PD replicas still renders; that case is pinned in the topology suite. Answers two P2 rechecks on hugegraph/hugegraph#221. --- helm/hugegraph/README.md | 56 ++++++++++++++----- helm/hugegraph/templates/_helpers.tpl | 13 +++++ .../hugegraph/tests/topology_quorum_test.yaml | 14 +++++ 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 185c9dde17..d5d0a4b273 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -955,20 +955,48 @@ back up when ready; the Servers wait, not-ready, until Stores register. `kubectl scale` changes only the live StatefulSet: the next `helm upgrade` renders `store.replicas` from values again and restores the full topology. -Scaling **down** PD or Store is not a values change. Raft and shard -membership are persisted, and deleting Pods does not reconfigure them: a -3-to-1 PD shrink permanently loses quorum, and removing a Store strands the -shard copies it holds. The chart therefore rejects an upgrade whose replica -count is below the live StatefulSet. The manual procedure: for Store, drain -the leaving Stores first (trigger `patrolPartitions` and -`balancePartitions`, then verify in Cluster Health that no shard lists -them); for PD, the persisted raft membership must be reduced through PD -itself before Pods are removed. Then scale the live StatefulSet with -`kubectl -n scale statefulset --replicas=` and run -`helm upgrade` with the matching value. The same applies after a manual -scale up: upgrade with the matching value, because the guard reads any -value below the live StatefulSet as a shrink. The guard needs the live -object, so a client-side `--dry-run` does not show it. +Changing PD or Store replicas on a live release is not a values change. +Raft and shard membership are persisted, and Pods alone do not reconfigure +them. The chart rejects both directions for PD and a shrink for Store, and +reads the live StatefulSet to do it, so a fresh install at any replica count +is unaffected and a client-side `--dry-run` does not show the guard. + +**PD, either direction.** The peer list the chart renders reaches raft only +as `NodeOptions.setInitialConf`, which jraft applies when a node bootstraps +without a configuration of its own. On an initialized group it is inert: a +3-to-5 upgrade starts two more PDs and changes the bootstrap list, while the +voting configuration stays at three, and a 3-to-1 shrink loses quorum +outright. Membership changes through `RaftEngine.changePeerList`, which the +PD client API reaches and no REST route exposes, so this is a client-side +operation the chart cannot perform and does not wrap. Change the membership +through PD, confirm the new configuration in `/v1/members`, scale the live +StatefulSet, then `helm upgrade` with the matching value. Until you have run +and verified that sequence on your own build, treat a PD replica change as +unsupported and install the PD count you intend to keep. + +**Store, shrinking.** Draining is a state transition, not a balance. +`patrolPartitions` reallocates groups whose shard count does not match the +configured replication factor and hands off the groups of Stores already in +`Tombstone`; `balancePartitions` spreads shards across the active Stores, +including the ones you mean to remove, so neither call retires a healthy +Store and the "no shard lists them" condition may never arrive. Retire the +leaving Store the same way the Disaster Recovery section retires a replaced +one: + +1. Check the remaining Stores can still hold the persisted replication + factor: after the shrink, live Stores must be at least + `pd.partition.shardCount`. +2. Map the ordinals the shrink will delete (the highest ones) to Store ids + through `/v1/stores`, matching on the Pod address. +3. `POST /v1/store/{id}` with `{"storeState":"Tombstone"}` for each leaving + id, which is what drives `storeTurnoff` and the reallocation. +4. Wait until `/v1/shardGroups` no longer lists those ids and every group + reports its full shard count with one leader. +5. Scale the live StatefulSet with `kubectl -n scale statefulset + --replicas=`, then `helm upgrade` with the matching value. + +Deleting the PersistentVolumeClaims of the removed ordinals is separate and +permanent; do it only after step 4 reports the data moved. ## Troubleshooting diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index ef5a3d3a03..6f3c2dc0c8 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -624,6 +624,19 @@ and the upgrade passes. {{- if and (gt $liveReplicas 0) (lt $desired $liveReplicas) -}} {{- fail (printf "%s.replicas cannot shrink from %d to %d through a helm upgrade: raft and shard membership are persisted, and removing Pods does not reconfigure them. Follow the manual scale-down procedure in the README (Scaling), which ends by scaling the live StatefulSet; the upgrade passes once the live replicas match the value" $comp $liveReplicas $desired) -}} {{- end -}} +{{/* +PD raft membership is the persisted voting configuration, and the peer list +the chart renders reaches it only as NodeOptions.setInitialConf, which jraft +applies when bootstrapping a node that has no configuration of its own. On an +initialized group, adding Pods adds non-voting strangers: the extra PD starts, +the peer list changes, and the voting configuration does not. PD exposes the +change through RaftEngine.changePeerList, reachable from the PD client API but +from no REST route, so the chart cannot perform it and does not pretend to. +Growing Store is ordinary scale-out and stays allowed. +*/}} +{{- if and (eq $comp "pd") (gt $liveReplicas 0) (gt $desired $liveReplicas) -}} +{{- fail (printf "pd.replicas cannot grow from %d to %d through a helm upgrade: the rendered peer list reaches raft only as the initial configuration, so new Pods would start without joining the voting configuration. Change the persisted membership through PD first, then scale the live StatefulSet, then upgrade with the matching value; the README (Scaling) has the procedure and its limits. A fresh install at any replica count is unaffected" $liveReplicas $desired) -}} +{{- end -}} {{- end -}} {{- end -}} {{/* diff --git a/helm/hugegraph/tests/topology_quorum_test.yaml b/helm/hugegraph/tests/topology_quorum_test.yaml index c375cde5ea..1069ed196d 100644 --- a/helm/hugegraph/tests/topology_quorum_test.yaml +++ b/helm/hugegraph/tests/topology_quorum_test.yaml @@ -93,3 +93,17 @@ tests: - equal: path: spec.clusterIP value: None + + # The PD replica guard reads the live StatefulSet, which a render never has, + # so a fresh install at any supported replica count must pass. The guarded + # cases, growing or shrinking an initialized group, need a cluster and + # cannot be reached from a render. + - it: leaves a fresh install at five PD replicas alone + template: pd-statefulset.yaml + set: + pd.replicas: 5 + pd.pdb.minAvailable: 3 + asserts: + - equal: + path: spec.replicas + value: 5 From 5a489698faf972571d608926b9e2a41808895609 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 01:07:53 +0530 Subject: [PATCH 55/61] feat(helm): derive the PD liveness path from the replica count PD's /v1/health returns 200 as soon as Jetty is up and never consults raft. With three PDs that is the right liveness signal, because restarting a follower for a normal election would turn one election into a rolling outage. With one PD there is no election to lose: a PD that steps down and cannot recover, as after a failed raft snapshot on a full disk (apache/hugegraph#3222), keeps answering /v1/health while serving no writes, and liveness never restarts it. Add pd.livenessPath, empty by default and derived: /v1/health above one replica, /v1/ready at one. The startup probe follows the same path, because Kubernetes suppresses liveness until startup succeeds, so leaving startup on /v1/health would give a single PD only the 60 s liveness budget to reach raft readiness after a restart and a slow log replay would crash-loop; following liveness charges that window to the 300 s startup budget instead. Multi-PD renders are unchanged. The value stops being needed if PD starts answering 503 from /v1/health in this state. Reported with measurements by @SebastianGruza on hugegraph/hugegraph#221. --- helm/hugegraph/README.md | 17 ++++++-- helm/hugegraph/templates/_helpers.tpl | 34 ++++++++++++++++ helm/hugegraph/templates/pd-statefulset.yaml | 4 +- .../tests/pd_readiness_path_test.yaml | 40 +++++++++++++++++++ helm/hugegraph/values.schema.json | 5 +++ helm/hugegraph/values.yaml | 7 ++++ 6 files changed, 101 insertions(+), 6 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index d5d0a4b273..64eb79c8d7 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -404,6 +404,7 @@ default values. | `pd.pdb.enabled` | Create a PodDisruptionBudget for PD | `true` | | `pd.pdb.minAvailable` | Must be strictly less than `pd.replicas`. No PDB is rendered when `pd.replicas` is 1 | `2` | | `pd.readinessPath` | Path the PD readinessProbe hits. `/v1/ready` is quorum-aware and returns 503 without a raft leader | `/v1/ready` | +| `pd.livenessPath` | Path the PD startup and liveness probes hit. Empty derives it from `pd.replicas`: `/v1/health` above one replica, `/v1/ready` at one | `""` | | `pd.auth.value` | Plaintext PD REST secret (`auth.secret-key`). Prefer `existingSecret` in shared clusters. Printable ASCII, no backslashes, no leading or trailing space (a properties read trims it) | `""` | | `pd.auth.existingSecret` | Pre-created Secret holding the PD REST secret under `pd.auth.key`. Wins over `value` and `autoGenerate`; the chart does not manage it. Its value must meet the same constraint as `pd.auth.value`: printable ASCII, no backslashes, no leading or trailing space | `""` | | `pd.auth.key` | Key inside the PD REST Secret | `secret-key` | @@ -1122,10 +1123,18 @@ independently of the release name. its one-shot resolution semantics (bring-up races and pod-IP-change rejections included) at the operator's own risk. - PD's `/v1/health` answers 200 as soon as the REST listener is up and never - consults raft, so it cannot see a lost quorum. The chart therefore uses it - only for PD startup and liveness (a PD that merely lost its leader is not - restarted) and puts readiness and the Store wait on `/v1/ready`, which - answers 503 without a raft leader. + consults raft, so it cannot see a lost quorum. With more than one PD the + chart uses it for startup and liveness on purpose, so that a follower which + merely lost its leader is not restarted, and puts readiness and the Store + wait on `/v1/ready`, which answers 503 without a raft leader. A single PD + is the exception: it has no election to lose, and a PD that steps down for + good, as after a failed raft snapshot on a full disk + ([apache/hugegraph#3222](https://github.com/apache/hugegraph/issues/3222)), + answers `/v1/health` forever while serving no writes. At `pd.replicas: 1` + startup and liveness therefore derive to `/v1/ready`, so the kubelet + restarts such a PD; `pd.livenessPath` overrides the derivation. If a future + PD answers 503 from `/v1/health` in that state, the value becomes + unnecessary. - Server discovery is a lease. Each Server re-registers its Pod IP with PD every 15 seconds and PD drops an entry after three missed heartbeats, so a replaced or evicted Server can stay in PD's list for up to 45 seconds after diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 6f3c2dc0c8..6141255277 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -241,6 +241,40 @@ renders emit a constant. {{- join "|" $parts | sha256sum -}} {{- end }} +{{/* +Path the PD startup and liveness probes hit. + +/v1/health answers 200 as soon as the REST listener is up and never consults +raft, which is what a multi-PD deployment wants: losing leadership is normal +during an election, and restarting a follower for it would turn one election +into a rolling outage. Readiness carries the raft-aware signal instead. + +A single PD has no election to lose. There, a PD that steps down and cannot +recover, as after a failed snapshot on a full disk +(apache/hugegraph#3222), keeps answering /v1/health forever and liveness +never restarts it, so the default moves to /v1/ready when pd.replicas is 1. + +The startup probe follows this path as well. Kubernetes suppresses liveness +until the startup probe succeeds, so leaving startup on /v1/health would give +a single PD only the liveness budget (60 s by default) to reach raft +readiness after a restart, and a slow log replay would crash-loop instead of +booting. Following the same path puts that window inside the startup budget +(300 s by default) instead. + +Setting pd.livenessPath overrides the choice in both places. If PD starts +answering 503 from /v1/health in this state, this value stops being needed. +*/}} +{{- define "hugegraph.pd.livenessPath" -}} +{{- $explicit := get .Values.pd "livenessPath" | default "" -}} +{{- if $explicit -}} +{{- $explicit -}} +{{- else if eq (int .Values.pd.replicas) 1 -}} +/v1/ready +{{- else -}} +/v1/health +{{- end -}} +{{- end }} + {{/* PD Raft peers list: pod-0.svc.ns.svc:8610,... Uses short headless DNS (cluster.local optional) resolvable inside the namespace. diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 62b78ee7a5..070945b208 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -136,7 +136,7 @@ spec: mountPath: {{ .Values.pd.dataPath }} startupProbe: httpGet: - path: /v1/health + path: {{ include "hugegraph.pd.livenessPath" . }} port: rest failureThreshold: {{ .Values.pd.probes.startup.failureThreshold }} periodSeconds: {{ .Values.pd.probes.startup.periodSeconds }} @@ -150,7 +150,7 @@ spec: {{- with include "hugegraph.probeTuning" .Values.pd.probes.readiness }}{{ . | trim | nindent 12 }}{{- end }} livenessProbe: httpGet: - path: /v1/health + path: {{ include "hugegraph.pd.livenessPath" . }} port: rest periodSeconds: {{ .Values.pd.probes.liveness.periodSeconds }} failureThreshold: {{ .Values.pd.probes.liveness.failureThreshold }} diff --git a/helm/hugegraph/tests/pd_readiness_path_test.yaml b/helm/hugegraph/tests/pd_readiness_path_test.yaml index 3025984985..33d4d230ad 100644 --- a/helm/hugegraph/tests/pd_readiness_path_test.yaml +++ b/helm/hugegraph/tests/pd_readiness_path_test.yaml @@ -75,3 +75,43 @@ tests: - equal: path: spec.template.spec.containers[0].readinessProbe.httpGet.path value: /v1/health + + - it: keeps PD startup and liveness off the raft-aware path with several PDs + template: pd-statefulset.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /v1/health + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /v1/health + + # A single PD has no election to lose, so a permanent step-down has to be + # restartable; /v1/health would pass forever. Startup follows liveness so the + # boot-to-ready window is charged to the startup budget, not the liveness one. + - it: moves PD startup and liveness to the raft-aware path at one replica + template: pd-statefulset.yaml + set: + pd.replicas: 1 + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /v1/ready + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /v1/ready + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /v1/ready + + - it: lets pd.livenessPath override the derived choice in both probes + template: pd-statefulset.yaml + set: + pd.livenessPath: /v1/ready + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /v1/ready + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /v1/ready diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index ece2dbc44e..ccd2265ef9 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -507,6 +507,11 @@ } } }, + "livenessPath": { + "type": "string", + "pattern": "^(/.*)?$", + "description": "HTTP path for the PD startup and liveness probes. Empty derives it from pd.replicas: /v1/health above one replica, /v1/ready at one (apache/hugegraph#3222)" + }, "readinessPath": { "type": "string", "pattern": "^/", diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index c84379a37f..5b393e975a 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -164,6 +164,13 @@ pd: # Startup and liveness stay on /v1/health so a PD that merely lost its # leader is not restarted. readinessPath: /v1/ready + # HTTP path the PD startup and liveness probes hit. Empty derives it from + # the replica count: /v1/health with more than one PD, so a normal election + # never restarts a follower, and /v1/ready with a single PD, which has no + # election to lose and would otherwise pass /v1/health forever after losing + # leadership for good (apache/hugegraph#3222). Startup follows liveness so + # the boot-to-ready window sits inside the startup budget. + livenessPath: "" probes: startup: failureThreshold: 30 From 23e88b4e8be37d0b3f7290aefefe0b087b48ae6b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 20:24:04 +0530 Subject: [PATCH 56/61] feat(helm): add per-component NetworkPolicy The chart turns PD's raft IP whitelist off in-cluster and until now had no replacement, so any Pod in the cluster could reach PD, Store and Server. networkPolicy.enabled renders one policy per component (PD, Store, Server, and Hubble when enabled). Each policy isolates its own Pods in both directions and carries all of their allows, so no apply order leaves a Pod denied before its allows exist; there is no separate default-deny object. The allows follow the traffic the images make: PD peers on raft and gRPC (followers forward to the leader), Store and Server to PD gRPC and REST, Store peers on raft only, Server to Store gRPC and REST (wait-partition.sh), Hubble in pd mode to PD gRPC and REST and Store REST, Hubble and the test hook to the Server, and DNS on port 53 from every component. Nothing outside the release is admitted unless networkPolicy..extraIngress names it, and the schema requires every such rule to name its peers. Exposing PD, Server or Hubble (a NodePort or LoadBalancer Service, an Ingress, or server.advertiseUrl) with an empty extraIngress fails the render instead of opening the port; the check runs after the existing exposure checks. Hubble also gets extraEgress for its optional outside endpoints. A Hubble policy without extra rules omits the ingress key, which the API server would otherwise drop and Helm would rewrite on every upgrade. Off in values.yaml, which cannot know the release's clients; on in values-cluster.yaml. The validateValues refusal of networkPolicy.enabled is removed. The README notes that the Store image downloads libjemalloc.so from github.com on each start, which times out under the policies. --- helm/hugegraph/README.md | 138 ++- helm/hugegraph/templates/NOTES.txt | 7 + helm/hugegraph/templates/_helpers.tpl | 46 +- helm/hugegraph/templates/networkpolicy.yaml | 199 ++++ .../__snapshot__/networkpolicy_test.yaml.snap | 269 ++++++ helm/hugegraph/tests/networkpolicy_test.yaml | 913 ++++++++++++++++++ .../hugegraph/tests/validate_values_test.yaml | 108 ++- helm/hugegraph/values-cluster.yaml | 5 + helm/hugegraph/values.schema.json | 78 ++ helm/hugegraph/values.yaml | 26 + 10 files changed, 1781 insertions(+), 8 deletions(-) create mode 100644 helm/hugegraph/templates/networkpolicy.yaml create mode 100644 helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap create mode 100644 helm/hugegraph/tests/networkpolicy_test.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 64eb79c8d7..40d329a0e9 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -185,7 +185,7 @@ helm test hugegraph --namespace hugegraph |---|---| | `values.yaml` | Default 3+3+3 topology with preferred anti-affinity, authentication on, and Hubble off | | `values-single.yaml` | Single-node 1+1+1 example with authentication on | -| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, and required anti-affinity for PD and Store; authentication on, Hubble still opt-in | +| `values-cluster.yaml` | Production 3+3+3 starting point with JVM/resources, PD/Store PDBs, required anti-affinity for PD and Store, and NetworkPolicy on; authentication on, Hubble still opt-in | `values-cluster.yaml` is a production starting point, not a capacity guarantee. Recalculate capacity for the graph size, traffic, failure budget, @@ -763,9 +763,141 @@ before anything reaches the cluster: `instance` or `component` keys on any workload. - A non-ClusterIP `pd.service.type` requires `pd.service.allowInsecureExposure=true`. +- With `networkPolicy.enabled`, exposing PD, Server or Hubble (a NodePort + or LoadBalancer Service, a Server or Hubble Ingress, or a set + `server.advertiseUrl`) requires a non-empty + `networkPolicy..extraIngress` naming who may connect. - An upgrade may not shrink `pd.replicas` or `store.replicas` below the live StatefulSet; see Scaling for the manual procedure. +### NetworkPolicy + +`networkPolicy.enabled` renders one NetworkPolicy per component (PD, Store, +Server, and Hubble when enabled). Each one isolates its own Pods in both +directions and lists the traffic they need, so the policies take effect in any +apply order. It is off in `values.yaml`, because the base values cannot know +who your clients are, and on in `values-cluster.yaml`. + +It only works when the cluster's network plugin enforces NetworkPolicy (kind +v0.25 or later, k3s, Calico, Cilium). Other plugins accept the objects and +enforce nothing. To check, run a Pod without chart labels in another +namespace and `curl` the PD client Service on the REST port: it must time out. + +With it on, the release admits only its own traffic: + +| To | From, ports | +|---|---| +| PD | PD: raft, gRPC. Store, Server, and Hubble in `pd` mode: gRPC, REST | +| Store | Store: raft. Server: gRPC, REST. Hubble in `pd` mode: REST | +| Server | Hubble and the `helm test` Pod: `server.port` | +| Hubble | nothing (port-forward uses loopback and needs no rule) | + +Every component may also resolve DNS on port 53. Nothing outside the release +is admitted unless it is listed in `networkPolicy..extraIngress`, +including the Ingress controller and clients of a NodePort or LoadBalancer +Service. Exposing PD, Server or Hubble that way, or setting +`server.advertiseUrl`, with an empty `extraIngress` fails the render instead +of opening the port. For PD this is the reachability restriction that +`pd.service.allowInsecureExposure` asks for. The check sees only exposure the +chart creates; a Service, Gateway route or proxy you add yourself needs its +own `extraIngress` entry. + +PD, Store and Server reach nothing outside the release except DNS, so a +feature that calls out (for example hugegraph-computer jobs through the +Kubernetes API, which this chart does not enable) does not work with the +policies on. One call is made by default: on every start the Store image +downloads `libjemalloc.so` from github.com. With the policies on that +connection times out after about two minutes, the Store starts without +jemalloc and continues (measured on kind: Ready after 151 s instead of 11 s). +The same happens on any cluster without internet access. The `helm test` Pod is selected by no chart policy, so its egress +is open only while nothing else selects it: under a namespace-wide +default-deny policy of your own, allow it egress to `server.port` and DNS. + +| Parameter | Description | Default | +|---|---|---| +| `networkPolicy.enabled` | Render the policies | `false` | +| `networkPolicy..extraIngress` | Extra NetworkPolicy ingress rules, appended as written | `[]` | +| `networkPolicy.hubble.extraEgress` | Extra egress rules for Hubble's optional outside endpoints (`es.urls`, `prometheus.url`) | `[]` | + +
+Letting other workloads in + +Anything outside the release is blocked until it is listed. Every rule must +name its peers in `from` (the schema rejects a rule without one); to admit any +address, write an `ipBlock` such as `0.0.0.0/0` explicitly. A +`namespaceSelector` and a `podSelector` in the same peer must both match; as +two separate peers, either one is enough. What a +NodePort or LoadBalancer client looks like from the Pod depends on the network +plugin, `externalTrafficPolicy` and the node the request arrives on. Measured +with a NodePort Server on two-node kind clusters: + +- kindnet, and Cilium with kube-proxy replacement: a call to the Server's own + node arrived with the client address; through the other node it arrived + with that node's address. +- Calico: through the other node the call arrived from that node's tunnel + address inside the Pod CIDR. +- Cilium with kube-proxy: no `ipBlock` rule admitted NodePort traffic, because + Cilium identifies node addresses by its own node identities rather than by + CIDR. + +Test with the plugin you run and name the CIDR you see arriving. + +```yaml +networkPolicy: + server: + extraIngress: + # The ingress-nginx controller, when server.ingress is enabled. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + podSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + ports: + - port: 8080 + # Applications in namespace "apps" call the Server API. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: apps + ports: + - port: 8080 + pd: + extraIngress: + # Prometheus scrapes /actuator/prometheus on PD REST. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - port: 8620 + # Vermeer reads partition metadata over PD gRPC. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vermeer + ports: + - port: 8686 + store: + extraIngress: + # Vermeer scans Store over gRPC; Prometheus scrapes Store REST. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vermeer + ports: + - port: 8500 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - port: 8520 +``` + +
+ ## Deep Dive ### Connecting to the Cluster @@ -1118,7 +1250,9 @@ independently of the release name. which under Kubernetes can block peers whose pod IPs were unpublished at that moment or change later. The chart therefore disables the whitelist in-cluster via the upstream `raft.ip-whitelist.enabled` switch, leaving - peer authentication to Kubernetes-level controls. Setting + peer authentication to Kubernetes-level controls: enable + `networkPolicy.enabled` (on in `values-cluster.yaml`) so that only PD Pods + reach the raft port. Setting `pd.raftIpWhitelistEnabled=true` restores the image default along with its one-shot resolution semantics (bring-up races and pod-IP-change rejections included) at the operator's own risk. diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 53bf833df8..421b88c901 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -76,6 +76,13 @@ Hubble Pod is replaced. Graph data is unaffected. Authentication is disabled. Do not expose this release to untrusted networks. {{- end }} +{{- if (get (get .Values "networkPolicy" | default dict) "enabled") }} + +NetworkPolicy is on: only this release's components, Hubble and the helm test +Pod reach each other. List other clients (applications, Prometheus, Vermeer) +in networkPolicy..extraIngress; see README, NetworkPolicy. The +policies are only enforced by a network plugin that implements NetworkPolicy. +{{- end }} {{- $advertiseUrl := trim (default "" .Values.server.advertiseUrl) }} {{- if $advertiseUrl }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 6141255277..105167bf54 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -621,10 +621,6 @@ Cross-field validation that JSON Schema draft-07 cannot express. {{- end -}} {{- end -}} {{- end -}} -{{- $networkPolicy := get .Values "networkPolicy" | default dict -}} -{{- if (get $networkPolicy "enabled" | default false) -}} -{{- fail "networkPolicy.enabled=true is unsupported because this chart does not implement NetworkPolicy resources" -}} -{{- end -}} {{- if and .Values.server.hpa.enabled (gt (int .Values.server.hpa.minReplicas) (int .Values.server.hpa.maxReplicas)) -}} {{- fail "server.hpa.minReplicas must be less than or equal to server.hpa.maxReplicas" -}} {{- end -}} @@ -806,12 +802,54 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- fail "server.auth.token requires existingSecret, value, or autoGenerate=true when auth is enabled" -}} {{- end -}} {{- end -}} +{{/* NetworkPolicy exposure check runs last, so an exposure the other checks + refuse (allowInsecureExposure, Ingress TLS) is reported first. */}} +{{- $networkPolicy := get .Values "networkPolicy" | default dict -}} +{{- if get $networkPolicy "enabled" -}} +{{- $exposed := dict + "pd" (ne .Values.pd.service.type "ClusterIP") + "server" (or (ne .Values.server.service.type "ClusterIP") .Values.server.ingress.enabled (ne (trim (default "" .Values.server.advertiseUrl)) "")) + "hubble" (and .Values.hubble.enabled (or (ne .Values.hubble.service.type "ClusterIP") .Values.hubble.ingress.enabled)) -}} +{{- range $comp := list "pd" "server" "hubble" -}} +{{- if and (get $exposed $comp) (empty (get (get $networkPolicy $comp | default dict) "extraIngress")) -}} +{{- fail (printf "networkPolicy.enabled admits nothing from outside the release, so the %s exposure (NodePort/LoadBalancer Service, Ingress%s) is unreachable; list its callers in networkPolicy.%s.extraIngress, for example the Ingress controller's namespace or a client CIDR" $comp (ternary ", server.advertiseUrl" "" (eq $comp "server")) $comp) -}} +{{- end -}} +{{- end -}} +{{- end -}} {{- end }} {{/* podAntiAffinity snippet for a component label key. mode: required | preferred | disabled */}} +{{/* +NetworkPolicy building blocks: a same-release peer by component, a TCP port +list, and DNS egress by port only, so it works wherever the cluster runs its +resolver (CoreDNS in any namespace, NodeLocal DNSCache). +*/}} +{{- define "hugegraph.netpol.peer" -}} +- podSelector: + matchLabels: + {{- include "hugegraph.selectorLabels" .root | nindent 6 }} + app.kubernetes.io/component: {{ .component }} +{{- end }} + +{{- define "hugegraph.netpol.ports" -}} +{{- $rules := list -}} +{{- range . -}} +{{- $rules = append $rules (printf "- protocol: TCP\n port: %d" (int .)) -}} +{{- end -}} +{{- join "\n" $rules -}} +{{- end }} + +{{- define "hugegraph.netpol.dns" -}} +- ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +{{- end }} + {{- define "hugegraph.antiAffinity" -}} {{- $mode := .mode -}} {{- $component := .component -}} diff --git a/helm/hugegraph/templates/networkpolicy.yaml b/helm/hugegraph/templates/networkpolicy.yaml new file mode 100644 index 0000000000..9b20c0afb1 --- /dev/null +++ b/helm/hugegraph/templates/networkpolicy.yaml @@ -0,0 +1,199 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- /* +One NetworkPolicy per component. Each object isolates its own Pods in both +directions and carries all of their allows, so no object ever denies a Pod +before that Pod's allows exist, whatever order the objects are applied in. +Peers match the release and component labels; ports are the values the +containers bind. The helm test Pod is selected by no policy, so its egress +stays open and the Server policy admits it. Nothing outside the release is +admitted unless .extraIngress names it; exposing a component +without such rules fails in validateValues instead of opening its ports. +*/}} +{{- $np := get .Values "networkPolicy" | default dict }} +{{- if get $np "enabled" }} +{{- $pd := .Values.pd.ports }} +{{- $store := .Values.store.ports }} +{{- $serverPort := .Values.server.port }} +{{- $hubbleEnabled := .Values.hubble.enabled }} +{{- $hubblePd := and $hubbleEnabled (eq .Values.hubble.mode "pd") }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "hugegraph.pd.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: pd +spec: + podSelector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: pd + policyTypes: + - Ingress + - Egress + ingress: + - from: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "pd") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $pd.raft $pd.grpc) | nindent 8 }} + - from: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "store") | nindent 8 }} + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "server") | nindent 8 }} + {{- if $hubblePd }} + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "hubble") | nindent 8 }} + {{- end }} + ports: + {{- include "hugegraph.netpol.ports" (list $pd.grpc $pd.rest) | nindent 8 }} + {{- with get (get $np "pd" | default dict) "extraIngress" }} + {{- toYaml . | nindent 4 }} + {{- end }} + egress: + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "pd") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $pd.raft $pd.grpc) | nindent 8 }} + {{- include "hugegraph.netpol.dns" . | nindent 4 }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "hugegraph.store.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: store +spec: + podSelector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: store + policyTypes: + - Ingress + - Egress + ingress: + - from: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "store") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $store.raft) | nindent 8 }} + - from: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "server") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $store.grpc $store.rest) | nindent 8 }} + {{- if $hubblePd }} + - from: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "hubble") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $store.rest) | nindent 8 }} + {{- end }} + {{- with get (get $np "store" | default dict) "extraIngress" }} + {{- toYaml . | nindent 4 }} + {{- end }} + egress: + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "pd") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $pd.grpc $pd.rest) | nindent 8 }} + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "store") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $store.raft) | nindent 8 }} + {{- include "hugegraph.netpol.dns" . | nindent 4 }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "hugegraph.server.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + podSelector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + policyTypes: + - Ingress + - Egress + ingress: + - from: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "test") | nindent 8 }} + {{- if $hubbleEnabled }} + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "hubble") | nindent 8 }} + {{- end }} + ports: + {{- include "hugegraph.netpol.ports" (list $serverPort) | nindent 8 }} + {{- with get (get $np "server" | default dict) "extraIngress" }} + {{- toYaml . | nindent 4 }} + {{- end }} + egress: + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "pd") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $pd.grpc $pd.rest) | nindent 8 }} + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "store") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $store.grpc $store.rest) | nindent 8 }} + {{- include "hugegraph.netpol.dns" . | nindent 4 }} +{{- if $hubbleEnabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "hugegraph.hubble.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: hubble +spec: + podSelector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: hubble + policyTypes: + - Ingress + - Egress + {{- with get (get $np "hubble" | default dict) "extraIngress" }} + ingress: + {{- toYaml . | nindent 4 }} + {{- else }} + # No ingress rules under policyTypes Ingress admits nothing. The key is left + # out rather than rendered empty, because the API server drops an empty list + # and Helm would then rewrite the object on every upgrade. kubectl + # port-forward reaches the Pod over loopback, which no policy filters. + {{- end }} + egress: + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "server") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $serverPort) | nindent 8 }} + {{- if $hubblePd }} + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "pd") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $pd.grpc $pd.rest) | nindent 8 }} + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "store") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list $store.rest) | nindent 8 }} + {{- end }} + {{- include "hugegraph.netpol.dns" . | nindent 4 }} + {{- with get (get $np "hubble" | default dict) "extraEgress" }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap b/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap new file mode 100644 index 0000000000..99d7d5e042 --- /dev/null +++ b/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap @@ -0,0 +1,269 @@ +matches the reviewed cluster plus Hubble render: + 1: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + labels: + app.kubernetes.io/component: pd + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: hugegraph + app.kubernetes.io/version: latest + helm.sh/chart: hugegraph-0.1.0 + name: RELEASE-NAME-hugegraph-pd + spec: + egress: + - ports: + - port: 8610 + protocol: TCP + - port: 8686 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: pd + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: pd + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + ports: + - port: 8610 + protocol: TCP + - port: 8686 + protocol: TCP + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: store + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - podSelector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - podSelector: + matchLabels: + app.kubernetes.io/component: hubble + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + ports: + - port: 8686 + protocol: TCP + - port: 8620 + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/component: pd + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + policyTypes: + - Ingress + - Egress + 2: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + labels: + app.kubernetes.io/component: store + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: hugegraph + app.kubernetes.io/version: latest + helm.sh/chart: hugegraph-0.1.0 + name: RELEASE-NAME-hugegraph-store + spec: + egress: + - ports: + - port: 8686 + protocol: TCP + - port: 8620 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: pd + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 8510 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: store + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: store + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + ports: + - port: 8510 + protocol: TCP + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + ports: + - port: 8500 + protocol: TCP + - port: 8520 + protocol: TCP + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: hubble + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + ports: + - port: 8520 + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/component: store + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + policyTypes: + - Ingress + - Egress + 3: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: hugegraph + app.kubernetes.io/version: latest + helm.sh/chart: hugegraph-0.1.0 + name: RELEASE-NAME-hugegraph-server + spec: + egress: + - ports: + - port: 8686 + protocol: TCP + - port: 8620 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: pd + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 8500 + protocol: TCP + - port: 8520 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: store + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: test + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - podSelector: + matchLabels: + app.kubernetes.io/component: hubble + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + ports: + - port: 8080 + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + policyTypes: + - Ingress + - Egress + 4: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + labels: + app.kubernetes.io/component: hubble + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: hugegraph + app.kubernetes.io/version: latest + helm.sh/chart: hugegraph-0.1.0 + name: RELEASE-NAME-hugegraph-hubble + spec: + egress: + - ports: + - port: 8080 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 8686 + protocol: TCP + - port: 8620 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: pd + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 8520 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/component: store + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/component: hubble + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/name: hugegraph + policyTypes: + - Ingress + - Egress diff --git a/helm/hugegraph/tests/networkpolicy_test.yaml b/helm/hugegraph/tests/networkpolicy_test.yaml new file mode 100644 index 0000000000..0bdcbc45c6 --- /dev/null +++ b/helm/hugegraph/tests/networkpolicy_test.yaml @@ -0,0 +1,913 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: NetworkPolicy +templates: + - networkpolicy.yaml +tests: + - it: renders nothing with the base values + asserts: + - hasDocuments: + count: 0 + + - it: renders nothing with the single preset as shipped + values: + - ../values-single.yaml + asserts: + - hasDocuments: + count: 0 + + - it: renders pd, store and server policies on the cluster preset + values: + - ../values-cluster.yaml + asserts: + - hasDocuments: + count: 3 + - isKind: + of: NetworkPolicy + + - it: renders pd, store and server policies on the single preset when enabled + values: + - ../values-single.yaml + set: + networkPolicy.enabled: true + asserts: + - hasDocuments: + count: 3 + + - it: adds a Hubble policy when Hubble is enabled + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + asserts: + - hasDocuments: + count: 4 + + - it: isolates PD in both directions and admits only its peers and clients + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + asserts: + - equal: + path: spec.podSelector.matchLabels + value: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + - equal: + path: spec.policyTypes + value: [Ingress, Egress] + - lengthEqual: + path: spec.ingress + count: 2 + - equal: + path: spec.ingress[0] + value: + from: + - podSelector: + matchLabels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: pd + ports: + - protocol: TCP + port: 8610 + - protocol: TCP + port: 8686 + - lengthEqual: + path: spec.ingress[1].from + count: 3 + - equal: + path: spec.ingress[1].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: store + - equal: + path: spec.ingress[1].from[1].podSelector.matchLabels["app.kubernetes.io/component"] + value: server + - equal: + path: spec.ingress[1].from[2].podSelector.matchLabels["app.kubernetes.io/component"] + value: hubble + - equal: + path: spec.ingress[1].ports + value: + - protocol: TCP + port: 8686 + - protocol: TCP + port: 8620 + - equal: + path: spec.egress[0].to[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: pd + - equal: + path: spec.egress[0].ports + value: + - protocol: TCP + port: 8610 + - protocol: TCP + port: 8686 + - equal: + path: spec.egress[1] + value: + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - lengthEqual: + path: spec.egress + count: 2 + + - it: lets Store peers use raft only and admits Server and Hubble on their ports + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-store + asserts: + - lengthEqual: + path: spec.ingress + count: 3 + - lengthEqual: + path: spec.ingress[0].from + count: 1 + - lengthEqual: + path: spec.ingress[1].from + count: 1 + - lengthEqual: + path: spec.ingress[2].from + count: 1 + - lengthEqual: + path: spec.egress[0].to + count: 1 + - lengthEqual: + path: spec.egress[1].to + count: 1 + - equal: + path: spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: store + - equal: + path: spec.ingress[0].ports + value: + - protocol: TCP + port: 8510 + - equal: + path: spec.ingress[1].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: server + - equal: + path: spec.ingress[1].ports + value: + - protocol: TCP + port: 8500 + - protocol: TCP + port: 8520 + - equal: + path: spec.ingress[2].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: hubble + - equal: + path: spec.ingress[2].ports + value: + - protocol: TCP + port: 8520 + - equal: + path: spec.egress[0].to[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: pd + - equal: + path: spec.egress[0].ports + value: + - protocol: TCP + port: 8686 + - protocol: TCP + port: 8620 + - equal: + path: spec.egress[1].to[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: store + - equal: + path: spec.egress[1].ports + value: + - protocol: TCP + port: 8510 + - equal: + path: spec.egress[2].ports + value: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + + - it: admits the test hook and Hubble to the Server and lets it reach PD and Store + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + asserts: + - lengthEqual: + path: spec.egress[0].to + count: 1 + - lengthEqual: + path: spec.egress[1].to + count: 1 + - lengthEqual: + path: spec.egress + count: 3 + - lengthEqual: + path: spec.ingress + count: 1 + - lengthEqual: + path: spec.ingress[0].from + count: 2 + - equal: + path: spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: test + - equal: + path: spec.ingress[0].from[1].podSelector.matchLabels["app.kubernetes.io/component"] + value: hubble + - equal: + path: spec.ingress[0].ports + value: + - protocol: TCP + port: 8080 + - equal: + path: spec.egress[0].ports + value: + - protocol: TCP + port: 8686 + - protocol: TCP + port: 8620 + - equal: + path: spec.egress[1].to[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: store + - equal: + path: spec.egress[1].ports + value: + - protocol: TCP + port: 8500 + - protocol: TCP + port: 8520 + - equal: + path: spec.egress[2].ports + value: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + + - it: lets PD-mode Hubble reach Server, PD and Store REST and nothing reach it + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-hubble + asserts: + - lengthEqual: + path: spec.egress + count: 4 + - notExists: + path: spec.ingress + - contains: + path: spec.policyTypes + content: Ingress + - equal: + path: spec.egress[0].to[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: server + - equal: + path: spec.egress[0].ports + value: + - protocol: TCP + port: 8080 + - equal: + path: spec.egress[1].to[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: pd + - equal: + path: spec.egress[1].ports + value: + - protocol: TCP + port: 8686 + - protocol: TCP + port: 8620 + - equal: + path: spec.egress[2].to[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: store + - equal: + path: spec.egress[2].ports + value: + - protocol: TCP + port: 8520 + - equal: + path: spec.egress[3].ports + value: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + + - it: keeps direct-mode Hubble to the Server only + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + hubble.mode: direct + asserts: + - lengthEqual: + path: spec.egress + count: 2 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-hubble + - lengthEqual: + path: spec.ingress[1].from + count: 2 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + - equal: + path: spec.ingress[1].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: store + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + - equal: + path: spec.ingress[1].from[1].podSelector.matchLabels["app.kubernetes.io/component"] + value: server + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + - lengthEqual: + path: spec.ingress + count: 2 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-store + + - it: leaves every Hubble peer out when Hubble is disabled + values: + - ../values-cluster.yaml + asserts: + - hasDocuments: + count: 3 + - notContains: + path: spec.egress + content: + to: + - podSelector: + matchLabels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: hubble + - notContains: + path: spec.ingress[1].from + content: + podSelector: + matchLabels: + app.kubernetes.io/name: hugegraph + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: hubble + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + - lengthEqual: + path: spec.ingress + count: 2 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-store + - lengthEqual: + path: spec.ingress[0].from + count: 1 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + - equal: + path: spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: test + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + + - it: follows port overrides in ingress and egress + values: + - ../values-cluster.yaml + set: + pd.ports.raft: 9610 + store.ports.rest: 9520 + server.port: 9080 + asserts: + - equal: + path: spec.ingress[0].ports + value: + - protocol: TCP + port: 9610 + - protocol: TCP + port: 8686 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + - equal: + path: spec.egress[1].ports + value: + - protocol: TCP + port: 8500 + - protocol: TCP + port: 9520 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + - equal: + path: spec.ingress[0].ports + value: + - protocol: TCP + port: 9080 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + + - it: appends the PD extraIngress when PD is exposed and adds nothing open + values: + - ../values-cluster.yaml + set: + pd.service.type: NodePort + pd.service.allowInsecureExposure: true + networkPolicy.pd.extraIngress: + - from: + - ipBlock: + cidr: 10.0.0.0/8 + ports: + - port: 8620 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + asserts: + - lengthEqual: + path: spec.ingress + count: 3 + - exists: + path: spec.ingress[0].from + - exists: + path: spec.ingress[1].from + - equal: + path: spec.ingress[2] + value: + from: + - ipBlock: + cidr: 10.0.0.0/8 + ports: + - port: 8620 + + - it: appends the Server extraIngress for a LoadBalancer Service and adds nothing open + values: + - ../values-cluster.yaml + set: + server.service.type: LoadBalancer + networkPolicy.server.extraIngress: + - from: + - ipBlock: + cidr: 192.168.0.0/16 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + asserts: + - lengthEqual: + path: spec.ingress + count: 2 + - exists: + path: spec.ingress[0].from + - equal: + path: spec.ingress[1].from[0].ipBlock.cidr + value: 192.168.0.0/16 + + - it: appends the Server extraIngress for an Ingress and keeps the test peer first + values: + - ../values-cluster.yaml + set: + server.ingress.enabled: true + server.ingress.allowPlainHttp: true + networkPolicy.server.extraIngress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + podSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + ports: + - port: 8080 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + asserts: + - lengthEqual: + path: spec.ingress + count: 2 + - equal: + path: spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/component"] + value: test + - equal: + path: spec.ingress[1] + value: + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + podSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + ports: + - port: 8080 + + - it: appends the Server extraIngress for an advertised URL and adds nothing open + values: + - ../values-cluster.yaml + set: + server.advertiseUrl: http://hg.example.com:8080 + networkPolicy.server.extraIngress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: proxy + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + asserts: + - lengthEqual: + path: spec.ingress + count: 2 + - exists: + path: spec.ingress[1].from + + - it: gives an exposed Hubble exactly the listed ingress rules + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + hubble.service.type: NodePort + networkPolicy.hubble.extraIngress: + - from: + - ipBlock: + cidr: 172.18.0.0/16 + ports: + - port: 8088 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-hubble + asserts: + - equal: + path: spec.ingress + value: + - from: + - ipBlock: + cidr: 172.18.0.0/16 + ports: + - port: 8088 + + - it: renders no rule without a peer on any document when every exposure is set + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + pd.service.type: LoadBalancer + pd.service.allowInsecureExposure: true + server.service.type: NodePort + server.advertiseUrl: http://hg.example.com:8080 + hubble.service.type: NodePort + networkPolicy.pd.extraIngress: + - from: + - ipBlock: + cidr: 10.0.0.0/8 + networkPolicy.server.extraIngress: + - from: + - ipBlock: + cidr: 10.0.0.0/8 + networkPolicy.hubble.extraIngress: + - from: + - ipBlock: + cidr: 10.0.0.0/8 + asserts: + - hasDocuments: + count: 4 + - notContains: + path: spec.ingress + content: + ports: + - protocol: TCP + port: 8686 + - protocol: TCP + port: 8620 + - notContains: + path: spec.ingress + content: + ports: + - protocol: TCP + port: 8080 + - notContains: + path: spec.ingress + content: + ports: + - protocol: TCP + port: 8088 + + - it: selects a component on every document and renders no default-deny object + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + asserts: + - hasDocuments: + count: 4 + - exists: + path: spec.podSelector.matchLabels["app.kubernetes.io/component"] + - exists: + path: spec.podSelector.matchLabels["app.kubernetes.io/instance"] + + - it: follows nameOverride in every selector and fullnameOverride in names + values: + - ../values-cluster.yaml + set: + nameOverride: graphdb + fullnameOverride: x + asserts: + - equal: + path: metadata.name + value: x-pd + documentIndex: 0 + - equal: + path: metadata.name + value: x-store + documentIndex: 1 + - equal: + path: metadata.name + value: x-server + documentIndex: 2 + - equal: + path: spec.podSelector.matchLabels["app.kubernetes.io/name"] + value: graphdb + - equal: + path: spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/name"] + value: graphdb + - equal: + path: spec.egress[0].to[0].podSelector.matchLabels["app.kubernetes.io/name"] + value: graphdb + + - it: keeps names within 63 characters and the full release name in selectors + release: + name: abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabc + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + asserts: + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.podSelector.matchLabels["app.kubernetes.io/instance"] + value: abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabc + - equal: + path: spec.egress[0].to[0].podSelector.matchLabels["app.kubernetes.io/instance"] + value: abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabc + + - it: keeps two extraIngress rules in order after the chart rules + values: + - ../values-cluster.yaml + set: + networkPolicy.pd.extraIngress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - port: 8620 + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vermeer + ports: + - port: 8686 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + asserts: + - lengthEqual: + path: spec.ingress + count: 4 + - equal: + path: spec.ingress[2].from[0].namespaceSelector.matchLabels["kubernetes.io/metadata.name"] + value: monitoring + - equal: + path: spec.ingress[3].from[0].namespaceSelector.matchLabels["kubernetes.io/metadata.name"] + value: vermeer + + - it: ignores Hubble extra rules while Hubble is disabled + values: + - ../values-cluster.yaml + set: + networkPolicy.hubble.extraIngress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ops + networkPolicy.hubble.extraEgress: + - to: + - ipBlock: + cidr: 10.0.0.10/32 + asserts: + - hasDocuments: + count: 3 + - notContains: + path: spec.ingress + content: + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ops + + - it: treats a whitespace-only advertiseUrl as unset + values: + - ../values-cluster.yaml + set: + server.advertiseUrl: " " + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-server + asserts: + - lengthEqual: + path: spec.ingress + count: 1 + + - it: renders a component whose networkPolicy key was removed + values: + - ../values-cluster.yaml + set: + networkPolicy.pd: null + asserts: + - hasDocuments: + count: 3 + + - it: matches the reviewed cluster plus Hubble render + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + asserts: + - matchSnapshot: {} + + - it: appends extra rules verbatim + values: + - ../values-cluster.yaml + set: + hubble.enabled: true + networkPolicy.store.extraIngress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - port: 8520 + networkPolicy.hubble.extraEgress: + - to: + - ipBlock: + cidr: 10.0.0.10/32 + ports: + - port: 9200 + asserts: + - contains: + path: spec.ingress + content: + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - port: 8520 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-store + - contains: + path: spec.egress + content: + to: + - ipBlock: + cidr: 10.0.0.10/32 + ports: + - port: 9200 + documentSelector: + path: metadata.name + value: RELEASE-NAME-hugegraph-hubble + + - it: rejects a non-boolean enabled flag through the schema + set: + networkPolicy.enabled: "yes" + asserts: + - failedTemplate: + errorPattern: "networkPolicy" + + - it: rejects egress rules on components that take none + set: + networkPolicy.store.extraEgress: + - to: + - ipBlock: + cidr: 10.0.0.0/8 + asserts: + - failedTemplate: + errorPattern: "(additional properties 'extraEgress' not allowed|Additional property extraEgress is not allowed)" + + - it: rejects a non-list extraIngress + set: + networkPolicy.pd.extraIngress: "abc" + asserts: + - failedTemplate: + errorPattern: "extraIngress" + + - it: rejects a null extraIngress rule + set: + networkPolicy.pd.extraIngress: + - null + asserts: + - failedTemplate: + errorPattern: "extraIngress" + + - it: rejects an unknown networkPolicy key + set: + networkPolicy.unknown: 1 + asserts: + - failedTemplate: + errorPattern: "(additional properties 'unknown' not allowed|Additional property unknown is not allowed)" + + - it: rejects the removed ingressControllers key + set: + networkPolicy.ingressControllers: [] + asserts: + - failedTemplate: + errorPattern: "(additional properties 'ingressControllers' not allowed|Additional property ingressControllers is not allowed)" + + - it: rejects a string enabled flag + set: + networkPolicy.enabled: "true" + asserts: + - failedTemplate: + errorPattern: "networkPolicy[./]enabled" + + - it: rejects an extraIngress rule without from + set: + networkPolicy.server.extraIngress: + - ports: + - port: 8080 + asserts: + - failedTemplate: + errorPattern: "(missing property 'from'|from is required)" + + - it: rejects an extraIngress rule with an empty from + set: + networkPolicy.server.extraIngress: + - from: [] + asserts: + - failedTemplate: + errorPattern: "(minItems|at least 1 items)" + + - it: rejects an empty extraIngress rule + set: + networkPolicy.pd.extraIngress: + - {} + asserts: + - failedTemplate: + errorPattern: "(missing property 'from'|from is required)" + + - it: rejects an empty peer + set: + networkPolicy.pd.extraIngress: + - from: + - {} + asserts: + - failedTemplate: + errorPattern: "(minProperties|at least 1 properties)" + + - it: rejects a Hubble egress rule without to + set: + networkPolicy.hubble.extraEgress: + - ports: + - port: 9200 + asserts: + - failedTemplate: + errorPattern: "(missing property 'to'|to is required)" diff --git a/helm/hugegraph/tests/validate_values_test.yaml b/helm/hugegraph/tests/validate_values_test.yaml index dbe36bd0c8..16d8453634 100644 --- a/helm/hugegraph/tests/validate_values_test.yaml +++ b/helm/hugegraph/tests/validate_values_test.yaml @@ -56,12 +56,116 @@ tests: - failedTemplate: errorPattern: "must not set the chart-managed variable JAVA_OPTIONS" - - it: rejects networkPolicy.enabled because the chart ships no policies + - it: rejects a PD NodePort under NetworkPolicy without pd extraIngress set: networkPolicy.enabled: true + pd.service.type: NodePort + pd.service.allowInsecureExposure: true + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the pd exposure" + + - it: rejects a PD LoadBalancer under NetworkPolicy without pd extraIngress + set: + networkPolicy.enabled: true + pd.service.type: LoadBalancer + pd.service.allowInsecureExposure: true + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the pd exposure" + + - it: rejects a Server NodePort under NetworkPolicy without server extraIngress + set: + networkPolicy.enabled: true + server.service.type: NodePort + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the server exposure" + + - it: rejects a Server LoadBalancer under NetworkPolicy without server extraIngress + set: + networkPolicy.enabled: true + server.service.type: LoadBalancer + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the server exposure" + + - it: rejects a Server Ingress under NetworkPolicy without server extraIngress + set: + networkPolicy.enabled: true + server.ingress.enabled: true + server.ingress.allowPlainHttp: true + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the server exposure" + + - it: rejects a Server advertiseUrl under NetworkPolicy without server extraIngress + set: + networkPolicy.enabled: true + server.advertiseUrl: http://hg.example.com:8080 + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the server exposure" + + - it: rejects a Hubble NodePort under NetworkPolicy without hubble extraIngress + set: + networkPolicy.enabled: true + hubble.enabled: true + hubble.service.type: NodePort + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the hubble exposure" + + - it: rejects a Hubble LoadBalancer under NetworkPolicy without hubble extraIngress + set: + networkPolicy.enabled: true + hubble.enabled: true + hubble.service.type: LoadBalancer + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the hubble exposure" + + - it: rejects a Hubble Ingress under NetworkPolicy without hubble extraIngress + set: + networkPolicy.enabled: true + hubble.enabled: true + hubble.ingress.enabled: true + hubble.ingress.allowPlainHttp: true + asserts: + - failedTemplate: + errorPattern: "admits nothing from outside the release, so the hubble exposure" + + - it: names advertiseUrl in the Server exposure message only + set: + networkPolicy.enabled: true + server.advertiseUrl: http://hg.example.com:8080 + asserts: + - failedTemplate: + errorPattern: "server exposure \\(NodePort/LoadBalancer Service, Ingress, server.advertiseUrl\\)" + + - it: reports the PD acknowledgement before the NetworkPolicy exposure check + set: + networkPolicy.enabled: true + pd.service.type: NodePort asserts: - failedTemplate: - errorPattern: "does not implement NetworkPolicy resources" + errorPattern: "pd.service.allowInsecureExposure=true" + + - it: leaves exposure alone while NetworkPolicy is off + set: + server.service.type: LoadBalancer + server.advertiseUrl: http://hg.example.com:8080 + asserts: + - hasDocuments: + count: 1 + + - it: ignores Hubble exposure settings while Hubble is disabled + set: + networkPolicy.enabled: true + hubble.service.type: NodePort + asserts: + - hasDocuments: + count: 1 - it: rejects Hubble without Server auth set: diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index 35c9d7a26d..e6751bdc51 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -131,3 +131,8 @@ server: # limits: # cpu: "1" # memory: 1536Mi + +# Production-shaped installs isolate the release; list outside clients in +# networkPolicy..extraIngress (see README, NetworkPolicy). +networkPolicy: + enabled: true diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index ccd2265ef9..159b8361b6 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -34,9 +34,87 @@ }, "hubble": { "$ref": "#/definitions/hubble" + }, + "networkPolicy": { + "$ref": "#/definitions/networkPolicy" } }, "definitions": { + "networkPolicyPeers": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "minProperties": 1 + } + }, + "networkPolicyIngressRules": { + "type": "array", + "items": { + "type": "object", + "required": [ + "from" + ], + "properties": { + "from": { + "$ref": "#/definitions/networkPolicyPeers" + } + } + } + }, + "networkPolicyEgressRules": { + "type": "array", + "items": { + "type": "object", + "required": [ + "to" + ], + "properties": { + "to": { + "$ref": "#/definitions/networkPolicyPeers" + } + } + } + }, + "networkPolicyComponent": { + "type": "object", + "additionalProperties": false, + "properties": { + "extraIngress": { + "$ref": "#/definitions/networkPolicyIngressRules" + } + } + }, + "networkPolicy": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "pd": { + "$ref": "#/definitions/networkPolicyComponent" + }, + "store": { + "$ref": "#/definitions/networkPolicyComponent" + }, + "server": { + "$ref": "#/definitions/networkPolicyComponent" + }, + "hubble": { + "type": "object", + "additionalProperties": false, + "properties": { + "extraIngress": { + "$ref": "#/definitions/networkPolicyIngressRules" + }, + "extraEgress": { + "$ref": "#/definitions/networkPolicyEgressRules" + } + } + } + } + }, "optionalPositiveInteger": { "oneOf": [ { diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 5b393e975a..a5a2b3355e 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -507,5 +507,31 @@ hubble: failureThreshold: 3 timeoutSeconds: 5 +# Kubernetes NetworkPolicy: one policy per component that admits only the +# traffic the components exchange, plus DNS. Off here because these values +# cannot know this release's clients; values-cluster.yaml turns it on. Only +# enforced when the network plugin implements NetworkPolicy (kind v0.25+, +# k3s, Calico, Cilium); elsewhere the objects are accepted and do nothing. +# Nothing outside the release is admitted unless it is listed in +# .extraIngress as standard NetworkPolicy ingress rules: apps in +# other namespaces, Prometheus, Vermeer, and the Ingress controller. Exposing +# a component (NodePort/LoadBalancer pd, server or hubble Service, a Server or +# Hubble Ingress, server.advertiseUrl) with an empty extraIngress fails the +# render. Every extra rule must name its peers ("from", or "to" for egress); +# to admit any address, say so with an ipBlock such as 0.0.0.0/0. +networkPolicy: + enabled: false + pd: + extraIngress: [] + store: + extraIngress: [] + server: + extraIngress: [] + hubble: + extraIngress: [] + # For Hubble's optional outside endpoints (es.urls, prometheus.url set + # through extraEnv). PD, Store and Server reach only the release and DNS. + extraEgress: [] + # No init Job; see README.md for the HStore initialization contract. # Install with: helm install ... --wait (no --wait-for-jobs) From 80e1ac8cd4bc69050f12f0d6a25f154de490b82f Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 21 Sep 2026 21:45:46 +0530 Subject: [PATCH 57/61] test(helm): add the license header to the NetworkPolicy snapshot The Apache RAT check fails on the helm-unittest snapshot file added with the NetworkPolicy tests because it carries no license header. helm-unittest ignores the comment when it reads the snapshot and leaves the file as is on a passing run (checked with 1.0.0 and 1.1.2). --- .../__snapshot__/networkpolicy_test.yaml.snap | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap b/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap index 99d7d5e042..447049801c 100644 --- a/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap +++ b/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + matches the reviewed cluster plus Hubble render: 1: | apiVersion: networking.k8s.io/v1 From 8ca50a2888423da7d54d343e89245df7e348bbae Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 23 Sep 2026 00:21:33 +0530 Subject: [PATCH 58/61] docs(helm): correct the Store replacement and recovery procedures A lifecycle run on a 3+3+3 kind install (chart d429dc462, :latest images) followed these procedures word for word and found three of them wrong or unusable as written. The OnDelete Store barrier could not see a Store that was down: PD keeps a Store Up and in every shard group until its keep-alive entry expires (300 s), so the documented check answered healthy on every sample through a 150 s replacement. The barrier now starts from the replaced Pod being Ready and a fresh lastHeartBeat, and points at the Store's own :8520/v1/partition/ for a closer look. It also named pd.partition.shardCount, which does not exist; the key is pd.partition.defaultShardCount, empty by default and derived as 3. Same name fixed in Scaling. Retiring a Store that was replaced with an empty PVC does not repair the cluster. The replacement keeps the Pod's raft address, so the reallocation adds a peer the group already has: 20 minutes and three patrols later all 12 groups still listed the retired id, the replacement held no partitions, and balancePartitions moved nothing. Nothing in /v1/stores, the cluster state or Hubble shows the lost redundancy. The section now says not to delete a Store PVC and what the failure looks like. The recovery runbook told operators to rerun the tasks on the new leader without mentioning that balancePartitions locks balanceLeaders for 180 s, which surfaces as a bare HTTP 500. It also gave no way to tell a real run from a no-op; the discriminating outputs are now written down. Two upgrade notes added from the same run: a Server that starts while PD is rolling can come up without its Gremlin binding and stays that way until the Pod is deleted (4 of 12 such starts), and dropping an inline credential back to the managed Secret rolls PD, Server and Hubble once more. Rotating the admin password through the Secret now says what actually happens, including the per-replica auth cache window measured at 9 to 21 minutes. --- helm/hugegraph/README.md | 155 ++++++++++++++++++++++++++++++++------- 1 file changed, 130 insertions(+), 25 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 40d329a0e9..71771d8c57 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -73,7 +73,17 @@ so operators do not have to: `auth.admin_pa` from the mounted Secret alongside `usePD=true` and `pd.peers`, then hands off to the image entrypoint. Two caveats: `auth.admin_pa` applies only when the admin is first created, so changing the - Secret does not rotate an existing cluster's password, and the value lands in + Secret does not rotate an existing cluster's password. Changing it anyway + rolls the Server Deployment, leaves the old password working, and makes the + Secret disagree with the live credential, so `helm test` (which reads the + Secret) fails until the two match again. To change the password on a running + cluster, change it through the Server API + (`PUT /graphspaces/DEFAULT/auth/users/admin` with `{"user_password": "..."}`) + and set the Secret to the same value. Each Server caches users and passwords + for `auth.cache_expire` (600 s by default) and nothing invalidates those + caches across replicas, so the other replicas keep accepting the old password + for a while: measured 9 to 21 minutes on a three-replica install. The value + also lands in `rest-server.properties` inside the container (file mode 600). Because the Java properties parser reinterprets them, the Secret value must not contain newlines, carriage returns, or backslashes; the wrapper refuses to start if @@ -263,27 +273,48 @@ Two cases are worth knowing about in advance: state `Up`. A Store is therefore `Up` before it has restored anything, and stays `Up` if restoring fails. - The strongest check the current images support is shard membership and - leadership per group, read from the PD leader: + Start with the Pod, not with PD. Wait for the replaced Pod to report + `Ready` (`kubectl -n wait --for=condition=Ready + pod/-hugegraph-store- --timeout=10m`), because PD alone + cannot tell you that the Store is running: PD marks a Store `Offline` only + after its keep-alive entry expires (`store.keepAlive-timeout`, 300 s on + current images) and the 60 s patrol notices, so a Pod that is deleted and + back inside that window never leaves `Up` and never leaves its shard + groups. Measured on a 3+3+3 install: a Store Pod was gone for 150 s and + every shard-group check below answered "healthy" on every sample for the + whole outage. + + Then check shard membership and leadership per group, read from the PD + leader: ```bash # PD leader, then its shard groups (see Disaster Recovery for the port-forward) curl -s -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/shardGroups | jq ' - .shardGroups[] | {id, + .shardGroups[] | {id: (.id // 0), shards: [.shards[] | {storeId, role}], leaders: [.shards[] | select(.role=="Leader")] | length}' ``` - Delete the next Store only when every group reports the full shard count - from `pd.partition.shardCount`, exactly one `Leader`, and the replaced - Store's id back in the groups it holds. `/v1/shardLeaders` gives the same - leadership view grouped by Store raft address. + (`id` is omitted for group 0 in the protobuf JSON, hence the `// 0`.) + + Delete the next Store only when the replaced Pod is `Ready`, its Store id + shows a `lastHeartBeat` newer than the restart in `/v1/stores`, and every + group reports the shard count in force (`pd.partition.defaultShardCount`; + empty derives 3 when `store.replicas` is at least 3), exactly one + `Leader`, and the replaced Store's id back in the groups it holds. + `/v1/shardLeaders` gives the same leadership view grouped by Store raft + address. Know what this does not prove. The shard list is PD's membership record, not a statement that the Store finished loading those partitions locally and caught up on the raft log. No endpoint in these images reports restoration-complete, so a group can list a Store whose local engine is - still behind. Leave a margin after the membership check rather than + still behind. For a closer look, port-forward the replaced Store Pod and + read its own view of each group: `GET :8520/v1/partition/` + returns the raft role, term and committed index that Store holds for that + group, and fails while the Store is down. (The plural `GET + :8520/v1/partitions` answers 500 on any Store that follows a group, so use + the per-group path.) Leave a margin after the membership check rather than deleting the next Pod on the same second, keep `store.pdb.minAvailable` at `replicas - 1` so an accidental second eviction is refused, and treat a group that is short a shard or has no leader as a stop. Closing that gap @@ -304,6 +335,24 @@ Two cases are worth knowing about in advance: restart the PD pods yourself. Rotating the PD REST Secret later rolls the same three workloads together, which keeps their copies of the secret in step. +- **A Server that starts while PD is rolling can come up without its Gremlin + binding.** Gremlin Server instantiates the graph once at startup; if the PD + client cannot connect at that moment the log says `Graph [DEFAULT-hugegraph] + ... could not be instantiated and will not be available in Gremlin Server`, + and the REST layer opens the graph seconds later anyway. The Pod then passes + readiness and serves REST while every Gremlin request on it fails with + `Could not rebind [graph]`, for the life of the Pod. Measured on current + images: 4 of 12 Server starts that overlapped a PD roll, none of 3 in a + Server-only roll. After an upgrade that rolls PD and Server together, check + Gremlin on each Server Pod and delete any Pod that fails; the replacement + binds normally once PD is stable (see Troubleshooting). +- **Dropping an inline credential back to the chart-managed Secret rolls PD, + Server and Hubble once more, with no credential change.** The rollout + checksum takes the inline value's digest while `pd.auth.value` or + `server.auth.token.value` is set, and the Secret's `resourceVersion` when it + is not, so removing the inline value changes the annotation although the + credential is unchanged (the chart never hashes Secret data). Expect one + extra roll on that upgrade. Every optional field stays optional, so a release created by an earlier revision continues to render under `--reuse-values`. Note that `--reuse-values` @@ -1037,7 +1086,21 @@ curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions Read `/v1/members` again after the tasks: if leadership moved mid-sequence, the later tasks ran on a follower and did nothing, so rerun them on the new -leader. +leader. Wait at least 180 s before rerunning `balanceLeaders` after a +`balancePartitions` call: `balancePartitions` sets a balance-shard flag for +180 s even when it moves nothing, and `balanceLeaders` inside that window +fails with a bare HTTP 500 whose reason (`balance shard is processing, +please try later!`) appears only in the PD log. + +Telling a real run from a no-op takes the PD leader's log, because the +responses do not. `patrolPartitions` answers `{"status": 0,"partitions": [ ]}` +on the leader and on a follower, whether or not it repaired anything; look +for `reallocShards`, `shardOffline` or `storeTurnoff` lines on the leader, +or diff `/v1/shardGroups` before and after. `balancePartitions` answers `{}` +on the leader and an empty body on a follower, which is a two-byte +difference. `balanceLeaders` is the one call whose body carries the work: a +JSON object of the groups whose leader moved, and `{}` when there was +nothing to move or when it reached a follower. The credential is required; PD answers 401 without it. The Secret name follows the release (`-pd-auth`) unless `pd.auth.existingSecret` @@ -1047,19 +1110,33 @@ Run `patrolPartitions` after replacing a Store that is not coming back, `balancePartitions` once the cluster is stable again, and `balanceLeaders` after restarts that skewed leader placement. -A Store replaced with an empty PVC registers under a **new Store ID**, even -though its Pod name and DNS address are unchanged, and the old ID stays -`Offline` in PD with its shard memberships intact; the patrol repairs only -`Tombstone` members, so it never touches the `Offline` entry. After such a -replacement, retire the old ID explicitly on the leader: find the -`Offline` entry in `/v1/stores` whose address matches the replaced Pod, -mark it `Tombstone` with `curl -u "hg:${PD_SECRET}" -X POST -H -'Content-Type: application/json' -d '{"storeState":"Tombstone"}' -http://127.0.0.1:8620/v1/store/` (this hands its shards to the -patrol), then run `patrolPartitions` and verify every shard group lists -only `Up` Stores. `DELETE /v1/store/` only erases the record and -strands the shard memberships; use it, if at all, as cleanup after the -patrol has finished. +**Do not delete a Store's PersistentVolumeClaim on current images.** A Store +replaced with an empty PVC registers under a **new Store ID**, while its Pod +name, DNS name and raft address are unchanged, and the cluster cannot be +brought back to full replication from there. Retiring the old ID runs, and +still does not repair the groups: PD accepts `{"storeState":"Tombstone"}` for +the old ID, `patrolPartitions` then logs `shardOffline` for every partition +and `reallocShards ShardGroup N, add shards from 2 to 3` with the new ID in +the computed list, and fires the configuration change; but the Store leader +sees that address already in the group (`changePeers start, old peer is [... + ...]`), so jraft has nothing to add and the group record keeps +the old ID. Measured on a 3+3+3 install: 20 minutes and three patrols later, +all 12 groups still listed the retired ID, the replacement Store held no +partitions at all (`:8520/v1/partition/` answered 500 on it for +every group), and `balancePartitions` refused to move anything +(`movedPartitions is empty`). `DELETE /v1/store/` erases the record +and leaves the groups naming an ID that no longer exists. + +Nothing in the documented health surface shows this: `/v1/stores` still +counts three `Up` Stores, cluster state stays `Cluster_OK`, Hubble lists +three Store nodes `UP`, and all Pods are `Ready`, while every shard group is +really running on two live replicas. The one check that shows it is the +replaced Store's own `:8520/v1/partition/`. + +So: replace a Store Pod, keep its PVC (the Store id lives in the data path, +and the Pod comes back under the same id). If a Store's volume is genuinely +lost, treat the cluster as degraded until an image-side fix lands, and expect +to rebuild rather than to recover in place. Periodic balancing and shard-sync progress metrics do not exist upstream yet and are out of scope for this chart. Periodic leader balancing is @@ -1117,8 +1194,9 @@ leaving Store the same way the Disaster Recovery section retires a replaced one: 1. Check the remaining Stores can still hold the persisted replication - factor: after the shrink, live Stores must be at least - `pd.partition.shardCount`. + factor: after the shrink, live Stores must be at least the shard count in + force (`pd.partition.defaultShardCount`; empty derives 3 when + `store.replicas` is at least 3). 2. Map the ordinals the shrink will delete (the highest ones) to Store ids through `/v1/stores`, matching on the Pod address. 3. `POST /v1/store/{id}` with `{"storeState":"Tombstone"}` for each leaving @@ -1193,6 +1271,24 @@ Cluster-wide readiness and PD-owned graph creation remain tracked in [#3139](https://github.com/apache/hugegraph/pull/3139); Phase 3: PD orchestration). +The same error has a second cause that does not close on its own: a Server +Pod that started while PD was rolling. Gremlin Server instantiates the graph +once at startup, so a PD client failure at that moment leaves the Pod without +a Gremlin binding for its whole life, while readiness passes and REST works. +The Pod's `hugegraph-server.log` names it: + +``` +Graph [DEFAULT-hugegraph] configured at [...] could not be instantiated and +will not be available in Gremlin Server +``` + +Check Gremlin on each Server Pod after any upgrade that rolled PD (a +port-forward to the Pod plus `POST /gremlin` with +`{"gremlin":"graph.traversal().V().limit(1).count()","aliases":{"graph":"DEFAULT-hugegraph"}}`), +and delete a Pod that fails. Its replacement binds normally as long as PD is +stable; measured on a 3+3+3 install, 4 of 12 Server starts that overlapped a +PD roll hit this, and both deletions recovered. + ### Pods OOM Killed or Restarting The default `values.yaml` sets **no** resource requests or limits and preserves @@ -1223,6 +1319,15 @@ independently of the release name. ## Limitations +- A Store cannot be recovered in place after its volume is lost. The + replacement keeps the Pod's DNS raft address, so PD's reallocation adds a + peer the raft group already has and the group keeps the old Store id; the + replacement stays empty and the group runs on the remaining replicas, with + no health surface reporting it. See Disaster Recovery for what was measured + and what to do instead. +- A Server Pod that starts while PD is rolling can lose its Gremlin binding + for the life of the Pod while passing readiness and serving REST; delete + that Pod. See Troubleshooting, "Could not rebind". - The default values set no container resources, so every pod is QoS class BestEffort and each JVM sizes its heap against total NODE memory rather than a cgroup limit. That is fine for a single-node or development install, but on From 26bbbcefb5df5cf5f2d1667d20be42899bae3a2b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 23 Sep 2026 11:17:20 +0530 Subject: [PATCH 59/61] docs(helm): fix resource names and retested recovery commands A retest of the procedures rewritten in 8ca50a288, run as written on a fresh 3+3+3 install with the release named hg, found five places where the text did not work. Workload and Service names were given as -hugegraph-*, which is only true when the release name does not contain "hugegraph"; for the README's own release it produced names like hugegraph-hugegraph-store. The naming paragraph now defines , and the three commands that used the wrong form use it. The Gremlin check answered 401 because it carried no credential, and it printed binary because the Server gzips the reply; it now reads the password the way Installing the Chart does and passes --compressed. Changing the admin password through the API left helm test passing or failing by replica while the auth caches expired. The README now says to restart the Server Pods, which applied it on every replica in 32 s. The Store's per-group view reports term and index 0 on a cluster with no writes, so the barrier text now says to compare them with a peer. --- helm/hugegraph/README.md | 45 ++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 71771d8c57..bad13f7d9f 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -82,8 +82,12 @@ so operators do not have to: and set the Secret to the same value. Each Server caches users and passwords for `auth.cache_expire` (600 s by default) and nothing invalidates those caches across replicas, so the other replicas keep accepting the old password - for a while: measured 9 to 21 minutes on a three-replica install. The value - also lands in + for a while: measured 9 to 21 minutes on a three-replica install, during + which `helm test` passes or fails depending on the replica it reaches. Restart + the Server Pods to apply the change everywhere at once + (`kubectl -n rollout restart deployment/-server`; + measured: 32 s, all replicas on the new password, `helm test` 4 of 4). The + value also lands in `rest-server.properties` inside the container (file mode 600). Because the Java properties parser reinterprets them, the Secret value must not contain newlines, carriage returns, or backslashes; the wrapper refuses to start if @@ -118,9 +122,12 @@ production use. The command examples in this document assume the release is named `hugegraph`. With a different release name, substitute the release-prefixed resource names (`kubectl get svc,secret -n ` lists them). -Workloads and Services are named `-hugegraph-*`, while the kept -Secrets are `-admin`, `-auth-token`, and -`-pd-auth`. +Workloads and Services are named `-*`, where `` is the +release name itself when it already contains `hugegraph` (release +`hugegraph` gives `hugegraph-pd`, `hugegraph-store`), `-hugegraph` +otherwise (release `hg` gives `hg-hugegraph-pd`), or `fullnameOverride` when +set. The kept Secrets always use the release name: `-admin`, +`-auth-token`, and `-pd-auth`. **Authentication is enabled by default.** The chart creates a kept Secret named `-admin` (for example `hugegraph-admin`) with a random @@ -275,7 +282,7 @@ Two cases are worth knowing about in advance: Start with the Pod, not with PD. Wait for the replaced Pod to report `Ready` (`kubectl -n wait --for=condition=Ready - pod/-hugegraph-store- --timeout=10m`), because PD alone + pod/-store- --timeout=10m`), because PD alone cannot tell you that the Store is running: PD marks a Store `Offline` only after its keep-alive entry expires (`store.keepAlive-timeout`, 300 s on current images) and the 60 s patrol notices, so a Pod that is deleted and @@ -312,7 +319,9 @@ Two cases are worth knowing about in advance: still behind. For a closer look, port-forward the replaced Store Pod and read its own view of each group: `GET :8520/v1/partition/` returns the raft role, term and committed index that Store holds for that - group, and fails while the Store is down. (The plural `GET + group, and fails while the Store is down. Compare term and index with the + same group on a peer Store rather than reading them alone: on a cluster + that has taken no writes they are 0 on every Store. (The plural `GET :8520/v1/partitions` answers 500 on any Store that follows a group, so use the per-group path.) Leave a margin after the membership check rather than deleting the next Pod on the same second, keep `store.pdb.minAvailable` at @@ -588,7 +597,7 @@ chart wires PD/Server for you. Open the UI with one port-forward: ```bash -kubectl -n port-forward svc/-hugegraph-hubble 8088:8088 +kubectl -n port-forward svc/-hubble 8088:8088 ``` Then open `http://127.0.0.1:8088`. For a shared environment, expose Hubble with @@ -1160,7 +1169,7 @@ overwrite the autoscaler's live replica count. `values.schema.json` requires at least one replica per component, so a staged rollout (PD and Server first, Stores later) cannot be written in a values file. Install the full topology and stage it with -`kubectl scale statefulset -hugegraph-store --replicas=0`, scaling +`kubectl scale statefulset -store --replicas=0`, scaling back up when ready; the Servers wait, not-ready, until Stores register. `kubectl scale` changes only the live StatefulSet: the next `helm upgrade` renders `store.replicas` from values again and restores the full topology. @@ -1282,10 +1291,20 @@ Graph [DEFAULT-hugegraph] configured at [...] could not be instantiated and will not be available in Gremlin Server ``` -Check Gremlin on each Server Pod after any upgrade that rolled PD (a -port-forward to the Pod plus `POST /gremlin` with -`{"gremlin":"graph.traversal().V().limit(1).count()","aliases":{"graph":"DEFAULT-hugegraph"}}`), -and delete a Pod that fails. Its replacement binds normally as long as PD is +Check Gremlin on each Server Pod after any upgrade that rolled PD: a +port-forward to the Pod, then `POST /gremlin` with the admin credential read +into `PASSWORD` as in Installing the Chart (authentication is on by default, so +the call answers 401 without it): + +```bash +kubectl port-forward -n hugegraph pod/ 8080:8080 +curl -s --compressed -u "admin:${PASSWORD}" -H 'Content-Type: application/json' \ + -X POST http://127.0.0.1:8080/gremlin \ + -d '{"gremlin":"graph.traversal().V().limit(1).count()","aliases":{"graph":"DEFAULT-hugegraph"}}' +``` + +A healthy Pod answers with `result.data`; delete a Pod that answers +`Could not rebind`. Its replacement binds normally as long as PD is stable; measured on a 3+3+3 install, 4 of 12 Server starts that overlapped a PD roll hit this, and both deletions recovered. From 8603cdbb3487ccd601d16d70d06072e7a85d9819 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 24 Sep 2026 20:59:47 +0530 Subject: [PATCH 60/61] fix(helm): add review guards, drop dead knobs, refresh the runbook Reserve the chart-owned checksum/ podAnnotations prefix on every component: user annotations render after the chart's own and the last duplicate key wins, so a fixed value would pin the rotation checksum. Refuse server.securityContext.readOnlyRootFilesystem=true, mirroring the Hubble guard, because the Server wrapper rewrites rest-server.properties inside the image. Reject updateStrategy.rollingUpdate options together with type OnDelete in the schema for PD and Store; Kubernetes refuses that combination at apply time, which would otherwise surface mid-upgrade. Unit tests cover the template guards and the CI reject-invalid-values step covers the schema constraint. Use the get-with-default pattern for the $exposed NetworkPolicy check, matching the rest of validateValues. Remove the constant $pdMeta and $wrapper indirection from the Server Deployment, and remove the unused server.restServer.minFreeMemory / batchMaxWriteThreads knobs end to end (template, values, schema, README); the chart is unreleased, so nothing depends on them. helm template output on the default, single and cluster presets is byte-identical before and after. Refresh the docs: the server.testResources default in the README matches values.yaml again, and the runbook follows apache/hugegraph#3232, #3233 and #3234 (merged 2026-09-24). The empty-PVC Store retirement was re-proven on images built from master at dbb6663a: all 12 groups converged onto the replacement 1 s after Tombstone and patrol, with 0 acknowledged writes lost, so the Disaster Recovery section documents the working procedure with a version caveat instead of a prohibition. --- .github/workflows/helm-chart-ci.yml | 10 ++ helm/hugegraph/README.md | 109 ++++++++++++------ helm/hugegraph/templates/_helpers.tpl | 19 ++- .../templates/server-deployment.yaml | 56 +-------- .../hugegraph/tests/validate_values_test.yaml | 23 ++++ helm/hugegraph/values.schema.json | 51 ++++---- helm/hugegraph/values.yaml | 6 - 7 files changed, 147 insertions(+), 127 deletions(-) diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 34e606698e..38c788db2a 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -173,6 +173,16 @@ jobs: helm template ci helm/hugegraph \ --set server.ingress.enabled=true \ --set server.ingress.allowPlainHttp=true > /dev/null + # Kubernetes refuses a StatefulSet that carries rollingUpdate + # options with updateStrategy OnDelete; the schema rejects the + # combination up front (a schema abort, so it lives here). + for component in pd store; do + must_fail \ + --set "${component}.updateStrategy.type=OnDelete" \ + --set "${component}.updateStrategy.rollingUpdate.partition=1" + helm template ci helm/hugegraph \ + --set "${component}.updateStrategy.type=OnDelete" > /dev/null + done # PD PDB must keep the Raft majority: floor(replicas/2)+1 must_fail --set pd.replicas=5 --set pd.pdb.minAvailable=2 must_fail --set pd.replicas=4 --set pd.pdb.minAvailable=2 diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index bad13f7d9f..0f44f37fd3 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -321,9 +321,14 @@ Two cases are worth knowing about in advance: returns the raft role, term and committed index that Store holds for that group, and fails while the Store is down. Compare term and index with the same group on a peer Store rather than reading them alone: on a cluster - that has taken no writes they are 0 on every Store. (The plural `GET - :8520/v1/partitions` answers 500 on any Store that follows a group, so use - the per-group path.) Leave a margin after the membership check rather than + that has taken no writes they are 0 on every Store. The plural `GET + :8520/v1/partitions` answers 500 on any Store that follows a group on + images built before + [apache/hugegraph#3232](https://github.com/apache/hugegraph/pull/3232) + (merged 2026-09-24); after it, the endpoint answers 200 on every Store, + with `conf` and `peers` null for the groups that Store follows. The + per-group path works on both. Leave a margin after the membership check + rather than deleting the next Pod on the same second, keep `store.pdb.minAvailable` at `replicas - 1` so an accidental second eviction is refused, and treat a group that is short a shard or has no leader as a stop. Closing that gap @@ -547,9 +552,7 @@ default values. | `server.serviceAccount.annotations` | Annotations on the created ServiceAccount | `{}` | | `server.serviceAccount.automountServiceAccountToken` | Mount an API token. The chart makes no API calls | `false` | | `server.waitImage` | Image for the Helm test hook | `curlimages/curl:8.5.0` | -| `server.testResources` | Resources for the Helm test hook container | `{}` | -| `server.restServer.minFreeMemory` | Empty preserves the image default | `""` | -| `server.restServer.batchMaxWriteThreads` | Empty preserves the image default | `""` | +| `server.testResources` | Resources for the Helm test hook container | requests `25m`/`32Mi`, limits `250m`/`64Mi` | | `server.initStoreEnabled` | Must remain `false` for distributed HStore | `false` | | `server.auth.enabled` | Enable admin authentication | `true` | | `server.auth.admin.password` | Optional inline admin password; prefer a Secret in shared clusters | `""` | @@ -813,12 +816,20 @@ before anything reaches the cluster: opts in. - `hubble.enabled` without `server.auth.enabled` is rejected unless `hubble.allowWithoutServerAuth=true`, and - `hubble.securityContext.readOnlyRootFilesystem=true` is rejected because - the Hubble wrapper writes its properties file inside the image at startup. + `securityContext.readOnlyRootFilesystem=true` is rejected for Server and + Hubble alike because each one's wrapper rewrites its properties file + inside the image at startup and the chart mounts no writable volume there. - `store.pdb.minAvailable` must be at least `store.replicas - 1`, so voluntary evictions cannot remove two copies of one shard at once. - `podLabels` may not override the chart-managed `app.kubernetes.io/name`, `instance` or `component` keys on any workload. +- `podAnnotations` may not set keys under the chart-owned `checksum/` + prefix on any workload: user annotations render after the chart's own + and the last duplicate key wins, so a fixed value would pin the checksum + and stop Secret or config rotation from rolling the pods. +- `updateStrategy.rollingUpdate` options are rejected together with + `updateStrategy.type: OnDelete` for PD and Store: Kubernetes refuses such + a StatefulSet at apply time, which would otherwise surface mid-upgrade. - A non-ClusterIP `pd.service.type` requires `pd.service.allowInsecureExposure=true`. - With `networkPolicy.enabled`, exposing PD, Server or Hubble (a NodePort @@ -1098,8 +1109,12 @@ the later tasks ran on a follower and did nothing, so rerun them on the new leader. Wait at least 180 s before rerunning `balanceLeaders` after a `balancePartitions` call: `balancePartitions` sets a balance-shard flag for 180 s even when it moves nothing, and `balanceLeaders` inside that window -fails with a bare HTTP 500 whose reason (`balance shard is processing, -please try later!`) appears only in the PD log. +is refused. On images built before +[apache/hugegraph#3233](https://github.com/apache/hugegraph/pull/3233) +(merged 2026-09-24) the refusal is a bare HTTP 500 whose reason (`balance +shard is processing, please try later!`) appears only in the PD log; after +it, the same reason comes back in the response body as +`{"status":1001,"error":"balance shard is processing, please try later!"}`. Telling a real run from a no-op takes the PD leader's log, because the responses do not. `patrolPartitions` answers `{"status": 0,"partitions": [ ]}` @@ -1119,13 +1134,36 @@ Run `patrolPartitions` after replacing a Store that is not coming back, `balancePartitions` once the cluster is stable again, and `balanceLeaders` after restarts that skewed leader placement. -**Do not delete a Store's PersistentVolumeClaim on current images.** A Store -replaced with an empty PVC registers under a **new Store ID**, while its Pod -name, DNS name and raft address are unchanged, and the cluster cannot be -brought back to full replication from there. Retiring the old ID runs, and -still does not repair the groups: PD accepts `{"storeState":"Tombstone"}` for -the old ID, `patrolPartitions` then logs `shardOffline` for every partition -and `reallocShards ShardGroup N, add shards from 2 to 3` with the new ID in +**A Store rebuilt with an empty PVC recovers in place on images carrying +[apache/hugegraph#3234](https://github.com/apache/hugegraph/pull/3234) +(merged 2026-09-24); on every earlier image, including all published release +images, it does not.** In both cases the replacement registers under a +**new Store ID** while its Pod name, DNS name and raft address are +unchanged, so `/v1/stores` lists two IDs at one address. + +On post-#3234 images the documented retirement then works: find the old ID +in `/v1/stores` (the row at the replaced Pod's address that is not the +newly registered one), `POST /v1/store/` with +`{"storeState":"Tombstone"}` on the PD leader, run +`GET /v1/task/patrolPartitions`, and wait; verify that every shard group +is back to full shard count with one leader, that no group names the old +ID, and that the replaced Store's own `:8520/v1/partition/` +answers 200 for every group; then `DELETE /v1/store/` to erase the +retired record. Measured 2026-09-24 on a 3+3+3 install with pd, store and +server built from `master` at `dbb6663a`: after deleting the Store's PVC +and Pod, the replacement was Ready in 156 s, every one of the 12 groups +converged onto the new ID 1 s after the Tombstone and patrol (the empty +Store caught up by raft snapshot install), the replaced Store answered 200 +on all 12 groups with its data directory back at full size, a later +restart with the kept PVC came back under the same ID with zero +registration rejections, the `DELETE` left no group naming the old ID, and +a continuous writer lost 0 of its 1,443 acknowledged vertices and 1,441 +acknowledged edges. + +On images without #3234 the same retirement runs and does not repair the +groups: PD accepts `{"storeState":"Tombstone"}` for the old ID, +`patrolPartitions` then logs `shardOffline` for every partition and +`reallocShards ShardGroup N, add shards from 2 to 3` with the new ID in the computed list, and fires the configuration change; but the Store leader sees that address already in the group (`changePeers start, old peer is [... ...]`), so jraft has nothing to add and the group record keeps @@ -1133,19 +1171,20 @@ the old ID. Measured on a 3+3+3 install: 20 minutes and three patrols later, all 12 groups still listed the retired ID, the replacement Store held no partitions at all (`:8520/v1/partition/` answered 500 on it for every group), and `balancePartitions` refused to move anything -(`movedPartitions is empty`). `DELETE /v1/store/` erases the record -and leaves the groups naming an ID that no longer exists. +(`movedPartitions is empty`). `DELETE /v1/store/` there erases the +record and leaves the groups naming an ID that no longer exists. -Nothing in the documented health surface shows this: `/v1/stores` still -counts three `Up` Stores, cluster state stays `Cluster_OK`, Hubble lists -three Store nodes `UP`, and all Pods are `Ready`, while every shard group is -really running on two live replicas. The one check that shows it is the -replaced Store's own `:8520/v1/partition/`. +Nothing in the documented health surface shows that failure: `/v1/stores` +still counts three `Up` Stores, cluster state stays `Cluster_OK`, Hubble +lists three Store nodes `UP`, and all Pods are `Ready`, while every shard +group is really running on two live replicas. The one check that shows it +is the replaced Store's own `:8520/v1/partition/`. -So: replace a Store Pod, keep its PVC (the Store id lives in the data path, -and the Pod comes back under the same id). If a Store's volume is genuinely -lost, treat the cluster as degraded until an image-side fix lands, and expect -to rebuild rather than to recover in place. +So the default remains: replace a Store Pod, keep its PVC (the Store id +lives in the data path, and the Pod comes back under the same id). Treat +the empty-PVC replacement above as a recovery procedure for post-#3234 +images only. On a released image a genuinely lost volume leaves the +cluster degraded; expect to rebuild rather than to recover in place. Periodic balancing and shard-sync progress metrics do not exist upstream yet and are out of scope for this chart. Periodic leader balancing is @@ -1338,12 +1377,14 @@ independently of the release name. ## Limitations -- A Store cannot be recovered in place after its volume is lost. The - replacement keeps the Pod's DNS raft address, so PD's reallocation adds a - peer the raft group already has and the group keeps the old Store id; the - replacement stays empty and the group runs on the remaining replicas, with - no health surface reporting it. See Disaster Recovery for what was measured - and what to do instead. +- On images predating + [apache/hugegraph#3234](https://github.com/apache/hugegraph/pull/3234) + (merged 2026-09-24, in no release yet), a Store cannot be recovered in + place after its volume is lost: the replacement keeps the Pod's DNS raft + address, PD's reallocation adds a peer the raft group already has, the + group keeps the old Store id, and no health surface reports it. Images + built from `master` at or after `dbb6663a` repair this through the + documented retirement; see Disaster Recovery for both measurements. - A Server Pod that starts while PD is rolling can lose its Gremlin binding for the life of the Pod while passing readiness and serving REST; delete that Pod. See Troubleshooting, "Could not rebind". diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 105167bf54..f73ce44a14 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -620,6 +620,16 @@ Cross-field validation that JSON Schema draft-07 cannot express. {{- fail (printf "%s.podLabels must not set %s: the chart manages it and the workload selectors, Services and PDBs match on it" $comp $reserved) -}} {{- end -}} {{- end -}} +{{/* User pod annotations render after the chart's own, and the Kubernetes + decoder keeps the last duplicate key, so a fixed checksum/* value would + replace the rendered checksum and pin it: rotating a Secret or changing + config would no longer roll the pods. */}} +{{- $compAnnotations := get (get $.Values $comp | default dict) "podAnnotations" | default dict -}} +{{- range $key, $_ := $compAnnotations -}} +{{- if hasPrefix "checksum/" $key -}} +{{- fail (printf "%s.podAnnotations must not set %s: the chart owns the checksum/ annotation prefix, which triggers pod rollouts when resolved Secrets or config change" $comp $key) -}} +{{- end -}} +{{- end -}} {{- end -}} {{- if and .Values.server.hpa.enabled (gt (int .Values.server.hpa.minReplicas) (int .Values.server.hpa.maxReplicas)) -}} {{- fail "server.hpa.minReplicas must be less than or equal to server.hpa.maxReplicas" -}} @@ -738,6 +748,9 @@ keys for releases stored before the values existed. {{- if and (get $serverIngress "enabled" | default false) (empty (get $serverIngress "tls")) (not (get $serverIngress "allowPlainHttp" | default false)) -}} {{- fail "server.ingress.enabled without tls publishes Basic-auth credentials and JWTs over plain HTTP; configure server.ingress.tls, or set server.ingress.allowPlainHttp=true to accept that on a trusted network" -}} {{- end -}} +{{- if get (get .Values.server "securityContext" | default dict) "readOnlyRootFilesystem" | default false -}} +{{- fail "server.securityContext.readOnlyRootFilesystem=true breaks Server: its wrapper rewrites conf/rest-server.properties inside the image at startup and the chart mounts no writable volume there" -}} +{{- end -}} {{/* extraEnv entries render after the chart-owned variables and Kubernetes lets the last duplicate win, so a duplicate name would silently override a @@ -807,9 +820,9 @@ start-hugegraph-pd.sh, start-hugegraph-store.sh, and hugegraph-server.sh). {{- $networkPolicy := get .Values "networkPolicy" | default dict -}} {{- if get $networkPolicy "enabled" -}} {{- $exposed := dict - "pd" (ne .Values.pd.service.type "ClusterIP") - "server" (or (ne .Values.server.service.type "ClusterIP") .Values.server.ingress.enabled (ne (trim (default "" .Values.server.advertiseUrl)) "")) - "hubble" (and .Values.hubble.enabled (or (ne .Values.hubble.service.type "ClusterIP") .Values.hubble.ingress.enabled)) -}} + "pd" (ne $pdSvcType "ClusterIP") + "server" (or (ne (get $svc "type" | default "ClusterIP") "ClusterIP") (get $serverIngress "enabled" | default false) (ne $advertiseUrl "")) + "hubble" (and (get $hubble "enabled" | default false) (or (ne (get (get $hubble "service" | default dict) "type" | default "ClusterIP") "ClusterIP") (get (get $hubble "ingress" | default dict) "enabled" | default false))) -}} {{- range $comp := list "pd" "server" "hubble" -}} {{- if and (get $exposed $comp) (empty (get (get $networkPolicy $comp | default dict) "extraIngress")) -}} {{- fail (printf "networkPolicy.enabled admits nothing from outside the release, so the %s exposure (NodePort/LoadBalancer Service, Ingress%s) is unreachable; list its callers in networkPolicy.%s.extraIngress, for example the Ingress controller's namespace or a client CIDR" $comp (ternary ", server.advertiseUrl" "" (eq $comp "server")) $comp) -}} diff --git a/helm/hugegraph/templates/server-deployment.yaml b/helm/hugegraph/templates/server-deployment.yaml index 93709f0c9b..2978b407dd 100644 --- a/helm/hugegraph/templates/server-deployment.yaml +++ b/helm/hugegraph/templates/server-deployment.yaml @@ -16,23 +16,13 @@ # {{- include "hugegraph.validateValues" . }} -{{- $restServer := .Values.server.restServer | default dict }} -{{- $minFreeMemory := "" }} -{{- $batchMaxWriteThreads := "" }} -{{- if hasKey $restServer "minFreeMemory" }} -{{- $minFreeMemory = toString (get $restServer "minFreeMemory") }} -{{- end }} -{{- if hasKey $restServer "batchMaxWriteThreads" }} -{{- $batchMaxWriteThreads = toString (get $restServer "batchMaxWriteThreads") }} -{{- end }} {{- $customPort := ne (int .Values.server.port) 8080 }} {{/* Distributed HStore requires every Server replica to share graph metadata through PD. The same registration properties let PD-mode Hubble discover the -Server and let the built-in authenticator create the admin on the PD path. +Server and let the built-in authenticator create the admin on the PD path, +so the config-rewrite wrapper below runs unconditionally. */}} -{{- $pdMeta := true }} -{{- $wrapper := or $pdMeta $customPort (ne $minFreeMemory "") (ne $batchMaxWriteThreads "") }} apiVersion: apps/v1 kind: Deployment metadata: @@ -106,7 +96,6 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - {{- if $wrapper }} command: - /usr/bin/dumb-init - -- @@ -137,7 +126,6 @@ spec: exit 1 fi TMP=$(mktemp) - {{- if $pdMeta }} FOUND_USE_PD=false FOUND_PD_PEERS=false FOUND_URLS_TO_PD=false @@ -148,22 +136,14 @@ spec: if [[ "${POD_IP:-}" == *:* && "${HG_SERVER_URLS_TO_PD:-}" == "http://${POD_IP}:"* ]]; then HG_SERVER_URLS_TO_PD="http://[${POD_IP}]:${HG_SERVER_URLS_TO_PD##*:}" fi - {{- end }} {{- if .Values.server.auth.enabled }} FOUND_AUTH_ADMIN_PA=false {{- end }} {{- if $customPort }} FOUND_RESTSERVER_URL=false {{- end }} - {{- if ne $minFreeMemory "" }} - FOUND_MIN_FREE_MEMORY=false - {{- end }} - {{- if ne $batchMaxWriteThreads "" }} - FOUND_BATCH_MAX_WRITE_THREADS=false - {{- end }} while IFS= read -r LINE || [[ -n "${LINE}" ]]; do case "${LINE}" in - {{- if $pdMeta }} usePD=*) printf 'usePD=true\n' >>"${TMP}" FOUND_USE_PD=true @@ -180,7 +160,6 @@ spec: printf 'server.deploy_in_k8s=true\n' >>"${TMP}" FOUND_DEPLOY_IN_K8S=true ;; - {{- end }} {{- if .Values.server.auth.enabled }} auth.admin_pa=*) printf 'auth.admin_pa=%s\n' "${PASSWORD}" >>"${TMP}" @@ -194,26 +173,11 @@ spec: FOUND_RESTSERVER_URL=true ;; {{- end }} - {{- if ne $minFreeMemory "" }} - restserver.min_free_memory=*) - printf 'restserver.min_free_memory=%s\n' \ - {{ $minFreeMemory | quote }} >>"${TMP}" - FOUND_MIN_FREE_MEMORY=true - ;; - {{- end }} - {{- if ne $batchMaxWriteThreads "" }} - batch.max_write_threads=*) - printf 'batch.max_write_threads=%s\n' \ - {{ $batchMaxWriteThreads | quote }} >>"${TMP}" - FOUND_BATCH_MAX_WRITE_THREADS=true - ;; - {{- end }} *) printf '%s\n' "${LINE}" >>"${TMP}" ;; esac done <"${CONF}" - {{- if $pdMeta }} if [[ "${FOUND_USE_PD}" == false ]]; then printf 'usePD=true\n' >>"${TMP}" fi @@ -232,7 +196,6 @@ spec: if [[ "${FOUND_DEPLOY_IN_K8S}" == false ]]; then printf 'server.deploy_in_k8s=true\n' >>"${TMP}" fi - {{- end }} {{- if .Values.server.auth.enabled }} if [[ "${FOUND_AUTH_ADMIN_PA}" == false ]]; then printf 'auth.admin_pa=%s\n' "${PASSWORD}" >>"${TMP}" @@ -244,22 +207,9 @@ spec: {{ .Values.server.port | quote }} >>"${TMP}" fi {{- end }} - {{- if ne $minFreeMemory "" }} - if [[ "${FOUND_MIN_FREE_MEMORY}" == false ]]; then - printf 'restserver.min_free_memory=%s\n' \ - {{ $minFreeMemory | quote }} >>"${TMP}" - fi - {{- end }} - {{- if ne $batchMaxWriteThreads "" }} - if [[ "${FOUND_BATCH_MAX_WRITE_THREADS}" == false ]]; then - printf 'batch.max_write_threads=%s\n' \ - {{ $batchMaxWriteThreads | quote }} >>"${TMP}" - fi - {{- end }} chmod 600 "${TMP}" mv "${TMP}" "${CONF}" exec ./docker-entrypoint.sh - {{- end }} ports: - name: http containerPort: {{ .Values.server.port }} @@ -290,10 +240,8 @@ spec: # coming up. Track the startup probe's own budget instead. - name: HG_SERVER_STARTUP_TIMEOUT_S value: {{ include "hugegraph.server.startupTimeoutSeconds" . | quote }} - {{- if $pdMeta }} - name: HG_SERVER_URLS_TO_PD value: {{ include "hugegraph.server.urlsToPd" . | quote }} - {{- end }} {{- with .Values.server.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} {{- with include "hugegraph.javaOptsEnv" .Values.server.javaOpts }}{{ . | trim | nindent 12 }}{{- end }} {{- if .Values.server.auth.enabled }} diff --git a/helm/hugegraph/tests/validate_values_test.yaml b/helm/hugegraph/tests/validate_values_test.yaml index 16d8453634..3c649b5029 100644 --- a/helm/hugegraph/tests/validate_values_test.yaml +++ b/helm/hugegraph/tests/validate_values_test.yaml @@ -227,3 +227,26 @@ tests: asserts: - failedTemplate: errorPattern: "readOnlyRootFilesystem=true breaks Hubble" + + - it: rejects a read-only root filesystem on Server + set: + server.securityContext.readOnlyRootFilesystem: true + asserts: + - failedTemplate: + errorPattern: "readOnlyRootFilesystem=true breaks Server" + + - it: rejects a Server pod annotation under the chart-owned checksum prefix + set: + server.podAnnotations: + checksum/auth: pinned + asserts: + - failedTemplate: + errorPattern: "server.podAnnotations must not set checksum/auth" + + - it: rejects a PD pod annotation under the chart-owned checksum prefix + set: + pd.podAnnotations: + checksum/pd-auth: pinned + asserts: + - failedTemplate: + errorPattern: "pd.podAnnotations must not set checksum/pd-auth" diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 159b8361b6..087688ae7e 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -172,6 +172,27 @@ "updateStrategy": { "type": "object", "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "OnDelete" + } + }, + "required": [ + "type" + ] + }, + "then": { + "not": { + "required": [ + "rollingUpdate" + ] + } + } + } + ], "properties": { "type": { "type": "string", @@ -773,36 +794,6 @@ "waitResources": { "$ref": "#/definitions/resources" }, - "restServer": { - "type": "object", - "additionalProperties": false, - "properties": { - "minFreeMemory": { - "oneOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string", - "const": "" - } - ] - }, - "batchMaxWriteThreads": { - "oneOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string", - "const": "" - } - ] - } - } - }, "initStoreEnabled": { "type": "boolean", "const": false diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index a5a2b3355e..5c4eeca8d4 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -331,7 +331,6 @@ server: minAvailable: 2 # Image used by the Helm test hook. waitImage: curlimages/curl:8.5.0 - # Optional resources for the Helm test hook container. # Resources for the Helm test hook container. Bounded by default so the hook # cannot run unlimited on a restricted or quota-managed namespace. testResources: @@ -341,11 +340,6 @@ server: limits: cpu: 250m memory: 64Mi - restServer: - # Empty preserves the image's restserver.min_free_memory default. - minFreeMemory: "" - # Empty preserves the image's batch.max_write_threads default. - batchMaxWriteThreads: "" # Distributed HStore: the init-store gate must be explicitly false, so that # concurrent Server replicas never initialize the same backend. No # HG_SERVER_SKIP_INIT and no init Job. From 3996bb12743c706a56fa4587abf016e38792b135 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 24 Sep 2026 22:24:07 +0530 Subject: [PATCH 61/61] docs(helm): slim the README against the docs-site pages Cut the README from 1,472 to 1,003 lines now that the docs site carries the operator walkthroughs. Each moved section keeps its load-bearing warning and commands and links its docs-site path: NetworkPolicy details, Cluster Health, Scheduling, Partition Sharding, the Disaster Recovery narrative (the keep-the-PVC rule and the retirement commands stay), the Scaling procedures, the outside-Hubble paths, and the Could-not-rebind measurements. The quickstart, presets, Kind flow, upgrade warnings with the OnDelete Store roll, the values tables, the validation list, every troubleshooting symptom and check command, and the Limitations stay. Deduplicate repeated passages to one home each: the ordinal-truncation explanation (Release Name Too Long), the anti-affinity trade (Installing), and the Hubble single-replica/H2 constraints (the Hubble section; the values.yaml comment now points there). README-only plus a values.yaml comment: helm template output is byte-identical, lint passes on the three presets, and all 139 unit tests pass. The docs-site links resolve once apache/hugegraph-doc#494 merges. --- helm/hugegraph/README.md | 945 ++++++++++--------------------------- helm/hugegraph/values.yaml | 11 +- 2 files changed, 242 insertions(+), 714 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 0f44f37fd3..50ec9bf59e 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -7,6 +7,12 @@ This chart deploys a distributed HugeGraph cluster - PD, Store, and Server - on Kubernetes. For HugeGraph itself see . +Two docs-site pages accompany this README: +[deploying with Helm](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm/) +and +[operating on Kubernetes](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/). +The operations page carries the walkthroughs this README links to below. + Note that this chart requires Helm 3. `--reset-then-reuse-values`, referenced under Upgrading, requires Helm 3.14 or later. @@ -32,12 +38,10 @@ A distributed HugeGraph cluster has a startup contract that this chart encodes so operators do not have to: - **Server does not run `init-store`.** The chart injects - `HG_SERVER_INIT_STORE_ENABLED=false`, and the image's `init-store` exits when - `init_store.enabled=false`, after which Server registers with PD normally. - This matters because nothing serializes Server replicas: without the gate, - every replica would initialize the same backend concurrently. The chart - creates no init Job and does not set `HG_SERVER_SKIP_INIT`. Standalone - behavior is unchanged, because the option defaults to `true` when unset. + `HG_SERVER_INIT_STORE_ENABLED=false`; nothing serializes Server replicas, + so without the gate every replica would initialize the same backend + concurrently. The chart creates no init Job and does not set + `HG_SERVER_SKIP_INIT`. - **Every Server uses PD for graph metadata.** The startup wrapper always writes `usePD=true` and the chart-derived `pd.peers` into `rest-server.properties`, so all Server replicas share the graph catalog @@ -56,46 +60,28 @@ so operators do not have to: to the Server storage wait as `PD_AUTH_PASSWORD`, and to Hubble as `operations.pd.password`; a `checksum/pd-auth` annotation rolls all three when the Secret changes. -- **The Server startup probe allows at least 450 seconds, and the image gets - the same budget.** The container may spend 300 seconds waiting for storage - and the rest in the start command, so the chart sets - `HG_SERVER_STARTUP_TIMEOUT_S` to the startup probe's own budget - (`failureThreshold` * `periodSeconds`, 450 seconds by default) rather than - leaving the image's 120-second default, which would self-kill a Server that - was still starting. A lower configured `failureThreshold` is raised to the - 450-second floor rather than being rejected, and raising the probe budget - raises the timeout with it. The variable is chart-managed, so - `server.extraEnv` may not set it; change the probe instead. +- **The Server startup probe allows at least 450 seconds, and the image + gets the same budget.** The chart sets `HG_SERVER_STARTUP_TIMEOUT_S` to + the startup probe's budget (`failureThreshold` * `periodSeconds`, 450 s + by default), so the image's 120-second default cannot self-kill a Server + that is still starting; a lower probe budget is raised to the floor, and + raising the probe raises the timeout. The variable is chart-managed; + change the probe, not `server.extraEnv`. - **The wrapper writes `auth.admin_pa` from the auth Secret.** With `init_store.enabled=false` the admin credential is created on the PD startup - path from `auth.admin_pa`, not from the Docker `PASSWORD` stdin path. When - authentication is enabled, the chart's wrapper therefore writes - `auth.admin_pa` from the mounted Secret alongside `usePD=true` and - `pd.peers`, then hands off to the image entrypoint. Two caveats: - `auth.admin_pa` applies only when the admin is first created, so changing the - Secret does not rotate an existing cluster's password. Changing it anyway - rolls the Server Deployment, leaves the old password working, and makes the - Secret disagree with the live credential, so `helm test` (which reads the - Secret) fails until the two match again. To change the password on a running - cluster, change it through the Server API - (`PUT /graphspaces/DEFAULT/auth/users/admin` with `{"user_password": "..."}`) - and set the Secret to the same value. Each Server caches users and passwords - for `auth.cache_expire` (600 s by default) and nothing invalidates those - caches across replicas, so the other replicas keep accepting the old password - for a while: measured 9 to 21 minutes on a three-replica install, during - which `helm test` passes or fails depending on the replica it reaches. Restart - the Server Pods to apply the change everywhere at once - (`kubectl -n rollout restart deployment/-server`; - measured: 32 s, all replicas on the new password, `helm test` 4 of 4). The - value also lands in - `rest-server.properties` inside the container (file mode 600). Because the - Java properties parser reinterprets them, the Secret value must not contain - newlines, carriage returns, or backslashes; the wrapper refuses to start if - it does. -- **Resource names reserve their suffix and StatefulSet ordinal before - truncation,** so a long release name cannot produce colliding or over-long - Pod and Service names, and PD/Store identities stay fixed when replicas - change. + path from `auth.admin_pa`, which applies only when the admin is first + created: changing the Secret later does not rotate a live cluster's + password, and only makes `helm test` disagree with the live credential. To + rotate, change the password through the Server API + (`PUT /graphspaces/DEFAULT/auth/users/admin` with `{"user_password": "..."}`), + set the Secret to the same value, and + `kubectl -n rollout restart deployment/-server` so + every replica's auth cache drops the old password at once (the caches + otherwise expire per replica over minutes). The Secret value lands in + `rest-server.properties` (mode 600) and must not contain newlines, carriage + returns, or backslashes; the wrapper refuses to start if it does. The + rotation caveats are also on the + [deployment page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm/#4-authentication-and-secrets). ## Installing the Chart @@ -121,13 +107,11 @@ production use. The command examples in this document assume the release is named `hugegraph`. With a different release name, substitute the release-prefixed -resource names (`kubectl get svc,secret -n ` lists them). -Workloads and Services are named `-*`, where `` is the -release name itself when it already contains `hugegraph` (release -`hugegraph` gives `hugegraph-pd`, `hugegraph-store`), `-hugegraph` -otherwise (release `hg` gives `hg-hugegraph-pd`), or `fullnameOverride` when -set. The kept Secrets always use the release name: `-admin`, -`-auth-token`, and `-pd-auth`. +resource names (`kubectl get svc,secret -n ` lists them): +workloads and Services are named `-*` (the release name itself +when it already contains `hugegraph`, `-hugegraph` otherwise, or +`fullnameOverride`), while the kept Secrets always use the release name: +`-admin`, `-auth-token`, and `-pd-auth`. **Authentication is enabled by default.** The chart creates a kept Secret named `-admin` (for example `hugegraph-admin`) with a random @@ -143,15 +127,12 @@ kubectl -n hugegraph create secret generic my-hugegraph-admin \ Then add `--set-string server.auth.admin.existingSecret=my-hugegraph-admin` to the install command. The Secret must contain a `password` key with no newlines, -carriage returns, backslashes, or surrounding whitespace. The last one bites -quietly: the Server wrapper writes the value into a properties file, and -Commons Configuration trims it when the Server reads it back, so a padded -Secret would create the account under the trimmed password and then fail to -authenticate with the value the Secret holds. The schema rejects padding on -inline values; for a bring-your-own Secret the chart cannot see the value, so -check it yourself. The JWT signing key uses -the same shape under `server.auth.token` (`value`, `existingSecret`, -`autoGenerate`), and its value must be at least 32 bytes. +carriage returns, backslashes, or surrounding whitespace (a properties read +trims padding, so a padded Secret creates the account under a different +password than it holds; the schema rejects padding on inline values but +cannot see a bring-your-own Secret). The JWT signing key uses the same shape +under `server.auth.token` (`value`, `existingSecret`, `autoGenerate`), and +its value must be at least 32 bytes. Read the password and exercise the API: ```bash @@ -271,28 +252,14 @@ Two cases are worth knowing about in advance: listener, not shard recovery: the controller can replace the next Store while the previous one is still rejoining its shard groups. For a production image roll, set `store.updateStrategy.type=OnDelete` and delete - Store Pods one at a time, checking shard membership between deletions. - - `Up` in PD is not that check. PD sets `StoreState.Up` and persists it in - `StoreNodeService.register()`, and only then does the notification reach - the Store, whose `HgStoreEngine.stateChanged` starts - `restoreLocalPartitionEngine()`; a failure there is logged and leaves the - state `Up`. A Store is therefore `Up` before it has restored anything, and - stays `Up` if restoring fails. - - Start with the Pod, not with PD. Wait for the replaced Pod to report - `Ready` (`kubectl -n wait --for=condition=Ready - pod/-store- --timeout=10m`), because PD alone - cannot tell you that the Store is running: PD marks a Store `Offline` only - after its keep-alive entry expires (`store.keepAlive-timeout`, 300 s on - current images) and the 60 s patrol notices, so a Pod that is deleted and - back inside that window never leaves `Up` and never leaves its shard - groups. Measured on a 3+3+3 install: a Store Pod was gone for 150 s and - every shard-group check below answered "healthy" on every sample for the - whole outage. - - Then check shard membership and leadership per group, read from the PD - leader: + Store Pods one at a time, checking between deletions. + + `Up` in PD is not that check: PD marks a Store `Up` at registration, + before anything is restored, and a stopped Store stays `Up` in every + shard group until its keep-alive entry expires (300 s on current images). + Start with the Pod (`kubectl -n wait --for=condition=Ready + pod/-store- --timeout=10m`), then check shard + membership and leadership per group, read from the PD leader: ```bash # PD leader, then its shard groups (see Disaster Recovery for the port-forward) @@ -302,39 +269,15 @@ Two cases are worth knowing about in advance: leaders: [.shards[] | select(.role=="Leader")] | length}' ``` - (`id` is omitted for group 0 in the protobuf JSON, hence the `// 0`.) - Delete the next Store only when the replaced Pod is `Ready`, its Store id - shows a `lastHeartBeat` newer than the restart in `/v1/stores`, and every - group reports the shard count in force (`pd.partition.defaultShardCount`; - empty derives 3 when `store.replicas` is at least 3), exactly one - `Leader`, and the replaced Store's id back in the groups it holds. - `/v1/shardLeaders` gives the same leadership view grouped by Store raft - address. - - Know what this does not prove. The shard list is PD's membership record, - not a statement that the Store finished loading those partitions locally - and caught up on the raft log. No endpoint in these images reports - restoration-complete, so a group can list a Store whose local engine is - still behind. For a closer look, port-forward the replaced Store Pod and - read its own view of each group: `GET :8520/v1/partition/` - returns the raft role, term and committed index that Store holds for that - group, and fails while the Store is down. Compare term and index with the - same group on a peer Store rather than reading them alone: on a cluster - that has taken no writes they are 0 on every Store. The plural `GET - :8520/v1/partitions` answers 500 on any Store that follows a group on - images built before - [apache/hugegraph#3232](https://github.com/apache/hugegraph/pull/3232) - (merged 2026-09-24); after it, the endpoint answers 200 on every Store, - with `conf` and `peers` null for the groups that Store follows. The - per-group path works on both. Leave a margin after the membership check - rather than - deleting the next Pod on the same second, keep `store.pdb.minAvailable` at - `replicas - 1` so an accidental second eviction is refused, and treat a - group that is short a shard or has no leader as a stop. Closing that gap - needs an image-side readiness signal for partition restoration, which is - the Store-side counterpart of the Server work in - [apache/hugegraph#3212](https://github.com/apache/hugegraph/issues/3212). + shows a fresh `lastHeartBeat` in `/v1/stores`, and every group reports its + full shard count with exactly one `Leader`. Leave a margin after the + membership check, keep `store.pdb.minAvailable` at `replicas - 1` so an + accidental second eviction is refused, and treat a group that is short a + shard or has no leader as a stop. What the membership record does not + prove, and the closer per-group check on the Store's own REST port, are on + the + [operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#6-rolling-store-images-safely). - **Server** rolls once on the first `helm upgrade` after a fresh install, when the `checksum/auth` annotation first observes the install-created Secrets. Template-only pipelines (`helm template`, GitOps renderers) never @@ -342,24 +285,15 @@ Two cases are worth knowing about in advance: does not roll pods. - **PD and Hubble** roll once on the first `helm upgrade` after a fresh install as well, when the `checksum/pd-auth` annotation first observes the - install-created PD REST Secret (same mechanism as the Server annotation - above; measured on a kind cluster: PD, Server and Hubble replaced, Store - untouched). A PD roll is a raft rolling restart, one pod at a time; for a - maintenance-window upgrade set `pd.updateStrategy.type=OnDelete` and - restart the PD pods yourself. Rotating the PD REST Secret later rolls the - same three workloads together, which keeps their copies of the secret in - step. + install-created PD REST Secret; Store is untouched. A PD roll is a raft + rolling restart, one pod at a time. Rotating the PD REST Secret later + rolls PD, Server and Hubble together, which keeps their copies in step. - **A Server that starts while PD is rolling can come up without its Gremlin - binding.** Gremlin Server instantiates the graph once at startup; if the PD - client cannot connect at that moment the log says `Graph [DEFAULT-hugegraph] - ... could not be instantiated and will not be available in Gremlin Server`, - and the REST layer opens the graph seconds later anyway. The Pod then passes - readiness and serves REST while every Gremlin request on it fails with - `Could not rebind [graph]`, for the life of the Pod. Measured on current - images: 4 of 12 Server starts that overlapped a PD roll, none of 3 in a - Server-only roll. After an upgrade that rolls PD and Server together, check - Gremlin on each Server Pod and delete any Pod that fails; the replacement - binds normally once PD is stable (see Troubleshooting). + binding** and then passes readiness and serves REST while every Gremlin + request on it fails with `Could not rebind [graph]`, for the life of the + Pod. After an upgrade that rolls PD and Server together, check Gremlin on + each Server Pod and delete any Pod that fails; the replacement binds + normally once PD is stable (see Troubleshooting). - **Dropping an inline credential back to the chart-managed Secret rolls PD, Server and Hubble once more, with no credential change.** The rollout checksum takes the inline value's digest while `pd.auth.value` or @@ -369,19 +303,13 @@ Two cases are worth knowing about in advance: extra roll on that upgrade. Every optional field stays optional, so a release created by an earlier -revision continues to render under `--reuse-values`. Note that `--reuse-values` -keeps the old release's values as the complete base, so a release created -before a field existed does **not** pick up its new default, including the -hardened `securityContext`, ServiceAccounts, and `terminationGracePeriodSeconds`. -Use `-f` with your own values, or `--reset-then-reuse-values`, to adopt them. -That rule covers values-sourced defaults only; the asymmetry is that -template-derived settings **are** applied even under `--reuse-values`, -because they are computed at render time from whatever values are in effect. -Pod-level token mounting (disabled unconditionally) and the derived -`-Dpartition.default-shard-count` in the PD `JAVA_OPTS`, plus the Server's -enforced PD metadata mode, are the current cases. The PD metadata change rolls -the Server Deployment. On an already-initialized cluster the seeded shard -count is inert either way; see Partition Sharding. +revision continues to render under `--reuse-values`. That flag keeps the old +values as the complete base, so such a release does **not** pick up new +values defaults (the hardened `securityContext`, ServiceAccounts, +`terminationGracePeriodSeconds`); use `-f` with your own values, or +`--reset-then-reuse-values`, to adopt them. Template-derived settings +**are** applied either way, because they are computed at render time from +whatever values are in effect. Upgrading an existing release to this chart version rolls the PD StatefulSet once: PD Pods now always carry a `JAVA_OPTS` environment variable with the @@ -394,10 +322,6 @@ effective value, but installs that relied on the old `required` default while supplying their own values files must now pin `antiAffinity: required` explicitly. -PD and Store resource names reserve room for their StatefulSet ordinal before truncation, so -identities stay fixed across replica changes and scaling never renames a -PersistentVolumeClaim. - ## Uninstalling the Chart ```bash @@ -594,85 +518,25 @@ PD. Store Operations metrics from outside the cluster are out of scope here. #### 1. In-cluster Hubble (recommended) -Set `hubble.enabled=true` (off by default so API-only clusters stay lean). The -chart wires PD/Server for you. - -Open the UI with one port-forward: +Set `hubble.enabled=true` (off by default so API-only clusters stay lean); +the chart wires PD/Server for you. Open the UI with one port-forward, then +open `http://127.0.0.1:8088` and log in with the chart admin password: ```bash kubectl -n port-forward svc/-hubble 8088:8088 ``` -Then open `http://127.0.0.1:8088`. For a shared environment, expose Hubble with -`hubble.service.type` NodePort/LoadBalancer or `hubble.ingress` instead of -port-forward. Log in with the chart admin password from the NOTES / admin -Secret. - -This is the average-user path: no Docker, no advertise URL, no PD peer list. - -#### 2. Outside Hubble, direct Server URL (simple external) - -Use this when Hubble runs on a host or VM outside the cluster, and you only -need graph / schema / data / Gremlin (not PD discovery). - -1. Leave in-chart Hubble off (`hubble.enabled=false`). -2. Expose Server (`server.service.type` NodePort/LoadBalancer, or Ingress). -3. Run a standalone Hubble image with `pd.enabled=false` and - `server.direct_url` set to that reachable Server URL (match Server auth). -4. Open the standalone Hubble port in a browser (or SSH tunnel to it). - -Use HTTPS (or a trusted channel such as a local port-forward) for -`server.direct_url`: login sends the Server credentials over that URL. - -Example property fragment for the standalone process: - -```properties -pd.enabled=false -server.direct_url=https://: -``` - -Mount the file at `/hubble/conf/hugegraph-hubble.properties` inside the -official image (workdir is `/hubble`). One Server URL is enough; you do not -need to expose PD. - -#### 3. Outside Hubble, PD discovery (advanced) - -Use this when an outside Hubble must ask PD for the Server address. - -In-cluster names such as `*.svc` are not reachable from outside. The chart -helps with two knobs: advertise a reachable Server URL to PD, and expose the -PD client Service. +For a shared environment, expose Hubble with `hubble.service.type` +NodePort/LoadBalancer or `hubble.ingress` instead of port-forward. -The chart always registers `server.urls_to_pd` with PD, so `server.advertiseUrl` -is honored whenever it is set. +#### 2 and 3. Outside Hubble (direct Server URL, or PD discovery) -1. Leave in-chart Hubble off if the bundled UI is not wanted. -2. Expose Server and set `server.advertiseUrl` to the absolute `http(s)://` - URL outside Hubble will use after discovery. The chart registers it via - `server.urls_to_pd` instead of the in-cluster Service URL. -3. Expose PD (`pd.service.type` NodePort/LoadBalancer, which needs - `pd.service.allowInsecureExposure=true`; PD gRPC has no authentication, - so restrict who can reach it first) so Hubble can dial PD - REST and gRPC. -4. Run standalone Hubble with `pd.enabled=true` and `pd.peers` / `pd.server` - pointed at those external PD addresses. Mount config at - `/hubble/conf/hugegraph-hubble.properties`. - -Example property fragment: - -```properties -pd.enabled=true -pd.peers=: -pd.server=: -``` - -Trade-off: when `server.advertiseUrl` is set, every Server replica registers that same logical URL and PD returns it to every discovery client, including an in-cluster Hubble. Leave it empty for the default in-cluster path, where each Server Pod registers its own IP and Hubble can retain the replica list. - -Local quick test (cluster and Hubble on the same machine): port-forward Server -`8080` and PD client `8620`/`8686`, set -`server.advertiseUrl=http://127.0.0.1:8080`, run standalone Hubble with -`--network host` and the PD properties above, then open Hubble on `8088` -(or SSH `-L 8088:127.0.0.1:8088` from a laptop). +A non-ClusterIP PD Service requires `pd.service.allowInsecureExposure=true` +(PD gRPC has no authentication; restrict who can reach it first), and a set +`server.advertiseUrl` registers that one URL with PD for every discovery +client, an in-cluster Hubble included. The walkthrough for both paths is on +the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#9-running-hubble-outside-the-cluster). | Parameter | Description | Default | |---|---|---| @@ -689,28 +553,18 @@ How to open Hubble (in-cluster vs outside) is under [Reaching Hubble](#reaching-hubble-pick-one-path) above. This section covers chart wiring and parameters. -Set `hubble.enabled=true` to deploy [HugeGraph Hubble](https://hugegraph.apache.org/docs/quickstart/toolchain/hugegraph-hubble/), -the web UI for graph management, schema browsing, Gremlin queries, and the -cluster operations view. A default install leaves Hubble off so API-only -clusters stay lean; authentication is already on, so enabling the UI is a -single flag (see Installing above). Login uses the admin credential from -`server.auth.admin.existingSecret` or the chart-managed `-admin` Secret. -`hubble.mode` selects the wiring. In the default -`pd` mode the chart points `pd.peers` at the PD gRPC peers, `pd.server` at -the PD client Service REST port, and the Store metrics allow-list at the -Store REST endpoints, so the cluster view works without manual wiring; the -Server is additionally configured to register each Server Pod IP with PD -(see below). In `direct` mode Hubble only receives `server.direct_url` -pointing at the Server client Service; there is no PD discovery and no -operations view. Everything else in `hugegraph-hubble.properties` keeps the -image default. - -The chart always runs the Server in PD meta mode (`usePD`, `pd.peers`, -`server.urls_to_pd`, `server.deploy_in_k8s`). In `pd` mode, Hubble uses that -registration so PD can hand it a resolvable Server address. The Store +`hubble.enabled=true` deploys [HugeGraph Hubble](https://hugegraph.apache.org/docs/quickstart/toolchain/hugegraph-hubble/), +the web UI; login uses the chart admin credential. `hubble.mode` selects +the wiring. The default `pd` mode points Hubble at the PD gRPC peers, the +PD client Service REST port, and the Store REST endpoints, so the cluster +operations view works without manual wiring (the Server's own PD +registration supplies a resolvable Server address); `direct` mode hands +Hubble only `server.direct_url` on the Server client Service, with no PD +discovery and no operations view. Everything else in +`hugegraph-hubble.properties` keeps the image default. The Store metrics allow-list is computed from `store.replicas` at render time, so scale Store -with `helm upgrade`, not `kubectl scale`, or the list goes stale until the next -upgrade. +with `helm upgrade`, not `kubectl scale`, or the list goes stale until the +next upgrade. Hubble is one replica by design: it keeps UI connection metadata, including any graph credentials entered in the UI, in an embedded per-instance H2 @@ -723,19 +577,16 @@ stored metadata), `size` and `storageClassName` apply at install time only, and a non-root `podSecurityContext` needs a matching `fsGroup` so H2 can write the volume. -**Current Hubble images still require `server.auth`.** The UI login -authenticates against the cluster; with authentication explicitly disabled -the login cannot complete (the server rejects `/auth/login` with -"Unconfigured authenticator"). The chart therefore refuses to render -`hubble.enabled=true` when `server.auth.enabled=false` unless +**Current Hubble images still require `server.auth`**: the UI login +authenticates against the cluster, so the chart refuses to render +`hubble.enabled=true` with `server.auth.enabled=false` unless `hubble.allowWithoutServerAuth=true` overrides it for images whose login does not need cluster authentication. -**Hubble serves plain HTTP.** Reach it with `kubectl port-forward` or behind -an HTTPS-terminating Ingress; never expose the port directly to an untrusted -network. An Ingress without `tls` is rejected at render time unless -`hubble.ingress.allowPlainHttp=true` explicitly accepts plain HTTP for a -trusted network. +**Hubble serves plain HTTP.** Reach it through a port-forward or an +HTTPS-terminating Ingress, never directly from an untrusted network; an +Ingress without `tls` is rejected unless `hubble.ingress.allowPlainHttp=true` +opts in. | Parameter | Description | Default | |---|---|---| @@ -852,35 +703,14 @@ v0.25 or later, k3s, Calico, Cilium). Other plugins accept the objects and enforce nothing. To check, run a Pod without chart labels in another namespace and `curl` the PD client Service on the REST port: it must time out. -With it on, the release admits only its own traffic: - -| To | From, ports | -|---|---| -| PD | PD: raft, gRPC. Store, Server, and Hubble in `pd` mode: gRPC, REST | -| Store | Store: raft. Server: gRPC, REST. Hubble in `pd` mode: REST | -| Server | Hubble and the `helm test` Pod: `server.port` | -| Hubble | nothing (port-forward uses loopback and needs no rule) | - -Every component may also resolve DNS on port 53. Nothing outside the release -is admitted unless it is listed in `networkPolicy..extraIngress`, -including the Ingress controller and clients of a NodePort or LoadBalancer -Service. Exposing PD, Server or Hubble that way, or setting -`server.advertiseUrl`, with an empty `extraIngress` fails the render instead -of opening the port. For PD this is the reachability restriction that -`pd.service.allowInsecureExposure` asks for. The check sees only exposure the -chart creates; a Service, Gateway route or proxy you add yourself needs its -own `extraIngress` entry. - -PD, Store and Server reach nothing outside the release except DNS, so a -feature that calls out (for example hugegraph-computer jobs through the -Kubernetes API, which this chart does not enable) does not work with the -policies on. One call is made by default: on every start the Store image -downloads `libjemalloc.so` from github.com. With the policies on that -connection times out after about two minutes, the Store starts without -jemalloc and continues (measured on kind: Ready after 151 s instead of 11 s). -The same happens on any cluster without internet access. The `helm test` Pod is selected by no chart policy, so its egress -is open only while nothing else selects it: under a namespace-wide -default-deny policy of your own, allow it egress to `server.port` and DNS. +With it on, the release admits only its own traffic plus DNS; anything +else, the Ingress controller and NodePort or LoadBalancer clients included, +must be listed in `networkPolicy..extraIngress`, and exposing +PD, Server or Hubble (or setting `server.advertiseUrl`) with an empty +`extraIngress` fails the render instead of opening the port. The +admitted-traffic matrix, the egress notes, and worked `extraIngress` +examples with the per-plugin NodePort client addresses are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#5-networkpolicy). | Parameter | Description | Default | |---|---|---| @@ -888,206 +718,41 @@ default-deny policy of your own, allow it egress to `server.port` and DNS. | `networkPolicy..extraIngress` | Extra NetworkPolicy ingress rules, appended as written | `[]` | | `networkPolicy.hubble.extraEgress` | Extra egress rules for Hubble's optional outside endpoints (`es.urls`, `prometheus.url`) | `[]` | -
-Letting other workloads in - -Anything outside the release is blocked until it is listed. Every rule must -name its peers in `from` (the schema rejects a rule without one); to admit any -address, write an `ipBlock` such as `0.0.0.0/0` explicitly. A -`namespaceSelector` and a `podSelector` in the same peer must both match; as -two separate peers, either one is enough. What a -NodePort or LoadBalancer client looks like from the Pod depends on the network -plugin, `externalTrafficPolicy` and the node the request arrives on. Measured -with a NodePort Server on two-node kind clusters: - -- kindnet, and Cilium with kube-proxy replacement: a call to the Server's own - node arrived with the client address; through the other node it arrived - with that node's address. -- Calico: through the other node the call arrived from that node's tunnel - address inside the Pod CIDR. -- Cilium with kube-proxy: no `ipBlock` rule admitted NodePort traffic, because - Cilium identifies node addresses by its own node identities rather than by - CIDR. - -Test with the plugin you run and name the CIDR you see arriving. - -```yaml -networkPolicy: - server: - extraIngress: - # The ingress-nginx controller, when server.ingress is enabled. - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: ingress-nginx - podSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - ports: - - port: 8080 - # Applications in namespace "apps" call the Server API. - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: apps - ports: - - port: 8080 - pd: - extraIngress: - # Prometheus scrapes /actuator/prometheus on PD REST. - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: monitoring - ports: - - port: 8620 - # Vermeer reads partition metadata over PD gRPC. - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: vermeer - ports: - - port: 8686 - store: - extraIngress: - # Vermeer scans Store over gRPC; Prometheus scrapes Store REST. - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: vermeer - ports: - - port: 8500 - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: monitoring - ports: - - port: 8520 -``` - -
- ## Deep Dive -### Connecting to the Cluster - -```bash -PASSWORD="$(kubectl get secret -n hugegraph hugegraph-admin \ - -o jsonpath='{.data.password}' | base64 --decode)" -kubectl port-forward -n hugegraph svc/hugegraph-server 8080:8080 -curl --user "admin:${PASSWORD}" http://127.0.0.1:8080/versions -curl --user "admin:${PASSWORD}" http://127.0.0.1:8080/graphs -``` - ### Cluster Health -| Component | Port | Purpose | -|------|-------------|---------| -| PD | `8686` | gRPC (Store and Server clients) | -| PD | `8620` | REST / health probes | -| PD | `8610` | Raft | -| Store | `8500` | gRPC | -| Store | `8510` | Raft | -| Store | `8520` | REST / health probes | -| Server | `8080` | Gremlin and REST API | - -All ports are configurable through `values.yaml`. Changing `server.port` updates -the listener, container port, and Service together. - -A stalled component (process alive but frozen) is ended by its liveness -probe, so the default 20 s period and 3-failure threshold bound the blast -radius of a stalled Store at roughly one minute; raft moves its partition -leaders within seconds of the restart. - ---- +Component ports are in the configuration tables above; a stalled component +is ended by its liveness probe within about a minute. The connection +commands and the health walkthrough are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#2-ports-and-health). ### Scheduling -Every component (`pd`, `store`, `server`, `hubble`) exposes the full set of -scheduling controls: `nodeSelector`, `tolerations`, `affinity`, -`topologySpreadConstraints`, and `priorityClassName`. For example, pinning -Store to labeled nodes is just: - -```yaml -store: - nodeSelector: - hugegraph/role: storage -``` - -`antiAffinity` (`required` | `preferred` | `disabled`) renders a hostname -pod-anti-affinity preset for `pd`, `store`, and `server`; Hubble has no -`antiAffinity` key because it is single-replica by design. Setting a raw -`affinity` replaces the preset entirely. All three default to `preferred` -(Server always did; the pd and store defaults changed from `required`), so -the chart schedules on clusters with fewer nodes than replicas (including -single-node development clusters). The trade: `preferred` lets the -scheduler co-locate replicas under node pressure, so a single node failure -can then take more than one PD or Store replica with it. Production -clusters with enough nodes should pin `pd.antiAffinity` and -`store.antiAffinity` to `required`, as `values-cluster.yaml` does. +Every component exposes the full set of scheduling controls, and +`antiAffinity` renders the hostname anti-affinity preset described under +Installing. Examples and the preferred-versus-required trade are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#3-scheduling). ### Partition Sharding -A fresh install seeds PD's persisted configuration with a partition shard -count of 3 when `store.replicas` is at least 3, and 1 otherwise. Without -this the PD image's `conf/application.yml` would pin -`partition.default-shard-count` to 1, leaving chart-deployed clusters -without store-level HA. The derivation never produces 2 because PD clamps a -shard count of 2 to 1: two shards cannot elect a leader. - -The chart renders the setting as `-Dpartition.default-shard-count` in the PD -container's `JAVA_OPTS`; system properties outrank the shipped config file, -and the PD start script appends `JAVA_OPTS` after its automatically computed -heap flags, so the image's JVM auto-sizing is unaffected. - -**The seed applies at first bootstrap only.** PD persists the shard count -into its own metadata the first time it starts with empty storage, and from -then on the stored value is authoritative: every PD leader change re-reads -it from storage, overwriting whatever the `-D` flag says. Changing -`pd.partition.defaultShardCount` later, or scaling `store.replicas` across -the derivation boundary, therefore has **no** effect on an initialized -cluster. Nor is the value frozen at partition creation: PD reconciles -existing shard groups toward the stored value whenever a partition patrol -runs. To change the shard count of a running cluster, use PD's own config -API (which accepts only odd values not exceeding the live store count) and -then trigger `GET /v1/task/patrolPartitions`; expect shard-group -reallocation when the counts differ. - -The shard count also fixes the initial partition count: -`store.replicas x storeMaxShardCount / shardCount`, computed once at -bootstrap. With the image's `store-max-shard-count` default of 12, the -derived shard count moves a default 3-store install from 36 partitions -(shard count 1) to 12 (shard count 3). Set -`pd.partition.storeMaxShardCount` higher to compensate when more partitions -are wanted; it is likewise seeded at first bootstrap only. - -An explicit `pd.partition.defaultShardCount` must be odd and at most -`store.replicas`. The chart rejects other values at render time: PD would -silently clamp a value above the live store count, clamp 2 to 1, and reject -even values at its config API, so an accepted render would not mean an -honored setting. +**The shard-count seed applies at first bootstrap only**: changing +`pd.partition.defaultShardCount`, or scaling `store.replicas` across the +derivation boundary, has no effect on an initialized cluster. Change a +running cluster through PD's own config API (odd values only, at most the +live store count), then trigger `GET /v1/task/patrolPartitions`. The +derivation, the initial partition count, and the constraints are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#4-partition-sharding). ### Disaster Recovery -What PD automates on current builds is narrow. A scheduled patrol runs on a -hardcoded 60-second cadence and only marks Stores that stopped sending -heartbeats as `Offline`; it does not touch partitions. There is **no -automatic re-replication**: re-placing the replicas of a lost Store, -reconciling shard groups against the stored shard count, and processing -tombstoned Stores all run only when a partition patrol is triggered -explicitly. PD's configuration binds `pd.patrol-interval` and -`store.max-down-time` keys, but no code path on current builds reads -either, which is why this chart does not expose them. - -Recovery and rebalancing are operator-triggered, and the task endpoints -execute **locally on the PD that receives them**: a follower answers with -an empty success and does no recovery work. Port-forwarding the client -Service selects an arbitrary PD, so identify the leader first and -port-forward that Pod: - -`kubectl port-forward` runs in the foreground, so use a second terminal -(or background the forward) for the curls, and stop the Service forward -before starting the leader one: +Recovery is operator-triggered on current builds: PD's own patrol only +marks silent Stores `Offline`, and there is **no automatic +re-replication**. The task endpoints execute locally on the PD that +receives them, and a follower answers with an empty success while doing +nothing, so identify the leader first and port-forward that Pod (the +forward runs in the foreground; use a second terminal for the curls). The +credential is required; PD answers 401 without it: ```bash kubectl port-forward -n hugegraph svc/hugegraph-pd-client 8620:8620 @@ -1105,90 +770,40 @@ curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/balancePartitions ``` Read `/v1/members` again after the tasks: if leadership moved mid-sequence, -the later tasks ran on a follower and did nothing, so rerun them on the new -leader. Wait at least 180 s before rerunning `balanceLeaders` after a -`balancePartitions` call: `balancePartitions` sets a balance-shard flag for -180 s even when it moves nothing, and `balanceLeaders` inside that window -is refused. On images built before -[apache/hugegraph#3233](https://github.com/apache/hugegraph/pull/3233) -(merged 2026-09-24) the refusal is a bare HTTP 500 whose reason (`balance -shard is processing, please try later!`) appears only in the PD log; after -it, the same reason comes back in the response body as -`{"status":1001,"error":"balance shard is processing, please try later!"}`. - -Telling a real run from a no-op takes the PD leader's log, because the -responses do not. `patrolPartitions` answers `{"status": 0,"partitions": [ ]}` -on the leader and on a follower, whether or not it repaired anything; look -for `reallocShards`, `shardOffline` or `storeTurnoff` lines on the leader, -or diff `/v1/shardGroups` before and after. `balancePartitions` answers `{}` -on the leader and an empty body on a follower, which is a two-byte -difference. `balanceLeaders` is the one call whose body carries the work: a -JSON object of the groups whose leader moved, and `{}` when there was -nothing to move or when it reached a follower. - -The credential is required; PD answers 401 without it. The Secret name -follows the release (`-pd-auth`) unless `pd.auth.existingSecret` -is set. - -Run `patrolPartitions` after replacing a Store that is not coming back, -`balancePartitions` once the cluster is stable again, and `balanceLeaders` -after restarts that skewed leader placement. - -**A Store rebuilt with an empty PVC recovers in place on images carrying +the later tasks ran on a follower and did nothing. Wait at least 180 s +before rerunning `balanceLeaders` after a `balancePartitions` call; the +refusal shapes, when to run which task, and telling a real run from a +no-op or a follower answer are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#7-disaster-recovery). + +**Replace a Store Pod, keep its PVC**: the Store id lives in the data path, +and the Pod comes back under the same id. A Store rebuilt with an empty PVC +registers under a **new Store ID** at the unchanged address, and recovers +in place only on images carrying [apache/hugegraph#3234](https://github.com/apache/hugegraph/pull/3234) -(merged 2026-09-24); on every earlier image, including all published release -images, it does not.** In both cases the replacement registers under a -**new Store ID** while its Pod name, DNS name and raft address are -unchanged, so `/v1/stores` lists two IDs at one address. - -On post-#3234 images the documented retirement then works: find the old ID -in `/v1/stores` (the row at the replaced Pod's address that is not the -newly registered one), `POST /v1/store/` with -`{"storeState":"Tombstone"}` on the PD leader, run -`GET /v1/task/patrolPartitions`, and wait; verify that every shard group -is back to full shard count with one leader, that no group names the old -ID, and that the replaced Store's own `:8520/v1/partition/` -answers 200 for every group; then `DELETE /v1/store/` to erase the -retired record. Measured 2026-09-24 on a 3+3+3 install with pd, store and -server built from `master` at `dbb6663a`: after deleting the Store's PVC -and Pod, the replacement was Ready in 156 s, every one of the 12 groups -converged onto the new ID 1 s after the Tombstone and patrol (the empty -Store caught up by raft snapshot install), the replaced Store answered 200 -on all 12 groups with its data directory back at full size, a later -restart with the kept PVC came back under the same ID with zero -registration rejections, the `DELETE` left no group naming the old ID, and -a continuous writer lost 0 of its 1,443 acknowledged vertices and 1,441 -acknowledged edges. - -On images without #3234 the same retirement runs and does not repair the -groups: PD accepts `{"storeState":"Tombstone"}` for the old ID, -`patrolPartitions` then logs `shardOffline` for every partition and -`reallocShards ShardGroup N, add shards from 2 to 3` with the new ID in -the computed list, and fires the configuration change; but the Store leader -sees that address already in the group (`changePeers start, old peer is [... - ...]`), so jraft has nothing to add and the group record keeps -the old ID. Measured on a 3+3+3 install: 20 minutes and three patrols later, -all 12 groups still listed the retired ID, the replacement Store held no -partitions at all (`:8520/v1/partition/` answered 500 on it for -every group), and `balancePartitions` refused to move anything -(`movedPartitions is empty`). `DELETE /v1/store/` there erases the -record and leaves the groups naming an ID that no longer exists. - -Nothing in the documented health surface shows that failure: `/v1/stores` -still counts three `Up` Stores, cluster state stays `Cluster_OK`, Hubble -lists three Store nodes `UP`, and all Pods are `Ready`, while every shard -group is really running on two live replicas. The one check that shows it -is the replaced Store's own `:8520/v1/partition/`. - -So the default remains: replace a Store Pod, keep its PVC (the Store id -lives in the data path, and the Pod comes back under the same id). Treat -the empty-PVC replacement above as a recovery procedure for post-#3234 -images only. On a released image a genuinely lost volume leaves the -cluster degraded; expect to rebuild rather than to recover in place. - -Periodic balancing and shard-sync progress metrics do not exist upstream -yet and are out of scope for this chart. Periodic leader balancing is -tracked in +(merged 2026-09-24); on every earlier image, including all published +release images, it does not. On post-#3234 images, retire the old ID on +the PD leader: + +```bash +# The old ID is the row at the replaced Pod's address that is not the +# newly registered one. +curl -su "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/stores +curl -u "hg:${PD_SECRET}" -X POST -H 'Content-Type: application/json' \ + -d '{"storeState":"Tombstone"}' http://127.0.0.1:8620/v1/store/ +curl -u "hg:${PD_SECRET}" http://127.0.0.1:8620/v1/task/patrolPartitions +# Verify: every group at full shard count with one leader, the old ID in +# no group, and the replaced Store answering 200 on :8520/v1/partition/. +curl -u "hg:${PD_SECRET}" -X DELETE http://127.0.0.1:8620/v1/store/ +``` + +On earlier images the same retirement runs and does not repair the groups, +and nothing in the health surface shows the loss: treat a genuinely lost +volume there as a degraded cluster and expect to rebuild rather than to +recover in place. Both measurements and the full walkthrough are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#7-disaster-recovery). + +Periodic leader balancing is tracked in [apache/hugegraph#3135](https://github.com/apache/hugegraph/issues/3135); disaster-recovery metrics are tracked in [apache/hugegraph#3136](https://github.com/apache/hugegraph/issues/3136). @@ -1197,65 +812,24 @@ disaster-recovery metrics are tracked in ### Scaling -PD and Store reserve the maximum StatefulSet ordinal in their resource names, -so scaling never renames a PersistentVolumeClaim or shifts a Pod identity. -Both are capped at 99 replicas. +Server scales through `server.replicas`, or by enabling `server.hpa` (the +Deployment then omits `spec.replicas`, so a Helm upgrade does not overwrite +the autoscaler). PD and Store are capped at 99 replicas. Stage a rollout +with `kubectl scale statefulset -store --replicas=0` and scale +back up when ready; the Servers wait, not-ready, until Stores register, and +the next `helm upgrade` restores the values topology. -Server scales through `server.replicas`, or by enabling `server.hpa`. With HPA -enabled the Deployment omits `spec.replicas`, so a Helm upgrade does not -overwrite the autoscaler's live replica count. - -`values.schema.json` requires at least one replica per component, so a -staged rollout (PD and Server first, Stores later) cannot be written in a -values file. Install the full topology and stage it with -`kubectl scale statefulset -store --replicas=0`, scaling -back up when ready; the Servers wait, not-ready, until Stores register. -`kubectl scale` changes only the live StatefulSet: the next `helm upgrade` -renders `store.replicas` from values again and restores the full topology. - -Changing PD or Store replicas on a live release is not a values change. +**Changing PD or Store replicas on a live release is not a values change.** Raft and shard membership are persisted, and Pods alone do not reconfigure -them. The chart rejects both directions for PD and a shrink for Store, and -reads the live StatefulSet to do it, so a fresh install at any replica count -is unaffected and a client-side `--dry-run` does not show the guard. - -**PD, either direction.** The peer list the chart renders reaches raft only -as `NodeOptions.setInitialConf`, which jraft applies when a node bootstraps -without a configuration of its own. On an initialized group it is inert: a -3-to-5 upgrade starts two more PDs and changes the bootstrap list, while the -voting configuration stays at three, and a 3-to-1 shrink loses quorum -outright. Membership changes through `RaftEngine.changePeerList`, which the -PD client API reaches and no REST route exposes, so this is a client-side -operation the chart cannot perform and does not wrap. Change the membership -through PD, confirm the new configuration in `/v1/members`, scale the live -StatefulSet, then `helm upgrade` with the matching value. Until you have run -and verified that sequence on your own build, treat a PD replica change as -unsupported and install the PD count you intend to keep. - -**Store, shrinking.** Draining is a state transition, not a balance. -`patrolPartitions` reallocates groups whose shard count does not match the -configured replication factor and hands off the groups of Stores already in -`Tombstone`; `balancePartitions` spreads shards across the active Stores, -including the ones you mean to remove, so neither call retires a healthy -Store and the "no shard lists them" condition may never arrive. Retire the -leaving Store the same way the Disaster Recovery section retires a replaced -one: - -1. Check the remaining Stores can still hold the persisted replication - factor: after the shrink, live Stores must be at least the shard count in - force (`pd.partition.defaultShardCount`; empty derives 3 when - `store.replicas` is at least 3). -2. Map the ordinals the shrink will delete (the highest ones) to Store ids - through `/v1/stores`, matching on the Pod address. -3. `POST /v1/store/{id}` with `{"storeState":"Tombstone"}` for each leaving - id, which is what drives `storeTurnoff` and the reallocation. -4. Wait until `/v1/shardGroups` no longer lists those ids and every group - reports its full shard count with one leader. -5. Scale the live StatefulSet with `kubectl -n scale statefulset - --replicas=`, then `helm upgrade` with the matching value. - -Deleting the PersistentVolumeClaims of the removed ordinals is separate and -permanent; do it only after step 4 reports the data moved. +them; the chart rejects both directions for PD and a shrink for Store by +reading the live StatefulSet (a client-side `--dry-run` does not show the +guard). Treat a PD replica change as unsupported and install the PD count +you intend to keep. The PD membership background and the Store +drain-then-scale procedure (Tombstone the leaving ids, wait for the groups, +then scale) are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#8-scaling). +Deleting the PVCs of removed ordinals is separate and permanent; do it only +after the groups no longer list the retired ids. ## Troubleshooting @@ -1296,34 +870,18 @@ curl -s --user "admin:${PASSWORD}" http://127.0.0.1:8080/graphs ### Queries Fail with "Could not rebind" Right After Creating a Graph -The Server that handles `CreateGraph` waits for its own Gremlin binding -before returning HTTP 200 -([#3138](https://github.com/apache/hugegraph/pull/3138)), so create-then-query -on the **same** Server (or sticky routing to that Pod) is reliable. - -Other Server replicas still converge independently through a PD metadata -watch plus a local graph open. Until they finish, a Gremlin query routed -through the load-balanced Service to a not-yet-converged replica can still -fail with a 400 error such as `Could not rebind [g]`. This is upstream -behavior, not a chart setting. Mitigations for multi-replica load-balanced -deployments: - -- Retry with backoff in the client; the window normally closes in seconds. -- Use sticky routing (or `kubectl port-forward` to one Pod) for - create-then-verify flows. -- Poll `/graphs` on each replica until the new graph appears everywhere - before opening query traffic. - -Cluster-wide readiness and PD-owned graph creation remain tracked in -[#3137](https://github.com/apache/hugegraph/issues/3137) (Phase 2: -[#3139](https://github.com/apache/hugegraph/pull/3139); Phase 3: PD -orchestration). - -The same error has a second cause that does not close on its own: a Server -Pod that started while PD was rolling. Gremlin Server instantiates the graph -once at startup, so a PD client failure at that moment leaves the Pod without -a Gremlin binding for its whole life, while readiness passes and REST works. -The Pod's `hugegraph-server.log` names it: +Two causes. Right after `CreateGraph`, the creating Server is consistent at +HTTP 200 ([#3138](https://github.com/apache/hugegraph/pull/3138)), but the +other replicas converge independently for a short window, and a Gremlin +query routed to a not-yet-converged replica fails with a 400 such as +`Could not rebind [g]`: retry with backoff, use sticky routing for +create-then-verify flows, or poll `/graphs` on each replica before opening +query traffic (cluster-wide readiness is tracked in +[#3137](https://github.com/apache/hugegraph/issues/3137)). + +The second cause does not close on its own: a Server Pod that started while +PD was rolling serves REST and passes readiness while every Gremlin call on +it fails, for the life of the Pod. Its `hugegraph-server.log` names it: ``` Graph [DEFAULT-hugegraph] configured at [...] could not be instantiated and @@ -1343,9 +901,9 @@ curl -s --compressed -u "admin:${PASSWORD}" -H 'Content-Type: application/json' ``` A healthy Pod answers with `result.data`; delete a Pod that answers -`Could not rebind`. Its replacement binds normally as long as PD is -stable; measured on a 3+3+3 install, 4 of 12 Server starts that overlapped a -PD roll hit this, and both deletions recovered. +`Could not rebind`, and its replacement binds normally once PD is stable. +The measurements behind both causes are on the +[operations page](https://hugegraph.apache.org/docs/quickstart/hugegraph/hugegraph-helm-operations/#10-when-gremlin-fails-with-could-not-rebind). ### Pods OOM Killed or Restarting @@ -1388,74 +946,49 @@ independently of the release name. - A Server Pod that starts while PD is rolling can lose its Gremlin binding for the life of the Pod while passing readiness and serving REST; delete that Pod. See Troubleshooting, "Could not rebind". -- The default values set no container resources, so every pod is QoS class - BestEffort and each JVM sizes its heap against total NODE memory rather than - a cgroup limit. That is fine for a single-node or development install, but on - a multi-node cluster where several pods share a node the heaps oversubscribe - it and pods abort. A measured example: on 7.6 GB workers the default install - gave PD `-Xmx3299m` and, with three to four pods per node, never converged. - Use `values-cluster.yaml`, or set your own `resources`, for any multi-node - deployment. -- The Store's memory ceiling is not its heap. The shipped - `conf/application.yml` includes the `pd` Spring profile, and - `conf/application-pd.yml` sets `rocksdb.total_memory_size` to - `32000000000`; `RaftRocksdbOptions` splits that number into a RocksDB - write cache and block cache, so those native caches are bounded by 32 GB - and not by the container. The jraft log storage also registers its own - 1 GiB LRU block cache once per process. With the cluster preset's - `-Xmx1024m -XX:MaxDirectMemorySize=512m`, a 4Gi limit sat below the - steady state and the kernel OOM-killed all three Stores after about 1 GB - of data; the preset now asks for 5Gi and limits at 8Gi, where a k3s run - measured 4.42 GiB anonymous RSS (2026-09-19). Scale both numbers with the - data size. The chart cannot lower the RocksDB budget itself: the Store - entrypoint rebuilds `SPRING_APPLICATION_JSON` from its own variables and - the chart mounts no config file, so `rocksdb.total_memory_size` can only - be changed in the image or through a custom config mount. -- PD's raft IP whitelist resolves peer hostnames to IPs once at startup, - which under Kubernetes can block peers whose pod IPs were unpublished at - that moment or change later. The chart therefore disables the whitelist - in-cluster via the upstream `raft.ip-whitelist.enabled` switch, leaving - peer authentication to Kubernetes-level controls: enable - `networkPolicy.enabled` (on in `values-cluster.yaml`) so that only PD Pods - reach the raft port. Setting - `pd.raftIpWhitelistEnabled=true` restores the image default along with - its one-shot resolution semantics (bring-up races and pod-IP-change - rejections included) at the operator's own risk. -- PD's `/v1/health` answers 200 as soon as the REST listener is up and never - consults raft, so it cannot see a lost quorum. With more than one PD the - chart uses it for startup and liveness on purpose, so that a follower which - merely lost its leader is not restarted, and puts readiness and the Store - wait on `/v1/ready`, which answers 503 without a raft leader. A single PD - is the exception: it has no election to lose, and a PD that steps down for - good, as after a failed raft snapshot on a full disk +- The default values set no container resources, so every pod is + BestEffort and each JVM sizes its heap against total node memory. Fine on + a single node; on a multi-node cluster the heaps oversubscribe the nodes + and pods abort. Use `values-cluster.yaml`, or set your own `resources`, + for any multi-node deployment. +- The Store's memory ceiling is not its heap: the image pins + `rocksdb.total_memory_size` at 32 GB of native caches outside the JVM, + and the chart cannot lower it (the entrypoint rebuilds its Spring config + and no file is mounted). The cluster preset requests 5Gi and limits at + 8Gi per Store for this reason; a 4Gi limit was OOM-killed after about + 1 GB of data. Scale both numbers with the data size; the full breakdown + is in the `values-cluster.yaml` comment. +- PD's raft IP whitelist resolves peer hostnames once at startup, which + under Kubernetes can block peers whose pod IPs were unpublished at that + moment or change later. The chart disables the whitelist in-cluster via + the upstream `raft.ip-whitelist.enabled` switch; enable + `networkPolicy.enabled` (on in `values-cluster.yaml`) so only PD Pods + reach the raft port. `pd.raftIpWhitelistEnabled=true` restores the image + default and its one-shot resolution races at the operator's own risk. +- PD's `/v1/health` answers 200 as soon as the REST listener is up and + cannot see a lost quorum. With more than one PD the chart uses it for + startup and liveness on purpose (a follower that merely lost its leader + is not restarted) and puts readiness and the Store wait on `/v1/ready`. + A single PD derives startup and liveness to `/v1/ready` instead: one + that steps down for good, as after a failed raft snapshot on a full disk ([apache/hugegraph#3222](https://github.com/apache/hugegraph/issues/3222)), - answers `/v1/health` forever while serving no writes. At `pd.replicas: 1` - startup and liveness therefore derive to `/v1/ready`, so the kubelet - restarts such a PD; `pd.livenessPath` overrides the derivation. If a future - PD answers 503 from `/v1/health` in that state, the value becomes - unnecessary. -- Server discovery is a lease. Each Server re-registers its Pod IP with PD - every 15 seconds and PD drops an entry after three missed heartbeats, so a - replaced or evicted Server can stay in PD's list for up to 45 seconds after - it stops (measured 30 to 35 seconds on a live rollout). Hubble's cluster - view and other discovery clients may show that stale address for the - duration; application traffic is unaffected because it reaches Servers - through the Service, which drops the Pod immediately. -- No TLS, backups, Operator, multi-cluster support, automatic leader transfer, - or a complete monitoring stack. Store recovery is manual on current builds: - re-replication after Store loss, leader balancing, and partition - rebalancing run only when triggered (see Disaster Recovery); periodic - balancing and shard-sync metrics are upstream feature work. + would answer `/v1/health` forever; `pd.livenessPath` overrides. +- Server discovery is a lease: a replaced Server can stay in PD's list for + up to 45 seconds after it stops, so Hubble's cluster view may briefly + show a stale address. Application traffic is unaffected, because it + reaches Servers through the Service, which drops the Pod immediately. +- No TLS, backups, Operator, multi-cluster support, automatic leader + transfer, or a complete monitoring stack. Store recovery is manual on + current builds (see Disaster Recovery); periodic balancing and shard-sync + metrics are upstream feature work. - After [#3138](https://github.com/apache/hugegraph/pull/3138), the creating Server is consistent at HTTP 200; other replicas may still lag for a short - window on load-balanced installs (see Troubleshooting: "Could not rebind"; - [#3137](https://github.com/apache/hugegraph/issues/3137) stays open for - cluster-wide and PD-owned creation). + window (see Troubleshooting: "Could not rebind"; + [#3137](https://github.com/apache/hugegraph/issues/3137) stays open). - The published images run as root, so `runAsNonRoot` and - `readOnlyRootFilesystem` are not chart defaults. The container - `securityContext` does default to `allowPrivilegeEscalation: false`, - `capabilities.drop: [ALL]`, and `seccompProfile: RuntimeDefault`, which are - valid for a root image; `podSecurityContext` and `securityContext` are fully + `readOnlyRootFilesystem` are not chart defaults; the container + `securityContext` is still hardened (`allowPrivilegeEscalation: false`, + `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault`) and fully configurable per component. - `values-cluster.yaml` is a starting point, not a capacity guarantee. - Authentication is on by default. The auth Secret sets the admin password @@ -1465,8 +998,6 @@ independently of the release name. `HG_SERVER_AUTH_TOKEN_SECRET` from `server.auth.token` (chart-managed by default) so Hubble login stays stable behind a multi-replica Service. -- Hubble is single-replica, serves plain HTTP, requires `server.auth` to be - enabled for its login to complete, and keeps UI connection metadata, - including any graph credentials entered in the UI, in an embedded H2 - database that is lost on Pod replacement unless `hubble.persistence` is - enabled. +- Hubble is single-replica, serves plain HTTP, and requires `server.auth` + for its login to complete; the persistence and H2 constraints are in the + Hubble section above. diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 5c4eeca8d4..4dc45d444d 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -433,13 +433,10 @@ hubble: digest: "" pullPolicy: Always port: 8088 - # Hubble keeps UI connection metadata, including any graph credentials - # entered in the UI, in an embedded per-instance H2 database, so the - # Deployment is fixed at a single replica (pointing SPRING_DATASOURCE_URL - # at an external database via extraEnv is not a supported configuration). - # Without persistence that metadata is lost on Pod replacement; graph data - # is unaffected. size and storageClassName apply at install time only, and - # the PVC is kept on helm uninstall. + # Keeps Hubble's per-instance H2 metadata across Pod replacement; see the + # README's Hubble section for the single-replica and H2 constraints. + # size and storageClassName apply at install time only; the PVC is kept + # on helm uninstall. persistence: enabled: false size: 1Gi