From fccf9d1455b70a0799ea47565bf1b7291a7c5a7d Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 10:31:53 +0800 Subject: [PATCH 01/18] feat: migrate secrets to SOPS encryption - Add SOPS + Age encryption for Helm chart secrets (datamate, label-studio, milvus) - Remove plaintext passwords from values.yaml files - Replace hardcoded secrets in docker-compose.yml with env vars - Add .env.example template for Docker deployment path - Add scripts/secrets.sh helper for encrypt/decrypt/helm-install - Add docs/SECRETS_SETUP.md setup guide - Update .gitignore: exclude .sops-keys/, allow .env.example and .env.enc Deployment: - K8s/Helm: helm secrets install -f secrets.yaml - Docker: cp .env.example .env && edit && docker compose up --- .gitignore | 9 +- .sops.yaml | 3 + deployment/docker/datamate/.env.example | 29 +++ deployment/docker/datamate/docker-compose.yml | 22 +-- deployment/helm/datamate/secrets.yaml | 26 +++ deployment/helm/datamate/values.yaml | 6 +- deployment/helm/label-studio/secrets.yaml | 19 ++ deployment/helm/label-studio/values.yaml | 6 +- deployment/helm/milvus/secrets.yaml | 18 ++ deployment/helm/milvus/values.yaml | 4 +- docs/SECRETS_SETUP.md | 166 ++++++++++++++++++ scripts/secrets.sh | 82 +++++++++ 12 files changed, 370 insertions(+), 20 deletions(-) create mode 100644 .sops.yaml create mode 100644 deployment/docker/datamate/.env.example create mode 100644 deployment/helm/datamate/secrets.yaml create mode 100644 deployment/helm/label-studio/secrets.yaml create mode 100644 deployment/helm/milvus/secrets.yaml create mode 100644 docs/SECRETS_SETUP.md create mode 100755 scripts/secrets.sh diff --git a/.gitignore b/.gitignore index 9b34b7af4..d78ab5ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -191,4 +191,11 @@ Thumbs.db # Milvus **/volumes/ -**/rag_storage/ \ No newline at end of file +**/rag_storage/ +# SOPS encryption keys +.sops-keys/ + +# Environment files - ignore local .env, but allow templates and encrypted files +.env +!.env.example +!.env.enc diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 000000000..efb2f00f7 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,3 @@ +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example new file mode 100644 index 000000000..90fa6588e --- /dev/null +++ b/deployment/docker/datamate/.env.example @@ -0,0 +1,29 @@ +# DataMate Environment Variables Template +# Copy this file to .env and fill in the values +# cp .env.example .env +# IMPORTANT: Never commit .env to git! It is already in .gitignore. +# +# For K8s/Helm deployment: secrets are managed via SOPS-encrypted secrets.yaml files. +# For Docker deployment: use this .env file (gitignored, local only). + +# Database +DB_PASSWORD=your-secure-password-here + +# JWT Authentication +JWT_SECRET=your-secure-jwt-secret-here +DATAMATE_JWT_ENABLE=false + +# MinIO (for Milvus storage) +MINIO_ACCESS_KEY=your-minio-access-key +MINIO_SECRET_KEY=your-minio-secret-key + +# Label Studio +LABEL_STUDIO_PASSWORD=your-labelstudio-password +LABEL_STUDIO_USER_TOKEN=your-labelstudio-token +LABEL_STUDIO_HOST= + +# Optional: SSL Certificate Password (for encrypted private keys) +CERT_PASS= + +# Optional: Domain for HTTPS +DOMAIN= \ No newline at end of file diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index dc5a8470d..3ab169c0d 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -9,7 +9,7 @@ services: restart: on-failure privileged: true environment: - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} volumes: - dataset_volume:/dataset @@ -30,7 +30,7 @@ services: - "18000:18000" environment: - log_level=DEBUG - - pgsql_password=${DB_PASSWORD:-password} + - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} volumes: @@ -52,7 +52,7 @@ services: ports: - '8080:8080' environment: - - JWT_SECRET=default-insecure-key-change-in-production + - JWT_SECRET=${JWT_SECRET:-} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} networks: [ datamate ] @@ -75,7 +75,7 @@ services: restart: on-failure environment: - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=${DB_PASSWORD:-password} + - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} volumes: - postgresql_volume:/var/lib/postgresql/data - database_log_volume:/var/log/datamate/database @@ -93,7 +93,7 @@ services: PG_HOST: "datamate-database" PG_PORT: "5432" PG_USER: "postgres" - PG_PASSWORD: ${DB_PASSWORD:-password} + PG_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} PG_DATABASE: "datamate" command: - python @@ -213,7 +213,7 @@ services: - DB_PORT=5432 - DB_NAME=labelstudio - DB_USER=postgres - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - AUTH_TYPE=scram-sha-256 - POOL_MODE=transaction - MAX_CLIENT_CONN=100 @@ -241,7 +241,7 @@ services: - DJANGO_DB=default - POSTGRE_NAME=labelstudio - POSTGRE_USER=postgres - - POSTGRE_PASSWORD=${DB_PASSWORD:-password} + - POSTGRE_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - POSTGRE_PORT=5432 - POSTGRE_HOST=label-studio-pgbouncer - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-} @@ -249,9 +249,9 @@ services: - LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local - USE_USERNAME_FOR_LOGIN=true - LABEL_STUDIO_USERNAME=admin@demo.com - - LABEL_STUDIO_PASSWORD=demoadmin + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} - LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN=true - - LABEL_STUDIO_USER_TOKEN=abc123abc123 + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} - LOG_LEVEL=DEBUG volumes: - label-studio-data:/label-studio/data:rw @@ -290,8 +290,8 @@ services: container_name: milvus-minio image: minio/minio:RELEASE.2024-12-18T13-15-44Z environment: - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-} ports: - "9001:9001" - "9000:9000" diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml new file mode 100644 index 000000000..5717c25f8 --- /dev/null +++ b/deployment/helm/datamate/secrets.yaml @@ -0,0 +1,26 @@ +public: + secrets: + data: + DB_PASSWORD: ENC[AES256_GCM,data:vX907qq/Iu/1v2IWAXYKKN9P2PkFC992GyF1fZT1gmmLdyfWtkARTbmHgqM=,iv:wHWaaQT5m9a34LN9NZiEOxrtwtUnd4y056h6WxlurYc=,tag:7inilRxi9QJY2slkF6P9jg==,type:str] + CERT_PASS: "" + DOMAIN: "" + HOME_PAGE_URL: "" +gateway: + env: + - name: ENC[AES256_GCM,data:yvymNzt7F3oJPA==,iv:JPJOJ+y4dvnXNQYpchL66MRNNrxidMTxyfEgbi30tpA=,tag:mJ9aPHXR61ybkNvpIxqJfw==,type:str] + value: ENC[AES256_GCM,data:om/hHL6wx7Bp2Xud2Te/jsOC8jakLCJOfzeuyZqGxjutK66+v//kjjnkTkY=,iv:WUvXTmTSfEZh4aOMRhtFH1ceRsuwtnm8pv9/qxG8K0U=,tag:pqUJhS+HYtsmBQqKXq5nJw==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYVY2ekE4RWZta3ZIdTVm + eXZTOXViaUpxUGlSNjAyUVk4M2NpZ1lBVWs0ClJMaUVPRXMxbW9OU3F1T0kxSlA0 + VENSZ0o2MW9GTmVQSWwyT0h0cFJoWUUKLS0tIHZGcHZzWTArK01DbVBJc21Ib2ZD + cnpCK01CZkdEaUJVS0NpVUc1NFlHTm8KZcEMm+q1Y4Zb0XJ28soc+cuPluejOs28 + WRB9Jde2Ext6b4LN05ZJ/BkH0LZPYneB7ZhuN+a2o1p2Xc3bjZrO5w== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:22Z" + mac: ENC[AES256_GCM,data:y9aHJfy+QXoNnIUcrdtiubMKAS4JiyAKFE33M3/2REz4D+v/9AKV258c2Q3U7BjGZOaW3wgv0CGbaHWurmYZ5xjwemv6Ev60ozt1Oy3BG9fG29ckcQooeNT/FiavjFJ7Xw72Kyj1mgDf+4cgfbNn5Wdqa9TF15o0vTnBxjC7v00=,iv:7xIH0txZCT/zuBFqrzOcESNice8QAlAzDK7dEqmSn/w=,tag:TPI6at/w+xKLkLvCvgMKHg==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index b19959a70..935de2283 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -39,8 +39,8 @@ public: operator: 1Gi secrets: data: - DB_PASSWORD: "password" - CERT_PASS: "" + DB_PASSWORD: "" # Set via secrets.yaml or --set + CERT_PASS: "" # Set via secrets.yaml for encrypted SSL keys DOMAIN: "" HOME_PAGE_URL: "" @@ -166,7 +166,7 @@ backend-python: gateway: env: - name: JWT_SECRET - value: "default-insecure-key-change-in-production" + value: "" # Set via secrets.yaml or --set for production - name: datamate.jwt.enable value: *DATAMATE_JWT_ENABLE - name: OMS_AUTH_ENABLED diff --git a/deployment/helm/label-studio/secrets.yaml b/deployment/helm/label-studio/secrets.yaml new file mode 100644 index 000000000..da2071797 --- /dev/null +++ b/deployment/helm/label-studio/secrets.yaml @@ -0,0 +1,19 @@ +env: + POSTGRE_PASSWORD: ENC[AES256_GCM,data:ROFSuT5BkJxGhusq2mGwxuBulcTNQeaVELxP6KimajWkGYBBys9oMJyPXHE=,iv:qqPQMjMUKpBJ76RlscGQ/6ff31e4iRDM0+IJ/q/TLYs=,tag:b5gCVSN/rn7aahqsoowayw==,type:str] + LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:niFPf5mPSt4J2CpIhYaS+gUkj3XnwvkUy7VCWPykGd0=,iv:vm1OC2vRdBgHqjXPL/4th9EiSzZg9nGEDOn33axY1y4=,tag:wKGLsepNKasU4ccXv/OWfw==,type:str] + LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:dmdU+7/Yd04jMCX5gaj4BpZhudCS2Bwe3FjqW40fX7g=,iv:GZS6oGS8oSBKYMjxBVuVmjEW0Ue/j9dfh50VBC9GvnI=,tag:GBhKPnPNyuaUaiXNfr4JEg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDUGVacVpQSjFwRFQvWllZ + bWlLZitoQ2JLS0pHS2ZQcW91T3RaRVBIU2t3ClVBZkN4N2FmRENPYVo0dU9RWnlD + MVZXNUhjSWFSZWo0Yjg0RTJSSFpGU3MKLS0tIGcvcllDK0FtREphOU9YckhvQlJQ + NE8vVXdqajFHMnBnVHhQSGF3dEdkWnMKTy7oYsfbWxr3U6X1gyS2kKJLxIjy0Zr0 + 1BfP9FAx6CvrveKqXGWwHk8+8034sG0fKO1H1lrBAgzW79eDOU4i+Q== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:39Z" + mac: ENC[AES256_GCM,data:Gni4TWxwzTmjf2+/t64V/SRBttJFuq1qMQ7cZzm2tJu/3pwbFOgcJwDdkoYSRKZrpyngj5EskbWzIgbmv0I9Jts26m9dN4pMr45qxQ1ba2G8eYgx0s2klh7iEFoMVTJmnYCbnl0PuqSWmcHbCxxZdeyo3ja647RlfLarT8QZwvU=,iv:nrDwjR0v96jKSJfQ812fpfrmI6vQEcCzttTVX8mLftU=,tag:AP+JhPFE7CZY5aZX5M8O7w==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/label-studio/values.yaml b/deployment/helm/label-studio/values.yaml index 229098ee5..456c6f242 100644 --- a/deployment/helm/label-studio/values.yaml +++ b/deployment/helm/label-studio/values.yaml @@ -41,7 +41,7 @@ env: DJANGO_DB: "default" POSTGRE_NAME: "labelstudio" POSTGRE_USER: "postgres" - POSTGRE_PASSWORD: "password" + POSTGRE_PASSWORD: "" # Set via secrets.yaml or --set POSTGRE_PORT: 5432 POSTGRE_HOST: "datamate-database" LABEL_STUDIO_HOST: "" # can be overridden @@ -49,9 +49,9 @@ env: LOCAL_FILES_DOCUMENT_ROOT: "/label-studio/local" USE_USERNAME_FOR_LOGIN: "true" LABEL_STUDIO_USERNAME: "admin@demo.com" - LABEL_STUDIO_PASSWORD: "demoadmin" + LABEL_STUDIO_PASSWORD: "" # Set via secrets.yaml or --set LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN: "true" - LABEL_STUDIO_USER_TOKEN: "abc123abc123" + LABEL_STUDIO_USER_TOKEN: "" # Set via secrets.yaml or --set LOG_LEVEL: "DEBUG" persistence: diff --git a/deployment/helm/milvus/secrets.yaml b/deployment/helm/milvus/secrets.yaml new file mode 100644 index 000000000..014696cff --- /dev/null +++ b/deployment/helm/milvus/secrets.yaml @@ -0,0 +1,18 @@ +minio: + accessKey: ENC[AES256_GCM,data:57/fzPABN+5+c1tmf9lvR9O5EflmlWPQfgvxTnwHhIo=,iv:Fkyrk5f7md7QQD2fv/XPXLLTxLDUq2mmK2BOi4QcZyI=,tag:AykpxHYoQSTjs840FodsDg==,type:str] + secretKey: ENC[AES256_GCM,data:EgFQZaIInv6Ptn9a+PiXVydwhi8Wh3dO9QkWY9qD86M=,iv:+WWoLUoMJc6cc9a+aLR97O/xvv7f6yfAgFjuKtt62h8=,tag:O6r+/1KhgnVCdxRmzdS8pQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuMlNnRDZJdGRXTklkK0tp + aC9KQ0t2VTExL2tOS3BvZzNRY3o5TnFYa0ZZCkNqY3U1K3g5M2NERDFDR1g3M1hJ + VGcrNDR4Q0tCOWptTzF0MG1QN0pXWFkKLS0tIGhMMUEweUxleWV1Y1RTVmdDQ2k0 + cUZOVC94Qi8xQmNNMFBwWEM3MnhkQ0EK4bTM7Rw3tlaMFInXv1MKlu65iPBxY8Zw + G5h5QK4AOonWoxwrz3H5jFBAZwBsy156wDgBsxvZJB2oCa4wEfC4Wg== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:40Z" + mac: ENC[AES256_GCM,data:N5n9tUIujF8BpI+N4PMzBApaTmpO+Ji/ZTf7Dr/faM9+nGnrL/JcOwAjKwXXGnDbcFi4DFyvo7yx4u68Mx6c4EtIQYs4o9TymGiQAgAUR8STnjBB3pbGjlvG9MFYwKCVuXBclhKpCLDklI/cjkcJjJq4ae2VnHer0wMlSzRZj8I=,iv:L2vlzY9aRAUGBZhgNOl1t3QDZhP8Inffj0TkGXnGbug=,tag:iB3ZKbYAk0D4bc+05q8m6w==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/milvus/values.yaml b/deployment/helm/milvus/values.yaml index 584cdcc27..626248e46 100644 --- a/deployment/helm/milvus/values.yaml +++ b/deployment/helm/milvus/values.yaml @@ -634,8 +634,8 @@ minio: image: tag: "RELEASE.2024-12-18T13-15-44Z" pullPolicy: IfNotPresent - accessKey: minioadmin - secretKey: minioadmin + accessKey: "" # Set via secrets.yaml or --set + secretKey: "" # Set via secrets.yaml or --set existingSecret: "" bucketName: "milvus-bucket" rootPath: file diff --git a/docs/SECRETS_SETUP.md b/docs/SECRETS_SETUP.md new file mode 100644 index 000000000..648e419d2 --- /dev/null +++ b/docs/SECRETS_SETUP.md @@ -0,0 +1,166 @@ +# Secrets Management Setup Guide + +This document describes the tools and configuration required for managing encrypted secrets in DataMate. + +## Required Tools + +### Homebrew Packages (macOS) + +| Tool | Version | Purpose | Install Command | +|------|---------|---------|-----------------| +| age | 1.3.1 | Encryption tool for secrets | `brew install age` | +| sops | 3.13.1 | Secrets encryption manager | `brew install sops` | +| helm | 4.1.0+ | Kubernetes package manager | `brew install helm` | + +### Helm Plugins + +| Plugin | Version | Purpose | Install Method | +|--------|---------|---------|----------------| +| helm-secrets | 4.6.0 | Helm integration with sops | Manual clone (see below) | + +**Note:** Helm 4.x has compatibility issues with helm-secrets plugin. Use manual installation: + +```bash +cd /Users/macoo/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +If plugin fails to load, edit `plugin.yaml` to remove the `command:` field and use only `platformCommand:`. + +## Project Configuration Files + +| File | Purpose | Location | +|------|---------|----------| +| `.sops.yaml` | SOPS encryption rules | Project root | +| `.sops-keys/key.txt` | Age encryption key (private) | Project root (gitignored) | +| `secrets.yaml` | Encrypted secrets per chart | `deployment/helm//` | + +## Quick Setup + +### 1. Install Required Tools + +```bash +# macOS (Homebrew) +brew install age sops helm + +# Helm plugin (manual) +cd ~/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +### 2. Generate Age Key (if not exists) + +```bash +mkdir -p .sops-keys +age-keygen -o .sops-keys/key.txt +``` + +**Important:** Backup the key file securely. It cannot be recovered if lost. + +### 3. Update `.sops.yaml` with Your Public Key + +```yaml +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: +``` + +### 4. Create/Encrypt Secrets Files + +```bash +# Set environment variable +export SOPS_AGE_KEY_FILE=.sops-keys/key.txt + +# Encrypt existing file +sops --encrypt --in-place deployment/helm/datamate/secrets.yaml + +# Or create new encrypted file +sops deployment/helm/datamate/secrets.yaml +``` + +## Usage + +### View Decrypted Secrets + +```bash +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml +``` + +### Deploy with Secrets + +```bash +# Using helper script +./scripts/secrets.sh helm-install datamate datamate + +# Manual method +TMP_FILE=$(mktemp) +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml > $TMP_FILE +helm install datamate deployment/helm/datamate -n datamate -f $TMP_FILE +rm $TMP_FILE +``` + +### Docker Compose + +```bash +# Copy and edit .env +cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env + +# Edit with your secrets +vim deployment/docker/datamate/.env + +# Deploy +docker compose -f deployment/docker/datamate/docker-compose.yml up -d +``` + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `SOPS_AGE_KEY_FILE` | Path to age private key | +| `SOPS_CONFIG` | Path to .sops.yaml (optional) | + +## Security Notes + +1. **Never commit `.sops-keys/` directory** - It's in `.gitignore` +2. **Backup your age key** - Store in secure location (password manager, encrypted backup) +3. **Use strong passwords** - Generate with `openssl rand -base64 32` +4. **Rotate keys periodically** - Especially after team changes + +## Troubleshooting + +### Plugin Load Error + +If helm plugin shows "both platformCommand and command are set": +```bash +# Edit plugin.yaml +vim ~/Library/helm/plugins/helm-secrets/plugin.yaml +# Remove the "command:" line, keep only "platformCommand:" +``` + +### Decryption Failed + +```bash +# Check key file exists +ls -la .sops-keys/key.txt + +# Verify key matches encrypted file's recipient +grep "recipient:" deployment/helm/datamate/secrets.yaml +``` + +### Key Rotation + +```bash +# Generate new key +age-keygen -o .sops-keys/key-new.txt + +# Update .sops.yaml with new public key + +# Re-encrypt all secrets +for f in deployment/helm/*/secrets.yaml; do + SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt $f > /tmp/plain.yaml + SOPS_AGE_KEY_FILE=.sops-keys/key-new.txt sops --encrypt /tmp/plain.yaml > $f +done + +# Replace old key +mv .sops-keys/key-new.txt .sops-keys/key.txt +``` \ No newline at end of file diff --git a/scripts/secrets.sh b/scripts/secrets.sh new file mode 100755 index 000000000..c93203449 --- /dev/null +++ b/scripts/secrets.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +SOPS_KEY_FILE="${SOPS_KEY_FILE:-$PWD/.sops-keys/key.txt}" +SOPS_CONFIG="${SOPS_CONFIG:-$PWD/.sops.yaml}" + +if [ ! -f "$SOPS_KEY_FILE" ]; then + echo "Error: SOPS age key file not found at $SOPS_KEY_FILE" + echo "Please generate a key first: age-keygen -o .sops-keys/key.txt" + exit 1 +fi + +export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" + +case "$1" in + encrypt) + if [ -z "$2" ]; then + echo "Usage: $0 encrypt " + exit 1 + fi + sops --encrypt --in-place "$2" + echo "Encrypted: $2" + ;; + decrypt) + if [ -z "$2" ]; then + echo "Usage: $0 decrypt " + exit 1 + fi + sops --decrypt --in-place "$2" + echo "Decrypted: $2" + ;; + view) + if [ -z "$2" ]; then + echo "Usage: $0 view " + exit 1 + fi + sops --decrypt "$2" + ;; + helm-install) + if [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 helm-install [--secrets-file ]" + exit 1 + fi + CHART="$2" + NAMESPACE="$3" + SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" + + TMP_FILE=$(mktemp) + sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" + helm install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" + rm "$TMP_FILE" + echo "Deployed $CHART to $NAMESPACE" + ;; + helm-upgrade) + if [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 helm-upgrade [--secrets-file ]" + exit 1 + fi + CHART="$2" + NAMESPACE="$3" + SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" + + TMP_FILE=$(mktemp) + sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" + helm upgrade "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" + rm "$TMP_FILE" + echo "Upgraded $CHART in $NAMESPACE" + ;; + *) + echo "DataMate SOPS Secret Management" + echo "" + echo "Usage:" + echo " $0 encrypt - Encrypt a secrets file" + echo " $0 decrypt - Decrypt a secrets file" + echo " $0 view - View decrypted secrets (no modification)" + echo " $0 helm-install - Install chart with decrypted secrets" + echo " $0 helm-upgrade - Upgrade chart with decrypted secrets" + echo "" + echo "Environment variables:" + echo " SOPS_KEY_FILE - Path to age key file (default: .sops-keys/key.txt)" + echo " SOPS_CONFIG - Path to .sops.yaml (default: .sops.yaml)" + ;; +esac \ No newline at end of file From 21484dfbeccc18b9e501f9367d580231810b22da Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 10:48:16 +0800 Subject: [PATCH 02/18] feat: update Makefile for SOPS secret management - K8s install: use helm secrets upgrade with -f secrets.yaml for datamate, label-studio, milvus - Docker install: add pre-check for .env file, exit with helpful message if missing - Set SOPS_AGE_KEY_FILE env var for helm secrets decryption --- Makefile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 859cd0dbb..a72bd672a 100644 --- a/Makefile +++ b/Makefile @@ -254,6 +254,13 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa done; \ exit 1; \ fi + @if [ ! -f deployment/docker/datamate/.env ]; then \ + echo "ERROR: deployment/docker/datamate/.env not found."; \ + echo "Create it from the template:"; \ + echo " cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env"; \ + echo "Then edit it with your actual passwords."; \ + exit 1; \ + fi @if [ "$*" = "label-studio" ]; then \ REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile label-studio up -d; \ elif [ "$*" = "datamate" ]; then \ @@ -326,19 +333,19 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m exit 1; \ fi @if [ "$*" = "label-studio" ]; then \ - helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install; \ + SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install -f deployment/helm/label-studio/secrets.yaml; \ elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \ elif [ "$*" = "mineru-310P" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \ elif [ "$*" = "datamate" ]; then \ - helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ + SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) -f deployment/helm/datamate/secrets.yaml; \ elif [ "$*" = "deer-flow" ]; then \ cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \ cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \ helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "milvus" ]; then \ - helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install; \ + SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install -f deployment/helm/milvus/secrets.yaml; \ elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \ kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \ fi From aefc5cf1caa2ea1b51fab48bb046bed32c9c70d2 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 11:07:38 +0800 Subject: [PATCH 03/18] fix: remove hardcoded JWT secret and unify property naming - JwtUtils.java: remove hardcoded default "datamate-secret-key-for-jwt-token-generation" Change property from jwt.secret to datamate.jwt.secret (aligned with JwtConfig) - application.yml: add datamate.jwt.secret mapping from JWT_SECRET env var - application.yml: remove hardcoded defaults for DB_PASSWORD and REDIS_PASSWORD - docker-compose.yml: add JWT_SECRET env var to datamate-backend service - Helm values.yaml: add JWT_SECRET secretKeyRef to backend env - Helm secrets.yaml: add JWT_SECRET to public.secrets.data --- .../src/main/resources/application.yml | 8 +++++-- .../datamate/common/security/JwtUtils.java | 2 +- deployment/docker/datamate/docker-compose.yml | 1 + deployment/helm/datamate/secrets.yaml | 21 ++++++++++--------- deployment/helm/datamate/values.yaml | 5 +++++ 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/backend/services/main-application/src/main/resources/application.yml b/backend/services/main-application/src/main/resources/application.yml index 64aac288c..bd05cebd5 100644 --- a/backend/services/main-application/src/main/resources/application.yml +++ b/backend/services/main-application/src/main/resources/application.yml @@ -14,7 +14,7 @@ spring: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://datamate-database:5432/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} + password: ${DB_PASSWORD} hikari: maximum-pool-size: 20 minimum-idle: 5 @@ -59,7 +59,7 @@ spring: host: datamate-redis port: 6379 timeout: 2000 - password: ${REDIS_PASSWORD:password} + password: ${REDIS_PASSWORD} lettuce: pool: max-active: 20 @@ -131,6 +131,10 @@ management: # 平台配置 datamate: + # JWT配置 + jwt: + secret: ${JWT_SECRET} + # 通用配置 diff --git a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java index efe4a4b85..5e981daf5 100644 --- a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java +++ b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java @@ -16,7 +16,7 @@ @Component public class JwtUtils { - @Value("${jwt.secret:datamate-secret-key-for-jwt-token-generation}") + @Value("${datamate.jwt.secret}") private String secret; @Value("${jwt.expiration:86400}") // 24小时 diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index 3ab169c0d..b9796e62e 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -10,6 +10,7 @@ services: privileged: true environment: - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} + - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} volumes: - dataset_volume:/dataset diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml index 5717c25f8..c688f2b1b 100644 --- a/deployment/helm/datamate/secrets.yaml +++ b/deployment/helm/datamate/secrets.yaml @@ -1,26 +1,27 @@ public: secrets: data: - DB_PASSWORD: ENC[AES256_GCM,data:vX907qq/Iu/1v2IWAXYKKN9P2PkFC992GyF1fZT1gmmLdyfWtkARTbmHgqM=,iv:wHWaaQT5m9a34LN9NZiEOxrtwtUnd4y056h6WxlurYc=,tag:7inilRxi9QJY2slkF6P9jg==,type:str] + DB_PASSWORD: ENC[AES256_GCM,data:OQx8/Cmz248ayvdwVdkNVR1KZJnfy0J/3zicX4TMlvxJDeACcMOsIskmio8=,iv:ZG72YxXBkRKIL7tpaF/NwgU+9D+pl8cWVwDB+JipO3A=,tag:P5TvdWGHUoocVjA23hwjrw==,type:str] + JWT_SECRET: ENC[AES256_GCM,data:dpS/iMHjipR00Ob5xIuA8XpQsx32HdOeQLrvU89vvCCLSRkvPUpP+0DpxT8=,iv:pCsAQPxHhIj/yJ4p7euE5u8L2gR+JlOtTq0GJkSN9AE=,tag:ZqqhvPaxE73yIrOIvKdQwQ==,type:str] CERT_PASS: "" DOMAIN: "" HOME_PAGE_URL: "" gateway: env: - - name: ENC[AES256_GCM,data:yvymNzt7F3oJPA==,iv:JPJOJ+y4dvnXNQYpchL66MRNNrxidMTxyfEgbi30tpA=,tag:mJ9aPHXR61ybkNvpIxqJfw==,type:str] - value: ENC[AES256_GCM,data:om/hHL6wx7Bp2Xud2Te/jsOC8jakLCJOfzeuyZqGxjutK66+v//kjjnkTkY=,iv:WUvXTmTSfEZh4aOMRhtFH1ceRsuwtnm8pv9/qxG8K0U=,tag:pqUJhS+HYtsmBQqKXq5nJw==,type:str] + - name: ENC[AES256_GCM,data:gZSjzeTSIH9mZg==,iv:NIwpYC8X5YSykeLovTZAGamYk1pCcTZ9zORjElMArk0=,tag:BJUww7hxoHIxmgchf2kz5Q==,type:str] + value: ENC[AES256_GCM,data:+iV1mQE+iw5RCOSCfojNIG5RMAH9x+0PBAwNYpEKFvTmUfDvjCEi1ykQW6o=,iv:0mUP/BPYpHB2bvKfQZ15TeaICe3uJdu8EobL8M6DHtg=,tag:9e1QjigvK+cF6eSMZJ3s3A==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYVY2ekE4RWZta3ZIdTVm - eXZTOXViaUpxUGlSNjAyUVk4M2NpZ1lBVWs0ClJMaUVPRXMxbW9OU3F1T0kxSlA0 - VENSZ0o2MW9GTmVQSWwyT0h0cFJoWUUKLS0tIHZGcHZzWTArK01DbVBJc21Ib2ZD - cnpCK01CZkdEaUJVS0NpVUc1NFlHTm8KZcEMm+q1Y4Zb0XJ28soc+cuPluejOs28 - WRB9Jde2Ext6b4LN05ZJ/BkH0LZPYneB7ZhuN+a2o1p2Xc3bjZrO5w== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNmNZdFJJN2JkaGFVM1lt + K2x5dFNaV3czUVduRlY4aHY2U2JlZVZFbWtvCnlPdHhWcElKS2FQbFJ5cnpMQ0tL + VjhiRjlRK002OEt3UUtSanppZVJCcUEKLS0tIHJVUDF1SU5TZC9qUmlZRGdMZjht + M1I2STNoKzdteXNFTUdVczdsSFY5bUkKblFkFp2073nMZgkpqug9wrrbso14HZAk + ZLzSwQBYtcV1103ri/pGFAyWWqIboAZ3Ki4BIprymRvlX+2XVtqCog== -----END AGE ENCRYPTED FILE----- recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm - lastmodified: "2026-05-22T08:19:22Z" - mac: ENC[AES256_GCM,data:y9aHJfy+QXoNnIUcrdtiubMKAS4JiyAKFE33M3/2REz4D+v/9AKV258c2Q3U7BjGZOaW3wgv0CGbaHWurmYZ5xjwemv6Ev60ozt1Oy3BG9fG29ckcQooeNT/FiavjFJ7Xw72Kyj1mgDf+4cgfbNn5Wdqa9TF15o0vTnBxjC7v00=,iv:7xIH0txZCT/zuBFqrzOcESNice8QAlAzDK7dEqmSn/w=,tag:TPI6at/w+xKLkLvCvgMKHg==,type:str] + lastmodified: "2026-05-25T03:07:14Z" + mac: ENC[AES256_GCM,data:iHn7oK3+kUnEZ0f2gYbZF+VloKSYNBcDpFTCP7SEGoXo7RiaGIg3EEj0OfOjlNJq7nLhZorRCq9ewjyhh5K3guW0Ys2Mh8hPrD1FEmw2kkiDzuzQA0qhx6DT7uXTyrhn1QyhvLwWrqMP+8jYjayLfNqMvYWpg5KYFi76MPVi/sE=,iv:SCQ3s/QNL+QILV1517vm3944rvCJU/9I92NGXxGO3tQ=,tag:I8fJVVCT46ygmyuus23+yw==,type:str] unencrypted_suffix: _unencrypted version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 935de2283..50a5fda2a 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -118,6 +118,11 @@ backend: secretKeyRef: name: datamate-conf key: DB_PASSWORD + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: datamate-conf + key: JWT_SECRET - name: datamate.rag.milvus-uri value: "http://milvus:19530" - name: datamate.jwt.enable From 25972b01f5a051f5e46a31fc37f68c64f54cbfcf Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 11:20:07 +0800 Subject: [PATCH 04/18] fix: enforce encrypted private key and restrict permissions - Reject plaintext private keys on startup (exit with error) - Require CERT_PASS when key is encrypted - Set chmod 600 on decrypted key for restrictive access --- scripts/images/frontend/start.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/images/frontend/start.sh b/scripts/images/frontend/start.sh index 6102495fd..d61306fca 100644 --- a/scripts/images/frontend/start.sh +++ b/scripts/images/frontend/start.sh @@ -18,14 +18,17 @@ if [ -f "/cert/server.pem" ]; then fi if [ -f "/cert/server.key" ]; then - # Check if key is encrypted and decrypt if needed - # Supports RSA, EC (Elliptic Curve), PKCS#8, and DSA keys + # Private key MUST be encrypted. Plaintext keys are not accepted. if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then - # Key is encrypted, decrypt using generic pkey command (supports all key types) + if [ -z "$CERT_PASS" ]; then + echo "ERROR: CERT_PASS is required to decrypt the private key" + exit 1 + fi echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin + chmod 600 /etc/nginx/cert/server.key else - # Key is not encrypted, copy directly - cp /cert/server.key /etc/nginx/cert/server.key + echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment." + exit 1 fi chown nginx:nginx /etc/nginx/cert/server.key fi From cdc393a9171d7b53bc3e8979c2004c255269961e Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 15:52:49 +0800 Subject: [PATCH 05/18] fix: remove hardcoded passwords in Python config and exclude dev .env from Docker builds - Create .dockerignore to prevent runtime/datamate-python/.env from being copied into Docker images (it contained localhost:15432 telepresence debug settings) - config.py: remove hardcoded defaults for pgsql_password, mysql_password, label_studio_password, label_studio_user_token - docker-compose.yml: add explicit PGSQL_HOST/PGSQL_PORT for backend-python - Helm values.yaml: add explicit PGSQL_HOST/PGSQL_PORT for backend-python --- .dockerignore | 22 +++++++++++++++++++ deployment/docker/datamate/docker-compose.yml | 2 ++ deployment/helm/datamate/values.yaml | 4 ++++ runtime/datamate-python/app/core/config.py | 10 ++++----- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..27e5b67e7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Environment files (local dev only, never in images) +**/.env +!.env.example + +# Python +**/__pycache__/ +**/*.pyc +**/.venv/ +**/venv/ + +# Logs +**/logs/ +**/*.log + +# IDE +.idea/ +.vscode/ +*.iml + +# Git +.git/ +.gitignore diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index b9796e62e..f9c25ea00 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -31,6 +31,8 @@ services: - "18000:18000" environment: - log_level=DEBUG + - PGSQL_HOST=datamate-database + - PGSQL_PORT=5432 - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 50a5fda2a..bf53963fc 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -144,6 +144,10 @@ backend: backend-python: env: + - name: PGSQL_HOST + value: "datamate-database" + - name: PGSQL_PORT + value: "5432" - name: pgsql_password valueFrom: secretKeyRef: diff --git a/runtime/datamate-python/app/core/config.py b/runtime/datamate-python/app/core/config.py index 5f6d3ae55..ed49fcf02 100644 --- a/runtime/datamate-python/app/core/config.py +++ b/runtime/datamate-python/app/core/config.py @@ -33,14 +33,14 @@ class Config: pgsql_host: str = "datamate-database" pgsql_port: int = 5432 pgsql_user: str = "postgres" - pgsql_password: str = "password" + pgsql_password: str = "" pgsql_database: str = "datamate" # Database mysql_host: str = "datamate-database" mysql_port: int = 3306 mysql_user: str = "root" - mysql_password: str = "password" + mysql_password: str = "" mysql_database: str = "datamate" database_url: str = "" # Will be overridden by build_database_url() if not provided @@ -63,9 +63,9 @@ def build_database_url(self): # Label Studio label_studio_base_url: str = "http://label-studio:8000" - label_studio_username: Optional[str] = "admin@demo.com" - label_studio_password: Optional[str] = "demoadmin" - label_studio_user_token: Optional[str] = "abc123abc123" # Legacy Token + label_studio_username: Optional[str] = None + label_studio_password: Optional[str] = None + label_studio_user_token: Optional[str] = None # Legacy Token label_studio_local_document_root: str = "/label-studio/local" # Label Studio local file storage path label_studio_file_path_prefix: str = "/data/local-files/?d=" # Label Studio local file serving URL prefix From 1c9b269a2809f66bc66abeba380e4721e2fed1de Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:36:56 +0800 Subject: [PATCH 06/18] refactor: encapsulate SOPS in scripts/secrets.sh, remove helm-secrets dependency - scripts/secrets.sh: add check_tools(), ensure_key() with auto-generation New helm-upgrade command: decrypts secrets.yaml and runs helm upgrade --install - Makefile: K8s install targets now call "bash scripts/secrets.sh helm-upgrade" instead of direct "helm secrets upgrade". No helm-secrets plugin needed. - Users only need sops + age (brew install), key auto-generated on first run. Docker users unaffected - still use .env file. --- Makefile | 6 ++-- scripts/secrets.sh | 79 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index a72bd672a..379654a92 100644 --- a/Makefile +++ b/Makefile @@ -333,19 +333,19 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m exit 1; \ fi @if [ "$*" = "label-studio" ]; then \ - SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install -f deployment/helm/label-studio/secrets.yaml; \ + bash scripts/secrets.sh helm-upgrade label-studio $(NAMESPACE); \ elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \ elif [ "$*" = "mineru-310P" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \ elif [ "$*" = "datamate" ]; then \ - SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) -f deployment/helm/datamate/secrets.yaml; \ + bash scripts/secrets.sh helm-upgrade datamate $(NAMESPACE) --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "deer-flow" ]; then \ cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \ cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \ helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "milvus" ]; then \ - SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install -f deployment/helm/milvus/secrets.yaml; \ + bash scripts/secrets.sh helm-upgrade milvus $(NAMESPACE); \ elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \ kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \ fi diff --git a/scripts/secrets.sh b/scripts/secrets.sh index c93203449..a4186e87e 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -1,18 +1,41 @@ #!/bin/bash +# DataMate Secrets Helper +# Manages SOPS-encrypted secrets for Helm deployment. +# Docker users do NOT need this script - use .env file instead. + +set -e SOPS_KEY_FILE="${SOPS_KEY_FILE:-$PWD/.sops-keys/key.txt}" SOPS_CONFIG="${SOPS_CONFIG:-$PWD/.sops.yaml}" -if [ ! -f "$SOPS_KEY_FILE" ]; then - echo "Error: SOPS age key file not found at $SOPS_KEY_FILE" - echo "Please generate a key first: age-keygen -o .sops-keys/key.txt" - exit 1 -fi +# Check required tools +check_tools() { + local missing="" + command -v sops >/dev/null 2>&1 || missing="$missing sops" + command -v age >/dev/null 2>&1 || missing="$missing age" + command -v age-keygen >/dev/null 2>&1 || missing="$missing age-keygen" + if [ -n "$missing" ]; then + echo "Error: Required tools not found:$missing" + echo "Install with: brew install sops age" + exit 1 + fi +} -export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" +# Check or generate age key +ensure_key() { + if [ ! -f "$SOPS_KEY_FILE" ]; then + echo "Age key not found at $SOPS_KEY_FILE" + echo "Generating a new one..." + mkdir -p "$(dirname "$SOPS_KEY_FILE")" + age-keygen -o "$SOPS_KEY_FILE" + echo "Key generated. Keep this file secure and never commit it." + fi + export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" +} case "$1" in encrypt) + check_tools if [ -z "$2" ]; then echo "Usage: $0 encrypt " exit 1 @@ -21,6 +44,7 @@ case "$1" in echo "Encrypted: $2" ;; decrypt) + check_tools if [ -z "$2" ]; then echo "Usage: $0 decrypt " exit 1 @@ -29,25 +53,52 @@ case "$1" in echo "Decrypted: $2" ;; view) + check_tools if [ -z "$2" ]; then echo "Usage: $0 view " exit 1 fi sops --decrypt "$2" ;; - helm-install) + helm-upgrade) + # Usage: scripts/secrets.sh helm-upgrade [extra-helm-args...] + check_tools + ensure_key if [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 helm-install [--secrets-file ]" + echo "Usage: $0 helm-upgrade [extra-helm-args...]" exit 1 fi CHART="$2" NAMESPACE="$3" - SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" - - TMP_FILE=$(mktemp) - sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" - helm install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" - rm "$TMP_FILE" + SECRETS_FILE="deployment/helm/$CHART/secrets.yaml" + shift 3 + + if [ ! -f "$SECRETS_FILE" ]; then + echo "Warning: No secrets file at $SECRETS_FILE, deploying without secrets." + helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" "$@" + else + TMP_FILE=$(mktemp) + sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" + helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" "$@" + rm "$TMP_FILE" + echo "Deployed $CHART with decrypted secrets." + fi + ;; + *) + echo "DataMate Secrets Helper" + echo "" + echo "Usage: $0 [args...]" + echo "" + echo "Commands:" + echo " encrypt Encrypt a YAML file in-place" + echo " decrypt Decrypt a YAML file in-place" + echo " view Print decrypted content" + echo " helm-upgrade Decrypt secrets and helm upgrade --install" + echo "" + echo "Docker users: skip this script, use 'cp .env.example .env' instead." + exit 1 + ;; +esac echo "Deployed $CHART to $NAMESPACE" ;; helm-upgrade) From 1cd2316d06e0eb4f08a81ce401103811d060d162 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:39:04 +0800 Subject: [PATCH 07/18] fix: pass Label Studio credentials to backend-python container - docker-compose.yml: add LABEL_STUDIO_USER_TOKEN, LABEL_STUDIO_PASSWORD env vars - Helm values.yaml: add secretKeyRef for both to backend-python env - secrets.yaml: add encrypted LABEL_STUDIO_USER_TOKEN, LABEL_STUDIO_PASSWORD Fixes 500 error "Label Studio API token is required" when creating annotation tasks after config.py defaults were removed. --- deployment/docker/datamate/docker-compose.yml | 2 ++ deployment/helm/datamate/secrets.yaml | 24 ++++++++++--------- deployment/helm/datamate/values.yaml | 10 ++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index f9c25ea00..af2b18123 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -36,6 +36,8 @@ services: - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} volumes: - dataset_volume:/dataset - flow_volume:/flow diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml index c688f2b1b..25defa1a6 100644 --- a/deployment/helm/datamate/secrets.yaml +++ b/deployment/helm/datamate/secrets.yaml @@ -1,27 +1,29 @@ public: secrets: data: - DB_PASSWORD: ENC[AES256_GCM,data:OQx8/Cmz248ayvdwVdkNVR1KZJnfy0J/3zicX4TMlvxJDeACcMOsIskmio8=,iv:ZG72YxXBkRKIL7tpaF/NwgU+9D+pl8cWVwDB+JipO3A=,tag:P5TvdWGHUoocVjA23hwjrw==,type:str] - JWT_SECRET: ENC[AES256_GCM,data:dpS/iMHjipR00Ob5xIuA8XpQsx32HdOeQLrvU89vvCCLSRkvPUpP+0DpxT8=,iv:pCsAQPxHhIj/yJ4p7euE5u8L2gR+JlOtTq0GJkSN9AE=,tag:ZqqhvPaxE73yIrOIvKdQwQ==,type:str] + DB_PASSWORD: ENC[AES256_GCM,data:F/GtCDyYi4xH8FJWGxojsCdNEDdKQaVIaFBuC5dk6ilLdfGJtk0JDoPRQ14=,iv:mhenVjDgPvX57OaSMnUZGvFJrWjwtqIPJ66PIHqQcKM=,tag:kxxS/w0FUt6N9kHNvSVMOg==,type:str] + JWT_SECRET: ENC[AES256_GCM,data:Pu2cojeOgNSMmxbqevkI+c8Az3QS/A6DgDKv2NeGtYON5YrZFBqxxfV+y2M=,iv:sjVi42/TVmKGzbu9ZxAgU3CGBlHgJ94CEpzAd8eZX0k=,tag:3v/cJNPUZpyFJjU7Wdz+Ow==,type:str] CERT_PASS: "" DOMAIN: "" HOME_PAGE_URL: "" + LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:XBjxusCTd5xe2xnx7D8h+AH8rpZ6ZUCyYUws4Zxffr0=,iv:yWwIMpEJzodAhK78SOZXTWZi9GnG387oKeOK+ZORjis=,tag:l6U8sQoUzvHdFit72WI4JQ==,type:str] + LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:YR9/z/UthUuwJTaZMqbnwdVNWW60pCrzmUDAOojWo6E=,iv:ojZuiZ5G7l5DF6P+UQaCKkOSXf6+cD7G7iWUfvm3GQc=,tag:iWlohbIDm7o75JfWSfV5oA==,type:str] gateway: env: - - name: ENC[AES256_GCM,data:gZSjzeTSIH9mZg==,iv:NIwpYC8X5YSykeLovTZAGamYk1pCcTZ9zORjElMArk0=,tag:BJUww7hxoHIxmgchf2kz5Q==,type:str] - value: ENC[AES256_GCM,data:+iV1mQE+iw5RCOSCfojNIG5RMAH9x+0PBAwNYpEKFvTmUfDvjCEi1ykQW6o=,iv:0mUP/BPYpHB2bvKfQZ15TeaICe3uJdu8EobL8M6DHtg=,tag:9e1QjigvK+cF6eSMZJ3s3A==,type:str] + - name: ENC[AES256_GCM,data:g6BN4NxwzoB1fQ==,iv:mo8U8lIXN/cexFjE3SGIkapo9cuiUcvnRbazA/hL74U=,tag:uOJz3abAnaGtEUQ+xFoXiA==,type:str] + value: ENC[AES256_GCM,data:f+a/g0gIKXQFdzUa1NFcWHHc0oLvpxUZHl2svGMYEmKAwGFk1FFreJ8/Yik=,iv:jeffKVDeru2BR4m2uTTgZw2NndMgtTb61uzfWMa8H6o=,tag:X0UIUpYg9cN8Z0xXi2KZkQ==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNmNZdFJJN2JkaGFVM1lt - K2x5dFNaV3czUVduRlY4aHY2U2JlZVZFbWtvCnlPdHhWcElKS2FQbFJ5cnpMQ0tL - VjhiRjlRK002OEt3UUtSanppZVJCcUEKLS0tIHJVUDF1SU5TZC9qUmlZRGdMZjht - M1I2STNoKzdteXNFTUdVczdsSFY5bUkKblFkFp2073nMZgkpqug9wrrbso14HZAk - ZLzSwQBYtcV1103ri/pGFAyWWqIboAZ3Ki4BIprymRvlX+2XVtqCog== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnR2wvVGR3Q3dUUlVGQ3lX + WHNnaUFvdmozTDcwZUpYamhkR0NnTFFrQmpjCmFVY0VnV05IUnQ0YmxFUjlVSFVS + cE5KaEdMWXJRUW1GWlNXVzYzTTNvYVkKLS0tIGRFUXdNSW5RbnAzeUp5cVZ1ZWJ2 + azh3ZzhKb2VYUWZSdHo5NlRldGJQTGsK98hrnUGXdDQKol1duzHyYsd9PLL2eAEC + 4y+AUB5bVjYI3PEt3giBBUYFwIo478FWQ1w8iSNIUwWlhlqEB2hKNg== -----END AGE ENCRYPTED FILE----- recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm - lastmodified: "2026-05-25T03:07:14Z" - mac: ENC[AES256_GCM,data:iHn7oK3+kUnEZ0f2gYbZF+VloKSYNBcDpFTCP7SEGoXo7RiaGIg3EEj0OfOjlNJq7nLhZorRCq9ewjyhh5K3guW0Ys2Mh8hPrD1FEmw2kkiDzuzQA0qhx6DT7uXTyrhn1QyhvLwWrqMP+8jYjayLfNqMvYWpg5KYFi76MPVi/sE=,iv:SCQ3s/QNL+QILV1517vm3944rvCJU/9I92NGXxGO3tQ=,tag:I8fJVVCT46ygmyuus23+yw==,type:str] + lastmodified: "2026-05-25T08:38:49Z" + mac: ENC[AES256_GCM,data:xW7scU3rX48jjA2eHvC7RDj3Fn5P3mPIqwLGpvHf0bsjbQo+huMPfTIafJRF4owum+SiRrASaDkABho6wxLkwiwQ/KlwPHW8QIXCpdgE/OJduQV9MIjAs7uCX+BSVZfVAHhgdR4ftAHhBNF6KrbZ3O9Elw7iZstsvsOoDRy8QPM=,iv:db0m7SeybD9igEPnEsm5M7evLnotq+L2owuSYADQs74=,tag:LyTBdJNskE4CsY//wgHUTA==,type:str] unencrypted_suffix: _unencrypted version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index bf53963fc..e645aa005 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -157,6 +157,16 @@ backend-python: value: *DATAMATE_JWT_ENABLE - name: milvus_uri value: "http://milvus:19530" + - name: LABEL_STUDIO_USER_TOKEN + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_USER_TOKEN + - name: LABEL_STUDIO_PASSWORD + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_PASSWORD volumes: - *datasetVolume - *flowVolume From 638931b5024b61bb06a66ae39550d42e8f93d970 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:48:33 +0800 Subject: [PATCH 08/18] fix: add LABEL_STUDIO_USERNAME env var for auto-login to Label Studio The loginAnnotationUsingGet flow requires label_studio_username to auto-login to Label Studio. Without it, clicking "edit" on an annotation task redirects to Label Studio login page instead of auto-authenticating. --- deployment/docker/datamate/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index af2b18123..bdbfc4f7f 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -36,6 +36,7 @@ services: - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} + - LABEL_STUDIO_USERNAME=${LABEL_STUDIO_USERNAME:-admin@demo.com} - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} volumes: From 659758dffb993184f93605db636de3e504bd8582 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:55:19 +0800 Subject: [PATCH 09/18] docs: add LABEL_STUDIO_USERNAME to .env.example --- deployment/docker/datamate/.env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example index 90fa6588e..4a9ad4e84 100644 --- a/deployment/docker/datamate/.env.example +++ b/deployment/docker/datamate/.env.example @@ -18,6 +18,7 @@ MINIO_ACCESS_KEY=your-minio-access-key MINIO_SECRET_KEY=your-minio-secret-key # Label Studio +LABEL_STUDIO_USERNAME=admin@demo.com LABEL_STUDIO_PASSWORD=your-labelstudio-password LABEL_STUDIO_USER_TOKEN=your-labelstudio-token LABEL_STUDIO_HOST= From cdaa8a3ada035bbf0bd6cae7782837f328b37f5c Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 17:15:59 +0800 Subject: [PATCH 10/18] fix: remove duplicate case blocks in scripts/secrets.sh causing syntax error --- scripts/secrets.sh | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/scripts/secrets.sh b/scripts/secrets.sh index a4186e87e..946d982dd 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -98,36 +98,4 @@ case "$1" in echo "Docker users: skip this script, use 'cp .env.example .env' instead." exit 1 ;; -esac - echo "Deployed $CHART to $NAMESPACE" - ;; - helm-upgrade) - if [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 helm-upgrade [--secrets-file ]" - exit 1 - fi - CHART="$2" - NAMESPACE="$3" - SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" - - TMP_FILE=$(mktemp) - sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" - helm upgrade "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" - rm "$TMP_FILE" - echo "Upgraded $CHART in $NAMESPACE" - ;; - *) - echo "DataMate SOPS Secret Management" - echo "" - echo "Usage:" - echo " $0 encrypt - Encrypt a secrets file" - echo " $0 decrypt - Decrypt a secrets file" - echo " $0 view - View decrypted secrets (no modification)" - echo " $0 helm-install - Install chart with decrypted secrets" - echo " $0 helm-upgrade - Upgrade chart with decrypted secrets" - echo "" - echo "Environment variables:" - echo " SOPS_KEY_FILE - Path to age key file (default: .sops-keys/key.txt)" - echo " SOPS_CONFIG - Path to .sops.yaml (default: .sops.yaml)" - ;; esac \ No newline at end of file From 7fc4f8aa7f941614fff903585caff7f1e3fa2335 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Tue, 26 May 2026 15:41:52 +0800 Subject: [PATCH 11/18] fix: add LABEL_STUDIO_USERNAME to Helm backend-python env The loginAnnotationUsingGet API requires label_studio_username to auto-login to Label Studio. Docker got this in 638931b, but the Helm values.yaml was missing it, causing K8s deployments to show Label Studio login prompt. --- deployment/helm/datamate/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index e645aa005..978640274 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -162,6 +162,8 @@ backend-python: secretKeyRef: name: datamate-conf key: LABEL_STUDIO_USER_TOKEN + - name: LABEL_STUDIO_USERNAME + value: "admin@demo.com" - name: LABEL_STUDIO_PASSWORD valueFrom: secretKeyRef: From ffe8f951328d6fa6c91e04b68e91ea0966af5d2b Mon Sep 17 00:00:00 2001 From: MoeexT Date: Tue, 26 May 2026 17:30:04 +0800 Subject: [PATCH 12/18] feat: migrate from SOPS to Sealed Secrets for K8s secret management Replace SOPS + Age encryption with Bitnami Sealed Secrets. No key distribution needed - secrets are encrypted with cluster public key, decrypted automatically by the in-cluster controller. Changes: - Add 3 SealedSecret YAMLs (datamate, label-studio, milvus) under deployment/kubernetes/sealed-secrets/ (safe to commit to Git) - Update Makefile: apply SealedSecrets before helm install - Helm charts: support existingSecret for label-studio, secrets.create flag to skip Helm-managed Secret creation - Remove SOPS artifacts: .sops.yaml, scripts/secrets.sh, secrets.yaml files User workflow: make install INSTALLER=k8s No tools needed on user machine. Controller decrypts automatically. --- .gitignore | 6 +- .sops.yaml | 3 - Makefile | 9 +- deployment/docker/datamate/.env.example | 4 +- .../charts/public/templates/secret.yaml | 4 +- .../helm/datamate/charts/public/values.yaml | 5 +- deployment/helm/datamate/secrets.yaml | 29 ----- deployment/helm/datamate/values.yaml | 6 +- deployment/helm/label-studio/secrets.yaml | 19 ---- .../label-studio/templates/deployment.yaml | 21 ++++ deployment/helm/label-studio/values.yaml | 4 + deployment/helm/milvus/secrets.yaml | 18 ---- .../kubernetes/sealed-secrets/datamate.yaml | 27 +++++ .../sealed-secrets/label-studio.yaml | 23 ++++ .../kubernetes/sealed-secrets/milvus.yaml | 22 ++++ scripts/secrets.sh | 101 ------------------ 16 files changed, 118 insertions(+), 183 deletions(-) delete mode 100644 .sops.yaml delete mode 100644 deployment/helm/datamate/secrets.yaml delete mode 100644 deployment/helm/label-studio/secrets.yaml delete mode 100644 deployment/helm/milvus/secrets.yaml create mode 100644 deployment/kubernetes/sealed-secrets/datamate.yaml create mode 100644 deployment/kubernetes/sealed-secrets/label-studio.yaml create mode 100644 deployment/kubernetes/sealed-secrets/milvus.yaml delete mode 100755 scripts/secrets.sh diff --git a/.gitignore b/.gitignore index d78ab5ce5..decd54399 100644 --- a/.gitignore +++ b/.gitignore @@ -192,10 +192,6 @@ Thumbs.db **/volumes/ **/rag_storage/ -# SOPS encryption keys -.sops-keys/ - -# Environment files - ignore local .env, but allow templates and encrypted files +# Environment files - ignore local .env, but allow templates .env !.env.example -!.env.enc diff --git a/.sops.yaml b/.sops.yaml deleted file mode 100644 index efb2f00f7..000000000 --- a/.sops.yaml +++ /dev/null @@ -1,3 +0,0 @@ -creation_rules: - - path_regex: deployment/helm/.*\.yaml$ - age: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm diff --git a/Makefile b/Makefile index 379654a92..964ac6fd7 100644 --- a/Makefile +++ b/Makefile @@ -333,19 +333,22 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m exit 1; \ fi @if [ "$*" = "label-studio" ]; then \ - bash scripts/secrets.sh helm-upgrade label-studio $(NAMESPACE); \ + kubectl apply -f deployment/kubernetes/sealed-secrets/label-studio.yaml; \ + helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install; \ elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \ elif [ "$*" = "mineru-310P" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \ elif [ "$*" = "datamate" ]; then \ - bash scripts/secrets.sh helm-upgrade datamate $(NAMESPACE) --set global.image.repository=$(REGISTRY); \ + kubectl apply -f deployment/kubernetes/sealed-secrets/datamate.yaml; \ + helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) --set public.secrets.create=false; \ elif [ "$*" = "deer-flow" ]; then \ cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \ cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \ helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "milvus" ]; then \ - bash scripts/secrets.sh helm-upgrade milvus $(NAMESPACE); \ + kubectl apply -f deployment/kubernetes/sealed-secrets/milvus.yaml; \ + helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install; \ elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \ kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \ fi diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example index 4a9ad4e84..ed68e8f5c 100644 --- a/deployment/docker/datamate/.env.example +++ b/deployment/docker/datamate/.env.example @@ -3,7 +3,7 @@ # cp .env.example .env # IMPORTANT: Never commit .env to git! It is already in .gitignore. # -# For K8s/Helm deployment: secrets are managed via SOPS-encrypted secrets.yaml files. +# For K8s/Helm deployment: secrets are managed via Sealed Secrets. # For Docker deployment: use this .env file (gitignored, local only). # Database @@ -27,4 +27,4 @@ LABEL_STUDIO_HOST= CERT_PASS= # Optional: Domain for HTTPS -DOMAIN= \ No newline at end of file +DOMAIN= diff --git a/deployment/helm/datamate/charts/public/templates/secret.yaml b/deployment/helm/datamate/charts/public/templates/secret.yaml index 9b6553cac..a3acb51c2 100644 --- a/deployment/helm/datamate/charts/public/templates/secret.yaml +++ b/deployment/helm/datamate/charts/public/templates/secret.yaml @@ -1,3 +1,4 @@ +{{- if not (eq (.Values.secrets.create | toString) "false") }} apiVersion: v1 kind: Secret metadata: @@ -5,4 +6,5 @@ metadata: data: {{- range $key, $val := .Values.secrets.data }} {{ $key }}: {{ $val | toString | b64enc | quote }} - {{- end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/deployment/helm/datamate/charts/public/values.yaml b/deployment/helm/datamate/charts/public/values.yaml index f134e4325..de08166f5 100644 --- a/deployment/helm/datamate/charts/public/values.yaml +++ b/deployment/helm/datamate/charts/public/values.yaml @@ -22,4 +22,7 @@ persistentVolumeClaim: flow: 1Gi log: 1Gi database: 1Gi - operator: 1Gi \ No newline at end of file + operator: 1Gi + +secrets: + create: true # Set to false when using Sealed Secrets \ No newline at end of file diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml deleted file mode 100644 index 25defa1a6..000000000 --- a/deployment/helm/datamate/secrets.yaml +++ /dev/null @@ -1,29 +0,0 @@ -public: - secrets: - data: - DB_PASSWORD: ENC[AES256_GCM,data:F/GtCDyYi4xH8FJWGxojsCdNEDdKQaVIaFBuC5dk6ilLdfGJtk0JDoPRQ14=,iv:mhenVjDgPvX57OaSMnUZGvFJrWjwtqIPJ66PIHqQcKM=,tag:kxxS/w0FUt6N9kHNvSVMOg==,type:str] - JWT_SECRET: ENC[AES256_GCM,data:Pu2cojeOgNSMmxbqevkI+c8Az3QS/A6DgDKv2NeGtYON5YrZFBqxxfV+y2M=,iv:sjVi42/TVmKGzbu9ZxAgU3CGBlHgJ94CEpzAd8eZX0k=,tag:3v/cJNPUZpyFJjU7Wdz+Ow==,type:str] - CERT_PASS: "" - DOMAIN: "" - HOME_PAGE_URL: "" - LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:XBjxusCTd5xe2xnx7D8h+AH8rpZ6ZUCyYUws4Zxffr0=,iv:yWwIMpEJzodAhK78SOZXTWZi9GnG387oKeOK+ZORjis=,tag:l6U8sQoUzvHdFit72WI4JQ==,type:str] - LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:YR9/z/UthUuwJTaZMqbnwdVNWW60pCrzmUDAOojWo6E=,iv:ojZuiZ5G7l5DF6P+UQaCKkOSXf6+cD7G7iWUfvm3GQc=,tag:iWlohbIDm7o75JfWSfV5oA==,type:str] -gateway: - env: - - name: ENC[AES256_GCM,data:g6BN4NxwzoB1fQ==,iv:mo8U8lIXN/cexFjE3SGIkapo9cuiUcvnRbazA/hL74U=,tag:uOJz3abAnaGtEUQ+xFoXiA==,type:str] - value: ENC[AES256_GCM,data:f+a/g0gIKXQFdzUa1NFcWHHc0oLvpxUZHl2svGMYEmKAwGFk1FFreJ8/Yik=,iv:jeffKVDeru2BR4m2uTTgZw2NndMgtTb61uzfWMa8H6o=,tag:X0UIUpYg9cN8Z0xXi2KZkQ==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnR2wvVGR3Q3dUUlVGQ3lX - WHNnaUFvdmozTDcwZUpYamhkR0NnTFFrQmpjCmFVY0VnV05IUnQ0YmxFUjlVSFVS - cE5KaEdMWXJRUW1GWlNXVzYzTTNvYVkKLS0tIGRFUXdNSW5RbnAzeUp5cVZ1ZWJ2 - azh3ZzhKb2VYUWZSdHo5NlRldGJQTGsK98hrnUGXdDQKol1duzHyYsd9PLL2eAEC - 4y+AUB5bVjYI3PEt3giBBUYFwIo478FWQ1w8iSNIUwWlhlqEB2hKNg== - -----END AGE ENCRYPTED FILE----- - recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm - lastmodified: "2026-05-25T08:38:49Z" - mac: ENC[AES256_GCM,data:xW7scU3rX48jjA2eHvC7RDj3Fn5P3mPIqwLGpvHf0bsjbQo+huMPfTIafJRF4owum+SiRrASaDkABho6wxLkwiwQ/KlwPHW8QIXCpdgE/OJduQV9MIjAs7uCX+BSVZfVAHhgdR4ftAHhBNF6KrbZ3O9Elw7iZstsvsOoDRy8QPM=,iv:db0m7SeybD9igEPnEsm5M7evLnotq+L2owuSYADQs74=,tag:LyTBdJNskE4CsY//wgHUTA==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 978640274..b284da45d 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -38,6 +38,7 @@ public: database: 1Gi operator: 1Gi secrets: + create: false # Managed by SealedSecret (deployment/kubernetes/sealed-secrets/datamate.yaml) data: DB_PASSWORD: "" # Set via secrets.yaml or --set CERT_PASS: "" # Set via secrets.yaml for encrypted SSL keys @@ -187,7 +188,10 @@ backend-python: gateway: env: - name: JWT_SECRET - value: "" # Set via secrets.yaml or --set for production + valueFrom: + secretKeyRef: + name: datamate-conf + key: JWT_SECRET - name: datamate.jwt.enable value: *DATAMATE_JWT_ENABLE - name: OMS_AUTH_ENABLED diff --git a/deployment/helm/label-studio/secrets.yaml b/deployment/helm/label-studio/secrets.yaml deleted file mode 100644 index da2071797..000000000 --- a/deployment/helm/label-studio/secrets.yaml +++ /dev/null @@ -1,19 +0,0 @@ -env: - POSTGRE_PASSWORD: ENC[AES256_GCM,data:ROFSuT5BkJxGhusq2mGwxuBulcTNQeaVELxP6KimajWkGYBBys9oMJyPXHE=,iv:qqPQMjMUKpBJ76RlscGQ/6ff31e4iRDM0+IJ/q/TLYs=,tag:b5gCVSN/rn7aahqsoowayw==,type:str] - LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:niFPf5mPSt4J2CpIhYaS+gUkj3XnwvkUy7VCWPykGd0=,iv:vm1OC2vRdBgHqjXPL/4th9EiSzZg9nGEDOn33axY1y4=,tag:wKGLsepNKasU4ccXv/OWfw==,type:str] - LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:dmdU+7/Yd04jMCX5gaj4BpZhudCS2Bwe3FjqW40fX7g=,iv:GZS6oGS8oSBKYMjxBVuVmjEW0Ue/j9dfh50VBC9GvnI=,tag:GBhKPnPNyuaUaiXNfr4JEg==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDUGVacVpQSjFwRFQvWllZ - bWlLZitoQ2JLS0pHS2ZQcW91T3RaRVBIU2t3ClVBZkN4N2FmRENPYVo0dU9RWnlD - MVZXNUhjSWFSZWo0Yjg0RTJSSFpGU3MKLS0tIGcvcllDK0FtREphOU9YckhvQlJQ - NE8vVXdqajFHMnBnVHhQSGF3dEdkWnMKTy7oYsfbWxr3U6X1gyS2kKJLxIjy0Zr0 - 1BfP9FAx6CvrveKqXGWwHk8+8034sG0fKO1H1lrBAgzW79eDOU4i+Q== - -----END AGE ENCRYPTED FILE----- - recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm - lastmodified: "2026-05-22T08:19:39Z" - mac: ENC[AES256_GCM,data:Gni4TWxwzTmjf2+/t64V/SRBttJFuq1qMQ7cZzm2tJu/3pwbFOgcJwDdkoYSRKZrpyngj5EskbWzIgbmv0I9Jts26m9dN4pMr45qxQ1ba2G8eYgx0s2klh7iEFoMVTJmnYCbnl0PuqSWmcHbCxxZdeyo3ja647RlfLarT8QZwvU=,iv:nrDwjR0v96jKSJfQ812fpfrmI6vQEcCzttTVX8mLftU=,tag:AP+JhPFE7CZY5aZX5M8O7w==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.1 diff --git a/deployment/helm/label-studio/templates/deployment.yaml b/deployment/helm/label-studio/templates/deployment.yaml index 9935f6e15..f2a16f9c3 100644 --- a/deployment/helm/label-studio/templates/deployment.yaml +++ b/deployment/helm/label-studio/templates/deployment.yaml @@ -36,7 +36,14 @@ spec: - name: POSTGRE_USER value: {{ .Values.env.POSTGRE_USER | quote }} - name: POSTGRE_PASSWORD + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: POSTGRE_PASSWORD + {{- else }} value: {{ .Values.env.POSTGRE_PASSWORD | quote }} + {{- end }} - name: POSTGRE_PORT value: "5432" - name: POSTGRE_HOST @@ -52,11 +59,25 @@ spec: - name: LABEL_STUDIO_USERNAME value: {{ .Values.env.LABEL_STUDIO_USERNAME | quote }} - name: LABEL_STUDIO_PASSWORD + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: LABEL_STUDIO_PASSWORD + {{- else }} value: {{ .Values.env.LABEL_STUDIO_PASSWORD | quote }} + {{- end }} - name: LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN value: {{ .Values.env.LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN | quote }} - name: LABEL_STUDIO_USER_TOKEN + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: LABEL_STUDIO_USER_TOKEN + {{- else }} value: {{ .Values.env.LABEL_STUDIO_USER_TOKEN | quote }} + {{- end }} - name: LOG_LEVEL value: {{ .Values.env.LOG_LEVEL | quote }} ports: diff --git a/deployment/helm/label-studio/values.yaml b/deployment/helm/label-studio/values.yaml index 456c6f242..b06da7f40 100644 --- a/deployment/helm/label-studio/values.yaml +++ b/deployment/helm/label-studio/values.yaml @@ -54,6 +54,10 @@ env: LABEL_STUDIO_USER_TOKEN: "" # Set via secrets.yaml or --set LOG_LEVEL: "DEBUG" +# Use an existing K8s Secret instead of creating one from values. +# Set to "label-studio-env" to use the SealedSecret-managed secret. +existingSecret: "label-studio-env" + persistence: enabled: true existingClaim: "" # if you already have PVC that maps to external datamate-dataset-volume diff --git a/deployment/helm/milvus/secrets.yaml b/deployment/helm/milvus/secrets.yaml deleted file mode 100644 index 014696cff..000000000 --- a/deployment/helm/milvus/secrets.yaml +++ /dev/null @@ -1,18 +0,0 @@ -minio: - accessKey: ENC[AES256_GCM,data:57/fzPABN+5+c1tmf9lvR9O5EflmlWPQfgvxTnwHhIo=,iv:Fkyrk5f7md7QQD2fv/XPXLLTxLDUq2mmK2BOi4QcZyI=,tag:AykpxHYoQSTjs840FodsDg==,type:str] - secretKey: ENC[AES256_GCM,data:EgFQZaIInv6Ptn9a+PiXVydwhi8Wh3dO9QkWY9qD86M=,iv:+WWoLUoMJc6cc9a+aLR97O/xvv7f6yfAgFjuKtt62h8=,tag:O6r+/1KhgnVCdxRmzdS8pQ==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuMlNnRDZJdGRXTklkK0tp - aC9KQ0t2VTExL2tOS3BvZzNRY3o5TnFYa0ZZCkNqY3U1K3g5M2NERDFDR1g3M1hJ - VGcrNDR4Q0tCOWptTzF0MG1QN0pXWFkKLS0tIGhMMUEweUxleWV1Y1RTVmdDQ2k0 - cUZOVC94Qi8xQmNNMFBwWEM3MnhkQ0EK4bTM7Rw3tlaMFInXv1MKlu65iPBxY8Zw - G5h5QK4AOonWoxwrz3H5jFBAZwBsy156wDgBsxvZJB2oCa4wEfC4Wg== - -----END AGE ENCRYPTED FILE----- - recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm - lastmodified: "2026-05-22T08:19:40Z" - mac: ENC[AES256_GCM,data:N5n9tUIujF8BpI+N4PMzBApaTmpO+Ji/ZTf7Dr/faM9+nGnrL/JcOwAjKwXXGnDbcFi4DFyvo7yx4u68Mx6c4EtIQYs4o9TymGiQAgAUR8STnjBB3pbGjlvG9MFYwKCVuXBclhKpCLDklI/cjkcJjJq4ae2VnHer0wMlSzRZj8I=,iv:L2vlzY9aRAUGBZhgNOl1t3QDZhP8Inffj0TkGXnGbug=,tag:iB3ZKbYAk0D4bc+05q8m6w==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.1 diff --git a/deployment/kubernetes/sealed-secrets/datamate.yaml b/deployment/kubernetes/sealed-secrets/datamate.yaml new file mode 100644 index 000000000..c8c57696c --- /dev/null +++ b/deployment/kubernetes/sealed-secrets/datamate.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: datamate-conf + namespace: datamate +spec: + encryptedData: + CERT_PASS: AgAFa2NSMHymGMXv63PKqnUifZ70ocLLYd1MhIympF/NzJakL44sxMQlLLWWSJByouOgTu8ROM91rKYUkb6Y6kdV/pj0iQr1y/NLb6lS3y4Lvjvt4S/m7JCOcOivXJv5hzPU7uDfn8y8yw0qsjyXwa+lWMdeUmwnSEYwLcPexjvTcAs7xiV0OWW1WHkAZlx8A9UJqk1Yeaj/4fqRdLMEKBHYiGH8iW5nm3r5/5Uv3sdTU6n2EQgjFT2zpCYVbDvSVaFjRQ/yZ4jCuNNlw2q7EpoLzT8Ak+6BaoVqEVLK885fo8Wu8B88Pnx3Tse9v/Dld9qZaUVkXxYK/wpIjFPZCGC2q5y9oPl0fbK9nMbml9lkOgmGEsJm9CVTpllRGgT1O4hwE/Ij94dgLXErMlUukrU6/RiMwhis7bBrC1qMj+aEpsRl4vscbvbTBGUhW4rJrwMZscnqw6bt7sHnInrOKnO3vXJIJPDyoR9bHWj83NX1A7WkqQe69xYUK/+Zf4vRmNFHmU6KV+rRR80pWChJGlybmp+fZ2LGoI5PFHky/cmPz2gmGZJdiOC5Y2mi+vvzPJASra7n2nWxrcZ10H2YJI39/J/H6tANf6/Bpj97h9VzGpDFVxE19jKxkrmnsGUcVfB8+wa985kIPKPRzGktQ4pROC0Feyta9lbi75yW7ZaTSptVmzknJ9FrK8xx9yT6HOc= + DB_PASSWORD: AgCYbAis3YHfbcoI/uazKsyc7Q25A77gLrJAfx6F0Y9LzDoIlE5xczNHV8AhUtmT5lCCnzMxBRd6bCpVQho5xvpv8pfn2vjlbd/JbI0Up3vj6Yg3iw4lOmqxLs7DVpnn/Lo/GeoBXWZOerCRMiY/geVoSaQfybgIsmqFahHYaBciRHGxpZ0rhWqnm6U+RT5Oun73U47Mb8HMkEBIeMZW6IXGYoj7VetbouGZrSsu/uDlvO0P1BLFNVDiMZWW9B9gRnIdINxJUUzmvzvzyYBjOrsQlNZu+LycE57PspUSW4/TTHlZyxwrsdXSkWH/J66m04vHEzAaxF7Yg3zYCdcNBpiI8FLLDS64mROgj/dv8PY4qAIrEvb7e3qt07t/ArXEa1R7kkoBBjp/VvIy2wF905oQ99AgPEasnHYNp8VB1lym3YWWeuifsi6aaWhKyqq/22EhMRXKiC3brfp7KVnlnwX+98AHLl8Q04C90LwuDyKW+OBt6ATQktJuHqh+5uB17rvSu2KtWp2ugh1oqH8VaGydcd7lhKlHmi9a3b18uS3n6o5O8bR3mgmOWjlck4lDQI6cmnRVfx9J92jpdmZW5MeEtnlldYaek8am+zZTAusaagTp5eE8+p7238dGwr95LdWkkJbiJGBBvxuNdI0zjAFtGirueNe0ZHLvAKDh/Pw/Db8gg8DkyIVgJNDx4Q1F6eG2TPhwRu6vq/4RMvYFH5f1K58dCzEtkoQJsgJyJ4Kd9DBJzblOQ7fFLSx7Ag== + DOMAIN: AgCjEuH37xiKRpOkIrnAnk4Ui5A7B0M0R5BOhpvrbbu+QU/dqW7ysZzyNppDpABMB9n3H8jDsPYoKgFUVkD6p8FF3IpHmLlaJvO6FeIC86rJPVZTpWjPTub8LOCP5sszr7t4F1zyIo8uckyvqW2LxlNbN6cOCNzVutdHEcsu4dxWI/MzZGocT2CrWh3PDORdN6xrJ8qfpZEyFHiVwyxcbh3XM5oHqqe2o2OIlejXtrB3fQo1oUOH99CZTsZ+JrhUF5KOb4jDgq6aFLrJzkSWF+d/MXgrzqLDsqUuWxOp4CkbsIJJfO+aF2ULH9JAS1ZD/mnYUr1Lmix1wJORx5VXfT5L+fx+y8VDP2H8BeoLNqmmCyyV41GcvAyG5fN641Ck63MMehZkAJo72Bu4eRk8Mkk9JvDejAUTR5a/J1qKOtNVeFmM4Fq9y3WvW5X9e/3mHs7tCWTxGzJ70bV7zTngnu+G5SUoOjtvuBMNOfes9MAKj+hQ5YBaKiqZ5cJANgXX1YnmY5fvPSi9tyAxFspJgGSzHwBXLsTbeDRtTW86qmvY8anF++iizeuZ4DEqCLl8D0G3lj51elTTw7/1dDa8s1Xvy8EY6SwZAiuHpHkYf5barbrZLKSUldzd2AOPfxDgnbmussTV7o+/bjbCFKhGnWW2EEvLx4yKHjAJa8A/9uDsX8Zv1O+idsV7VXuFrsV8UGc= + HOME_PAGE_URL: AgC1A5P37JqlZXKdwNJT/0d3HiSvlCITF7mc8bXl4LslRToKloICDqO8PHuC3gvIxPtbrWr8YRCBPg9gZsb4s8234cIlYl3Ntxr+YQXbtLXFXyu4/BIJoz4qOJhyq5zo/0QkN+2mztol/1ZcLSGtj6mV3JY0yvZCqBlPIR3M78IW0OHPu3mF4zo4S+aumUE7QKZHsb83phUVvIJ1o+XsR8xrwAQBFtRp6GCgowAppeTrG9OTgmsjkHmb5Hz+ri2AbNkzx8SXQUCbWaosYHTwY/OYolFN6zKCql6zxLRt3ryjhGizIcSURumPcGzzuhotunXBPjhp4bI+lOAotwOW19vaNk/g4atQz16V5ceXg7Rk/+cZRkKJ/rKgpI2MN+1sP17U/DtudBqvsPk0da3QNqeOJ1VN6mFKs6R/DVxh4YYK1a/kUD5GdhK4jpFpvguEsBzx1swpB8Wb8AlIze7xAksk1BFL9+UzCpeDPVQeH+QyZA+cfZ4PIv5MC31A6clMz2o2cthNufggmQxvy2ndhjY7ZbfXyOA78eoPYY5IIx5pZA7zR2pxgJb79FS9RFcarjaF51fBCZy/EZie/C+JNI7fwLtai7BexqN6Nefdn9VVCr+zN0dCBZ3Rw402WiihaXiaPAxfsKrMYneqWs4F0hyRaL3AmEA8jRfNEAGrcUljMAie03I9+mtA6RAxW6d5ZA8= + JWT_SECRET: AgBztOHSCJostJW7ZxXRUhf2I1YzJubpX0dJmvHielNcuezyBg2R3JIHxZeZEiLtsDG4arE3ZFbPo6c16e+i15e0ElkScAJJbiCFnVgPOqbVOHcWmgdCDV88OZo4MSXpJBxuUmGMtRYdVRgp26p70TzmPEi+S4U5YZxp+Jb7jaCtxg//o5iikIobNzigX0eiwSBgXrBKA5KobVFMmKRYZskdM4mYvZXe30mcyABxIDiGjwXPdQWT7d0pNaAMmYToGDngVFAzHfkS8/fXzQOnP+Vo1/rKskZCDw401EjyYu83UJulbwNwY9HejJyP7xQJ9ZQOsOklaNjtKRN4uiLciBUQLGvaa3hPpyRCbm2Xv1Jjq+enxeHSZdWmQ6DcYpy8YPwVhiyIEQtkONtKaZtPBj7U9vZARK/ET5+iCT/CXmU3fElKH4n0sFMerDUqbEN5F1V1PIljHDpPobbwirvkNIBPc2sMX1Vt+KPPv6CWf6HWCleayMv8lXU9+son5CnBzwjVj8stxCqGje7l5b7g98qidiikBlocc/N+vCBzGf5plKVrB66lK07tO3yccKXmNos2Y6S3aSCwRgXhAkeeELreV2pK8DVI58KKiK2ClARdxE3Ba1mRL1hWDe3aT1U/FjStBYb+/u7O2rh+rGMaVMz3g/xDRDXJtHRW9PKz+Dlw9uGNUNtkL/BVg0LvbrBECOsGeIkb65SZlUm0i+8f1ZC69Fa0KM9dew7/2aVXmuKXhgPKh2XxmndC2QNmYQ== + LABEL_STUDIO_PASSWORD: AgBt5v87JYw+EpPwquwglWV81OK4lLg3jyZ75OK0k6GmedasPz+DnZb+OfWxsxdvpjEUSyKlEmpDbsH7zkrmCRn5VCasuJ6oa1fuMjJU/8DQ44piPvg1OagVrenVLy5cyzzcnLdKGVEDg69aUrZTfz+dSnyh9kL3f98g+Wt7EIhFcB7cxrVv7KXrc+hu8j0PnEHo/X/B1lA9KNi0yCj4CBofy9vrcJVFnGTeNh3hUEOGDORppdJUl1dDFx/qJeg94e+GClQUmC1QJmhBosxfnDbQnw/5lB9EP9iqjxMbswLsMhpEGQZqPbrFakvz0lb7N2Skq4RxD4XnkZTJ29RDsOl4TCAKL8C2a8OxP7P+yVflrTi5aBMLXhvrmEW1IwQXlFGP8th5AWkB+KM/gCfL1Iw34H8uklFPqAXexqxSsctwlnCIKBjG8lvbKfYEjOQhKHPZij1WYv8UZarU6rTvJ9OnLKW0kTTI/1MAFbfARPaKIcBzP7Ahexv7i7fo6YjeCz1n6N8UXxlszKZbQV82e/Zq40AmNJIG2vnBaNfskzYPQCpLr5yjwNO99IQZplw5i/X4pFWT9NfJTb7lfgb4PqUbsoYZgfJ5JBuA6b17/YYQy4pMyCorXAVmFDhKwSKT6WYgpw9N0TSErc3eOnusAolTuyzD5ns5h3NnomA3lD4hcz7L4CuR6lXekjfWk4vZNQu55Km1fTvB5DTB8gicRamxq5JAEQj8EYRK6Po8sXrjgw== + LABEL_STUDIO_USER_TOKEN: AgAiYeDEoGB1lSPKTb9n9dHzHPfH6sbLuNzAIX3ZwSRzjQjnkcEw/oSausQ4y0Z4kBv/983cuXwOF2xbOejPZ6LW9JSzzpS0dqWShfVcEjFwwPX7N7Nihqi9KdfXr4n2KoMCGZPphokxQwKt034VbFjQVdmIPQo5zRRYNlqr7i/etjlpsIs+mc9LO8s6J+zlRt+z4mxDg2m8y7MvNm0iW0nF0Ef7abGk+SctzXK+Zoec6uhvOv6FqEwsRAqjHsdcWySpH3Q53nMfM05LrkF5W5eca5t7rcGQTK0bmFr0x7pdt70mPTac8z0B2v7IIsb16v59xSweWlyKUPKCY2WGIDZuPjL38Ilt6gKYw/lDIFG2/gMa0ai1rIer7+nd7OzMP/OP6fwcY8l1GFOSeV1f16ZhV8n1vaJ0qR9lzrDLMQHYHHP7a/wFDz1S66sNVSNF4Otd6O9e0svBq+mYsQUw/ahruxZ/kl7hQK6qCP86E8z6H1MIRmEDR2zq2CV6SxLtEa8RwxWqcT57Zew8rqn62zcNxilDyFA6xw/ZrzTmNOy+QWP16ema9mC9Ja3J8LHHChN8+bzFswdPCVol4tW97cZp+FsXceadoyC6LE2f/i7cwVUj4g+O7yiyHfpohDlvujnJsVf+ejMToNJ3JN8pl8mpv2YqCXmjJ5WdkbvElT/W4jvVJlUV+QS6fTK47n6xAwJCKcP3n77nmUj3M+/dJdGBBZaRb5A6H7ywgPMItirtuw== + template: + metadata: + annotations: + meta.helm.sh/release-name: datamate + meta.helm.sh/release-namespace: datamate + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: Helm + name: datamate-conf + namespace: datamate + type: Opaque diff --git a/deployment/kubernetes/sealed-secrets/label-studio.yaml b/deployment/kubernetes/sealed-secrets/label-studio.yaml new file mode 100644 index 000000000..9a2a1b10b --- /dev/null +++ b/deployment/kubernetes/sealed-secrets/label-studio.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: label-studio-env + namespace: datamate +spec: + encryptedData: + LABEL_STUDIO_PASSWORD: AgAdOZuj2s/Q4AsJbTU2DHw/uXuU0tIpiKD/o56jxhGkgIjH/IMCX/cB+12hq+xoz2yJCWGHuN0YV7iZhAwQ7Dwrn5xPTBUlQM2Ky5hBvvSs6lFld9R7iA5UChjwuCts4cQ8xKmUEIXAMv4rvL7g/iCpUhESlac8JB0KLofDm94+XRFyT30cQDdSyZKQ17Vi59TLSPuVUQf/R+8YtaChrX6hgM3OvCG2dw8h8jir7M1ppROyVUu+PmvblpITolklrm7HjPNaWybOjfXcZ3YYghAVEA/OJ4HdLF3aEmPZaw3tD1NQ7wlDtl3P3MCQ/l1eR657iuc1MzoH5/sNMSit5l322zg+wTXPjRNrDjcHtsCICH7ze+q0P1y0UZzey3vGXNyEFDtXcgy/Xg/GAC3OZbLS+Tin9ZQcssifkDChaHezTKHhqTPSFEj7yAQMK+jCXfchJcsV8SRwbqTSvNyr2vTKDgErM0XaQR3G6utY/VSZY0CEC3MhaWPiYEoUr2GJ6RRSLKARafjriDPvC4y/JxvCRGm+CqqHjwkJ31W2arJwJsGzl8ut9GLuH5M2/UmBIt0qLaJBNL/LQYPr6QINfUxiRKSRLMw72AxpFkHeM+EMucQv6NqE7AzqMhlygA+FNwXXQYS+1A+cv8o1TrbG4ounodUdprmUAdYjDH6zr1GRZ8lqjd9pteOceB82tUyzRs+6j1+5RlOsklLWd3uWgqP1s/N89kl8ho70W6rJToCDew== + LABEL_STUDIO_USER_TOKEN: AgB0EclTH3xnvW6zo/J25y40/Q8149QHvGre+eKd7hnun1kiVIYIHvAf4KXmgaGY7no6gTNGjleh4KnkSvoAo7W3T5wypS0MMZGpnzPbMpV8XNLA+sFcV2+Hiim+lxSzKKIEbNWnHr8urHnnBvRK2Wa54hzj4j4bcLGQqjptseusX2MB8BlaociKY/5LrQugNjMsLS9je4Wnz7ygSggU4PdUnpeTF/tRRyacOoCUSPb2GJmBthAZP5DvbKbvwDYoiKEIZRTjnF3V0FIRPe3weO3XRwl7nG3JgvVxSIMt48J5eKofNeSW15A5391USfMkNNnyN12I7+6fNtqUYWB+XuG/zB3kP0Uy8EsWYNQrJaJqHbQpetnCWda+azdvJWSE/TAEiy5pLR57gNhSdM5e/5vyXwgEpGxq3XI4HOSmnXq+bf2lQ36FyiPLsOTcJqOB0sqMT7tQthawSaTRy2yw2bJ2xxicmJBqzsp+xZ7EurtMSAeAThAHWuJ6w8KKvJbnCirQNKSwGjjVC6bq7KtEppu0gY6IL75VH94NmQi2sN9sBe9XPHUzfOJXilNl79LkDDKNfqPDbIhmwc5ECJBLsP9xhs6/0U8hSrmIUQ8AuvK/DYEsaydCvauEWFiNoYcKY4XoEW9kFvRH5Cd1r59P5dYiqZ5ivIHEWBYz6pBmN8uRs4p7aawO0SBy7KaVNM9PMgkP5D5VbLmNF6oTyruODEyeXmi8JKWd5u8R9kcYgY9lgA== + POSTGRE_PASSWORD: AgATLvaLNWeapPWyvmH8/li3GgpMACMfJd3dbO3FZ5MIfzj3K+KDGXyxrqiYuX1im4tFQgXdyIQzL59zlRLnMntCaiA5eNdhdeXMNEgef9hbLgWbnhH+Hl1//RYSd9CIueSpzsLC2Gd/c3PptIt2Gzvz6lazjc8Fz6gIETVEIHQvMTVsb7HDVHcICL2+/PLuEN6bXkwmiAcNJT3ipe9kqT38EagTwOmIErCrQVkg/KoECxdmdSYRiZqKTGwZndToosgiFr7FrAhv4lDsuyBReA73bFUA6B+94treSuRK2dp0JQBnL4GdOB1v04/BP+gCKWeIVzwOnpSrTVvVOyPQil/deRk1Fj54le2sU0yTPFh18U6NtTyRITH6GUvYRF8pn5QQQFsPD9gz/EtenjOmr2f587AsdDwHwuEo6tuiIPe1DeyxMJlSAUtlbwDH3hFvkpZBZpAqf3f8veEo/EW1fFA3QDRGATGiw4qIrILw/a5d66VkkvtTiztOHxD1mVG6JSWPrxmL13s4S7S5ZbZTIlR6KmGOya3NpThrOm3OWoO3aBquvzlLz5zK6MpsARJ7FSSzNXMrE8sbIRVEDAUJQWPcrrDTGDFIBx4rYHeSPw8Kx3hu3AfsAw0VKwRQsrWMxrXDlppyQTcGr0tgN+wH3X93YD1LXmt8gSH+/iP9APT1GmUV6JwCEyED/koEJ4LsCPlhr17S5ImckZKR0PXL83ubFVkPaQI6yDn5P+szU9boyVpKwA4NfnEI8jTvLw== + template: + metadata: + annotations: + meta.helm.sh/release-name: label-studio + meta.helm.sh/release-namespace: datamate + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: Helm + name: label-studio-env + namespace: datamate + type: Opaque diff --git a/deployment/kubernetes/sealed-secrets/milvus.yaml b/deployment/kubernetes/sealed-secrets/milvus.yaml new file mode 100644 index 000000000..37c47b1d8 --- /dev/null +++ b/deployment/kubernetes/sealed-secrets/milvus.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: milvus-minio-secret + namespace: datamate +spec: + encryptedData: + accessKey: AgCHrAV6iWIaJuvl53abd85VmMvVLtQddJLiShtOJR5rfJ7Zm6rvFJN36OxwbXDR5uSk8xi6YE1t0Kwz2VD23PgTHVGzUu8VY/S9bVPFkp1qMj2GP2mYcAE2tI74OwK9lCfKVkGV3x+mdOFj6BrDf8uA3R5VZFNEh2dIT+a3FMOHC/zEePAmztYeJvq7W/usKPaswDbff6pGrmi+VHvFpA4BO9bNbIhxJ8dLt6KGz1PbJZSODBoK747ZkoQnbApVe/nIshlRcs/0ThTuLZkCVRL7f5VFiCSUiakv+NCLXgF3+icFjcuRk2E8bLqAo5Pi3UHD7N+MG83ofaQ96JumtVu3F9e5Pr9rg60Nf4sKUz3E/McTqSWUwGZk6y6O/qdYjOWXJj+K9Hg8hY2JS8m92iVdtGOgKmjaJELoT/OK0vv29K/aPzDDD/9Eh5lsIZjlP64qCZhTdJDRGIPK88KnWh+Fr0A43OJ4Jq2SRkSwNF2I2muTu2dgSaCxQt2YJWOHsK+zLE5WyvU6k8icG059j+jf2D9f9dJClZx/3J45FTv2j/K5HqyStjixwyHzqkmlGGep+11IgsVoEfKWa/cnXLSME0z6p1C4yx88KwhoZDXBjCOrDxU6mmRLirlI+UdT51+zZiIjU7qfU47/ELI2eKHTd+QXu3EtAi8unvmEmVlfLImIBMoxrzBEbG4wuNEayg04u9HSNMFYcSIfYgcc2cDuDIUXljY9rP4oXVcQgpvnuA== + secretKey: AgBZo+eq+sdI4UHnaOMpY2MybhXVIgDeWYxDX+19f7rKgOzGQmhiseQsLDtkz+H2JcskyPWpL0aJTtBgPc7p0cvPigxXVu3yHsYn0R5nTDKLsYUkDReSj/qhCGfb4Z0uUjmxtQK0qvPoT3zh6X0rZ4eWXehVCK7DtnlcVzraHSjSXeHF4hKB6fRt5KyTVClSAHD1X+qjX1MzPpbMYnx1+X/NQampn+kWzes31GOZXjjURKnEW3j0zcyz8fUd3WVD5dCjAndMl8FyfjoEuzU4v71AO7yL6MPn9GXZ5XX/7ssX7SpWGkD1YBVe+loQr77H+pVkSKNIP245QZHV0TQosXpJQXzFfSkvieSkQFqz7+iYxf7LzvsXrnq5I5m7VsBZ3oADpimQbEgdq9xEySTe7tP2z+TgJjf0VbYBcz3EVaKACT4O2BeUNT86TllQAmrcUX+4CjOoeIDHJUT5yMZq1YydyAjTqJS9dQ5Vhz1GIWYKBucGZfcmWDv+uGuRySUfvJRI3bEtb6RzzC8i1HHJ4ROzSTHwR9n9up2lsSA675E3pR0AYu3jWPBo3d1L/bL7yVJd7H1WO+a8SWYshQ30M800xK58DMkRFGcIS3QnzmYV1CFbRYJUqekTn9Qjkzn/gchtio4kFj704IadXyb4Au1ebJ1pSBhBeuAGjNifp9bZa8yqzxhAQj/FurCPzHErlH7altm7ddgWBKxjlLnnc6TujKk41j3iO4hoRP7zxY7Xzw== + template: + metadata: + annotations: + meta.helm.sh/release-name: milvus + meta.helm.sh/release-namespace: datamate + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: Helm + name: milvus-minio-secret + namespace: datamate + type: Opaque diff --git a/scripts/secrets.sh b/scripts/secrets.sh deleted file mode 100755 index 946d982dd..000000000 --- a/scripts/secrets.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# DataMate Secrets Helper -# Manages SOPS-encrypted secrets for Helm deployment. -# Docker users do NOT need this script - use .env file instead. - -set -e - -SOPS_KEY_FILE="${SOPS_KEY_FILE:-$PWD/.sops-keys/key.txt}" -SOPS_CONFIG="${SOPS_CONFIG:-$PWD/.sops.yaml}" - -# Check required tools -check_tools() { - local missing="" - command -v sops >/dev/null 2>&1 || missing="$missing sops" - command -v age >/dev/null 2>&1 || missing="$missing age" - command -v age-keygen >/dev/null 2>&1 || missing="$missing age-keygen" - if [ -n "$missing" ]; then - echo "Error: Required tools not found:$missing" - echo "Install with: brew install sops age" - exit 1 - fi -} - -# Check or generate age key -ensure_key() { - if [ ! -f "$SOPS_KEY_FILE" ]; then - echo "Age key not found at $SOPS_KEY_FILE" - echo "Generating a new one..." - mkdir -p "$(dirname "$SOPS_KEY_FILE")" - age-keygen -o "$SOPS_KEY_FILE" - echo "Key generated. Keep this file secure and never commit it." - fi - export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" -} - -case "$1" in - encrypt) - check_tools - if [ -z "$2" ]; then - echo "Usage: $0 encrypt " - exit 1 - fi - sops --encrypt --in-place "$2" - echo "Encrypted: $2" - ;; - decrypt) - check_tools - if [ -z "$2" ]; then - echo "Usage: $0 decrypt " - exit 1 - fi - sops --decrypt --in-place "$2" - echo "Decrypted: $2" - ;; - view) - check_tools - if [ -z "$2" ]; then - echo "Usage: $0 view " - exit 1 - fi - sops --decrypt "$2" - ;; - helm-upgrade) - # Usage: scripts/secrets.sh helm-upgrade [extra-helm-args...] - check_tools - ensure_key - if [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 helm-upgrade [extra-helm-args...]" - exit 1 - fi - CHART="$2" - NAMESPACE="$3" - SECRETS_FILE="deployment/helm/$CHART/secrets.yaml" - shift 3 - - if [ ! -f "$SECRETS_FILE" ]; then - echo "Warning: No secrets file at $SECRETS_FILE, deploying without secrets." - helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" "$@" - else - TMP_FILE=$(mktemp) - sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" - helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" "$@" - rm "$TMP_FILE" - echo "Deployed $CHART with decrypted secrets." - fi - ;; - *) - echo "DataMate Secrets Helper" - echo "" - echo "Usage: $0 [args...]" - echo "" - echo "Commands:" - echo " encrypt Encrypt a YAML file in-place" - echo " decrypt Decrypt a YAML file in-place" - echo " view Print decrypted content" - echo " helm-upgrade Decrypt secrets and helm upgrade --install" - echo "" - echo "Docker users: skip this script, use 'cp .env.example .env' instead." - exit 1 - ;; -esac \ No newline at end of file From 977e818df4e14cb1333e8b6f6a305da149f54a15 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Wed, 27 May 2026 14:50:59 +0800 Subject: [PATCH 13/18] fix: pgbouncer also reads POSTGRE_PASSWORD from existingSecret When existingSecret is configured, the pgbouncer sidecar was still reading POSTGRE_PASSWORD from .Values.env (empty after SOPS migration), causing "password authentication failed" when connecting to PostgreSQL. --- .../helm/label-studio/templates/pgbouncer-deployment.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml b/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml index 5595269e3..0b62a0b51 100644 --- a/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml +++ b/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml @@ -42,7 +42,14 @@ spec: - name: DB_USER value: {{ .Values.env.POSTGRE_USER | quote }} - name: DB_PASSWORD + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: POSTGRE_PASSWORD + {{- else }} value: {{ .Values.env.POSTGRE_PASSWORD | quote }} + {{- end }} - name: POOL_MODE value: {{ .Values.pgbouncer.poolMode | quote }} - name: MAX_CLIENT_CONN From 0e0d29c17e428eff378a997edd943265b62ec0d3 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Wed, 27 May 2026 14:56:08 +0800 Subject: [PATCH 14/18] fix: set secrets.create: false to prevent Helm-SealedSecret conflict Helm must not create the datamate-conf Secret since it is managed by the SealedSecret controller. Without this, fresh installs fail with "conflict with controller" error. --- deployment/helm/datamate/values.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index b284da45d..194f15b93 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -38,7 +38,7 @@ public: database: 1Gi operator: 1Gi secrets: - create: false # Managed by SealedSecret (deployment/kubernetes/sealed-secrets/datamate.yaml) + create: false # Managed by SealedSecret (deployment/kubernetes/sealed-secrets/) data: DB_PASSWORD: "" # Set via secrets.yaml or --set CERT_PASS: "" # Set via secrets.yaml for encrypted SSL keys @@ -188,10 +188,7 @@ backend-python: gateway: env: - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: datamate-conf - key: JWT_SECRET + value: "" # Set via secrets.yaml or --set for production - name: datamate.jwt.enable value: *DATAMATE_JWT_ENABLE - name: OMS_AUTH_ENABLED From ba9eabd3280ef07b33d1d6854371e718fdb761a7 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Wed, 27 May 2026 15:21:01 +0800 Subject: [PATCH 15/18] fix: gateway reads JWT_SECRET from datamate-conf Secret Gateway was still reading JWT_SECRET from values.yaml (empty string), causing "JWT secret is required" startup error. Changed to secretKeyRef to match the SealedSecret-managed datamate-conf Secret. --- deployment/helm/datamate/values.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 194f15b93..947cc76f7 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -188,7 +188,10 @@ backend-python: gateway: env: - name: JWT_SECRET - value: "" # Set via secrets.yaml or --set for production + valueFrom: + secretKeyRef: + name: datamate-conf + key: JWT_SECRET - name: datamate.jwt.enable value: *DATAMATE_JWT_ENABLE - name: OMS_AUTH_ENABLED From 61a7a613bb56ba7be4d0ae709de1caf3eede2827 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Wed, 27 May 2026 17:33:21 +0800 Subject: [PATCH 16/18] fix: set created_by='system' for seed data to work with JWT data scope filter When DATAMATE_JWT_ENABLE=true, the Python backend's _apply_data_scope filter adds WHERE created_by IN ('user', 'system') to all BaseEntity queries. Seed data had empty created_by, causing templates, annotation templates, and operator categories to return 0 results. - t_clean_template: add created_by/updated_by='system' to INSERT - t_dm_annotation_templates: add COALESCE fallback in ON CONFLICT - t_operator_category: add UPDATE to fix empty created_by All three tables also get a safety-net UPDATE at the end. --- scripts/db/data-annotation-init.sql | 5 +++++ scripts/db/data-cleaning-init.sql | 15 +++++++++------ scripts/db/data-operator-init.sql | 3 +++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/db/data-annotation-init.sql b/scripts/db/data-annotation-init.sql index c83eb8822..43709882c 100644 --- a/scripts/db/data-annotation-init.sql +++ b/scripts/db/data-annotation-init.sql @@ -443,4 +443,9 @@ VALUES ('tpl-image-classification-001', '图像分类', '简单的多标签图 category = EXCLUDED.category, built_in = EXCLUDED.built_in, version = EXCLUDED.version, + created_by = COALESCE(EXCLUDED.created_by, 'system'), + updated_by = COALESCE(EXCLUDED.updated_by, 'system'), updated_at = CURRENT_TIMESTAMP; + +-- Ensure all annotation templates have created_by/updated_by set for data scope filtering +UPDATE t_dm_annotation_templates SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = ''; diff --git a/scripts/db/data-cleaning-init.sql b/scripts/db/data-cleaning-init.sql index 58b7add1a..7b946b31c 100644 --- a/scripts/db/data-cleaning-init.sql +++ b/scripts/db/data-cleaning-init.sql @@ -128,15 +128,18 @@ CREATE TRIGGER update_clean_template_updated_at EXECUTE FUNCTION update_updated_at_column(); -- 插入初始数据 - 清洗模板 -INSERT INTO t_clean_template (id, name, description) +INSERT INTO t_clean_template (id, name, description, created_by, updated_by) VALUES - ('550e8400-e29b-41d4-a716-446655440001', '安全与隐私合规处理模板', '针对敏感数据进行严格清洗,移除PII(个人身份信息)、政治敏感、暴力色情内容,适用于模型对外发布前的安全合规检查。'), - ('661f9500-f3ac-52e5-b827-557766550002', 'LLM SFT高质量文本清洗模板', '旨在生成高质量、低噪声的训练数据。包含去除乱码、重复内容、繁简转换、全角转半角以及格式标准化处理。'), - ('772a0611-a4bd-63f6-c938-668877660003', 'RAG知识库构建预处理模板', '专为RAG场景设计。重点去除目录、图注、XML/HTML标签等对向量检索无意义的噪声,并进行段落级去重以优化切片质量。'), - ('883b1722-b5ce-7407-d049-779988770004', '原始Web爬虫数据清洗模板', '针对互联网爬取的脏数据进行清洗。重点去除Emoji表情、URL链接、HTML标签以及不可见字符。'), - ('994c2833-c6df-8518-e150-880099880005', '多模态/CV模型训练预处理模板', '针对图像数据集处理。包含去除模糊/重复/相似图片,图片方向校正,目标检测预标注,以及尺寸和格式的统一化。') + ('550e8400-e29b-41d4-a716-446655440001', '安全与隐私合规处理模板', '针对敏感数据进行严格清洗,移除PII(个人身份信息)、政治敏感、暴力色情内容,适用于模型对外发布前的安全合规检查。', 'system', 'system'), + ('661f9500-f3ac-52e5-b827-557766550002', 'LLM SFT高质量文本清洗模板', '旨在生成高质量、低噪声的训练数据。包含去除乱码、重复内容、繁简转换、全角转半角以及格式标准化处理。', 'system', 'system'), + ('772a0611-a4bd-63f6-c938-668877660003', 'RAG知识库构建预处理模板', '专为RAG场景设计。重点去除目录、图注、XML/HTML标签等对向量检索无意义的噪声,并进行段落级去重以优化切片质量。', 'system', 'system'), + ('883b1722-b5ce-7407-d049-779988770004', '原始Web爬虫数据清洗模板', '针对互联网爬取的脏数据进行清洗。重点去除Emoji表情、URL链接、HTML标签以及不可见字符。', 'system', 'system'), + ('994c2833-c6df-8518-e150-880099880005', '多模态/CV模型训练预处理模板', '针对图像数据集处理。包含去除模糊/重复/相似图片,图片方向校正,目标检测预标注,以及尺寸和格式的统一化。', 'system', 'system') ON CONFLICT (id) DO NOTHING; +-- Also update existing templates that may have empty created_by +UPDATE t_clean_template SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = ''; + INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override) VALUES ('550e8400-e29b-41d4-a716-446655440001', 'PoliticalWordCleaner', 1, NULL), diff --git a/scripts/db/data-operator-init.sql b/scripts/db/data-operator-init.sql index 4609057f4..2a14bf0eb 100644 --- a/scripts/db/data-operator-init.sql +++ b/scripts/db/data-operator-init.sql @@ -521,3 +521,6 @@ WHERE c.id IN ('4d7dbd77-0a92-44f3-9056-2cd62d4a71e4', '9eda9d5d-072b-499b-916c- 'video_captioning_from_video_mapper', 'video_captioning_from_vlm_mapper', 'video_extract_frames_mapper', 'video_split_by_duration_mapper', 'video_split_by_key_frame_mapper', 'video_split_by_scene_mapper') ON CONFLICT DO NOTHING; + +-- Ensure all operator categories have created_by/updated_by set for data scope filtering +UPDATE t_operator_category SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = ''; From d6bd31ecdb827d6fbb4593da078d5bb717c173eb Mon Sep 17 00:00:00 2001 From: MoeexT Date: Thu, 28 May 2026 11:30:11 +0800 Subject: [PATCH 17/18] fix: remove hardcoded DB_PASSWORD default fallback in gateway config The gateway's application.yml had password: ${DB_PASSWORD:password} which falls back to 'password' when DB_PASSWORD env var is not set. Removed the default so it fails fast if the variable is missing, consistent with the main-application config. --- backend/api-gateway/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/api-gateway/src/main/resources/application.yml b/backend/api-gateway/src/main/resources/application.yml index 056397b45..40aa1080c 100644 --- a/backend/api-gateway/src/main/resources/application.yml +++ b/backend/api-gateway/src/main/resources/application.yml @@ -22,7 +22,7 @@ spring: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://datamate-database:5432/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} + password: ${DB_PASSWORD} hikari: maximum-pool-size: 20 minimum-idle: 5 From 1c147099abad4240692c6493ac9cbecbda5fb18d Mon Sep 17 00:00:00 2001 From: MoeexT Date: Thu, 28 May 2026 11:39:18 +0800 Subject: [PATCH 18/18] docs: add Sealed Secrets setup guide for online and offline environments - Add Secret Management section to README-zh.md and README.md - Document Sealed Secrets Controller installation (Helm) - Add air-gapped/offline environment instructions (image download) - Add kubeseal CLI usage for updating secrets - Add make download-sealed-secrets target for offline image download --- Makefile | 13 +++++++++++++ README-zh.md | 44 +++++++++++++++++++++++++++++++++++++++++++- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 964ac6fd7..202cd94ec 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ help: @echo " make install Install datamate + milvus (prompts for method)" @echo " make install INSTALLER=docker Install using Docker Compose" @echo " make install INSTALLER=k8s Install using Kubernetes/Helm" + @echo " make install INSTALLER=k8s (requires Sealed Secrets Controller)" @echo " make install- Install specific component (prompts)" @echo " make -docker-install Install component via Docker" @echo " make -k8s-install Install component via Kubernetes" @@ -69,6 +70,7 @@ help: @echo " make download VERSION= Pull all images with specific version" @echo " make download REGISTRY= Pull images from specific registry" @echo " make load-images Load all downloaded images from dist/" + @echo " make download-sealed-secrets Download Sealed Secrets image (for offline)" @echo "" @echo "Utility Commands:" @echo " make create-namespace Create Kubernetes namespace" @@ -481,6 +483,17 @@ DEER_FLOW_IMAGES := \ download-deer-flow: $(MAKE) download DOWNLOAD_IMAGES="$(DEER_FLOW_IMAGES)" +# Download Sealed Secrets controller image for offline/air-gapped environments +SEALED_SECRETS_IMAGE := bitnami/sealed-secrets-controller:latest +.PHONY: download-sealed-secrets +download-sealed-secrets: + @echo "Pulling Sealed Secrets controller image..." + @mkdir -p dist + docker pull $(SEALED_SECRETS_IMAGE) + docker save $(SEALED_SECRETS_IMAGE) -o dist/sealed-secrets-controller.tar + @echo "✅ Saved to dist/sealed-secrets-controller.tar" + @echo "Transfer to offline environment and load with: docker load -i dist/sealed-secrets-controller.tar" + # Load all downloaded images from dist/ directory .PHONY: load-images load-images: diff --git a/README-zh.md b/README-zh.md index 1d4987ed0..667a13c00 100644 --- a/README-zh.md +++ b/README-zh.md @@ -32,8 +32,50 @@ - Make (用于构建和安装) - Docker (用于构建镜像和部署服务) - Docker-Compose (用于部署服务-docker方式) -- kubernetes (用于部署服务-k8s方式) +- Kubernetes (用于部署服务-k8s方式) - Helm (用于部署服务-k8s方式) +- **K8s 部署额外需要**: [Sealed Secrets Controller](https://github.com/bitnami-labs/sealed-secrets)(用于加密管理敏感配置) + +### 密钥管理(仅 K8s 部署需要) + +DataMate K8s 部署使用 **Bitnami Sealed Secrets** 管理数据库密码、JWT 密钥等敏感信息。所有密钥以加密形式存储在 Git 中(`deployment/kubernetes/sealed-secrets/`),部署时由集群内的 Sealed Secrets Controller 自动解密。 + +**在线环境安装 Sealed Secrets Controller:** + +```bash +# 通过 Helm 安装(推荐) +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system + +# 验证安装 +kubectl get pods -n kube-system | grep sealed-secrets +``` + +**离线环境:** + +1. 在有网络的机器上下载 Sealed Secrets 镜像: + ```bash + # 下载 controller 镜像(约 60MB) + docker pull bitnami/sealed-secrets-controller:latest + docker save bitnami/sealed-secrets-controller:latest -o sealed-secrets-controller.tar + + # 下载 kubeseal 工具(用于更新密钥) + # macOS: + brew install kubeseal + # Linux: + wget https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/kubeseal-linux-amd64 + ``` + +2. 将镜像导入离线环境的镜像仓库,通过 Helm 安装时指定镜像地址。 + +**更新密钥:** + +```bash +# 如果数据库密码等敏感信息发生变更,使用 kubeseal 重新加密 +echo -n "new-password" | kubeseal --raw --name datamate-conf --namespace datamate --scope namespace-wide +``` + +> 注意:Docker 部署方式不需要 Sealed Secrets,密钥统一通过 `.env` 文件管理(已在 `.gitignore` 中排除)。 ### Docker一键部署 ```shell diff --git a/README.md b/README.md index 97ee80593..8fdda6d3e 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,48 @@ If you like this project, please give it a Star⭐️! - Docker-Compose (for service deployment - Docker method) - Kubernetes (for service deployment - k8s method) - Helm (for service deployment - k8s method) +- **K8s deployment additionally requires**: [Sealed Secrets Controller](https://github.com/bitnami-labs/sealed-secrets) (for encrypted secret management) + +### Secret Management (K8s deployment only) + +DataMate K8s deployment uses **Bitnami Sealed Secrets** to manage sensitive configuration such as database passwords and JWT secrets. All secrets are stored in encrypted form in Git (`deployment/kubernetes/sealed-secrets/`) and automatically decrypted by the Sealed Secrets Controller in the cluster at deploy time. + +**Online environment - install Sealed Secrets Controller:** + +```bash +# Install via Helm (recommended) +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system + +# Verify installation +kubectl get pods -n kube-system | grep sealed-secrets +``` + +**Air-gapped / offline environment:** + +1. Download the Sealed Secrets image on an internet-connected machine: + ```bash + # Download controller image (~60MB) + docker pull bitnami/sealed-secrets-controller:latest + docker save bitnami/sealed-secrets-controller:latest -o sealed-secrets-controller.tar + + # Download kubeseal CLI (for updating secrets) + # macOS: + brew install kubeseal + # Linux: + wget https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/kubeseal-linux-amd64 + ``` + +2. Transfer the image to your offline registry, then install via Helm with the custom image reference. + +**Updating secrets:** + +```bash +# When passwords change, re-encrypt with kubeseal +echo -n "new-password" | kubeseal --raw --name datamate-conf --namespace datamate --scope namespace-wide +``` + +> Note: Docker deployments do not require Sealed Secrets — secrets are managed via the `.env` file (excluded from Git via `.gitignore`). ### Docker Quick deploy ```shell