diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..27e5b67e7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Environment files (local dev only, never in images) +**/.env +!.env.example + +# Python +**/__pycache__/ +**/*.pyc +**/.venv/ +**/venv/ + +# Logs +**/logs/ +**/*.log + +# IDE +.idea/ +.vscode/ +*.iml + +# Git +.git/ +.gitignore diff --git a/.gitignore b/.gitignore index 9b34b7af4..decd54399 100644 --- a/.gitignore +++ b/.gitignore @@ -191,4 +191,7 @@ Thumbs.db # Milvus **/volumes/ -**/rag_storage/ \ No newline at end of file +**/rag_storage/ +# Environment files - ignore local .env, but allow templates +.env +!.env.example diff --git a/Makefile b/Makefile index 859cd0dbb..202cd94ec 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ help: @echo " make install Install datamate + milvus (prompts for method)" @echo " make install INSTALLER=docker Install using Docker Compose" @echo " make install INSTALLER=k8s Install using Kubernetes/Helm" + @echo " make install INSTALLER=k8s (requires Sealed Secrets Controller)" @echo " make install- Install specific component (prompts)" @echo " make -docker-install Install component via Docker" @echo " make -k8s-install Install component via Kubernetes" @@ -69,6 +70,7 @@ help: @echo " make download VERSION= Pull all images with specific version" @echo " make download REGISTRY= Pull images from specific registry" @echo " make load-images Load all downloaded images from dist/" + @echo " make download-sealed-secrets Download Sealed Secrets image (for offline)" @echo "" @echo "Utility Commands:" @echo " make create-namespace Create Kubernetes namespace" @@ -254,6 +256,13 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa done; \ exit 1; \ fi + @if [ ! -f deployment/docker/datamate/.env ]; then \ + echo "ERROR: deployment/docker/datamate/.env not found."; \ + echo "Create it from the template:"; \ + echo " cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env"; \ + echo "Then edit it with your actual passwords."; \ + exit 1; \ + fi @if [ "$*" = "label-studio" ]; then \ REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile label-studio up -d; \ elif [ "$*" = "datamate" ]; then \ @@ -326,18 +335,21 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m exit 1; \ fi @if [ "$*" = "label-studio" ]; then \ + kubectl apply -f deployment/kubernetes/sealed-secrets/label-studio.yaml; \ helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install; \ elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \ elif [ "$*" = "mineru-310P" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \ elif [ "$*" = "datamate" ]; then \ - helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ + kubectl apply -f deployment/kubernetes/sealed-secrets/datamate.yaml; \ + helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) --set public.secrets.create=false; \ elif [ "$*" = "deer-flow" ]; then \ cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \ cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \ helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "milvus" ]; then \ + kubectl apply -f deployment/kubernetes/sealed-secrets/milvus.yaml; \ helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install; \ elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \ kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \ @@ -471,6 +483,17 @@ DEER_FLOW_IMAGES := \ download-deer-flow: $(MAKE) download DOWNLOAD_IMAGES="$(DEER_FLOW_IMAGES)" +# Download Sealed Secrets controller image for offline/air-gapped environments +SEALED_SECRETS_IMAGE := bitnami/sealed-secrets-controller:latest +.PHONY: download-sealed-secrets +download-sealed-secrets: + @echo "Pulling Sealed Secrets controller image..." + @mkdir -p dist + docker pull $(SEALED_SECRETS_IMAGE) + docker save $(SEALED_SECRETS_IMAGE) -o dist/sealed-secrets-controller.tar + @echo "✅ Saved to dist/sealed-secrets-controller.tar" + @echo "Transfer to offline environment and load with: docker load -i dist/sealed-secrets-controller.tar" + # Load all downloaded images from dist/ directory .PHONY: load-images load-images: diff --git a/README-zh.md b/README-zh.md index 1d4987ed0..667a13c00 100644 --- a/README-zh.md +++ b/README-zh.md @@ -32,8 +32,50 @@ - Make (用于构建和安装) - Docker (用于构建镜像和部署服务) - Docker-Compose (用于部署服务-docker方式) -- kubernetes (用于部署服务-k8s方式) +- Kubernetes (用于部署服务-k8s方式) - Helm (用于部署服务-k8s方式) +- **K8s 部署额外需要**: [Sealed Secrets Controller](https://github.com/bitnami-labs/sealed-secrets)(用于加密管理敏感配置) + +### 密钥管理(仅 K8s 部署需要) + +DataMate K8s 部署使用 **Bitnami Sealed Secrets** 管理数据库密码、JWT 密钥等敏感信息。所有密钥以加密形式存储在 Git 中(`deployment/kubernetes/sealed-secrets/`),部署时由集群内的 Sealed Secrets Controller 自动解密。 + +**在线环境安装 Sealed Secrets Controller:** + +```bash +# 通过 Helm 安装(推荐) +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system + +# 验证安装 +kubectl get pods -n kube-system | grep sealed-secrets +``` + +**离线环境:** + +1. 在有网络的机器上下载 Sealed Secrets 镜像: + ```bash + # 下载 controller 镜像(约 60MB) + docker pull bitnami/sealed-secrets-controller:latest + docker save bitnami/sealed-secrets-controller:latest -o sealed-secrets-controller.tar + + # 下载 kubeseal 工具(用于更新密钥) + # macOS: + brew install kubeseal + # Linux: + wget https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/kubeseal-linux-amd64 + ``` + +2. 将镜像导入离线环境的镜像仓库,通过 Helm 安装时指定镜像地址。 + +**更新密钥:** + +```bash +# 如果数据库密码等敏感信息发生变更,使用 kubeseal 重新加密 +echo -n "new-password" | kubeseal --raw --name datamate-conf --namespace datamate --scope namespace-wide +``` + +> 注意:Docker 部署方式不需要 Sealed Secrets,密钥统一通过 `.env` 文件管理(已在 `.gitignore` 中排除)。 ### Docker一键部署 ```shell diff --git a/README.md b/README.md index 97ee80593..8fdda6d3e 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,48 @@ If you like this project, please give it a Star⭐️! - Docker-Compose (for service deployment - Docker method) - Kubernetes (for service deployment - k8s method) - Helm (for service deployment - k8s method) +- **K8s deployment additionally requires**: [Sealed Secrets Controller](https://github.com/bitnami-labs/sealed-secrets) (for encrypted secret management) + +### Secret Management (K8s deployment only) + +DataMate K8s deployment uses **Bitnami Sealed Secrets** to manage sensitive configuration such as database passwords and JWT secrets. All secrets are stored in encrypted form in Git (`deployment/kubernetes/sealed-secrets/`) and automatically decrypted by the Sealed Secrets Controller in the cluster at deploy time. + +**Online environment - install Sealed Secrets Controller:** + +```bash +# Install via Helm (recommended) +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets +helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system + +# Verify installation +kubectl get pods -n kube-system | grep sealed-secrets +``` + +**Air-gapped / offline environment:** + +1. Download the Sealed Secrets image on an internet-connected machine: + ```bash + # Download controller image (~60MB) + docker pull bitnami/sealed-secrets-controller:latest + docker save bitnami/sealed-secrets-controller:latest -o sealed-secrets-controller.tar + + # Download kubeseal CLI (for updating secrets) + # macOS: + brew install kubeseal + # Linux: + wget https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/kubeseal-linux-amd64 + ``` + +2. Transfer the image to your offline registry, then install via Helm with the custom image reference. + +**Updating secrets:** + +```bash +# When passwords change, re-encrypt with kubeseal +echo -n "new-password" | kubeseal --raw --name datamate-conf --namespace datamate --scope namespace-wide +``` + +> Note: Docker deployments do not require Sealed Secrets — secrets are managed via the `.env` file (excluded from Git via `.gitignore`). ### Docker Quick deploy ```shell diff --git a/backend/api-gateway/src/main/resources/application.yml b/backend/api-gateway/src/main/resources/application.yml index 056397b45..40aa1080c 100644 --- a/backend/api-gateway/src/main/resources/application.yml +++ b/backend/api-gateway/src/main/resources/application.yml @@ -22,7 +22,7 @@ spring: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://datamate-database:5432/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} + password: ${DB_PASSWORD} hikari: maximum-pool-size: 20 minimum-idle: 5 diff --git a/backend/services/main-application/src/main/resources/application.yml b/backend/services/main-application/src/main/resources/application.yml index 64aac288c..bd05cebd5 100644 --- a/backend/services/main-application/src/main/resources/application.yml +++ b/backend/services/main-application/src/main/resources/application.yml @@ -14,7 +14,7 @@ spring: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://datamate-database:5432/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} + password: ${DB_PASSWORD} hikari: maximum-pool-size: 20 minimum-idle: 5 @@ -59,7 +59,7 @@ spring: host: datamate-redis port: 6379 timeout: 2000 - password: ${REDIS_PASSWORD:password} + password: ${REDIS_PASSWORD} lettuce: pool: max-active: 20 @@ -131,6 +131,10 @@ management: # 平台配置 datamate: + # JWT配置 + jwt: + secret: ${JWT_SECRET} + # 通用配置 diff --git a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java index efe4a4b85..5e981daf5 100644 --- a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java +++ b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java @@ -16,7 +16,7 @@ @Component public class JwtUtils { - @Value("${jwt.secret:datamate-secret-key-for-jwt-token-generation}") + @Value("${datamate.jwt.secret}") private String secret; @Value("${jwt.expiration:86400}") // 24小时 diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example new file mode 100644 index 000000000..ed68e8f5c --- /dev/null +++ b/deployment/docker/datamate/.env.example @@ -0,0 +1,30 @@ +# DataMate Environment Variables Template +# Copy this file to .env and fill in the values +# cp .env.example .env +# IMPORTANT: Never commit .env to git! It is already in .gitignore. +# +# For K8s/Helm deployment: secrets are managed via Sealed Secrets. +# For Docker deployment: use this .env file (gitignored, local only). + +# Database +DB_PASSWORD=your-secure-password-here + +# JWT Authentication +JWT_SECRET=your-secure-jwt-secret-here +DATAMATE_JWT_ENABLE=false + +# MinIO (for Milvus storage) +MINIO_ACCESS_KEY=your-minio-access-key +MINIO_SECRET_KEY=your-minio-secret-key + +# Label Studio +LABEL_STUDIO_USERNAME=admin@demo.com +LABEL_STUDIO_PASSWORD=your-labelstudio-password +LABEL_STUDIO_USER_TOKEN=your-labelstudio-token +LABEL_STUDIO_HOST= + +# Optional: SSL Certificate Password (for encrypted private keys) +CERT_PASS= + +# Optional: Domain for HTTPS +DOMAIN= diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index dc5a8470d..bdbfc4f7f 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -9,7 +9,8 @@ services: restart: on-failure privileged: true environment: - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} + - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} volumes: - dataset_volume:/dataset @@ -30,9 +31,14 @@ services: - "18000:18000" environment: - log_level=DEBUG - - pgsql_password=${DB_PASSWORD:-password} + - PGSQL_HOST=datamate-database + - PGSQL_PORT=5432 + - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} + - LABEL_STUDIO_USERNAME=${LABEL_STUDIO_USERNAME:-admin@demo.com} + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} volumes: - dataset_volume:/dataset - flow_volume:/flow @@ -52,7 +58,7 @@ services: ports: - '8080:8080' environment: - - JWT_SECRET=default-insecure-key-change-in-production + - JWT_SECRET=${JWT_SECRET:-} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} networks: [ datamate ] @@ -75,7 +81,7 @@ services: restart: on-failure environment: - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=${DB_PASSWORD:-password} + - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} volumes: - postgresql_volume:/var/lib/postgresql/data - database_log_volume:/var/log/datamate/database @@ -93,7 +99,7 @@ services: PG_HOST: "datamate-database" PG_PORT: "5432" PG_USER: "postgres" - PG_PASSWORD: ${DB_PASSWORD:-password} + PG_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} PG_DATABASE: "datamate" command: - python @@ -213,7 +219,7 @@ services: - DB_PORT=5432 - DB_NAME=labelstudio - DB_USER=postgres - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - AUTH_TYPE=scram-sha-256 - POOL_MODE=transaction - MAX_CLIENT_CONN=100 @@ -241,7 +247,7 @@ services: - DJANGO_DB=default - POSTGRE_NAME=labelstudio - POSTGRE_USER=postgres - - POSTGRE_PASSWORD=${DB_PASSWORD:-password} + - POSTGRE_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - POSTGRE_PORT=5432 - POSTGRE_HOST=label-studio-pgbouncer - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-} @@ -249,9 +255,9 @@ services: - LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local - USE_USERNAME_FOR_LOGIN=true - LABEL_STUDIO_USERNAME=admin@demo.com - - LABEL_STUDIO_PASSWORD=demoadmin + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} - LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN=true - - LABEL_STUDIO_USER_TOKEN=abc123abc123 + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} - LOG_LEVEL=DEBUG volumes: - label-studio-data:/label-studio/data:rw @@ -290,8 +296,8 @@ services: container_name: milvus-minio image: minio/minio:RELEASE.2024-12-18T13-15-44Z environment: - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-} ports: - "9001:9001" - "9000:9000" diff --git a/deployment/helm/datamate/charts/public/templates/secret.yaml b/deployment/helm/datamate/charts/public/templates/secret.yaml index 9b6553cac..a3acb51c2 100644 --- a/deployment/helm/datamate/charts/public/templates/secret.yaml +++ b/deployment/helm/datamate/charts/public/templates/secret.yaml @@ -1,3 +1,4 @@ +{{- if not (eq (.Values.secrets.create | toString) "false") }} apiVersion: v1 kind: Secret metadata: @@ -5,4 +6,5 @@ metadata: data: {{- range $key, $val := .Values.secrets.data }} {{ $key }}: {{ $val | toString | b64enc | quote }} - {{- end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/deployment/helm/datamate/charts/public/values.yaml b/deployment/helm/datamate/charts/public/values.yaml index f134e4325..de08166f5 100644 --- a/deployment/helm/datamate/charts/public/values.yaml +++ b/deployment/helm/datamate/charts/public/values.yaml @@ -22,4 +22,7 @@ persistentVolumeClaim: flow: 1Gi log: 1Gi database: 1Gi - operator: 1Gi \ No newline at end of file + operator: 1Gi + +secrets: + create: true # Set to false when using Sealed Secrets \ No newline at end of file diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index b19959a70..947cc76f7 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -38,9 +38,10 @@ public: database: 1Gi operator: 1Gi secrets: + create: false # Managed by SealedSecret (deployment/kubernetes/sealed-secrets/) data: - DB_PASSWORD: "password" - CERT_PASS: "" + DB_PASSWORD: "" # Set via secrets.yaml or --set + CERT_PASS: "" # Set via secrets.yaml for encrypted SSL keys DOMAIN: "" HOME_PAGE_URL: "" @@ -118,6 +119,11 @@ backend: secretKeyRef: name: datamate-conf key: DB_PASSWORD + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: datamate-conf + key: JWT_SECRET - name: datamate.rag.milvus-uri value: "http://milvus:19530" - name: datamate.jwt.enable @@ -139,6 +145,10 @@ backend: backend-python: env: + - name: PGSQL_HOST + value: "datamate-database" + - name: PGSQL_PORT + value: "5432" - name: pgsql_password valueFrom: secretKeyRef: @@ -148,6 +158,18 @@ backend-python: value: *DATAMATE_JWT_ENABLE - name: milvus_uri value: "http://milvus:19530" + - name: LABEL_STUDIO_USER_TOKEN + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_USER_TOKEN + - name: LABEL_STUDIO_USERNAME + value: "admin@demo.com" + - name: LABEL_STUDIO_PASSWORD + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_PASSWORD volumes: - *datasetVolume - *flowVolume @@ -166,7 +188,10 @@ backend-python: gateway: env: - name: JWT_SECRET - value: "default-insecure-key-change-in-production" + valueFrom: + secretKeyRef: + name: datamate-conf + key: JWT_SECRET - name: datamate.jwt.enable value: *DATAMATE_JWT_ENABLE - name: OMS_AUTH_ENABLED diff --git a/deployment/helm/label-studio/templates/deployment.yaml b/deployment/helm/label-studio/templates/deployment.yaml index 9935f6e15..f2a16f9c3 100644 --- a/deployment/helm/label-studio/templates/deployment.yaml +++ b/deployment/helm/label-studio/templates/deployment.yaml @@ -36,7 +36,14 @@ spec: - name: POSTGRE_USER value: {{ .Values.env.POSTGRE_USER | quote }} - name: POSTGRE_PASSWORD + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: POSTGRE_PASSWORD + {{- else }} value: {{ .Values.env.POSTGRE_PASSWORD | quote }} + {{- end }} - name: POSTGRE_PORT value: "5432" - name: POSTGRE_HOST @@ -52,11 +59,25 @@ spec: - name: LABEL_STUDIO_USERNAME value: {{ .Values.env.LABEL_STUDIO_USERNAME | quote }} - name: LABEL_STUDIO_PASSWORD + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: LABEL_STUDIO_PASSWORD + {{- else }} value: {{ .Values.env.LABEL_STUDIO_PASSWORD | quote }} + {{- end }} - name: LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN value: {{ .Values.env.LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN | quote }} - name: LABEL_STUDIO_USER_TOKEN + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: LABEL_STUDIO_USER_TOKEN + {{- else }} value: {{ .Values.env.LABEL_STUDIO_USER_TOKEN | quote }} + {{- end }} - name: LOG_LEVEL value: {{ .Values.env.LOG_LEVEL | quote }} ports: diff --git a/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml b/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml index 5595269e3..0b62a0b51 100644 --- a/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml +++ b/deployment/helm/label-studio/templates/pgbouncer-deployment.yaml @@ -42,7 +42,14 @@ spec: - name: DB_USER value: {{ .Values.env.POSTGRE_USER | quote }} - name: DB_PASSWORD + {{- if .Values.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.existingSecret | quote }} + key: POSTGRE_PASSWORD + {{- else }} value: {{ .Values.env.POSTGRE_PASSWORD | quote }} + {{- end }} - name: POOL_MODE value: {{ .Values.pgbouncer.poolMode | quote }} - name: MAX_CLIENT_CONN diff --git a/deployment/helm/label-studio/values.yaml b/deployment/helm/label-studio/values.yaml index 229098ee5..b06da7f40 100644 --- a/deployment/helm/label-studio/values.yaml +++ b/deployment/helm/label-studio/values.yaml @@ -41,7 +41,7 @@ env: DJANGO_DB: "default" POSTGRE_NAME: "labelstudio" POSTGRE_USER: "postgres" - POSTGRE_PASSWORD: "password" + POSTGRE_PASSWORD: "" # Set via secrets.yaml or --set POSTGRE_PORT: 5432 POSTGRE_HOST: "datamate-database" LABEL_STUDIO_HOST: "" # can be overridden @@ -49,11 +49,15 @@ env: LOCAL_FILES_DOCUMENT_ROOT: "/label-studio/local" USE_USERNAME_FOR_LOGIN: "true" LABEL_STUDIO_USERNAME: "admin@demo.com" - LABEL_STUDIO_PASSWORD: "demoadmin" + LABEL_STUDIO_PASSWORD: "" # Set via secrets.yaml or --set LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN: "true" - LABEL_STUDIO_USER_TOKEN: "abc123abc123" + LABEL_STUDIO_USER_TOKEN: "" # Set via secrets.yaml or --set LOG_LEVEL: "DEBUG" +# Use an existing K8s Secret instead of creating one from values. +# Set to "label-studio-env" to use the SealedSecret-managed secret. +existingSecret: "label-studio-env" + persistence: enabled: true existingClaim: "" # if you already have PVC that maps to external datamate-dataset-volume diff --git a/deployment/helm/milvus/values.yaml b/deployment/helm/milvus/values.yaml index 584cdcc27..626248e46 100644 --- a/deployment/helm/milvus/values.yaml +++ b/deployment/helm/milvus/values.yaml @@ -634,8 +634,8 @@ minio: image: tag: "RELEASE.2024-12-18T13-15-44Z" pullPolicy: IfNotPresent - accessKey: minioadmin - secretKey: minioadmin + accessKey: "" # Set via secrets.yaml or --set + secretKey: "" # Set via secrets.yaml or --set existingSecret: "" bucketName: "milvus-bucket" rootPath: file diff --git a/deployment/kubernetes/sealed-secrets/datamate.yaml b/deployment/kubernetes/sealed-secrets/datamate.yaml new file mode 100644 index 000000000..c8c57696c --- /dev/null +++ b/deployment/kubernetes/sealed-secrets/datamate.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: datamate-conf + namespace: datamate +spec: + encryptedData: + CERT_PASS: AgAFa2NSMHymGMXv63PKqnUifZ70ocLLYd1MhIympF/NzJakL44sxMQlLLWWSJByouOgTu8ROM91rKYUkb6Y6kdV/pj0iQr1y/NLb6lS3y4Lvjvt4S/m7JCOcOivXJv5hzPU7uDfn8y8yw0qsjyXwa+lWMdeUmwnSEYwLcPexjvTcAs7xiV0OWW1WHkAZlx8A9UJqk1Yeaj/4fqRdLMEKBHYiGH8iW5nm3r5/5Uv3sdTU6n2EQgjFT2zpCYVbDvSVaFjRQ/yZ4jCuNNlw2q7EpoLzT8Ak+6BaoVqEVLK885fo8Wu8B88Pnx3Tse9v/Dld9qZaUVkXxYK/wpIjFPZCGC2q5y9oPl0fbK9nMbml9lkOgmGEsJm9CVTpllRGgT1O4hwE/Ij94dgLXErMlUukrU6/RiMwhis7bBrC1qMj+aEpsRl4vscbvbTBGUhW4rJrwMZscnqw6bt7sHnInrOKnO3vXJIJPDyoR9bHWj83NX1A7WkqQe69xYUK/+Zf4vRmNFHmU6KV+rRR80pWChJGlybmp+fZ2LGoI5PFHky/cmPz2gmGZJdiOC5Y2mi+vvzPJASra7n2nWxrcZ10H2YJI39/J/H6tANf6/Bpj97h9VzGpDFVxE19jKxkrmnsGUcVfB8+wa985kIPKPRzGktQ4pROC0Feyta9lbi75yW7ZaTSptVmzknJ9FrK8xx9yT6HOc= + DB_PASSWORD: AgCYbAis3YHfbcoI/uazKsyc7Q25A77gLrJAfx6F0Y9LzDoIlE5xczNHV8AhUtmT5lCCnzMxBRd6bCpVQho5xvpv8pfn2vjlbd/JbI0Up3vj6Yg3iw4lOmqxLs7DVpnn/Lo/GeoBXWZOerCRMiY/geVoSaQfybgIsmqFahHYaBciRHGxpZ0rhWqnm6U+RT5Oun73U47Mb8HMkEBIeMZW6IXGYoj7VetbouGZrSsu/uDlvO0P1BLFNVDiMZWW9B9gRnIdINxJUUzmvzvzyYBjOrsQlNZu+LycE57PspUSW4/TTHlZyxwrsdXSkWH/J66m04vHEzAaxF7Yg3zYCdcNBpiI8FLLDS64mROgj/dv8PY4qAIrEvb7e3qt07t/ArXEa1R7kkoBBjp/VvIy2wF905oQ99AgPEasnHYNp8VB1lym3YWWeuifsi6aaWhKyqq/22EhMRXKiC3brfp7KVnlnwX+98AHLl8Q04C90LwuDyKW+OBt6ATQktJuHqh+5uB17rvSu2KtWp2ugh1oqH8VaGydcd7lhKlHmi9a3b18uS3n6o5O8bR3mgmOWjlck4lDQI6cmnRVfx9J92jpdmZW5MeEtnlldYaek8am+zZTAusaagTp5eE8+p7238dGwr95LdWkkJbiJGBBvxuNdI0zjAFtGirueNe0ZHLvAKDh/Pw/Db8gg8DkyIVgJNDx4Q1F6eG2TPhwRu6vq/4RMvYFH5f1K58dCzEtkoQJsgJyJ4Kd9DBJzblOQ7fFLSx7Ag== + DOMAIN: AgCjEuH37xiKRpOkIrnAnk4Ui5A7B0M0R5BOhpvrbbu+QU/dqW7ysZzyNppDpABMB9n3H8jDsPYoKgFUVkD6p8FF3IpHmLlaJvO6FeIC86rJPVZTpWjPTub8LOCP5sszr7t4F1zyIo8uckyvqW2LxlNbN6cOCNzVutdHEcsu4dxWI/MzZGocT2CrWh3PDORdN6xrJ8qfpZEyFHiVwyxcbh3XM5oHqqe2o2OIlejXtrB3fQo1oUOH99CZTsZ+JrhUF5KOb4jDgq6aFLrJzkSWF+d/MXgrzqLDsqUuWxOp4CkbsIJJfO+aF2ULH9JAS1ZD/mnYUr1Lmix1wJORx5VXfT5L+fx+y8VDP2H8BeoLNqmmCyyV41GcvAyG5fN641Ck63MMehZkAJo72Bu4eRk8Mkk9JvDejAUTR5a/J1qKOtNVeFmM4Fq9y3WvW5X9e/3mHs7tCWTxGzJ70bV7zTngnu+G5SUoOjtvuBMNOfes9MAKj+hQ5YBaKiqZ5cJANgXX1YnmY5fvPSi9tyAxFspJgGSzHwBXLsTbeDRtTW86qmvY8anF++iizeuZ4DEqCLl8D0G3lj51elTTw7/1dDa8s1Xvy8EY6SwZAiuHpHkYf5barbrZLKSUldzd2AOPfxDgnbmussTV7o+/bjbCFKhGnWW2EEvLx4yKHjAJa8A/9uDsX8Zv1O+idsV7VXuFrsV8UGc= + HOME_PAGE_URL: AgC1A5P37JqlZXKdwNJT/0d3HiSvlCITF7mc8bXl4LslRToKloICDqO8PHuC3gvIxPtbrWr8YRCBPg9gZsb4s8234cIlYl3Ntxr+YQXbtLXFXyu4/BIJoz4qOJhyq5zo/0QkN+2mztol/1ZcLSGtj6mV3JY0yvZCqBlPIR3M78IW0OHPu3mF4zo4S+aumUE7QKZHsb83phUVvIJ1o+XsR8xrwAQBFtRp6GCgowAppeTrG9OTgmsjkHmb5Hz+ri2AbNkzx8SXQUCbWaosYHTwY/OYolFN6zKCql6zxLRt3ryjhGizIcSURumPcGzzuhotunXBPjhp4bI+lOAotwOW19vaNk/g4atQz16V5ceXg7Rk/+cZRkKJ/rKgpI2MN+1sP17U/DtudBqvsPk0da3QNqeOJ1VN6mFKs6R/DVxh4YYK1a/kUD5GdhK4jpFpvguEsBzx1swpB8Wb8AlIze7xAksk1BFL9+UzCpeDPVQeH+QyZA+cfZ4PIv5MC31A6clMz2o2cthNufggmQxvy2ndhjY7ZbfXyOA78eoPYY5IIx5pZA7zR2pxgJb79FS9RFcarjaF51fBCZy/EZie/C+JNI7fwLtai7BexqN6Nefdn9VVCr+zN0dCBZ3Rw402WiihaXiaPAxfsKrMYneqWs4F0hyRaL3AmEA8jRfNEAGrcUljMAie03I9+mtA6RAxW6d5ZA8= + JWT_SECRET: AgBztOHSCJostJW7ZxXRUhf2I1YzJubpX0dJmvHielNcuezyBg2R3JIHxZeZEiLtsDG4arE3ZFbPo6c16e+i15e0ElkScAJJbiCFnVgPOqbVOHcWmgdCDV88OZo4MSXpJBxuUmGMtRYdVRgp26p70TzmPEi+S4U5YZxp+Jb7jaCtxg//o5iikIobNzigX0eiwSBgXrBKA5KobVFMmKRYZskdM4mYvZXe30mcyABxIDiGjwXPdQWT7d0pNaAMmYToGDngVFAzHfkS8/fXzQOnP+Vo1/rKskZCDw401EjyYu83UJulbwNwY9HejJyP7xQJ9ZQOsOklaNjtKRN4uiLciBUQLGvaa3hPpyRCbm2Xv1Jjq+enxeHSZdWmQ6DcYpy8YPwVhiyIEQtkONtKaZtPBj7U9vZARK/ET5+iCT/CXmU3fElKH4n0sFMerDUqbEN5F1V1PIljHDpPobbwirvkNIBPc2sMX1Vt+KPPv6CWf6HWCleayMv8lXU9+son5CnBzwjVj8stxCqGje7l5b7g98qidiikBlocc/N+vCBzGf5plKVrB66lK07tO3yccKXmNos2Y6S3aSCwRgXhAkeeELreV2pK8DVI58KKiK2ClARdxE3Ba1mRL1hWDe3aT1U/FjStBYb+/u7O2rh+rGMaVMz3g/xDRDXJtHRW9PKz+Dlw9uGNUNtkL/BVg0LvbrBECOsGeIkb65SZlUm0i+8f1ZC69Fa0KM9dew7/2aVXmuKXhgPKh2XxmndC2QNmYQ== + LABEL_STUDIO_PASSWORD: AgBt5v87JYw+EpPwquwglWV81OK4lLg3jyZ75OK0k6GmedasPz+DnZb+OfWxsxdvpjEUSyKlEmpDbsH7zkrmCRn5VCasuJ6oa1fuMjJU/8DQ44piPvg1OagVrenVLy5cyzzcnLdKGVEDg69aUrZTfz+dSnyh9kL3f98g+Wt7EIhFcB7cxrVv7KXrc+hu8j0PnEHo/X/B1lA9KNi0yCj4CBofy9vrcJVFnGTeNh3hUEOGDORppdJUl1dDFx/qJeg94e+GClQUmC1QJmhBosxfnDbQnw/5lB9EP9iqjxMbswLsMhpEGQZqPbrFakvz0lb7N2Skq4RxD4XnkZTJ29RDsOl4TCAKL8C2a8OxP7P+yVflrTi5aBMLXhvrmEW1IwQXlFGP8th5AWkB+KM/gCfL1Iw34H8uklFPqAXexqxSsctwlnCIKBjG8lvbKfYEjOQhKHPZij1WYv8UZarU6rTvJ9OnLKW0kTTI/1MAFbfARPaKIcBzP7Ahexv7i7fo6YjeCz1n6N8UXxlszKZbQV82e/Zq40AmNJIG2vnBaNfskzYPQCpLr5yjwNO99IQZplw5i/X4pFWT9NfJTb7lfgb4PqUbsoYZgfJ5JBuA6b17/YYQy4pMyCorXAVmFDhKwSKT6WYgpw9N0TSErc3eOnusAolTuyzD5ns5h3NnomA3lD4hcz7L4CuR6lXekjfWk4vZNQu55Km1fTvB5DTB8gicRamxq5JAEQj8EYRK6Po8sXrjgw== + LABEL_STUDIO_USER_TOKEN: AgAiYeDEoGB1lSPKTb9n9dHzHPfH6sbLuNzAIX3ZwSRzjQjnkcEw/oSausQ4y0Z4kBv/983cuXwOF2xbOejPZ6LW9JSzzpS0dqWShfVcEjFwwPX7N7Nihqi9KdfXr4n2KoMCGZPphokxQwKt034VbFjQVdmIPQo5zRRYNlqr7i/etjlpsIs+mc9LO8s6J+zlRt+z4mxDg2m8y7MvNm0iW0nF0Ef7abGk+SctzXK+Zoec6uhvOv6FqEwsRAqjHsdcWySpH3Q53nMfM05LrkF5W5eca5t7rcGQTK0bmFr0x7pdt70mPTac8z0B2v7IIsb16v59xSweWlyKUPKCY2WGIDZuPjL38Ilt6gKYw/lDIFG2/gMa0ai1rIer7+nd7OzMP/OP6fwcY8l1GFOSeV1f16ZhV8n1vaJ0qR9lzrDLMQHYHHP7a/wFDz1S66sNVSNF4Otd6O9e0svBq+mYsQUw/ahruxZ/kl7hQK6qCP86E8z6H1MIRmEDR2zq2CV6SxLtEa8RwxWqcT57Zew8rqn62zcNxilDyFA6xw/ZrzTmNOy+QWP16ema9mC9Ja3J8LHHChN8+bzFswdPCVol4tW97cZp+FsXceadoyC6LE2f/i7cwVUj4g+O7yiyHfpohDlvujnJsVf+ejMToNJ3JN8pl8mpv2YqCXmjJ5WdkbvElT/W4jvVJlUV+QS6fTK47n6xAwJCKcP3n77nmUj3M+/dJdGBBZaRb5A6H7ywgPMItirtuw== + template: + metadata: + annotations: + meta.helm.sh/release-name: datamate + meta.helm.sh/release-namespace: datamate + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: Helm + name: datamate-conf + namespace: datamate + type: Opaque diff --git a/deployment/kubernetes/sealed-secrets/label-studio.yaml b/deployment/kubernetes/sealed-secrets/label-studio.yaml new file mode 100644 index 000000000..9a2a1b10b --- /dev/null +++ b/deployment/kubernetes/sealed-secrets/label-studio.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: label-studio-env + namespace: datamate +spec: + encryptedData: + LABEL_STUDIO_PASSWORD: AgAdOZuj2s/Q4AsJbTU2DHw/uXuU0tIpiKD/o56jxhGkgIjH/IMCX/cB+12hq+xoz2yJCWGHuN0YV7iZhAwQ7Dwrn5xPTBUlQM2Ky5hBvvSs6lFld9R7iA5UChjwuCts4cQ8xKmUEIXAMv4rvL7g/iCpUhESlac8JB0KLofDm94+XRFyT30cQDdSyZKQ17Vi59TLSPuVUQf/R+8YtaChrX6hgM3OvCG2dw8h8jir7M1ppROyVUu+PmvblpITolklrm7HjPNaWybOjfXcZ3YYghAVEA/OJ4HdLF3aEmPZaw3tD1NQ7wlDtl3P3MCQ/l1eR657iuc1MzoH5/sNMSit5l322zg+wTXPjRNrDjcHtsCICH7ze+q0P1y0UZzey3vGXNyEFDtXcgy/Xg/GAC3OZbLS+Tin9ZQcssifkDChaHezTKHhqTPSFEj7yAQMK+jCXfchJcsV8SRwbqTSvNyr2vTKDgErM0XaQR3G6utY/VSZY0CEC3MhaWPiYEoUr2GJ6RRSLKARafjriDPvC4y/JxvCRGm+CqqHjwkJ31W2arJwJsGzl8ut9GLuH5M2/UmBIt0qLaJBNL/LQYPr6QINfUxiRKSRLMw72AxpFkHeM+EMucQv6NqE7AzqMhlygA+FNwXXQYS+1A+cv8o1TrbG4ounodUdprmUAdYjDH6zr1GRZ8lqjd9pteOceB82tUyzRs+6j1+5RlOsklLWd3uWgqP1s/N89kl8ho70W6rJToCDew== + LABEL_STUDIO_USER_TOKEN: AgB0EclTH3xnvW6zo/J25y40/Q8149QHvGre+eKd7hnun1kiVIYIHvAf4KXmgaGY7no6gTNGjleh4KnkSvoAo7W3T5wypS0MMZGpnzPbMpV8XNLA+sFcV2+Hiim+lxSzKKIEbNWnHr8urHnnBvRK2Wa54hzj4j4bcLGQqjptseusX2MB8BlaociKY/5LrQugNjMsLS9je4Wnz7ygSggU4PdUnpeTF/tRRyacOoCUSPb2GJmBthAZP5DvbKbvwDYoiKEIZRTjnF3V0FIRPe3weO3XRwl7nG3JgvVxSIMt48J5eKofNeSW15A5391USfMkNNnyN12I7+6fNtqUYWB+XuG/zB3kP0Uy8EsWYNQrJaJqHbQpetnCWda+azdvJWSE/TAEiy5pLR57gNhSdM5e/5vyXwgEpGxq3XI4HOSmnXq+bf2lQ36FyiPLsOTcJqOB0sqMT7tQthawSaTRy2yw2bJ2xxicmJBqzsp+xZ7EurtMSAeAThAHWuJ6w8KKvJbnCirQNKSwGjjVC6bq7KtEppu0gY6IL75VH94NmQi2sN9sBe9XPHUzfOJXilNl79LkDDKNfqPDbIhmwc5ECJBLsP9xhs6/0U8hSrmIUQ8AuvK/DYEsaydCvauEWFiNoYcKY4XoEW9kFvRH5Cd1r59P5dYiqZ5ivIHEWBYz6pBmN8uRs4p7aawO0SBy7KaVNM9PMgkP5D5VbLmNF6oTyruODEyeXmi8JKWd5u8R9kcYgY9lgA== + POSTGRE_PASSWORD: AgATLvaLNWeapPWyvmH8/li3GgpMACMfJd3dbO3FZ5MIfzj3K+KDGXyxrqiYuX1im4tFQgXdyIQzL59zlRLnMntCaiA5eNdhdeXMNEgef9hbLgWbnhH+Hl1//RYSd9CIueSpzsLC2Gd/c3PptIt2Gzvz6lazjc8Fz6gIETVEIHQvMTVsb7HDVHcICL2+/PLuEN6bXkwmiAcNJT3ipe9kqT38EagTwOmIErCrQVkg/KoECxdmdSYRiZqKTGwZndToosgiFr7FrAhv4lDsuyBReA73bFUA6B+94treSuRK2dp0JQBnL4GdOB1v04/BP+gCKWeIVzwOnpSrTVvVOyPQil/deRk1Fj54le2sU0yTPFh18U6NtTyRITH6GUvYRF8pn5QQQFsPD9gz/EtenjOmr2f587AsdDwHwuEo6tuiIPe1DeyxMJlSAUtlbwDH3hFvkpZBZpAqf3f8veEo/EW1fFA3QDRGATGiw4qIrILw/a5d66VkkvtTiztOHxD1mVG6JSWPrxmL13s4S7S5ZbZTIlR6KmGOya3NpThrOm3OWoO3aBquvzlLz5zK6MpsARJ7FSSzNXMrE8sbIRVEDAUJQWPcrrDTGDFIBx4rYHeSPw8Kx3hu3AfsAw0VKwRQsrWMxrXDlppyQTcGr0tgN+wH3X93YD1LXmt8gSH+/iP9APT1GmUV6JwCEyED/koEJ4LsCPlhr17S5ImckZKR0PXL83ubFVkPaQI6yDn5P+szU9boyVpKwA4NfnEI8jTvLw== + template: + metadata: + annotations: + meta.helm.sh/release-name: label-studio + meta.helm.sh/release-namespace: datamate + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: Helm + name: label-studio-env + namespace: datamate + type: Opaque diff --git a/deployment/kubernetes/sealed-secrets/milvus.yaml b/deployment/kubernetes/sealed-secrets/milvus.yaml new file mode 100644 index 000000000..37c47b1d8 --- /dev/null +++ b/deployment/kubernetes/sealed-secrets/milvus.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: milvus-minio-secret + namespace: datamate +spec: + encryptedData: + accessKey: AgCHrAV6iWIaJuvl53abd85VmMvVLtQddJLiShtOJR5rfJ7Zm6rvFJN36OxwbXDR5uSk8xi6YE1t0Kwz2VD23PgTHVGzUu8VY/S9bVPFkp1qMj2GP2mYcAE2tI74OwK9lCfKVkGV3x+mdOFj6BrDf8uA3R5VZFNEh2dIT+a3FMOHC/zEePAmztYeJvq7W/usKPaswDbff6pGrmi+VHvFpA4BO9bNbIhxJ8dLt6KGz1PbJZSODBoK747ZkoQnbApVe/nIshlRcs/0ThTuLZkCVRL7f5VFiCSUiakv+NCLXgF3+icFjcuRk2E8bLqAo5Pi3UHD7N+MG83ofaQ96JumtVu3F9e5Pr9rg60Nf4sKUz3E/McTqSWUwGZk6y6O/qdYjOWXJj+K9Hg8hY2JS8m92iVdtGOgKmjaJELoT/OK0vv29K/aPzDDD/9Eh5lsIZjlP64qCZhTdJDRGIPK88KnWh+Fr0A43OJ4Jq2SRkSwNF2I2muTu2dgSaCxQt2YJWOHsK+zLE5WyvU6k8icG059j+jf2D9f9dJClZx/3J45FTv2j/K5HqyStjixwyHzqkmlGGep+11IgsVoEfKWa/cnXLSME0z6p1C4yx88KwhoZDXBjCOrDxU6mmRLirlI+UdT51+zZiIjU7qfU47/ELI2eKHTd+QXu3EtAi8unvmEmVlfLImIBMoxrzBEbG4wuNEayg04u9HSNMFYcSIfYgcc2cDuDIUXljY9rP4oXVcQgpvnuA== + secretKey: AgBZo+eq+sdI4UHnaOMpY2MybhXVIgDeWYxDX+19f7rKgOzGQmhiseQsLDtkz+H2JcskyPWpL0aJTtBgPc7p0cvPigxXVu3yHsYn0R5nTDKLsYUkDReSj/qhCGfb4Z0uUjmxtQK0qvPoT3zh6X0rZ4eWXehVCK7DtnlcVzraHSjSXeHF4hKB6fRt5KyTVClSAHD1X+qjX1MzPpbMYnx1+X/NQampn+kWzes31GOZXjjURKnEW3j0zcyz8fUd3WVD5dCjAndMl8FyfjoEuzU4v71AO7yL6MPn9GXZ5XX/7ssX7SpWGkD1YBVe+loQr77H+pVkSKNIP245QZHV0TQosXpJQXzFfSkvieSkQFqz7+iYxf7LzvsXrnq5I5m7VsBZ3oADpimQbEgdq9xEySTe7tP2z+TgJjf0VbYBcz3EVaKACT4O2BeUNT86TllQAmrcUX+4CjOoeIDHJUT5yMZq1YydyAjTqJS9dQ5Vhz1GIWYKBucGZfcmWDv+uGuRySUfvJRI3bEtb6RzzC8i1HHJ4ROzSTHwR9n9up2lsSA675E3pR0AYu3jWPBo3d1L/bL7yVJd7H1WO+a8SWYshQ30M800xK58DMkRFGcIS3QnzmYV1CFbRYJUqekTn9Qjkzn/gchtio4kFj704IadXyb4Au1ebJ1pSBhBeuAGjNifp9bZa8yqzxhAQj/FurCPzHErlH7altm7ddgWBKxjlLnnc6TujKk41j3iO4hoRP7zxY7Xzw== + template: + metadata: + annotations: + meta.helm.sh/release-name: milvus + meta.helm.sh/release-namespace: datamate + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: Helm + name: milvus-minio-secret + namespace: datamate + type: Opaque diff --git a/docs/SECRETS_SETUP.md b/docs/SECRETS_SETUP.md new file mode 100644 index 000000000..648e419d2 --- /dev/null +++ b/docs/SECRETS_SETUP.md @@ -0,0 +1,166 @@ +# Secrets Management Setup Guide + +This document describes the tools and configuration required for managing encrypted secrets in DataMate. + +## Required Tools + +### Homebrew Packages (macOS) + +| Tool | Version | Purpose | Install Command | +|------|---------|---------|-----------------| +| age | 1.3.1 | Encryption tool for secrets | `brew install age` | +| sops | 3.13.1 | Secrets encryption manager | `brew install sops` | +| helm | 4.1.0+ | Kubernetes package manager | `brew install helm` | + +### Helm Plugins + +| Plugin | Version | Purpose | Install Method | +|--------|---------|---------|----------------| +| helm-secrets | 4.6.0 | Helm integration with sops | Manual clone (see below) | + +**Note:** Helm 4.x has compatibility issues with helm-secrets plugin. Use manual installation: + +```bash +cd /Users/macoo/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +If plugin fails to load, edit `plugin.yaml` to remove the `command:` field and use only `platformCommand:`. + +## Project Configuration Files + +| File | Purpose | Location | +|------|---------|----------| +| `.sops.yaml` | SOPS encryption rules | Project root | +| `.sops-keys/key.txt` | Age encryption key (private) | Project root (gitignored) | +| `secrets.yaml` | Encrypted secrets per chart | `deployment/helm//` | + +## Quick Setup + +### 1. Install Required Tools + +```bash +# macOS (Homebrew) +brew install age sops helm + +# Helm plugin (manual) +cd ~/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +### 2. Generate Age Key (if not exists) + +```bash +mkdir -p .sops-keys +age-keygen -o .sops-keys/key.txt +``` + +**Important:** Backup the key file securely. It cannot be recovered if lost. + +### 3. Update `.sops.yaml` with Your Public Key + +```yaml +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: +``` + +### 4. Create/Encrypt Secrets Files + +```bash +# Set environment variable +export SOPS_AGE_KEY_FILE=.sops-keys/key.txt + +# Encrypt existing file +sops --encrypt --in-place deployment/helm/datamate/secrets.yaml + +# Or create new encrypted file +sops deployment/helm/datamate/secrets.yaml +``` + +## Usage + +### View Decrypted Secrets + +```bash +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml +``` + +### Deploy with Secrets + +```bash +# Using helper script +./scripts/secrets.sh helm-install datamate datamate + +# Manual method +TMP_FILE=$(mktemp) +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml > $TMP_FILE +helm install datamate deployment/helm/datamate -n datamate -f $TMP_FILE +rm $TMP_FILE +``` + +### Docker Compose + +```bash +# Copy and edit .env +cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env + +# Edit with your secrets +vim deployment/docker/datamate/.env + +# Deploy +docker compose -f deployment/docker/datamate/docker-compose.yml up -d +``` + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `SOPS_AGE_KEY_FILE` | Path to age private key | +| `SOPS_CONFIG` | Path to .sops.yaml (optional) | + +## Security Notes + +1. **Never commit `.sops-keys/` directory** - It's in `.gitignore` +2. **Backup your age key** - Store in secure location (password manager, encrypted backup) +3. **Use strong passwords** - Generate with `openssl rand -base64 32` +4. **Rotate keys periodically** - Especially after team changes + +## Troubleshooting + +### Plugin Load Error + +If helm plugin shows "both platformCommand and command are set": +```bash +# Edit plugin.yaml +vim ~/Library/helm/plugins/helm-secrets/plugin.yaml +# Remove the "command:" line, keep only "platformCommand:" +``` + +### Decryption Failed + +```bash +# Check key file exists +ls -la .sops-keys/key.txt + +# Verify key matches encrypted file's recipient +grep "recipient:" deployment/helm/datamate/secrets.yaml +``` + +### Key Rotation + +```bash +# Generate new key +age-keygen -o .sops-keys/key-new.txt + +# Update .sops.yaml with new public key + +# Re-encrypt all secrets +for f in deployment/helm/*/secrets.yaml; do + SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt $f > /tmp/plain.yaml + SOPS_AGE_KEY_FILE=.sops-keys/key-new.txt sops --encrypt /tmp/plain.yaml > $f +done + +# Replace old key +mv .sops-keys/key-new.txt .sops-keys/key.txt +``` \ No newline at end of file diff --git a/runtime/datamate-python/app/core/config.py b/runtime/datamate-python/app/core/config.py index 5f6d3ae55..ed49fcf02 100644 --- a/runtime/datamate-python/app/core/config.py +++ b/runtime/datamate-python/app/core/config.py @@ -33,14 +33,14 @@ class Config: pgsql_host: str = "datamate-database" pgsql_port: int = 5432 pgsql_user: str = "postgres" - pgsql_password: str = "password" + pgsql_password: str = "" pgsql_database: str = "datamate" # Database mysql_host: str = "datamate-database" mysql_port: int = 3306 mysql_user: str = "root" - mysql_password: str = "password" + mysql_password: str = "" mysql_database: str = "datamate" database_url: str = "" # Will be overridden by build_database_url() if not provided @@ -63,9 +63,9 @@ def build_database_url(self): # Label Studio label_studio_base_url: str = "http://label-studio:8000" - label_studio_username: Optional[str] = "admin@demo.com" - label_studio_password: Optional[str] = "demoadmin" - label_studio_user_token: Optional[str] = "abc123abc123" # Legacy Token + label_studio_username: Optional[str] = None + label_studio_password: Optional[str] = None + label_studio_user_token: Optional[str] = None # Legacy Token label_studio_local_document_root: str = "/label-studio/local" # Label Studio local file storage path label_studio_file_path_prefix: str = "/data/local-files/?d=" # Label Studio local file serving URL prefix diff --git a/scripts/db/data-annotation-init.sql b/scripts/db/data-annotation-init.sql index c83eb8822..43709882c 100644 --- a/scripts/db/data-annotation-init.sql +++ b/scripts/db/data-annotation-init.sql @@ -443,4 +443,9 @@ VALUES ('tpl-image-classification-001', '图像分类', '简单的多标签图 category = EXCLUDED.category, built_in = EXCLUDED.built_in, version = EXCLUDED.version, + created_by = COALESCE(EXCLUDED.created_by, 'system'), + updated_by = COALESCE(EXCLUDED.updated_by, 'system'), updated_at = CURRENT_TIMESTAMP; + +-- Ensure all annotation templates have created_by/updated_by set for data scope filtering +UPDATE t_dm_annotation_templates SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = ''; diff --git a/scripts/db/data-cleaning-init.sql b/scripts/db/data-cleaning-init.sql index 58b7add1a..7b946b31c 100644 --- a/scripts/db/data-cleaning-init.sql +++ b/scripts/db/data-cleaning-init.sql @@ -128,15 +128,18 @@ CREATE TRIGGER update_clean_template_updated_at EXECUTE FUNCTION update_updated_at_column(); -- 插入初始数据 - 清洗模板 -INSERT INTO t_clean_template (id, name, description) +INSERT INTO t_clean_template (id, name, description, created_by, updated_by) VALUES - ('550e8400-e29b-41d4-a716-446655440001', '安全与隐私合规处理模板', '针对敏感数据进行严格清洗,移除PII(个人身份信息)、政治敏感、暴力色情内容,适用于模型对外发布前的安全合规检查。'), - ('661f9500-f3ac-52e5-b827-557766550002', 'LLM SFT高质量文本清洗模板', '旨在生成高质量、低噪声的训练数据。包含去除乱码、重复内容、繁简转换、全角转半角以及格式标准化处理。'), - ('772a0611-a4bd-63f6-c938-668877660003', 'RAG知识库构建预处理模板', '专为RAG场景设计。重点去除目录、图注、XML/HTML标签等对向量检索无意义的噪声,并进行段落级去重以优化切片质量。'), - ('883b1722-b5ce-7407-d049-779988770004', '原始Web爬虫数据清洗模板', '针对互联网爬取的脏数据进行清洗。重点去除Emoji表情、URL链接、HTML标签以及不可见字符。'), - ('994c2833-c6df-8518-e150-880099880005', '多模态/CV模型训练预处理模板', '针对图像数据集处理。包含去除模糊/重复/相似图片,图片方向校正,目标检测预标注,以及尺寸和格式的统一化。') + ('550e8400-e29b-41d4-a716-446655440001', '安全与隐私合规处理模板', '针对敏感数据进行严格清洗,移除PII(个人身份信息)、政治敏感、暴力色情内容,适用于模型对外发布前的安全合规检查。', 'system', 'system'), + ('661f9500-f3ac-52e5-b827-557766550002', 'LLM SFT高质量文本清洗模板', '旨在生成高质量、低噪声的训练数据。包含去除乱码、重复内容、繁简转换、全角转半角以及格式标准化处理。', 'system', 'system'), + ('772a0611-a4bd-63f6-c938-668877660003', 'RAG知识库构建预处理模板', '专为RAG场景设计。重点去除目录、图注、XML/HTML标签等对向量检索无意义的噪声,并进行段落级去重以优化切片质量。', 'system', 'system'), + ('883b1722-b5ce-7407-d049-779988770004', '原始Web爬虫数据清洗模板', '针对互联网爬取的脏数据进行清洗。重点去除Emoji表情、URL链接、HTML标签以及不可见字符。', 'system', 'system'), + ('994c2833-c6df-8518-e150-880099880005', '多模态/CV模型训练预处理模板', '针对图像数据集处理。包含去除模糊/重复/相似图片,图片方向校正,目标检测预标注,以及尺寸和格式的统一化。', 'system', 'system') ON CONFLICT (id) DO NOTHING; +-- Also update existing templates that may have empty created_by +UPDATE t_clean_template SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = ''; + INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override) VALUES ('550e8400-e29b-41d4-a716-446655440001', 'PoliticalWordCleaner', 1, NULL), diff --git a/scripts/db/data-operator-init.sql b/scripts/db/data-operator-init.sql index 4609057f4..2a14bf0eb 100644 --- a/scripts/db/data-operator-init.sql +++ b/scripts/db/data-operator-init.sql @@ -521,3 +521,6 @@ WHERE c.id IN ('4d7dbd77-0a92-44f3-9056-2cd62d4a71e4', '9eda9d5d-072b-499b-916c- 'video_captioning_from_video_mapper', 'video_captioning_from_vlm_mapper', 'video_extract_frames_mapper', 'video_split_by_duration_mapper', 'video_split_by_key_frame_mapper', 'video_split_by_scene_mapper') ON CONFLICT DO NOTHING; + +-- Ensure all operator categories have created_by/updated_by set for data scope filtering +UPDATE t_operator_category SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = ''; diff --git a/scripts/images/frontend/start.sh b/scripts/images/frontend/start.sh index 6102495fd..d61306fca 100644 --- a/scripts/images/frontend/start.sh +++ b/scripts/images/frontend/start.sh @@ -18,14 +18,17 @@ if [ -f "/cert/server.pem" ]; then fi if [ -f "/cert/server.key" ]; then - # Check if key is encrypted and decrypt if needed - # Supports RSA, EC (Elliptic Curve), PKCS#8, and DSA keys + # Private key MUST be encrypted. Plaintext keys are not accepted. if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then - # Key is encrypted, decrypt using generic pkey command (supports all key types) + if [ -z "$CERT_PASS" ]; then + echo "ERROR: CERT_PASS is required to decrypt the private key" + exit 1 + fi echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin + chmod 600 /etc/nginx/cert/server.key else - # Key is not encrypted, copy directly - cp /cert/server.key /etc/nginx/cert/server.key + echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment." + exit 1 fi chown nginx:nginx /etc/nginx/cert/server.key fi