From 54bd91b17f9eca9498c4f12dd10e20ffa8acd1ba Mon Sep 17 00:00:00 2001 From: nfebe Date: Sun, 2 Aug 2026 02:35:13 +0100 Subject: [PATCH 1/9] feat(object-stores): Add self-hosted object stores with auto-registration Object stores are now a first-class resource. A self-hosted, S3-compatible store (MinIO) can be deployed straight from the Object Stores page and connected automatically: FlatRun issues the store's credentials, creates its bucket, and registers it as a backup target. An already-running deployment can instead be connected as a store by supplying its S3 details. Self-hosted stores run as the host user so their data stays manageable without root, and they join a shared storage network (the database network by default, or a dedicated one) so the agent and apps can reach them by address. Deploying an app from the marketplace now hands off to the standard deployment flow rather than a separate one-field install. --- internal/api/backup_destinations.go | 41 ++++- internal/api/object_store_list_test.go | 68 +++++++ internal/api/object_stores.go | 244 +++++++++++++++++++++++++ internal/api/object_stores_test.go | 97 ++++++++++ internal/api/server.go | 90 ++++++--- internal/backup/store_s3.go | 21 +++ internal/docker/api.go | 36 ++++ internal/docker/compose.go | 10 +- internal/docker/manager.go | 12 ++ pkg/config/config.go | 15 +- templates/minio/docker-compose.yml | 6 +- templates/minio/metadata.yml | 8 +- templates/templates.go | 1 + 13 files changed, 617 insertions(+), 32 deletions(-) create mode 100644 internal/api/object_store_list_test.go create mode 100644 internal/api/object_stores.go diff --git a/internal/api/backup_destinations.go b/internal/api/backup_destinations.go index 0be4b45..88da8ea 100644 --- a/internal/api/backup_destinations.go +++ b/internal/api/backup_destinations.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "net/http" + "net/url" "strings" "github.com/flatrun/agent/internal/backup" @@ -13,6 +14,12 @@ import ( "github.com/gin-gonic/gin" ) +// managedStorePort is the fallback S3 API port used when a managed +// destination's stored endpoint carries none. A managed object store is reached +// over the shared object-storage network (see objectStorageNetworkName); the +// agent, a host process, dials the container's address on that network. +const managedStorePort = "9000" + // buildStore resolves a destination plus its referenced credential into a // ready-to-use remote Store. func (s *Server) buildStore(dest config.BackupDestination) (backup.Store, error) { @@ -25,9 +32,15 @@ func (s *Server) buildStore(dest config.BackupDestination) (backup.Store, error) if cred.Kind != models.CredentialKindS3 { return nil, fmt.Errorf("destination %q: credential %q is not an s3 credential", dest.Name, dest.CredentialID) } + endpoint := dest.Endpoint + if dest.Kind == "managed" && dest.Deployment != "" { + if resolved, err := s.resolveManagedEndpoint(dest); err == nil && resolved != "" { + endpoint = resolved + } + } return backup.NewS3Store(backup.S3Config{ Name: dest.Name, - Endpoint: dest.Endpoint, + Endpoint: endpoint, Region: dest.Region, Bucket: dest.Bucket, Prefix: dest.Prefix, @@ -40,6 +53,32 @@ func (s *Server) buildStore(dest config.BackupDestination) (backup.Store, error) } } +// resolveManagedEndpoint returns the address a managed store is reachable at +// right now. A managed object store only exposes its port on an internal +// compose network and its container IP changes across recreates, so the +// endpoint is resolved live from the deployment rather than trusted from +// stored config. The scheme and port of the stored endpoint are preserved. +func (s *Server) resolveManagedEndpoint(dest config.BackupDestination) (string, error) { + if s.manager == nil { + return "", fmt.Errorf("docker manager unavailable") + } + ip, err := s.manager.ContainerPrimaryIP(dest.Deployment, objectStorageNetworkName(s.config)) + if err != nil { + return "", err + } + + u, err := url.Parse(dest.Endpoint) + if err != nil || u.Host == "" { + return "http://" + ip + ":" + managedStorePort, nil + } + port := u.Port() + if port == "" { + port = managedStorePort + } + u.Host = ip + ":" + port + return u.String(), nil +} + // applyBackupDestinations rebuilds the backup manager's remote stores from the // current config. It is called at startup and by the runtime applier when the // destinations config changes. diff --git a/internal/api/object_store_list_test.go b/internal/api/object_store_list_test.go new file mode 100644 index 0000000..d04d433 --- /dev/null +++ b/internal/api/object_store_list_test.go @@ -0,0 +1,68 @@ +package api + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/flatrun/agent/pkg/config" + "github.com/gin-gonic/gin" +) + +// The object store template must carry its declared S3 bootstrap contract +// through the list API, which is what the picker forwards to auto-register it. +func TestListTemplates_ObjectStoreCarriesContract(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := &Server{config: &config.Config{DeploymentsPath: t.TempDir()}} + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest("GET", "/templates", nil) + srv.listTemplates(c) + + var resp struct { + Templates []Template `json:"templates"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + var minio *Template + for i := range resp.Templates { + if resp.Templates[i].ID == "minio" { + minio = &resp.Templates[i] + } + } + if minio == nil { + t.Fatal("minio template missing from type=all listing") + } + if minio.ObjectStore == nil { + t.Fatal("minio template does not carry an object_store contract") + } + if minio.ObjectStore.APIPort != 9000 || minio.ObjectStore.AccessKeyEnv != "MINIO_ROOT_USER" { + t.Fatalf("unexpected contract: %+v", *minio.ObjectStore) + } +} + +// A storage template is a normal, standalone template: it must appear in the +// default catalog listing, not only under type=all. +func TestListTemplates_ObjectStoreVisibleInDefault(t *testing.T) { + gin.SetMode(gin.TestMode) + srv := &Server{config: &config.Config{DeploymentsPath: t.TempDir()}} + + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest("GET", "/templates", nil) + srv.listTemplates(c) + + var resp struct { + Templates []Template `json:"templates"` + } + _ = json.Unmarshal(w.Body.Bytes(), &resp) + for _, tpl := range resp.Templates { + if tpl.ID == "minio" { + return + } + } + t.Fatal("minio (storage category) should appear in the default listing") +} diff --git a/internal/api/object_stores.go b/internal/api/object_stores.go new file mode 100644 index 0000000..93420f7 --- /dev/null +++ b/internal/api/object_stores.go @@ -0,0 +1,244 @@ +package api + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/flatrun/agent/internal/backup" + "github.com/flatrun/agent/pkg/config" + "github.com/flatrun/agent/pkg/models" + "github.com/gin-gonic/gin" +) + +const ( + defaultManagedBucket = "backups" + defaultManagedRegion = "us-east-1" + managedProvisionAttempts = 20 + managedProvisionInterval = 500 * time.Millisecond +) + +// provisionManagedObjectStoreRequest carries the S3 bootstrap contract for a +// deployment FlatRun runs. Credentials come either literally (a custom store +// whose secrets the caller knows) or by naming the deployment env vars that +// hold them (a template store, where the UI forwards the template's declared +// object_store contract). The agent stays free of any per-image knowledge. +type provisionManagedObjectStoreRequest struct { + Deployment string `json:"deployment" binding:"required"` + StoreName string `json:"store_name"` + Bucket string `json:"bucket"` + + AccessKeyEnv string `json:"access_key_env"` + SecretKeyEnv string `json:"secret_key_env"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + + APIPort int `json:"api_port"` + Region string `json:"region"` + UsePathStyle *bool `json:"use_path_style"` +} + +// provisionManagedObjectStore turns a deployment FlatRun runs into a connected +// store: it resolves the store's S3 credentials, stores a credential, ensures +// the bucket exists, and registers a managed backup destination pointing at the +// deployment. It is the auto-register step behind the "Deploy a local store" +// and "use an existing deployment" flows. +func (s *Server) provisionManagedObjectStore(c *gin.Context) { + if s.backupManager == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Backup manager not enabled"}) + return + } + + var req provisionManagedObjectStoreRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if req.APIPort <= 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "api_port is required"}) + return + } + + accessKey, secretKey, err := s.resolveStoreCredentials(req) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + storeName := strings.TrimSpace(req.StoreName) + if storeName == "" { + storeName = req.Deployment + } + bucket := strings.TrimSpace(req.Bucket) + if bucket == "" { + bucket = defaultManagedBucket + } + region := strings.TrimSpace(req.Region) + if region == "" { + region = defaultManagedRegion + } + usePathStyle := true + if req.UsePathStyle != nil { + usePathStyle = *req.UsePathStyle + } + + // A create returns before the store finishes starting, so wait for the + // container to become reachable before making the bucket on it. + endpoint, err := s.waitForManagedEndpoint(req.Deployment, req.APIPort) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("object store %q is not reachable yet: %v", req.Deployment, err)}) + return + } + + cred, err := s.credentialsManager.CreateGenericCredential(storeName+"-keys", models.CredentialKindS3, map[string]string{ + "access_key_id": accessKey, + "secret_access_key": secretKey, + }) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + enabled := true + dest := config.BackupDestination{ + Name: storeName, + Type: "s3", + Kind: "managed", + Deployment: req.Deployment, + Endpoint: endpoint, + Region: region, + Bucket: bucket, + CredentialID: cred.ID, + UsePathStyle: usePathStyle, + Enabled: &enabled, + } + + if err := s.ensureManagedBucket(c, dest); err != nil { + _ = s.credentialsManager.DeleteGenericCredential(cred.ID) + c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("could not create bucket on %q: %v", storeName, err)}) + return + } + + updated := append(append([]config.BackupDestination{}, s.config.Backup.Destinations...), dest) + outcome, err := s.applyConfigUpdate("backup.destinations", updated) + if err != nil { + _ = s.credentialsManager.DeleteGenericCredential(cred.ID) + respondAPIError(c, err) + return + } + + resp := gin.H{ + "message": "Managed object store connected", + "destination": dest, + "credential": cred, + "applied": outcome.Applied, + } + if outcome.ApplyErr != nil { + resp["apply_error"] = outcome.ApplyErr.Error() + } + c.JSON(http.StatusCreated, resp) +} + +// resolveStoreCredentials returns the S3 access key and secret for a store, +// taken from the request when supplied directly, otherwise read from the +// deployment env vars the request names. +func (s *Server) resolveStoreCredentials(req provisionManagedObjectStoreRequest) (string, string, error) { + if req.AccessKey != "" && req.SecretKey != "" { + return req.AccessKey, req.SecretKey, nil + } + if req.AccessKeyEnv == "" || req.SecretKeyEnv == "" { + return "", "", fmt.Errorf("provide S3 credentials, or the env keys that hold them") + } + + env, err := s.readDeploymentEnvMap(req.Deployment) + if err != nil { + return "", "", err + } + access, secret := env[req.AccessKeyEnv], env[req.SecretKeyEnv] + if access == "" || secret == "" { + return "", "", fmt.Errorf("object store %q has no S3 credentials in %s / %s", req.Deployment, req.AccessKeyEnv, req.SecretKeyEnv) + } + return access, secret, nil +} + +// readDeploymentEnvMap reads a deployment's generated env into a key/value map. +// A template writes its env to whatever file it declares (MinIO uses .env, +// others use .env.flatrun), so every env file in the deployment is read and +// merged rather than assuming one fixed name. The FlatRun-managed .env.flatrun +// wins where a key appears in more than one. +func (s *Server) readDeploymentEnvMap(name string) (map[string]string, error) { + dir := filepath.Join(s.config.DeploymentsPath, name) + matches, _ := filepath.Glob(filepath.Join(dir, ".env*")) + + env := make(map[string]string) + found := false + for _, path := range matches { + base := filepath.Base(path) + if strings.HasSuffix(base, ".example") || strings.HasSuffix(base, ".sample") { + continue + } + content, err := os.ReadFile(path) + if err != nil { + continue + } + found = true + for _, v := range parseEnvContent(string(content)) { + if base == ".env.flatrun" || env[v.Key] == "" { + env[v.Key] = v.Value + } + } + } + if !found { + return nil, fmt.Errorf("could not read object store environment in %s", dir) + } + return env, nil +} + +// waitForManagedEndpoint polls for the deployment's container to become +// reachable and returns the address its S3 API is served on. +func (s *Server) waitForManagedEndpoint(deployment string, apiPort int) (string, error) { + if s.manager == nil { + return "", fmt.Errorf("docker manager unavailable") + } + var lastErr error + for i := 0; i < managedProvisionAttempts; i++ { + ip, err := s.manager.ContainerPrimaryIP(deployment, objectStorageNetworkName(s.config)) + if err == nil && ip != "" { + return fmt.Sprintf("http://%s:%d", ip, apiPort), nil + } + lastErr = err + time.Sleep(managedProvisionInterval) + } + if lastErr == nil { + lastErr = fmt.Errorf("container did not report an address") + } + return "", lastErr +} + +// ensureManagedBucket makes the destination's bucket if it is missing. The +// container reports an address a moment before its S3 API is actually serving, +// so the bucket call is retried over a short warmup window. +func (s *Server) ensureManagedBucket(c *gin.Context, dest config.BackupDestination) error { + store, err := s.buildStore(dest) + if err != nil { + return err + } + s3store, ok := store.(*backup.S3Store) + if !ok { + return nil + } + + var lastErr error + for i := 0; i < managedProvisionAttempts; i++ { + if err := s3store.EnsureBucket(c.Request.Context()); err == nil { + return nil + } else { + lastErr = err + } + time.Sleep(managedProvisionInterval) + } + return lastErr +} diff --git a/internal/api/object_stores_test.go b/internal/api/object_stores_test.go index 19b9906..aaccfb0 100644 --- a/internal/api/object_stores_test.go +++ b/internal/api/object_stores_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "testing" "github.com/flatrun/agent/internal/auth" @@ -65,6 +66,7 @@ func setupObjectStoreTestServer(t *testing.T) (*Server, *gin.Engine, func()) { protected.PUT("/storage-credentials/:id", mw.RequirePermission(auth.PermBackupsWrite), server.updateStorageCredential) protected.DELETE("/storage-credentials/:id", mw.RequirePermission(auth.PermBackupsDelete), server.deleteStorageCredential) protected.GET("/backup-destinations", mw.RequirePermission(auth.PermBackupsRead), server.listBackupDestinations) + protected.POST("/object-stores/provision-managed", mw.RequirePermission(auth.PermBackupsWrite), server.provisionManagedObjectStore) cleanup := func() { authManager.Close() @@ -178,6 +180,101 @@ func TestStorageCredential_DeleteInUseConflicts(t *testing.T) { } } +func TestProvisionManagedObjectStore_RequiresApiPort(t *testing.T) { + _, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + res := osReq(t, router, http.MethodPost, "/api/object-stores/provision-managed", token, map[string]any{ + "deployment": "some-store", + }) + if res.Code != http.StatusBadRequest { + t.Fatalf("expected 400 without an api_port, got %d %s", res.Code, res.Body.String()) + } +} + +func TestProvisionManagedObjectStore_MissingEnvReturns400(t *testing.T) { + _, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + res := osReq(t, router, http.MethodPost, "/api/object-stores/provision-managed", token, map[string]any{ + "deployment": "no-such-store", + "access_key_env": "MINIO_ROOT_USER", + "secret_key_env": "MINIO_ROOT_PASSWORD", + "api_port": 9000, + }) + if res.Code != http.StatusBadRequest { + t.Fatalf("expected 400 for a deployment with no environment, got %d %s", res.Code, res.Body.String()) + } +} + +// Credentials must be read from whatever env file the template wrote. The MinIO +// template writes .env (not .env.flatrun), so a deployment with only a .env must +// still resolve credentials (getting past resolution to the reachability step, +// 502, rather than failing at 400 for missing env). +func TestProvisionManagedObjectStore_ReadsPlainEnvFile(t *testing.T) { + server, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + deployment := "minio-store" + deployDir := filepath.Join(server.config.DeploymentsPath, deployment) + if err := os.MkdirAll(deployDir, 0o755); err != nil { + t.Fatalf("mkdir deployment: %v", err) + } + env := "MINIO_ROOT_USER=flatrun\nMINIO_ROOT_PASSWORD=generated-secret\n" + if err := os.WriteFile(filepath.Join(deployDir, ".env"), []byte(env), 0o600); err != nil { + t.Fatalf("write env: %v", err) + } + + res := osReq(t, router, http.MethodPost, "/api/object-stores/provision-managed", token, map[string]any{ + "deployment": deployment, + "access_key_env": "MINIO_ROOT_USER", + "secret_key_env": "MINIO_ROOT_PASSWORD", + "api_port": 9000, + }) + if res.Code != http.StatusBadGateway { + t.Fatalf("expected 502 (creds resolved from .env, store unreachable), got %d %s", res.Code, res.Body.String()) + } +} + +// A store whose credentials resolve but that is not reachable (no docker in the +// unit environment) must fail before any credential or destination is created, +// so a failed provision leaves no orphaned state behind. This holds for the +// template path (credentials read from named env vars). +func TestProvisionManagedObjectStore_UnreachableLeavesNoState(t *testing.T) { + server, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + deployment := "my-store" + deployDir := filepath.Join(server.config.DeploymentsPath, deployment) + if err := os.MkdirAll(deployDir, 0o755); err != nil { + t.Fatalf("mkdir deployment: %v", err) + } + env := "MINIO_ROOT_USER=flatrun\nMINIO_ROOT_PASSWORD=generated-secret\n" + if err := os.WriteFile(filepath.Join(deployDir, ".env.flatrun"), []byte(env), 0o600); err != nil { + t.Fatalf("write env: %v", err) + } + + res := osReq(t, router, http.MethodPost, "/api/object-stores/provision-managed", token, map[string]any{ + "deployment": deployment, + "access_key_env": "MINIO_ROOT_USER", + "secret_key_env": "MINIO_ROOT_PASSWORD", + "api_port": 9000, + }) + if res.Code != http.StatusBadGateway { + t.Fatalf("expected 502 when the store is unreachable, got %d %s", res.Code, res.Body.String()) + } + if creds := server.credentialsManager.ListGenericCredentials(models.CredentialKindS3); len(creds) != 0 { + t.Fatalf("failed provision left a credential behind: %#v", creds) + } + if len(server.config.Backup.Destinations) != 0 { + t.Fatalf("failed provision left a destination behind: %#v", server.config.Backup.Destinations) + } +} + func TestBackupDestinations_ListReportsKind(t *testing.T) { server, router, cleanup := setupObjectStoreTestServer(t) defer cleanup() diff --git a/internal/api/server.go b/internal/api/server.go index a24f6ff..9be07b1 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -118,6 +118,17 @@ type Server struct { statsAt time.Time } +// objectStorageNetworkName resolves the shared network self-hosted object +// stores join. It reuses the database network unless a dedicated one is +// configured, so object storage rides the same data-backend network as +// databases by default. +func objectStorageNetworkName(cfg *config.Config) string { + if n := cfg.Infrastructure.DefaultObjectStorageNetwork; n != "" { + return n + } + return cfg.Infrastructure.DefaultDatabaseNetwork +} + func New(cfg *config.Config, configPath string) *Server { if cfg.Logging.Level == "debug" { gin.SetMode(gin.DebugMode) @@ -125,6 +136,10 @@ func New(cfg *config.Config, configPath string) *Server { gin.SetMode(gin.ReleaseMode) } + // Exposed to compose substitution so an object-store template joins the + // configured network (${FLATRUN_OBJECT_NETWORK:-database}) on every up. + os.Setenv("FLATRUN_OBJECT_NETWORK", objectStorageNetworkName(cfg)) + router := gin.Default() if cfg.API.EnableCORS { @@ -733,6 +748,9 @@ func (s *Server) setupRoutes() { protected.GET("/backup-destinations", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listBackupDestinations) protected.POST("/backup-destinations/test", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.testBackupDestination) + // Object stores + protected.POST("/object-stores/provision-managed", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.provisionManagedObjectStore) + // Scheduler endpoints protected.GET("/scheduler/tasks", s.authMiddleware.RequirePermission(auth.PermSchedulerRead), s.listScheduledTasks) protected.GET("/scheduler/tasks/:id", s.authMiddleware.RequirePermission(auth.PermSchedulerRead), s.getScheduledTask) @@ -1197,6 +1215,19 @@ func (s *Server) createDeployment(c *gin.Context) { s.processTemplateFiles(req.Name, req.TemplateID, allEnvVars) s.processTemplateEnv(req.Name, req.TemplateID, req.ComposeContent, allEnvVars) s.applyTemplateMountOwnership(req.Name, req.TemplateID) + + // An object store joins the shared object-storage network; ensure it + // exists so the container can start (it is declared external). + if md, err := s.loadTemplateMetadata(req.TemplateID); err == nil && md.ObjectStore != nil { + if objNet := objectStorageNetworkName(s.config); objNet != "" && s.networksManager != nil { + if err := s.networksManager.EnsureNetwork(objNet); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "error": fmt.Sprintf("Failed to ensure object storage network %q exists: %v", objNet, err), + }) + return + } + } + } } // Seeding reads the images, so it runs before the deployment is started and @@ -3341,17 +3372,30 @@ type TemplateFile struct { } type TemplateMetadata struct { - Name string `yaml:"name"` - Description string `yaml:"description"` - Icon string `yaml:"icon"` - Logo string `yaml:"logo"` - Category string `yaml:"category"` - Type string `yaml:"type"` - Priority int `yaml:"priority"` - ContainerPort int `yaml:"container_port"` - Mounts []TemplateMount `yaml:"mounts"` - Files []TemplateFile `yaml:"files"` - Env TemplateEnv `yaml:"env"` + Name string `yaml:"name"` + Description string `yaml:"description"` + Icon string `yaml:"icon"` + Logo string `yaml:"logo"` + Category string `yaml:"category"` + Type string `yaml:"type"` + ObjectStore *TemplateObjectStore `yaml:"object_store"` + Priority int `yaml:"priority"` + ContainerPort int `yaml:"container_port"` + Mounts []TemplateMount `yaml:"mounts"` + Files []TemplateFile `yaml:"files"` + Env TemplateEnv `yaml:"env"` +} + +// TemplateObjectStore is a template's declaration that it runs an S3-compatible +// object store, and how to bootstrap it: which generated env vars carry the +// root credentials and which port serves the S3 API. Declaring this keeps the +// agent free of any per-image (MinIO, Garage, ...) knowledge. +type TemplateObjectStore struct { + AccessKeyEnv string `json:"access_key_env" yaml:"access_key_env"` + SecretKeyEnv string `json:"secret_key_env" yaml:"secret_key_env"` + APIPort int `json:"api_port" yaml:"api_port"` + Region string `json:"region,omitempty" yaml:"region,omitempty"` + UsePathStyle bool `json:"use_path_style" yaml:"use_path_style"` } // TemplateEnv describes how a platform's environment file is produced. The @@ -3398,17 +3442,18 @@ func templateMountHostPath(m TemplateMount) string { } type Template struct { - ID string `json:"id"` - Name string `json:"name" yaml:"name"` - Description string `json:"description" yaml:"description"` - Icon string `json:"icon" yaml:"icon"` - Logo string `json:"logo" yaml:"logo"` - Category string `json:"category" yaml:"category"` - Priority int `json:"priority" yaml:"priority"` - ContainerPort int `json:"container_port" yaml:"container_port"` - Mounts []TemplateMount `json:"mounts" yaml:"mounts"` - Files []TemplateFile `json:"files" yaml:"files"` - Content string `json:"content"` + ID string `json:"id"` + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + Icon string `json:"icon" yaml:"icon"` + Logo string `json:"logo" yaml:"logo"` + Category string `json:"category" yaml:"category"` + ObjectStore *TemplateObjectStore `json:"object_store,omitempty" yaml:"object_store,omitempty"` + Priority int `json:"priority" yaml:"priority"` + ContainerPort int `json:"container_port" yaml:"container_port"` + Mounts []TemplateMount `json:"mounts" yaml:"mounts"` + Files []TemplateFile `json:"files" yaml:"files"` + Content string `json:"content"` } func (s *Server) listTemplates(c *gin.Context) { @@ -3491,6 +3536,7 @@ func (s *Server) listTemplates(c *gin.Context) { Icon: metadata.Icon, Logo: metadata.Logo, Category: metadata.Category, + ObjectStore: metadata.ObjectStore, Priority: metadata.Priority, ContainerPort: metadata.ContainerPort, Mounts: metadata.Mounts, diff --git a/internal/backup/store_s3.go b/internal/backup/store_s3.go index 046c20c..d6c2841 100644 --- a/internal/backup/store_s3.go +++ b/internal/backup/store_s3.go @@ -63,6 +63,27 @@ func NewS3Store(cfg S3Config) (*S3Store, error) { func (s *S3Store) Name() string { return s.cfg.Name } +// EnsureBucket creates the store's bucket when it does not already exist. A +// freshly deployed object store starts empty, so a managed store needs its +// bucket made before the first backup can be mirrored to it. +func (s *S3Store) EnsureBucket(ctx context.Context) error { + _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.cfg.Bucket)}) + if err == nil { + return nil + } + + _, err = s.client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.cfg.Bucket)}) + if err != nil { + var owned *s3types.BucketAlreadyOwnedByYou + var exists *s3types.BucketAlreadyExists + if errors.As(err, &owned) || errors.As(err, &exists) { + return nil + } + return fmt.Errorf("create bucket %q: %w", s.cfg.Bucket, err) + } + return nil +} + // fullKey prepends the destination prefix to a relative backup key. func (s *S3Store) fullKey(key string) string { if s.cfg.Prefix == "" { diff --git a/internal/docker/api.go b/internal/docker/api.go index d7b17bb..d28cd6e 100644 --- a/internal/docker/api.go +++ b/internal/docker/api.go @@ -58,6 +58,42 @@ func (a *APIClient) FindContainer(ctx context.Context, project, service string) return containers[0].ID, nil } +// ContainerPrimaryIP returns the IP address of a project's first running +// container on the given docker network. The agent runs on the host, so a +// service that only exposes ports on an internal compose network (a self-hosted +// object store, say) is reached by dialing the container's address directly. +// When network is empty, the first attached network with an address is used. +func (a *APIClient) ContainerPrimaryIP(ctx context.Context, project, network string) (string, error) { + f := filters.NewArgs( + filters.Arg("label", fmt.Sprintf("%s=%s", composeProjectLabel, project)), + filters.Arg("status", "running"), + ) + + containers, err := a.cli.ContainerList(ctx, container.ListOptions{Filters: f}) + if err != nil { + return "", fmt.Errorf("failed to list containers: %w", err) + } + if len(containers) == 0 { + return "", fmt.Errorf("no running container found for project %q", project) + } + + ns := containers[0].NetworkSettings + if ns == nil { + return "", fmt.Errorf("container for project %q has no network settings", project) + } + if network != "" { + if n, ok := ns.Networks[network]; ok && n.IPAddress != "" { + return n.IPAddress, nil + } + } + for _, n := range ns.Networks { + if n.IPAddress != "" { + return n.IPAddress, nil + } + } + return "", fmt.Errorf("container for project %q has no network address yet", project) +} + func (a *APIClient) ExecInContainer(ctx context.Context, containerID string, command string) (string, error) { execConfig := container.ExecOptions{ Cmd: []string{"sh", "-c", command}, diff --git a/internal/docker/compose.go b/internal/docker/compose.go index 0aa24de..f52cdcb 100644 --- a/internal/docker/compose.go +++ b/internal/docker/compose.go @@ -420,8 +420,16 @@ func (c *ComposeExecutor) runCompose(deploymentPath string, opts []RunOption, ar for _, opt := range opts { opt(&ro) } + + // Expose the agent's own uid/gid to compose substitution so a template can + // run its container as the user that owns the deployment directory, keeping + // bind-mounted data host-manageable (deletable) instead of root-owned. + cmd.Env = append(os.Environ(), + fmt.Sprintf("FLATRUN_UID=%d", os.Getuid()), + fmt.Sprintf("FLATRUN_GID=%d", os.Getgid()), + ) if len(ro.extraEnv) > 0 { - cmd.Env = append(os.Environ(), ro.extraEnv...) + cmd.Env = append(cmd.Env, ro.extraEnv...) } if ro.lineSink != nil { diff --git a/internal/docker/manager.go b/internal/docker/manager.go index 9755c3c..579a00c 100644 --- a/internal/docker/manager.go +++ b/internal/docker/manager.go @@ -54,6 +54,18 @@ func (m *Manager) indexContainersByProject(ctx context.Context) (containerIndex, return index, nil } +// ContainerPrimaryIP returns the first running container's address for a +// deployment on the given docker network. A flatrun deploy names its compose +// project after the deployment, so the project name is the deployment name. +func (m *Manager) ContainerPrimaryIP(project, network string) (string, error) { + if m.apiClient == nil { + return "", fmt.Errorf("docker api client unavailable") + } + ctx, cancel := context.WithTimeout(context.Background(), statusReadTimeout) + defer cancel() + return m.apiClient.ContainerPrimaryIP(ctx, project, network) +} + // projectFor resolves a deployment's compose project name without shelling out. // It mirrors ComposeExecutor.getProjectName, except that the fallback probe for // an existing project reads the already-fetched index instead of running diff --git a/pkg/config/config.go b/pkg/config/config.go index 5f05151..0a9ecdf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -180,11 +180,16 @@ type HealthConfig struct { } type InfrastructureConfig struct { - DefaultProxyNetwork string `yaml:"default_proxy_network" json:"default_proxy_network"` - DefaultDatabaseNetwork string `yaml:"default_database_network" json:"default_database_network"` - Database SharedDatabaseConfig `yaml:"database" json:"database"` - Redis SharedRedisConfig `yaml:"redis" json:"redis"` - PowerDNS PowerDNSConfig `yaml:"powerdns" json:"powerdns"` + DefaultProxyNetwork string `yaml:"default_proxy_network" json:"default_proxy_network"` + DefaultDatabaseNetwork string `yaml:"default_database_network" json:"default_database_network"` + // DefaultObjectStorageNetwork is the shared network self-hosted object + // stores join so apps can reach them by name. Empty reuses the database + // network (object storage is a data backend like a database); set it to run + // object stores on a dedicated network instead. + DefaultObjectStorageNetwork string `yaml:"default_object_storage_network" json:"default_object_storage_network"` + Database SharedDatabaseConfig `yaml:"database" json:"database"` + Redis SharedRedisConfig `yaml:"redis" json:"redis"` + PowerDNS PowerDNSConfig `yaml:"powerdns" json:"powerdns"` } type PowerDNSConfig struct { diff --git a/templates/minio/docker-compose.yml b/templates/minio/docker-compose.yml index 22d15ab..00cfc9d 100644 --- a/templates/minio/docker-compose.yml +++ b/templates/minio/docker-compose.yml @@ -3,6 +3,7 @@ services: minio: image: minio/minio:latest container_name: ${NAME} + user: "${FLATRUN_UID:-1000}:${FLATRUN_GID:-1000}" command: server /data --console-address ":9001" environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-flatrun} @@ -13,9 +14,10 @@ services: - "9000" - "9001" networks: - - proxy + - objstore restart: unless-stopped networks: - proxy: + objstore: external: true + name: ${FLATRUN_OBJECT_NETWORK:-database} diff --git a/templates/minio/metadata.yml b/templates/minio/metadata.yml index f1fb4f0..0285244 100644 --- a/templates/minio/metadata.yml +++ b/templates/minio/metadata.yml @@ -2,7 +2,13 @@ name: MinIO description: S3-compatible object storage you host yourself icon: pi pi-database logo: https://cdn.simpleicons.org/minio -category: infrastructure +category: storage +object_store: + access_key_env: MINIO_ROOT_USER + secret_key_env: MINIO_ROOT_PASSWORD + api_port: 9000 + region: us-east-1 + use_path_style: true priority: 68 container_port: 9000 mounts: diff --git a/templates/templates.go b/templates/templates.go index 5168682..4900b4e 100644 --- a/templates/templates.go +++ b/templates/templates.go @@ -24,6 +24,7 @@ var Categories = []Category{ {ID: "framework", Name: "Frameworks", Icon: "pi pi-code", Priority: 90}, {ID: "runtime", Name: "Runtimes", Icon: "pi pi-cog", Priority: 80}, {ID: "infrastructure", Name: "Infrastructure", Icon: "pi pi-server", Priority: 70}, + {ID: "storage", Name: "Storage", Icon: "pi pi-box", Priority: 65}, {ID: "database", Name: "Databases", Icon: "pi pi-database", Priority: 60}, {ID: "basic", Name: "Basic", Icon: "pi pi-file", Priority: 50}, } From f26c1de2e02424cefe03056cebfbac7e94bbc92d Mon Sep 17 00:00:00 2001 From: nfebe Date: Sun, 2 Aug 2026 02:42:57 +0100 Subject: [PATCH 2/9] feat(object-stores): Browse, upload, download and delete objects A connected store's full contents can now be listed and managed directly: every object in the store is shown (not just backup archives), and individual objects can be uploaded, downloaded, and deleted. --- internal/api/object_browser.go | 119 +++++++++++++++++++++++++++++ internal/api/object_stores_test.go | 15 ++++ internal/api/server.go | 4 + internal/backup/store_s3.go | 25 ++++++ 4 files changed, 163 insertions(+) create mode 100644 internal/api/object_browser.go diff --git a/internal/api/object_browser.go b/internal/api/object_browser.go new file mode 100644 index 0000000..91b7649 --- /dev/null +++ b/internal/api/object_browser.go @@ -0,0 +1,119 @@ +package api + +import ( + "fmt" + "net/http" + "path" + + "github.com/flatrun/agent/internal/backup" + "github.com/gin-gonic/gin" +) + +// storeS3ByName resolves a registered object store by name into a usable S3 +// client, writing an error response and returning false on failure. +func (s *Server) storeS3ByName(c *gin.Context) (*backup.S3Store, bool) { + dest, ok := s.findDestinationByName(c.Param("name")) + if !ok { + c.JSON(http.StatusNotFound, gin.H{"error": "object store not found"}) + return nil, false + } + store, err := s.buildStore(dest) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return nil, false + } + s3store, ok := store.(*backup.S3Store) + if !ok { + c.JSON(http.StatusBadRequest, gin.H{"error": "store is not S3-compatible"}) + return nil, false + } + return s3store, true +} + +// listStoreObjects returns every object in a store, optionally filtered by a +// key prefix. +func (s *Server) listStoreObjects(c *gin.Context) { + store, ok := s.storeS3ByName(c) + if !ok { + return + } + objects, err := store.ListObjects(c.Request.Context(), c.Query("prefix")) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + if objects == nil { + objects = []backup.ObjectInfo{} + } + c.JSON(http.StatusOK, gin.H{"objects": objects}) +} + +// uploadStoreObject stores an uploaded file at the given key (defaulting to the +// file's name). +func (s *Server) uploadStoreObject(c *gin.Context) { + store, ok := s.storeS3ByName(c) + if !ok { + return + } + fileHeader, err := c.FormFile("file") + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "file is required"}) + return + } + key := c.PostForm("key") + if key == "" { + key = fileHeader.Filename + } + f, err := fileHeader.Open() + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + defer f.Close() + + if err := store.Put(c.Request.Context(), key, f, fileHeader.Size); err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusCreated, gin.H{"message": "Object uploaded", "key": key}) +} + +// downloadStoreObject streams an object back to the caller as an attachment. +func (s *Server) downloadStoreObject(c *gin.Context) { + store, ok := s.storeS3ByName(c) + if !ok { + return + } + key := c.Query("key") + if key == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "key is required"}) + return + } + reader, err := store.Open(c.Request.Context(), key) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + defer reader.Close() + + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%q", path.Base(key))) + c.DataFromReader(http.StatusOK, -1, "application/octet-stream", reader, nil) +} + +// deleteStoreObject removes a single object from a store. +func (s *Server) deleteStoreObject(c *gin.Context) { + store, ok := s.storeS3ByName(c) + if !ok { + return + } + key := c.Query("key") + if key == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "key is required"}) + return + } + if err := store.Delete(c.Request.Context(), key); err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"message": "Object deleted", "key": key}) +} diff --git a/internal/api/object_stores_test.go b/internal/api/object_stores_test.go index aaccfb0..512c448 100644 --- a/internal/api/object_stores_test.go +++ b/internal/api/object_stores_test.go @@ -67,6 +67,10 @@ func setupObjectStoreTestServer(t *testing.T) (*Server, *gin.Engine, func()) { protected.DELETE("/storage-credentials/:id", mw.RequirePermission(auth.PermBackupsDelete), server.deleteStorageCredential) protected.GET("/backup-destinations", mw.RequirePermission(auth.PermBackupsRead), server.listBackupDestinations) protected.POST("/object-stores/provision-managed", mw.RequirePermission(auth.PermBackupsWrite), server.provisionManagedObjectStore) + protected.GET("/object-stores/:name/objects", mw.RequirePermission(auth.PermBackupsRead), server.listStoreObjects) + protected.POST("/object-stores/:name/objects", mw.RequirePermission(auth.PermBackupsWrite), server.uploadStoreObject) + protected.GET("/object-stores/:name/objects/download", mw.RequirePermission(auth.PermBackupsRead), server.downloadStoreObject) + protected.DELETE("/object-stores/:name/objects", mw.RequirePermission(auth.PermBackupsWrite), server.deleteStoreObject) cleanup := func() { authManager.Close() @@ -275,6 +279,17 @@ func TestProvisionManagedObjectStore_UnreachableLeavesNoState(t *testing.T) { } } +func TestListStoreObjects_UnknownStore404(t *testing.T) { + _, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + res := osReq(t, router, http.MethodGet, "/api/object-stores/nope/objects", token, nil) + if res.Code != http.StatusNotFound { + t.Fatalf("expected 404 for an unknown store, got %d %s", res.Code, res.Body.String()) + } +} + func TestBackupDestinations_ListReportsKind(t *testing.T) { server, router, cleanup := setupObjectStoreTestServer(t) defer cleanup() diff --git a/internal/api/server.go b/internal/api/server.go index 9be07b1..a564aaf 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -750,6 +750,10 @@ func (s *Server) setupRoutes() { // Object stores protected.POST("/object-stores/provision-managed", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.provisionManagedObjectStore) + protected.GET("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listStoreObjects) + protected.POST("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.uploadStoreObject) + protected.GET("/object-stores/:name/objects/download", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.downloadStoreObject) + protected.DELETE("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.deleteStoreObject) // Scheduler endpoints protected.GET("/scheduler/tasks", s.authMiddleware.RequirePermission(auth.PermSchedulerRead), s.listScheduledTasks) diff --git a/internal/backup/store_s3.go b/internal/backup/store_s3.go index d6c2841..382f14c 100644 --- a/internal/backup/store_s3.go +++ b/internal/backup/store_s3.go @@ -154,6 +154,31 @@ func (s *S3Store) List(ctx context.Context, prefix string) ([]ObjectInfo, error) return out, nil } +// ListObjects returns every object under prefix, unlike List which is scoped to +// backup archives (.tar.gz). It backs the object browser, where a store's full +// contents are shown. +func (s *S3Store) ListObjects(ctx context.Context, prefix string) ([]ObjectInfo, error) { + var out []ObjectInfo + paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.cfg.Bucket), + Prefix: aws.String(s.fullKey(prefix)), + }) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("s3 list %s: %w", prefix, err) + } + for _, obj := range page.Contents { + out = append(out, ObjectInfo{ + Key: s.relKey(aws.ToString(obj.Key)), + Size: aws.ToInt64(obj.Size), + ModTime: aws.ToTime(obj.LastModified), + }) + } + } + return out, nil +} + func (s *S3Store) Delete(ctx context.Context, key string) error { _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s.cfg.Bucket), From 088b258518256a4b897bd0dd80c4299bb101f355 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sun, 2 Aug 2026 02:50:21 +0100 Subject: [PATCH 3/9] feat(object-stores): Attach a store to a deployment's environment A store can now be attached to another deployment: its endpoint, bucket, region, credentials, and path-style flag are written into that deployment's environment under a chosen prefix, and the service is wired to load them. A self-hosted store is addressed by its in-cluster name and the app is joined to its network; an external store uses its public URL. The deployment is restarted to pick it up. --- internal/api/object_consume.go | 127 +++++++++++++++++++++++++++++ internal/api/object_stores_test.go | 51 ++++++++++++ internal/api/server.go | 1 + internal/docker/compose_yaml.go | 60 ++++++++++++++ 4 files changed, 239 insertions(+) create mode 100644 internal/api/object_consume.go diff --git a/internal/api/object_consume.go b/internal/api/object_consume.go new file mode 100644 index 0000000..7c542bc --- /dev/null +++ b/internal/api/object_consume.go @@ -0,0 +1,127 @@ +package api + +import ( + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + + "github.com/flatrun/agent/internal/docker" + "github.com/gin-gonic/gin" +) + +// attachStoreToDeployment injects a store's connection details into another +// deployment's environment so its app can use the store directly. A managed +// store is reached inside the cluster by its container name on the shared +// object-storage network (which the app is joined to); an external store by its +// public URL. The deployment must be restarted for the change to take effect. +func (s *Server) attachStoreToDeployment(c *gin.Context) { + dest, ok := s.findDestinationByName(c.Param("name")) + if !ok { + c.JSON(http.StatusNotFound, gin.H{"error": "object store not found"}) + return + } + + var req struct { + Deployment string `json:"deployment" binding:"required"` + Prefix string `json:"prefix"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + prefix := req.Prefix + if prefix == "" { + prefix = "S3_" + } + + deployDir := filepath.Join(s.config.DeploymentsPath, req.Deployment) + composePath := filepath.Join(deployDir, "docker-compose.yml") + composeContent, err := os.ReadFile(composePath) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "could not read deployment compose: " + err.Error()}) + return + } + + cred, err := s.credentialsManager.GetGenericCredential(dest.CredentialID) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "store has no usable credential: " + err.Error()}) + return + } + + endpoint := dest.Endpoint + joinNetwork := "" + if dest.Kind == "managed" && dest.Deployment != "" { + port := managedStorePort + if u, perr := url.Parse(dest.Endpoint); perr == nil && u.Port() != "" { + port = u.Port() + } + endpoint = fmt.Sprintf("http://%s:%s", dest.Deployment, port) + joinNetwork = objectStorageNetworkName(s.config) + } + + vars := map[string]string{ + prefix + "ENDPOINT": endpoint, + prefix + "BUCKET": dest.Bucket, + prefix + "REGION": dest.Region, + prefix + "ACCESS_KEY_ID": cred.Data["access_key_id"], + prefix + "SECRET_ACCESS_KEY": cred.Data["secret_access_key"], + prefix + "USE_PATH_STYLE": strconv.FormatBool(dest.UsePathStyle), + } + + // Upsert into the deployment's .env.flatrun, preserving anything already there. + var envVars []EnvVar + if existing, rerr := os.ReadFile(filepath.Join(deployDir, ".env.flatrun")); rerr == nil { + envVars = parseEnvContent(string(existing)) + } + index := make(map[string]int, len(envVars)) + for i, e := range envVars { + index[e.Key] = i + } + for k, v := range vars { + if i, ok := index[k]; ok { + envVars[i].Value = v + } else { + envVars = append(envVars, EnvVar{Key: k, Value: v}) + } + } + if err := s.writeEnvFile(req.Deployment, envVars); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write environment: " + err.Error()}) + return + } + + // Wire the service to load that env file, and (managed) join the store's network. + updated, err := docker.EnsureServiceEnvFile(string(composeContent), ".env.flatrun") + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to update compose: " + err.Error()}) + return + } + if joinNetwork != "" { + if s.networksManager != nil { + _ = s.networksManager.EnsureNetwork(joinNetwork) + } + if withNet, nerr := docker.AddNetworkToCompose(updated, joinNetwork); nerr == nil { + updated = withNet + } + } + if err := os.WriteFile(composePath, []byte(updated), 0644); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write compose: " + err.Error()}) + return + } + + keys := make([]string, 0, len(vars)) + for k := range vars { + keys = append(keys, k) + } + sort.Strings(keys) + + c.JSON(http.StatusOK, gin.H{ + "message": "Store attached. Restart the deployment to apply.", + "keys": keys, + "endpoint": endpoint, + "network": joinNetwork, + }) +} diff --git a/internal/api/object_stores_test.go b/internal/api/object_stores_test.go index 512c448..8f729b7 100644 --- a/internal/api/object_stores_test.go +++ b/internal/api/object_stores_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strings" "testing" "github.com/flatrun/agent/internal/auth" @@ -71,6 +72,7 @@ func setupObjectStoreTestServer(t *testing.T) (*Server, *gin.Engine, func()) { protected.POST("/object-stores/:name/objects", mw.RequirePermission(auth.PermBackupsWrite), server.uploadStoreObject) protected.GET("/object-stores/:name/objects/download", mw.RequirePermission(auth.PermBackupsRead), server.downloadStoreObject) protected.DELETE("/object-stores/:name/objects", mw.RequirePermission(auth.PermBackupsWrite), server.deleteStoreObject) + protected.POST("/object-stores/:name/attach", mw.RequirePermission(auth.PermDeploymentsWrite), server.attachStoreToDeployment) cleanup := func() { authManager.Close() @@ -290,6 +292,55 @@ func TestListStoreObjects_UnknownStore404(t *testing.T) { } } +func TestAttachStoreToDeployment_WritesEnvAndWiresCompose(t *testing.T) { + server, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + cred, err := server.credentialsManager.CreateGenericCredential("app-keys", models.CredentialKindS3, map[string]string{ + "access_key_id": "AKIATEST", "secret_access_key": "shh", + }) + if err != nil { + t.Fatalf("create credential: %v", err) + } + server.config.Backup.Destinations = []config.BackupDestination{{ + Name: "r2", Type: "s3", Kind: "external", Endpoint: "https://s3.example.com", + Region: "us-east-1", Bucket: "assets", CredentialID: cred.ID, UsePathStyle: true, + }} + + app := "webapp" + appDir := filepath.Join(server.config.DeploymentsPath, app) + if err := os.MkdirAll(appDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + compose := "services:\n app:\n image: node:20-alpine\n" + if err := os.WriteFile(filepath.Join(appDir, "docker-compose.yml"), []byte(compose), 0o644); err != nil { + t.Fatalf("write compose: %v", err) + } + + res := osReq(t, router, http.MethodPost, "/api/object-stores/r2/attach", token, map[string]any{ + "deployment": app, + }) + if res.Code != http.StatusOK { + t.Fatalf("attach: %d %s", res.Code, res.Body.String()) + } + + env, err := os.ReadFile(filepath.Join(appDir, ".env.flatrun")) + if err != nil { + t.Fatalf("read env: %v", err) + } + for _, want := range []string{"S3_ENDPOINT=https://s3.example.com", "S3_BUCKET=assets", "S3_ACCESS_KEY_ID=AKIATEST", "S3_SECRET_ACCESS_KEY=shh"} { + if !strings.Contains(string(env), want) { + t.Fatalf("env missing %q:\n%s", want, env) + } + } + + updated, _ := os.ReadFile(filepath.Join(appDir, "docker-compose.yml")) + if !strings.Contains(string(updated), "env_file") || !strings.Contains(string(updated), ".env.flatrun") { + t.Fatalf("compose not wired to env file:\n%s", updated) + } +} + func TestBackupDestinations_ListReportsKind(t *testing.T) { server, router, cleanup := setupObjectStoreTestServer(t) defer cleanup() diff --git a/internal/api/server.go b/internal/api/server.go index a564aaf..fd64daa 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -754,6 +754,7 @@ func (s *Server) setupRoutes() { protected.POST("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.uploadStoreObject) protected.GET("/object-stores/:name/objects/download", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.downloadStoreObject) protected.DELETE("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.deleteStoreObject) + protected.POST("/object-stores/:name/attach", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.attachStoreToDeployment) // Scheduler endpoints protected.GET("/scheduler/tasks", s.authMiddleware.RequirePermission(auth.PermSchedulerRead), s.listScheduledTasks) diff --git a/internal/docker/compose_yaml.go b/internal/docker/compose_yaml.go index 3e00e1a..8c7a7fa 100644 --- a/internal/docker/compose_yaml.go +++ b/internal/docker/compose_yaml.go @@ -157,6 +157,66 @@ func ParseComposeYAML(content string) (map[string]interface{}, error) { return compose, nil } +// EnsureServiceEnvFile makes every service load envFile, so env written there +// (e.g. an attached object store's connection details) actually reaches the +// containers. An existing env_file (string or list form) is preserved and +// envFile appended only when missing. +func EnsureServiceEnvFile(content, envFile string) (string, error) { + compose, err := ParseComposeYAML(content) + if err != nil { + return content, err + } + services, ok := compose["services"].(map[string]interface{}) + if !ok { + return content, nil + } + + changed := false + for name, raw := range services { + service, ok := raw.(map[string]interface{}) + if !ok { + continue + } + + var files []string + switch ef := service["env_file"].(type) { + case string: + files = []string{ef} + case []interface{}: + for _, f := range ef { + if s, ok := f.(string); ok { + files = append(files, s) + } + } + } + + found := false + for _, f := range files { + if f == envFile { + found = true + break + } + } + if found { + continue + } + + files = append(files, envFile) + service["env_file"] = files + services[name] = service + changed = true + } + + if !changed { + return content, nil + } + out, err := yaml.Marshal(compose) + if err != nil { + return content, err + } + return string(out), nil +} + // ContainerNameForService returns the DNS-resolvable container name for a service in // a deployment, matching what EnsureContainerNames assigns. An explicit container_name in // the compose wins; otherwise the EnsureContainerNames rule applies (primary service -> From 655da746a2b4b96f251fb6c2b1ed93c16af2b6b5 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sun, 2 Aug 2026 02:57:06 +0100 Subject: [PATCH 4/9] feat(object-stores): Replicate a store's objects to another store A store's contents can now be copied to another store, for an offsite copy of a self-hosted store or a local cache of a remote one. Replication is incremental: objects already present at the same size in the target are skipped, so re-running only moves what changed. --- internal/api/object_replicate.go | 128 +++++++++++++++++++++++++++++++ internal/api/server.go | 1 + 2 files changed, 129 insertions(+) create mode 100644 internal/api/object_replicate.go diff --git a/internal/api/object_replicate.go b/internal/api/object_replicate.go new file mode 100644 index 0000000..2f9ac33 --- /dev/null +++ b/internal/api/object_replicate.go @@ -0,0 +1,128 @@ +package api + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + + "github.com/flatrun/agent/internal/backup" + "github.com/gin-gonic/gin" +) + +// s3StoreByName resolves a registered store by name into an S3 client without +// writing a response, for callers that handle their own errors. +func (s *Server) s3StoreByName(name string) (*backup.S3Store, error) { + dest, ok := s.findDestinationByName(name) + if !ok { + return nil, fmt.Errorf("object store %q not found", name) + } + store, err := s.buildStore(dest) + if err != nil { + return nil, err + } + s3store, ok := store.(*backup.S3Store) + if !ok { + return nil, fmt.Errorf("store %q is not S3-compatible", name) + } + return s3store, nil +} + +// copyObject streams a source object through a temp file before uploading it. +// The S3 client must sign the request body, which requires a seekable reader; +// an object read stream is not seekable, so it is buffered to disk first. Disk +// (not memory) keeps large backup archives from exhausting RAM. +func copyObject(ctx context.Context, src, dst *backup.S3Store, key string) (int64, error) { + reader, err := src.Open(ctx, key) + if err != nil { + return 0, err + } + defer reader.Close() + + tmp, err := os.CreateTemp("", "flatrun-replicate-*") + if err != nil { + return 0, err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + + n, err := io.Copy(tmp, reader) + if err != nil { + return 0, err + } + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return 0, err + } + if err := dst.Put(ctx, key, tmp, n); err != nil { + return 0, err + } + return n, nil +} + +// replicateStore copies every object from the store named in the path to a +// target store. It is incremental: an object already present in the target at +// the same size is skipped, so re-running only moves what changed. This backs +// offsite copies (managed to external) and local caches (external to managed). +func (s *Server) replicateStore(c *gin.Context) { + src, ok := s.storeS3ByName(c) + if !ok { + return + } + + var req struct { + Target string `json:"target" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if req.Target == c.Param("name") { + c.JSON(http.StatusBadRequest, gin.H{"error": "a store cannot replicate to itself"}) + return + } + + dst, err := s.s3StoreByName(req.Target) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + ctx := c.Request.Context() + if err := dst.EnsureBucket(ctx); err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": "target bucket unavailable: " + err.Error()}) + return + } + + objects, err := src.ListObjects(ctx, "") + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": "could not list source: " + err.Error()}) + return + } + + var copied, skipped, failed int + var bytes int64 + for _, o := range objects { + if info, err := dst.Stat(ctx, o.Key); err == nil && info.Size == o.Size { + skipped++ + continue + } + n, err := copyObject(ctx, src, dst, o.Key) + if err != nil { + failed++ + continue + } + copied++ + bytes += n + } + + c.JSON(http.StatusOK, gin.H{ + "message": fmt.Sprintf("Replicated %d object(s) to %s", copied, req.Target), + "target": req.Target, + "copied": copied, + "skipped": skipped, + "failed": failed, + "bytes_copied": bytes, + "total_objects": len(objects), + }) +} diff --git a/internal/api/server.go b/internal/api/server.go index fd64daa..9f259e0 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -755,6 +755,7 @@ func (s *Server) setupRoutes() { protected.GET("/object-stores/:name/objects/download", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.downloadStoreObject) protected.DELETE("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.deleteStoreObject) protected.POST("/object-stores/:name/attach", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.attachStoreToDeployment) + protected.POST("/object-stores/:name/replicate", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.replicateStore) // Scheduler endpoints protected.GET("/scheduler/tasks", s.authMiddleware.RequirePermission(auth.PermSchedulerRead), s.listScheduledTasks) From 96f24acb913e1f4125c3f705e7847c9f905b014e Mon Sep 17 00:00:00 2001 From: nfebe Date: Sun, 2 Aug 2026 12:53:39 +0100 Subject: [PATCH 5/9] feat(object-stores): Browse buckets and protect the backup bucket A store's server can host many buckets, so a store is no longer limited to its one configured bucket: its buckets can be listed and created, and objects can be browsed and uploaded within any of them. Deleting objects in the store's backup bucket is refused so browsing cannot destroy backups; other buckets stay fully editable. --- internal/api/object_browser.go | 99 ++++++++++++++++++++++++++------ internal/api/object_replicate.go | 2 +- internal/api/server.go | 2 + internal/backup/store_s3.go | 27 +++++++++ 4 files changed, 111 insertions(+), 19 deletions(-) diff --git a/internal/api/object_browser.go b/internal/api/object_browser.go index 91b7649..54c79b2 100644 --- a/internal/api/object_browser.go +++ b/internal/api/object_browser.go @@ -6,38 +6,84 @@ import ( "path" "github.com/flatrun/agent/internal/backup" + "github.com/flatrun/agent/pkg/config" "github.com/gin-gonic/gin" ) // storeS3ByName resolves a registered object store by name into a usable S3 -// client, writing an error response and returning false on failure. -func (s *Server) storeS3ByName(c *gin.Context) (*backup.S3Store, bool) { +// client (plus its destination record), writing an error response and returning +// false on failure. +func (s *Server) storeS3ByName(c *gin.Context) (*backup.S3Store, config.BackupDestination, bool) { dest, ok := s.findDestinationByName(c.Param("name")) if !ok { c.JSON(http.StatusNotFound, gin.H{"error": "object store not found"}) - return nil, false + return nil, config.BackupDestination{}, false } store, err := s.buildStore(dest) if err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) - return nil, false + return nil, config.BackupDestination{}, false } s3store, ok := store.(*backup.S3Store) if !ok { c.JSON(http.StatusBadRequest, gin.H{"error": "store is not S3-compatible"}) - return nil, false + return nil, config.BackupDestination{}, false } - return s3store, true + return s3store, dest, true } -// listStoreObjects returns every object in a store, optionally filtered by a -// key prefix. +// scopedStore returns the store scoped to the request's target bucket, which is +// the ?bucket query when given, otherwise the store's configured bucket. +func scopedStore(store *backup.S3Store, dest config.BackupDestination, c *gin.Context) *backup.S3Store { + bucket := c.Query("bucket") + if bucket == "" { + bucket = dest.Bucket + } + return store.WithBucket(bucket) +} + +// listStoreBuckets lists every bucket on the store's server. +func (s *Server) listStoreBuckets(c *gin.Context) { + store, dest, ok := s.storeS3ByName(c) + if !ok { + return + } + buckets, err := store.ListBuckets(c.Request.Context()) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"buckets": buckets, "backup_bucket": dest.Bucket}) +} + +// createStoreBucket creates a new bucket on the store's server. +func (s *Server) createStoreBucket(c *gin.Context) { + store, _, ok := s.storeS3ByName(c) + if !ok { + return + } + var req struct { + Bucket string `json:"bucket" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if err := store.WithBucket(req.Bucket).EnsureBucket(c.Request.Context()); err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusCreated, gin.H{"message": "Bucket created", "bucket": req.Bucket}) +} + +// listStoreObjects returns every object in the target bucket, optionally +// filtered by a key prefix. func (s *Server) listStoreObjects(c *gin.Context) { - store, ok := s.storeS3ByName(c) + store, dest, ok := s.storeS3ByName(c) if !ok { return } - objects, err := store.ListObjects(c.Request.Context(), c.Query("prefix")) + objects, err := scopedStore(store, dest, c).ListObjects(c.Request.Context(), c.Query("prefix")) if err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) return @@ -49,12 +95,13 @@ func (s *Server) listStoreObjects(c *gin.Context) { } // uploadStoreObject stores an uploaded file at the given key (defaulting to the -// file's name). +// file's name) in the target bucket. func (s *Server) uploadStoreObject(c *gin.Context) { - store, ok := s.storeS3ByName(c) + store, dest, ok := s.storeS3ByName(c) if !ok { return } + target := scopedStore(store, dest, c) fileHeader, err := c.FormFile("file") if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "file is required"}) @@ -71,7 +118,7 @@ func (s *Server) uploadStoreObject(c *gin.Context) { } defer f.Close() - if err := store.Put(c.Request.Context(), key, f, fileHeader.Size); err != nil { + if err := target.Put(c.Request.Context(), key, f, fileHeader.Size); err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) return } @@ -80,7 +127,7 @@ func (s *Server) uploadStoreObject(c *gin.Context) { // downloadStoreObject streams an object back to the caller as an attachment. func (s *Server) downloadStoreObject(c *gin.Context) { - store, ok := s.storeS3ByName(c) + store, dest, ok := s.storeS3ByName(c) if !ok { return } @@ -89,29 +136,45 @@ func (s *Server) downloadStoreObject(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "key is required"}) return } - reader, err := store.Open(c.Request.Context(), key) + reader, err := scopedStore(store, dest, c).Open(c.Request.Context(), key) if err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) return } defer reader.Close() + if c.Query("inline") == "true" { + c.DataFromReader(http.StatusOK, -1, "application/octet-stream", reader, nil) + return + } c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%q", path.Base(key))) c.DataFromReader(http.StatusOK, -1, "application/octet-stream", reader, nil) } -// deleteStoreObject removes a single object from a store. +// deleteStoreObject removes a single object from the target bucket. Deletes in +// the store's backup bucket are refused, so browsing cannot destroy backups; +// objects in any other bucket are freely removable. func (s *Server) deleteStoreObject(c *gin.Context) { - store, ok := s.storeS3ByName(c) + store, dest, ok := s.storeS3ByName(c) if !ok { return } + bucket := c.Query("bucket") + if bucket == "" { + bucket = dest.Bucket + } + if bucket == dest.Bucket { + c.JSON(http.StatusForbidden, gin.H{ + "error": fmt.Sprintf("deletes are disabled in the backup bucket %q to protect backups; use another bucket", dest.Bucket), + }) + return + } key := c.Query("key") if key == "" { c.JSON(http.StatusBadRequest, gin.H{"error": "key is required"}) return } - if err := store.Delete(c.Request.Context(), key); err != nil { + if err := store.WithBucket(bucket).Delete(c.Request.Context(), key); err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) return } diff --git a/internal/api/object_replicate.go b/internal/api/object_replicate.go index 2f9ac33..088f6d9 100644 --- a/internal/api/object_replicate.go +++ b/internal/api/object_replicate.go @@ -65,7 +65,7 @@ func copyObject(ctx context.Context, src, dst *backup.S3Store, key string) (int6 // the same size is skipped, so re-running only moves what changed. This backs // offsite copies (managed to external) and local caches (external to managed). func (s *Server) replicateStore(c *gin.Context) { - src, ok := s.storeS3ByName(c) + src, _, ok := s.storeS3ByName(c) if !ok { return } diff --git a/internal/api/server.go b/internal/api/server.go index 9f259e0..ac35c29 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -750,6 +750,8 @@ func (s *Server) setupRoutes() { // Object stores protected.POST("/object-stores/provision-managed", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.provisionManagedObjectStore) + protected.GET("/object-stores/:name/buckets", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listStoreBuckets) + protected.POST("/object-stores/:name/buckets", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.createStoreBucket) protected.GET("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listStoreObjects) protected.POST("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.uploadStoreObject) protected.GET("/object-stores/:name/objects/download", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.downloadStoreObject) diff --git a/internal/backup/store_s3.go b/internal/backup/store_s3.go index 382f14c..6cc6fa2 100644 --- a/internal/backup/store_s3.go +++ b/internal/backup/store_s3.go @@ -154,6 +154,33 @@ func (s *S3Store) List(ctx context.Context, prefix string) ([]ObjectInfo, error) return out, nil } +// ListBuckets returns the names of every bucket reachable with the store's +// credentials. A store is configured with one bucket, but the server it points +// at (a self-hosted MinIO, say) can host many. +func (s *S3Store) ListBuckets(ctx context.Context) ([]string, error) { + out, err := s.client.ListBuckets(ctx, &s3.ListBucketsInput{}) + if err != nil { + return nil, fmt.Errorf("s3 list buckets: %w", err) + } + names := make([]string, 0, len(out.Buckets)) + for _, b := range out.Buckets { + names = append(names, aws.ToString(b.Name)) + } + return names, nil +} + +// WithBucket returns a view of the store scoped to another bucket at its root +// (no prefix), so the object browser can work across buckets on the same server. +func (s *S3Store) WithBucket(bucket string) *S3Store { + cfg := s.cfg + cfg.Bucket = bucket + cfg.Prefix = "" + return &S3Store{client: s.client, cfg: cfg} +} + +// Bucket returns the bucket this store is scoped to. +func (s *S3Store) Bucket() string { return s.cfg.Bucket } + // ListObjects returns every object under prefix, unlike List which is scoped to // backup archives (.tar.gz). It backs the object browser, where a store's full // contents are shown. From c47d76d21b6375bbb9c6101bb77d527264402b10 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sun, 2 Aug 2026 16:37:00 +0100 Subject: [PATCH 6/9] feat(object-stores): Paginate objects and report bucket stats The bucket list now reports each bucket's object count and total size (bounded so large buckets stay responsive), and objects are listed a page at a time so a bucket of any size can be browsed. An empty, non-backup bucket can be deleted. --- internal/api/object_browser.go | 68 ++++++++++++++++++++++++++++++---- internal/api/server.go | 1 + internal/backup/store_s3.go | 63 +++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/internal/api/object_browser.go b/internal/api/object_browser.go index 54c79b2..86cc6e9 100644 --- a/internal/api/object_browser.go +++ b/internal/api/object_browser.go @@ -4,12 +4,20 @@ import ( "fmt" "net/http" "path" + "strconv" "github.com/flatrun/agent/internal/backup" "github.com/flatrun/agent/pkg/config" "github.com/gin-gonic/gin" ) +// bucketStatsCap bounds how many objects are counted per bucket for the bucket +// list, so a very large bucket does not stall the page. +const bucketStatsCap = 10000 + +// defaultObjectPage is the object-listing page size when none is requested. +const defaultObjectPage = 200 + // storeS3ByName resolves a registered object store by name into a usable S3 // client (plus its destination record), writing an error response and returning // false on failure. @@ -42,18 +50,59 @@ func scopedStore(store *backup.S3Store, dest config.BackupDestination, c *gin.Co return store.WithBucket(bucket) } -// listStoreBuckets lists every bucket on the store's server. +type bucketInfo struct { + Name string `json:"name"` + Objects int `json:"objects"` + Size int64 `json:"size"` + Truncated bool `json:"truncated"` + IsBackup bool `json:"is_backup"` +} + +// listStoreBuckets lists every bucket on the store's server with its object +// count and total size, and marks the backup bucket. func (s *Server) listStoreBuckets(c *gin.Context) { store, dest, ok := s.storeS3ByName(c) if !ok { return } - buckets, err := store.ListBuckets(c.Request.Context()) + ctx := c.Request.Context() + names, err := store.ListBuckets(ctx) if err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) return } - c.JSON(http.StatusOK, gin.H{"buckets": buckets, "backup_bucket": dest.Bucket}) + + out := make([]bucketInfo, 0, len(names)) + for _, n := range names { + count, size, truncated, _ := store.WithBucket(n).BucketStats(ctx, bucketStatsCap) + out = append(out, bucketInfo{ + Name: n, + Objects: count, + Size: size, + Truncated: truncated, + IsBackup: n == dest.Bucket, + }) + } + c.JSON(http.StatusOK, gin.H{"buckets": out, "backup_bucket": dest.Bucket}) +} + +// deleteStoreBucket removes an empty bucket. The store's backup bucket is +// protected, so browsing cannot destroy the backup target. +func (s *Server) deleteStoreBucket(c *gin.Context) { + store, dest, ok := s.storeS3ByName(c) + if !ok { + return + } + bucket := c.Param("bucket") + if bucket == dest.Bucket { + c.JSON(http.StatusForbidden, gin.H{"error": fmt.Sprintf("the backup bucket %q cannot be deleted", dest.Bucket)}) + return + } + if err := store.WithBucket(bucket).DeleteBucket(c.Request.Context()); err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error() + " (a bucket must be empty to delete)"}) + return + } + c.JSON(http.StatusOK, gin.H{"message": "Bucket deleted", "bucket": bucket}) } // createStoreBucket creates a new bucket on the store's server. @@ -76,14 +125,19 @@ func (s *Server) createStoreBucket(c *gin.Context) { c.JSON(http.StatusCreated, gin.H{"message": "Bucket created", "bucket": req.Bucket}) } -// listStoreObjects returns every object in the target bucket, optionally -// filtered by a key prefix. +// listStoreObjects returns one page of objects in the target bucket, with a +// continuation token for the next page. func (s *Server) listStoreObjects(c *gin.Context) { store, dest, ok := s.storeS3ByName(c) if !ok { return } - objects, err := scopedStore(store, dest, c).ListObjects(c.Request.Context(), c.Query("prefix")) + limit := int32(defaultObjectPage) + if v, err := strconv.Atoi(c.Query("limit")); err == nil && v > 0 && v <= 1000 { + limit = int32(v) + } + + objects, next, err := scopedStore(store, dest, c).ListObjectsPage(c.Request.Context(), c.Query("prefix"), c.Query("token"), limit) if err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) return @@ -91,7 +145,7 @@ func (s *Server) listStoreObjects(c *gin.Context) { if objects == nil { objects = []backup.ObjectInfo{} } - c.JSON(http.StatusOK, gin.H{"objects": objects}) + c.JSON(http.StatusOK, gin.H{"objects": objects, "next_token": next}) } // uploadStoreObject stores an uploaded file at the given key (defaulting to the diff --git a/internal/api/server.go b/internal/api/server.go index ac35c29..7287893 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -752,6 +752,7 @@ func (s *Server) setupRoutes() { protected.POST("/object-stores/provision-managed", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.provisionManagedObjectStore) protected.GET("/object-stores/:name/buckets", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listStoreBuckets) protected.POST("/object-stores/:name/buckets", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.createStoreBucket) + protected.DELETE("/object-stores/:name/buckets/:bucket", s.authMiddleware.RequirePermission(auth.PermBackupsDelete), s.deleteStoreBucket) protected.GET("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listStoreObjects) protected.POST("/object-stores/:name/objects", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.uploadStoreObject) protected.GET("/object-stores/:name/objects/download", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.downloadStoreObject) diff --git a/internal/backup/store_s3.go b/internal/backup/store_s3.go index 6cc6fa2..ac2cbf7 100644 --- a/internal/backup/store_s3.go +++ b/internal/backup/store_s3.go @@ -181,6 +181,69 @@ func (s *S3Store) WithBucket(bucket string) *S3Store { // Bucket returns the bucket this store is scoped to. func (s *S3Store) Bucket() string { return s.cfg.Bucket } +// ListObjectsPage returns one page of objects and a continuation token for the +// next page (empty when there are no more), so a browser can page through a +// bucket of any size instead of loading it whole. +func (s *S3Store) ListObjectsPage(ctx context.Context, prefix, token string, limit int32) ([]ObjectInfo, string, error) { + in := &s3.ListObjectsV2Input{ + Bucket: aws.String(s.cfg.Bucket), + Prefix: aws.String(s.fullKey(prefix)), + } + if limit > 0 { + in.MaxKeys = aws.Int32(limit) + } + if token != "" { + in.ContinuationToken = aws.String(token) + } + + out, err := s.client.ListObjectsV2(ctx, in) + if err != nil { + return nil, "", fmt.Errorf("s3 list page: %w", err) + } + objects := make([]ObjectInfo, 0, len(out.Contents)) + for _, o := range out.Contents { + objects = append(objects, ObjectInfo{ + Key: s.relKey(aws.ToString(o.Key)), + Size: aws.ToInt64(o.Size), + ModTime: aws.ToTime(o.LastModified), + }) + } + next := "" + if aws.ToBool(out.IsTruncated) { + next = aws.ToString(out.NextContinuationToken) + } + return objects, next, nil +} + +// BucketStats returns the object count and total size of the store's bucket. +// Counting stops at limit (when > 0), reporting truncated=true, so the bucket +// list stays responsive on very large buckets. +func (s *S3Store) BucketStats(ctx context.Context, limit int) (count int, size int64, truncated bool, err error) { + paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{Bucket: aws.String(s.cfg.Bucket)}) + for paginator.HasMorePages() { + page, perr := paginator.NextPage(ctx) + if perr != nil { + return count, size, truncated, fmt.Errorf("s3 stats: %w", perr) + } + for _, o := range page.Contents { + count++ + size += aws.ToInt64(o.Size) + if limit > 0 && count >= limit { + return count, size, true, nil + } + } + } + return count, size, false, nil +} + +// DeleteBucket removes the store's bucket. S3 requires it to be empty. +func (s *S3Store) DeleteBucket(ctx context.Context) error { + if _, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(s.cfg.Bucket)}); err != nil { + return fmt.Errorf("s3 delete bucket %q: %w", s.cfg.Bucket, err) + } + return nil +} + // ListObjects returns every object under prefix, unlike List which is scoped to // backup archives (.tar.gz). It backs the object browser, where a store's full // contents are shown. From fec87dbb1107a718e1e9c051f7e8fa50a11fa882 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sun, 2 Aug 2026 23:54:12 +0100 Subject: [PATCH 7/9] feat(backups): Auto-detect database servers and dump them in full A deployment whose service is a database server (including the shared infrastructure database) is now detected and backed up with a full logical dump of every database it holds, using its root credentials, instead of only copying its live data directory. This makes database backups consistent and complete with no per-deployment configuration; a shared database captures all app data in one dump, and a standalone or sidecar database captures its own. --- internal/api/backup_detect.go | 150 +++++++++++++++++++++++++++++ internal/api/backup_detect_test.go | 45 +++++++++ internal/api/backup_handlers.go | 15 +-- internal/backup/manager.go | 55 +++++++---- pkg/models/deployment.go | 31 +++--- 5 files changed, 254 insertions(+), 42 deletions(-) create mode 100644 internal/api/backup_detect.go create mode 100644 internal/api/backup_detect_test.go diff --git a/internal/api/backup_detect.go b/internal/api/backup_detect.go new file mode 100644 index 0000000..43e8294 --- /dev/null +++ b/internal/api/backup_detect.go @@ -0,0 +1,150 @@ +package api + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/flatrun/agent/internal/backup" + "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/pkg/models" +) + +// effectiveBackupSpec returns the deployment's configured backup spec, or, when +// no databases are configured, one with databases auto-detected from the +// deployment. This is what makes a database-server deployment (including the +// shared infrastructure database) get a dump without any manual setup. +func (s *Server) effectiveBackupSpec(d *models.Deployment) *backup.BackupSpec { + var spec *backup.BackupSpec + if d.Metadata != nil && d.Metadata.Backup != nil { + spec = d.Metadata.Backup + } + if spec != nil && len(spec.Databases) > 0 { + return spec + } + + detected := s.detectBackupDatabases(d) + if len(detected) == 0 { + return spec + } + + out := backup.BackupSpec{} + if spec != nil { + out = *spec + } + out.Databases = detected + return &out +} + +// detectBackupDatabases finds database-server services in a deployment's compose +// and returns a spec to dump each in full (pg_dumpall / --all-databases) using +// its root credentials. A shared database deployment thus backs up every app's +// data in one dump; a standalone or sidecar database backs up its own. +func (s *Server) detectBackupDatabases(d *models.Deployment) []models.DatabaseBackupSpec { + content, err := os.ReadFile(filepath.Join(d.Path, "docker-compose.yml")) + if err != nil { + return nil + } + compose, err := docker.ParseComposeYAML(string(content)) + if err != nil { + return nil + } + services, ok := compose["services"].(map[string]interface{}) + if !ok { + return nil + } + + env, _ := s.readDeploymentEnvMap(d.Name) + + var specs []models.DatabaseBackupSpec + for name, raw := range services { + svc, ok := raw.(map[string]interface{}) + if !ok { + continue + } + image, _ := svc["image"].(string) + dbType := dbTypeFromImage(image) + if dbType == "" { + continue + } + user, password := dbRootCreds(dbType, svc, env) + specs = append(specs, models.DatabaseBackupSpec{ + Service: name, + Type: dbType, + Container: docker.ContainerNameForService(string(content), d.Name, name), + AllDatabases: true, + User: user, + Password: password, + }) + } + return specs +} + +func dbTypeFromImage(image string) string { + img := strings.ToLower(image) + switch { + case strings.Contains(img, "postgres") || strings.Contains(img, "postgis"): + return "postgres" + case strings.Contains(img, "mariadb") || strings.Contains(img, "mysql") || strings.Contains(img, "percona"): + return "mysql" + } + return "" +} + +func dbRootCreds(dbType string, svc map[string]interface{}, env map[string]string) (user, password string) { + switch dbType { + case "postgres": + user = serviceEnvValue(svc, env, "POSTGRES_USER") + if user == "" { + user = "postgres" + } + password = serviceEnvValue(svc, env, "POSTGRES_PASSWORD") + case "mysql": + user = "root" + password = serviceEnvValue(svc, env, "MYSQL_ROOT_PASSWORD") + } + return user, password +} + +// serviceEnvValue resolves an env var for a service: the deployment's env file +// wins, then the service's own environment block (resolving a ${VAR} reference +// back against the env file). +func serviceEnvValue(svc map[string]interface{}, env map[string]string, key string) string { + if v, ok := env[key]; ok && v != "" { + return v + } + switch e := svc["environment"].(type) { + case map[string]interface{}: + if raw, ok := e[key]; ok { + return resolveEnvRef(fmt.Sprint(raw), env) + } + case []interface{}: + for _, item := range e { + str, ok := item.(string) + if !ok { + continue + } + if k, val, found := strings.Cut(str, "="); found && k == key { + return resolveEnvRef(val, env) + } + } + } + return "" +} + +func resolveEnvRef(v string, env map[string]string) string { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, "${") && strings.HasSuffix(v, "}") { + inner := v[2 : len(v)-1] + name, def := inner, "" + if i := strings.Index(inner, ":-"); i >= 0 { + name, def = inner[:i], inner[i+2:] + } + if val, ok := env[name]; ok && val != "" { + return val + } + return def + } + return v +} diff --git a/internal/api/backup_detect_test.go b/internal/api/backup_detect_test.go new file mode 100644 index 0000000..c0e6c5c --- /dev/null +++ b/internal/api/backup_detect_test.go @@ -0,0 +1,45 @@ +package api + +import ( + "os" + "path/filepath" + "testing" + + "github.com/flatrun/agent/pkg/config" + "github.com/flatrun/agent/pkg/models" +) + +func TestDetectBackupDatabases_DBServerGlobalDump(t *testing.T) { + dir := t.TempDir() + srv := &Server{config: &config.Config{DeploymentsPath: dir}} + + name := "shared-db" + dep := filepath.Join(dir, name) + if err := os.MkdirAll(dep, 0o755); err != nil { + t.Fatal(err) + } + compose := "name: shared-db\nservices:\n postgres:\n image: postgres:16\n environment:\n POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}\n" + os.WriteFile(filepath.Join(dep, "docker-compose.yml"), []byte(compose), 0o644) + os.WriteFile(filepath.Join(dep, ".env"), []byte("POSTGRES_PASSWORD=topsecret\n"), 0o600) + + specs := srv.detectBackupDatabases(&models.Deployment{Name: name, Path: dep}) + if len(specs) != 1 { + t.Fatalf("want 1 detected db, got %d: %#v", len(specs), specs) + } + s := specs[0] + if s.Type != "postgres" || !s.AllDatabases || s.User != "postgres" || s.Password != "topsecret" || s.Container == "" { + t.Fatalf("unexpected detected spec: %#v", s) + } +} + +func TestDbTypeFromImage(t *testing.T) { + cases := map[string]string{ + "postgres:16": "postgres", "postgis/postgis": "postgres", + "mariadb:11": "mysql", "mysql:8": "mysql", "redis:7": "", "nginx": "", + } + for img, want := range cases { + if got := dbTypeFromImage(img); got != want { + t.Errorf("%s: got %q want %q", img, got, want) + } + } +} diff --git a/internal/api/backup_handlers.go b/internal/api/backup_handlers.go index 697a651..3bc92f2 100644 --- a/internal/api/backup_handlers.go +++ b/internal/api/backup_handlers.go @@ -91,10 +91,7 @@ func (s *Server) createBackup(c *gin.Context) { return } - var spec *backup.BackupSpec - if deployment.Metadata != nil && deployment.Metadata.Backup != nil { - spec = deployment.Metadata.Backup - } + spec := s.effectiveBackupSpec(deployment) jobID := s.backupManager.StartBackupJob(req.DeploymentName, spec) c.JSON(http.StatusAccepted, gin.H{"job_id": jobID, "message": "Backup job started"}) @@ -113,10 +110,7 @@ func (s *Server) createDeploymentBackup(c *gin.Context) { return } - var spec *backup.BackupSpec - if deployment.Metadata != nil && deployment.Metadata.Backup != nil { - spec = deployment.Metadata.Backup - } + spec := s.effectiveBackupSpec(deployment) jobID := s.backupManager.StartBackupJob(deploymentName, spec) c.JSON(http.StatusAccepted, gin.H{"job_id": jobID, "message": "Backup job started"}) @@ -209,10 +203,7 @@ func (s *Server) getDeploymentBackupConfig(c *gin.Context) { return } - var spec *backup.BackupSpec - if deployment.Metadata != nil && deployment.Metadata.Backup != nil { - spec = deployment.Metadata.Backup - } + spec := s.effectiveBackupSpec(deployment) c.JSON(http.StatusOK, gin.H{"backup_config": spec}) } diff --git a/internal/backup/manager.go b/internal/backup/manager.go index 72c7980..9f741a4 100644 --- a/internal/backup/manager.go +++ b/internal/backup/manager.go @@ -303,12 +303,18 @@ func (m *Manager) backupDatabases(ctx context.Context, deploymentName string, da } func (m *Manager) dumpMySQL(ctx context.Context, deploymentName string, db *DatabaseSpec, dbDir string) (string, error) { - containerName := fmt.Sprintf("%s-%s", deploymentName, db.Service) - if db.Service == deploymentName || db.Service == "" { - containerName = deploymentName + containerName := db.Container + if containerName == "" { + containerName = fmt.Sprintf("%s-%s", deploymentName, db.Service) + if db.Service == deploymentName || db.Service == "" { + containerName = deploymentName + } } - dumpFile := filepath.Join(dbDir, fmt.Sprintf("%s_mysql.sql", db.Service)) + label := db.Service + if label == "" { + label = deploymentName + } host := db.Host if host == "" { @@ -324,15 +330,19 @@ func (m *Manager) dumpMySQL(ctx context.Context, deploymentName string, db *Data } args := []string{"exec"} - if db.Password != "" { args = append(args, "-e", "MYSQL_PWD="+db.Password) } + args = append(args, containerName, "mysqldump", "-h", host, "-u", user, + "--single-transaction", "--routines", "--triggers") - args = append(args, containerName, "mysqldump", - "-h", host, - "-u", user, - "--single-transaction", "--routines", "--triggers", database) + dumpFile := filepath.Join(dbDir, fmt.Sprintf("%s_mysql.sql", label)) + if db.AllDatabases { + args = append(args, "--all-databases") + dumpFile = filepath.Join(dbDir, fmt.Sprintf("%s_mysql_all.sql", label)) + } else { + args = append(args, database) + } cmd := exec.CommandContext(ctx, "docker", args...) output, err := cmd.Output() @@ -348,12 +358,18 @@ func (m *Manager) dumpMySQL(ctx context.Context, deploymentName string, db *Data } func (m *Manager) dumpPostgres(ctx context.Context, deploymentName string, db *DatabaseSpec, dbDir string) (string, error) { - containerName := fmt.Sprintf("%s-%s", deploymentName, db.Service) - if db.Service == deploymentName || db.Service == "" { - containerName = deploymentName + containerName := db.Container + if containerName == "" { + containerName = fmt.Sprintf("%s-%s", deploymentName, db.Service) + if db.Service == deploymentName || db.Service == "" { + containerName = deploymentName + } } - dumpFile := filepath.Join(dbDir, fmt.Sprintf("%s_postgres.sql", db.Service)) + label := db.Service + if label == "" { + label = deploymentName + } user := db.User if user == "" { @@ -364,15 +380,18 @@ func (m *Manager) dumpPostgres(ctx context.Context, deploymentName string, db *D database = deploymentName } - args := []string{ - "exec", - } - + args := []string{"exec"} if db.Password != "" { args = append(args, "-e", fmt.Sprintf("PGPASSWORD=%s", db.Password)) } - args = append(args, containerName, "pg_dump", "-U", user, database) + dumpFile := filepath.Join(dbDir, fmt.Sprintf("%s_postgres.sql", label)) + if db.AllDatabases { + args = append(args, containerName, "pg_dumpall", "-U", user) + dumpFile = filepath.Join(dbDir, fmt.Sprintf("%s_postgres_all.sql", label)) + } else { + args = append(args, containerName, "pg_dump", "-U", user, database) + } cmd := exec.CommandContext(ctx, "docker", args...) output, err := cmd.Output() diff --git a/pkg/models/deployment.go b/pkg/models/deployment.go index 2fd6049..b2f6110 100644 --- a/pkg/models/deployment.go +++ b/pkg/models/deployment.go @@ -176,18 +176,25 @@ type ContainerBackupPath struct { } type DatabaseBackupSpec struct { - Service string `yaml:"service" json:"service"` - Type string `yaml:"type" json:"type"` - HostEnv string `yaml:"host_env,omitempty" json:"host_env,omitempty"` - PortEnv string `yaml:"port_env,omitempty" json:"port_env,omitempty"` - UserEnv string `yaml:"user_env,omitempty" json:"user_env,omitempty"` - PasswordEnv string `yaml:"password_env,omitempty" json:"password_env,omitempty"` - DatabaseEnv string `yaml:"database_env,omitempty" json:"database_env,omitempty"` - Host string `yaml:"host,omitempty" json:"host,omitempty"` - Port int `yaml:"port,omitempty" json:"port,omitempty"` - User string `yaml:"user,omitempty" json:"user,omitempty"` - Password string `yaml:"password,omitempty" json:"password,omitempty"` - Database string `yaml:"database,omitempty" json:"database,omitempty"` + Service string `yaml:"service" json:"service"` + Type string `yaml:"type" json:"type"` + // Container, when set, is the exact container to exec the dump in, overriding + // the name derived from the deployment and service. + Container string `yaml:"container,omitempty" json:"container,omitempty"` + // AllDatabases dumps the whole server (pg_dumpall / mysqldump --all-databases) + // with root credentials, used to back up a database-server deployment such as + // the shared infrastructure database. + AllDatabases bool `yaml:"all_databases,omitempty" json:"all_databases,omitempty"` + HostEnv string `yaml:"host_env,omitempty" json:"host_env,omitempty"` + PortEnv string `yaml:"port_env,omitempty" json:"port_env,omitempty"` + UserEnv string `yaml:"user_env,omitempty" json:"user_env,omitempty"` + PasswordEnv string `yaml:"password_env,omitempty" json:"password_env,omitempty"` + DatabaseEnv string `yaml:"database_env,omitempty" json:"database_env,omitempty"` + Host string `yaml:"host,omitempty" json:"host,omitempty"` + Port int `yaml:"port,omitempty" json:"port,omitempty"` + User string `yaml:"user,omitempty" json:"user,omitempty"` + Password string `yaml:"password,omitempty" json:"password,omitempty"` + Database string `yaml:"database,omitempty" json:"database,omitempty"` } type BackupHookSpec struct { From de46e66bb5b8b4d588444795a7894df52ae9b836 Mon Sep 17 00:00:00 2001 From: nfebe Date: Mon, 3 Aug 2026 00:13:53 +0100 Subject: [PATCH 8/9] feat(backups): Dump an app's own database from the shared server An app that uses the shared database now also dumps just its own database into its own archive, resolved automatically from the app's recorded database usage. This keeps an app's (often more frequent) backups self-contained rather than depending on the shared database's own, possibly less frequent, full backup. --- internal/api/backup_detect.go | 65 ++++++++++++++++++++++++++++++ internal/api/backup_detect_test.go | 27 +++++++++++++ 2 files changed, 92 insertions(+) diff --git a/internal/api/backup_detect.go b/internal/api/backup_detect.go index 43e8294..3cbeda2 100644 --- a/internal/api/backup_detect.go +++ b/internal/api/backup_detect.go @@ -78,9 +78,74 @@ func (s *Server) detectBackupDatabases(d *models.Deployment) []models.DatabaseBa Password: password, }) } + + // Per-app slice: a deployment using the shared database dumps just its own + // database from the shared server, so its (typically more frequent) backups + // are self-contained rather than depending on the shared database's own, + // possibly less frequent, global backup. + specs = append(specs, s.detectSharedDatabaseSlices(d, env)...) return specs } +func (s *Server) detectSharedDatabaseSlices(d *models.Deployment, env map[string]string) []models.DatabaseBackupSpec { + if d.Metadata == nil { + return nil + } + shared := s.config.Infrastructure.Database + if shared.Container == "" { + return nil + } + + var specs []models.DatabaseBackupSpec + for _, dbc := range d.Metadata.Databases { + if !dbc.IsShared && dbc.Mode != "shared" { + continue + } + dbName := "" + if dbc.DatabaseName != "" { + dbName = dbc.DatabaseName + } else if dbc.EnvPrefix != "" { + dbName = env[dbc.EnvPrefix+"_DATABASE"] + } + if dbName == "" { + dbName = env["DB_DATABASE"] + } + if dbName == "" { + continue + } + dbType := normalizeDBType(dbc.Type) + if dbType == "" { + dbType = normalizeDBType(shared.Type) + } + if dbType == "" { + continue + } + label := dbc.Alias + if label == "" { + label = "app" + } + specs = append(specs, models.DatabaseBackupSpec{ + Service: label, + Type: dbType, + Container: shared.Container, + Database: dbName, + User: shared.RootUser, + Password: shared.RootPassword, + }) + } + return specs +} + +func normalizeDBType(t string) string { + switch strings.ToLower(t) { + case "mysql", "mariadb": + return "mysql" + case "postgres", "postgresql": + return "postgres" + } + return "" +} + func dbTypeFromImage(image string) string { img := strings.ToLower(image) switch { diff --git a/internal/api/backup_detect_test.go b/internal/api/backup_detect_test.go index c0e6c5c..53191d8 100644 --- a/internal/api/backup_detect_test.go +++ b/internal/api/backup_detect_test.go @@ -43,3 +43,30 @@ func TestDbTypeFromImage(t *testing.T) { } } } + +func TestDetectBackupDatabases_SharedAppSlice(t *testing.T) { + dir := t.TempDir() + srv := &Server{config: &config.Config{ + DeploymentsPath: dir, + Infrastructure: config.InfrastructureConfig{Database: config.SharedDatabaseConfig{ + Enabled: true, Type: "postgres", Container: "shared-pg", RootUser: "postgres", RootPassword: "rootpw", + }}, + }} + name := "wordpress" + dep := filepath.Join(dir, name) + os.MkdirAll(dep, 0o755) + os.WriteFile(filepath.Join(dep, "docker-compose.yml"), []byte("services:\n app:\n image: wordpress:6\n"), 0o644) + os.WriteFile(filepath.Join(dep, ".env"), []byte("DB_DATABASE=wordpress_db\nDB_PASSWORD=apppw\n"), 0o600) + + specs := srv.detectBackupDatabases(&models.Deployment{ + Name: name, Path: dep, + Metadata: &models.ServiceMetadata{Databases: []models.DatabaseConfig{{Alias: "primary", Type: "postgres", Mode: "shared", IsShared: true}}}, + }) + if len(specs) != 1 { + t.Fatalf("want 1 slice, got %d: %#v", len(specs), specs) + } + s := specs[0] + if s.Container != "shared-pg" || s.Database != "wordpress_db" || s.User != "postgres" || s.Password != "rootpw" || s.AllDatabases { + t.Fatalf("unexpected slice: %#v", s) + } +} From 8ab5aac5345860574660a63d9d75b5e72cc76969 Mon Sep 17 00:00:00 2001 From: nfebe Date: Mon, 3 Aug 2026 00:30:16 +0100 Subject: [PATCH 9/9] feat(backups): Exclude a database's live data directory from file backup When a deployment's database is dumped, its live data directory is no longer also copied into the archive: that copy duplicates the dump and, taken from a running database, can be inconsistent. Other files such as app data and config are still included. Backups also gain honoured exclude patterns, which were previously declared but ignored. --- internal/api/backup_detect.go | 86 ++++++++++++++++++++++++++++++ internal/api/backup_detect_test.go | 25 +++++++++ internal/backup/exclude_test.go | 42 +++++++++++++++ internal/backup/manager.go | 26 ++++++++- 4 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 internal/backup/exclude_test.go diff --git a/internal/api/backup_detect.go b/internal/api/backup_detect.go index 3cbeda2..8d03773 100644 --- a/internal/api/backup_detect.go +++ b/internal/api/backup_detect.go @@ -34,9 +34,95 @@ func (s *Server) effectiveBackupSpec(d *models.Deployment) *backup.BackupSpec { out = *spec } out.Databases = detected + // A database server's live data directory is captured by its dump, so drop + // it from the file copy: it is redundant and, taken hot, potentially + // inconsistent. Other data (app files, config) is still copied. + out.ExcludePatterns = mergeUnique(out.ExcludePatterns, s.dbServerDataDirs(d)) return &out } +// dbServerDataDirs returns the bind-mount directory names of the deployment's +// own database services, so the live database files can be excluded from the +// file backup in favour of the logical dump. +func (s *Server) dbServerDataDirs(d *models.Deployment) []string { + content, err := os.ReadFile(filepath.Join(d.Path, "docker-compose.yml")) + if err != nil { + return nil + } + compose, err := docker.ParseComposeYAML(string(content)) + if err != nil { + return nil + } + services, ok := compose["services"].(map[string]interface{}) + if !ok { + return nil + } + + seen := map[string]bool{} + var dirs []string + for _, raw := range services { + svc, ok := raw.(map[string]interface{}) + if !ok { + continue + } + image, _ := svc["image"].(string) + if dbTypeFromImage(image) == "" { + continue + } + for _, host := range serviceBindHostPaths(svc) { + base := filepath.Base(host) + if base == "." || base == "/" || base == "" || seen[base] { + continue + } + seen[base] = true + dirs = append(dirs, base) + } + } + return dirs +} + +func serviceBindHostPaths(svc map[string]interface{}) []string { + vols, ok := svc["volumes"].([]interface{}) + if !ok { + return nil + } + var out []string + for _, v := range vols { + switch vv := v.(type) { + case string: + host := vv + if i := strings.Index(vv, ":"); i >= 0 { + host = vv[:i] + } + if strings.HasPrefix(host, ".") || strings.Contains(host, "/") { + out = append(out, host) + } + case map[string]interface{}: + if t, _ := vv["type"].(string); t == "bind" { + if src, _ := vv["source"].(string); src != "" { + out = append(out, src) + } + } + } + } + return out +} + +func mergeUnique(a, b []string) []string { + seen := map[string]bool{} + var out []string + for _, list := range [][]string{a, b} { + for _, s := range list { + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + } + } + return out +} + // detectBackupDatabases finds database-server services in a deployment's compose // and returns a spec to dump each in full (pg_dumpall / --all-databases) using // its root credentials. A shared database deployment thus backs up every app's diff --git a/internal/api/backup_detect_test.go b/internal/api/backup_detect_test.go index 53191d8..65b2a8f 100644 --- a/internal/api/backup_detect_test.go +++ b/internal/api/backup_detect_test.go @@ -70,3 +70,28 @@ func TestDetectBackupDatabases_SharedAppSlice(t *testing.T) { t.Fatalf("unexpected slice: %#v", s) } } + +func TestEffectiveBackupSpec_ExcludesDBDataDir(t *testing.T) { + dir := t.TempDir() + srv := &Server{config: &config.Config{DeploymentsPath: dir}} + name := "pg" + dep := filepath.Join(dir, name) + os.MkdirAll(dep, 0o755) + compose := "name: pg\nservices:\n postgres:\n image: postgres:16\n volumes:\n - ./data:/var/lib/postgresql/data\n environment:\n POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}\n" + os.WriteFile(filepath.Join(dep, "docker-compose.yml"), []byte(compose), 0o644) + os.WriteFile(filepath.Join(dep, ".env"), []byte("POSTGRES_PASSWORD=pw\n"), 0o600) + + spec := srv.effectiveBackupSpec(&models.Deployment{Name: name, Path: dep}) + if spec == nil || len(spec.Databases) != 1 { + t.Fatalf("expected a detected database: %#v", spec) + } + found := false + for _, e := range spec.ExcludePatterns { + if e == "data" { + found = true + } + } + if !found { + t.Fatalf("expected 'data' excluded, got %#v", spec.ExcludePatterns) + } +} diff --git a/internal/backup/exclude_test.go b/internal/backup/exclude_test.go new file mode 100644 index 0000000..d1c9130 --- /dev/null +++ b/internal/backup/exclude_test.go @@ -0,0 +1,42 @@ +package backup + +import ( + "os" + "path/filepath" + "testing" +) + +func TestMatchesExclude(t *testing.T) { + if !matchesExclude("data", []string{"data"}) { + t.Fatal("exact match failed") + } + if !matchesExclude("data", []string{"da*"}) { + t.Fatal("glob match failed") + } + if matchesExclude("uploads", []string{"data"}) { + t.Fatal("should not match") + } +} + +func TestBackupMountedData_SkipsExcluded(t *testing.T) { + m, err := NewManager(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dep := t.TempDir() + for _, d := range []string{"data", "uploads"} { + os.MkdirAll(filepath.Join(dep, d), 0o755) + os.WriteFile(filepath.Join(dep, d, "f"), []byte("x"), 0o644) + } + out := t.TempDir() + var md BackupMetadata + if err := m.backupMountedData(dep, out, &md, []string{"data"}); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(out, "data", "data")); err == nil { + t.Fatal("excluded 'data' dir should not be copied") + } + if _, err := os.Stat(filepath.Join(out, "data", "uploads")); err != nil { + t.Fatal("'uploads' should be copied") + } +} diff --git a/internal/backup/manager.go b/internal/backup/manager.go index 9f741a4..4d14c29 100644 --- a/internal/backup/manager.go +++ b/internal/backup/manager.go @@ -100,7 +100,11 @@ func (m *Manager) CreateBackup(ctx context.Context, deploymentName string, spec backup.Components = append(backup.Components, "metadata") } - if err := m.backupMountedData(deploymentPath, tempDir, &metadata); err != nil { + var excludes []string + if spec != nil { + excludes = spec.ExcludePatterns + } + if err := m.backupMountedData(deploymentPath, tempDir, &metadata, excludes); err != nil { log.Printf("Backup: mounted data warning: %v", err) } if len(metadata.Components.MountedData) > 0 { @@ -217,7 +221,7 @@ func (m *Manager) backupMetadataFile(deploymentPath, tempDir string, metadata *B return nil } -func (m *Manager) backupMountedData(deploymentPath, tempDir string, metadata *BackupMetadata) error { +func (m *Manager) backupMountedData(deploymentPath, tempDir string, metadata *BackupMetadata, excludes []string) error { dataDir := filepath.Join(tempDir, "data") if err := os.MkdirAll(dataDir, 0755); err != nil { return fmt.Errorf("failed to create data backup directory: %w", err) @@ -225,6 +229,10 @@ func (m *Manager) backupMountedData(deploymentPath, tempDir string, metadata *Ba commonDataDirs := []string{"data", "uploads", "storage", "config", "logs"} for _, dir := range commonDataDirs { + if matchesExclude(dir, excludes) { + log.Printf("Backup: skipping %s (excluded, e.g. a database's live files captured by its dump)", dir) + continue + } srcPath := filepath.Join(deploymentPath, dir) if info, err := os.Stat(srcPath); err == nil && info.IsDir() { destPath := filepath.Join(dataDir, dir) @@ -239,6 +247,20 @@ func (m *Manager) backupMountedData(deploymentPath, tempDir string, metadata *Ba return nil } +// matchesExclude reports whether a mounted-data directory name matches any +// exclude pattern (glob or exact). +func matchesExclude(name string, patterns []string) bool { + for _, p := range patterns { + if p == name { + return true + } + if ok, err := filepath.Match(p, name); err == nil && ok { + return true + } + } + return false +} + func (m *Manager) backupContainerData(ctx context.Context, deploymentName string, paths []ContainerPath, tempDir string, metadata *BackupMetadata) error { containerDir := filepath.Join(tempDir, "container_data") if err := os.MkdirAll(containerDir, 0755); err != nil {