From 13c066d05c4bc050990e624019d310c4f235b183 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 16:24:40 +0200 Subject: [PATCH 01/24] fix: report stopped status for stacks with all-stopped containers --- app/src/routes/(app)/resource-groups/[id]/+page.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/routes/(app)/resource-groups/[id]/+page.svelte b/app/src/routes/(app)/resource-groups/[id]/+page.svelte index ab863fe..4dc5945 100644 --- a/app/src/routes/(app)/resource-groups/[id]/+page.svelte +++ b/app/src/routes/(app)/resource-groups/[id]/+page.svelte @@ -741,6 +741,7 @@ function stackStatus(children: Workload[]): string { if (children.every((c) => c.status === "running")) return "running"; if (children.some((c) => c.status === "failed" || c.status === "error")) return "failed"; + if (children.every((c) => c.status === "stopped")) return "stopped"; return "pending"; } From fe06ba8b2504a0e277be49dea2dbdf7f290b04dd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 16 Aug 2026 14:27:57 +0000 Subject: [PATCH 02/24] style: apply cargo fmt and clippy fixes --- agent/src/firecracker/rootfs.rs | 5 +---- agent/src/server.rs | 12 +++++++----- control-plane/api-gateway/src/routes/agent_proxy.rs | 12 +++++++----- control-plane/api-gateway/src/routes/users.rs | 4 +++- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/agent/src/firecracker/rootfs.rs b/agent/src/firecracker/rootfs.rs index 462c4ed..adbc22a 100644 --- a/agent/src/firecracker/rootfs.rs +++ b/agent/src/firecracker/rootfs.rs @@ -68,10 +68,7 @@ impl RootfsBuilder { (Some(tag), None) => format!(":{tag}"), (None, None) => String::new(), }; - let mirrored = format!( - "{mirror}/{}{tag_or_digest}", - reference.repository() - ); + let mirrored = format!("{mirror}/{}{tag_or_digest}", reference.repository()); mirrored.parse().context("Invalid mirrored image reference") } diff --git a/agent/src/server.rs b/agent/src/server.rs index 1c962e8..72df7f2 100644 --- a/agent/src/server.rs +++ b/agent/src/server.rs @@ -187,13 +187,15 @@ async fn logs_handler( })?; info!(workload_id = %workload_id, container_id = %container_id, "opening log stream to guest"); - let stream = futures_util::stream::once(async { - Ok::<_, std::io::Error>(axum::body::Bytes::new()) - }) - .chain(state.firecracker.logs(&container_id)); + let stream = + futures_util::stream::once(async { Ok::<_, std::io::Error>(axum::body::Bytes::new()) }) + .chain(state.firecracker.logs(&container_id)); Ok(( - [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")], + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], axum::body::Body::from_stream(stream), )) } diff --git a/control-plane/api-gateway/src/routes/agent_proxy.rs b/control-plane/api-gateway/src/routes/agent_proxy.rs index 2f16793..405887d 100644 --- a/control-plane/api-gateway/src/routes/agent_proxy.rs +++ b/control-plane/api-gateway/src/routes/agent_proxy.rs @@ -199,13 +199,15 @@ pub async fn stream_workload_logs( )); } - let stream = futures_util::stream::once(async { - Ok::<_, reqwest::Error>(axum::body::Bytes::new()) - }) - .chain(resp.bytes_stream()); + let stream = + futures_util::stream::once(async { Ok::<_, reqwest::Error>(axum::body::Bytes::new()) }) + .chain(resp.bytes_stream()); Ok(( - [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")], + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], Body::from_stream(stream), )) } diff --git a/control-plane/api-gateway/src/routes/users.rs b/control-plane/api-gateway/src/routes/users.rs index 1875aee..f953e0c 100644 --- a/control-plane/api-gateway/src/routes/users.rs +++ b/control-plane/api-gateway/src/routes/users.rs @@ -564,7 +564,9 @@ pub async fn change_gravatar_email( .change_gravatar_email(claims.user_id, payload.gravatar_email) .await { - Ok(_) => Ok(Json(json!({ "message": "Gravatar email changed successfully" }))), + Ok(_) => Ok(Json( + json!({ "message": "Gravatar email changed successfully" }), + )), Err(err) => { tracing::error!("Failed to change gravatar email: {}", err); Err(StatusCode::INTERNAL_SERVER_ERROR) From c20964c2e76f856bf7376b4281fa84997d8d3465 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 16:48:27 +0200 Subject: [PATCH 03/24] feat: add data model for object storage buckets and garage nodes --- .../entity/src/entities/bucket_access_keys.rs | 35 +++ .../shared/entity/src/entities/buckets.rs | 48 +++ .../entity/src/entities/garage_nodes.rs | 38 +++ .../shared/entity/src/entities/mod.rs | 6 + .../entity/src/entities/resource_groups.rs | 8 + control-plane/shared/migration/src/lib.rs | 2 + .../m20260816_000000_add_object_storage.rs | 290 ++++++++++++++++++ 7 files changed, 427 insertions(+) create mode 100644 control-plane/shared/entity/src/entities/bucket_access_keys.rs create mode 100644 control-plane/shared/entity/src/entities/buckets.rs create mode 100644 control-plane/shared/entity/src/entities/garage_nodes.rs create mode 100644 control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs diff --git a/control-plane/shared/entity/src/entities/bucket_access_keys.rs b/control-plane/shared/entity/src/entities/bucket_access_keys.rs new file mode 100644 index 0000000..3b4f75c --- /dev/null +++ b/control-plane/shared/entity/src/entities/bucket_access_keys.rs @@ -0,0 +1,35 @@ +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "bucket_access_keys")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + pub bucket_id: Uuid, + pub name: String, + pub garage_key_id: String, + pub permissions: String, + pub expires_at: Option, + pub last_rotated_at: Option, + pub created_at: chrono::NaiveDateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::buckets::Entity", + from = "Column::BucketId", + to = "super::buckets::Column::Id", + on_delete = "Cascade" + )] + Bucket, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Bucket.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/entity/src/entities/buckets.rs b/control-plane/shared/entity/src/entities/buckets.rs new file mode 100644 index 0000000..2cb8090 --- /dev/null +++ b/control-plane/shared/entity/src/entities/buckets.rs @@ -0,0 +1,48 @@ +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "buckets")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + pub name: String, + pub garage_bucket_id: Option, + pub global_alias: String, + pub exposure: String, + pub quota_max_size: Option, + pub quota_max_objects: Option, + pub status: String, + pub organization_id: Option, + pub resource_group_id: Option, + pub created_at: chrono::NaiveDateTime, + pub updated_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::bucket_access_keys::Entity")] + AccessKeys, + #[sea_orm( + belongs_to = "super::resource_groups::Entity", + from = "Column::ResourceGroupId", + to = "super::resource_groups::Column::Id", + on_update = "NoAction", + on_delete = "SetNull" + )] + ResourceGroup, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::AccessKeys.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::ResourceGroup.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/entity/src/entities/garage_nodes.rs b/control-plane/shared/entity/src/entities/garage_nodes.rs new file mode 100644 index 0000000..06da9b7 --- /dev/null +++ b/control-plane/shared/entity/src/entities/garage_nodes.rs @@ -0,0 +1,38 @@ +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "garage_nodes")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + pub agent_id: Uuid, + pub garage_node_id: Option, + pub zone: String, + pub capacity_bytes: Option, + pub role: String, + pub status: String, + pub layout_version: Option, + pub last_seen_at: Option, + pub created_at: chrono::NaiveDateTime, + pub updated_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::agents::Entity", + from = "Column::AgentId", + to = "super::agents::Column::Id", + on_delete = "Cascade" + )] + Agent, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Agent.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/entity/src/entities/mod.rs b/control-plane/shared/entity/src/entities/mod.rs index 343c9f6..f5decf3 100644 --- a/control-plane/shared/entity/src/entities/mod.rs +++ b/control-plane/shared/entity/src/entities/mod.rs @@ -3,8 +3,11 @@ pub mod agent_certificates; pub mod agent_metrics; pub mod agents; pub mod bootstrap_tokens; +pub mod bucket_access_keys; +pub mod buckets; pub mod certificate_revocations; pub mod failover_events; +pub mod garage_nodes; pub mod invalid_jwt; pub mod key; pub mod logs; @@ -32,8 +35,11 @@ pub use agent_certificates::Entity as AgentCertificates; pub use agent_metrics::Entity as AgentMetrics; pub use agents::Entity as Agents; pub use bootstrap_tokens::Entity as BootstrapTokens; +pub use bucket_access_keys::Entity as BucketAccessKeys; +pub use buckets::Entity as Buckets; pub use certificate_revocations::Entity as CertificateRevocations; pub use failover_events::Entity as FailoverEvents; +pub use garage_nodes::Entity as GarageNodes; pub use invalid_jwt::Entity as InvalidJwt; pub use key::Entity as Key; pub use logs::Entity as Logs; diff --git a/control-plane/shared/entity/src/entities/resource_groups.rs b/control-plane/shared/entity/src/entities/resource_groups.rs index 3994ea4..d3e0c15 100644 --- a/control-plane/shared/entity/src/entities/resource_groups.rs +++ b/control-plane/shared/entity/src/entities/resource_groups.rs @@ -37,6 +37,8 @@ pub enum Relation { WorkloadStacks, #[sea_orm(has_many = "super::resource_group_vpn_peers::Entity")] ResourceGroupVpnPeers, + #[sea_orm(has_many = "super::buckets::Entity")] + Buckets, } impl Related for Entity { @@ -75,4 +77,10 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Buckets.def() + } +} + impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/migration/src/lib.rs b/control-plane/shared/migration/src/lib.rs index aeed875..961cef2 100644 --- a/control-plane/shared/migration/src/lib.rs +++ b/control-plane/shared/migration/src/lib.rs @@ -32,6 +32,7 @@ mod m20260712_010000_add_resource_group_vpn_peers; mod m20260723_000000_add_resource_group_appearance; mod m20260723_010000_runtime_class_default_firecracker; mod m20260809_000000_add_user_gravatar_email; +mod m20260816_000000_add_object_storage; pub struct Migrator; @@ -71,6 +72,7 @@ impl MigratorTrait for Migrator { Box::new(m20260723_000000_add_resource_group_appearance::Migration), Box::new(m20260723_010000_runtime_class_default_firecracker::Migration), Box::new(m20260809_000000_add_user_gravatar_email::Migration), + Box::new(m20260816_000000_add_object_storage::Migration), ] } } diff --git a/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs b/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs new file mode 100644 index 0000000..505227c --- /dev/null +++ b/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs @@ -0,0 +1,290 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Buckets::Table) + .if_not_exists() + .col(ColumnDef::new(Buckets::Id).uuid().not_null().primary_key()) + .col(ColumnDef::new(Buckets::Name).string().not_null()) + .col(ColumnDef::new(Buckets::GarageBucketId).string().null()) + .col(ColumnDef::new(Buckets::GlobalAlias).string().not_null()) + .col( + ColumnDef::new(Buckets::Exposure) + .string() + .not_null() + .default("internal"), + ) + .col(ColumnDef::new(Buckets::QuotaMaxSize).big_integer().null()) + .col(ColumnDef::new(Buckets::QuotaMaxObjects).big_integer().null()) + .col( + ColumnDef::new(Buckets::Status) + .string() + .not_null() + .default("provisioning"), + ) + .col(ColumnDef::new(Buckets::OrganizationId).uuid().null()) + .col(ColumnDef::new(Buckets::ResourceGroupId).uuid().null()) + .col(ColumnDef::new(Buckets::CreatedAt).date_time().not_null()) + .col(ColumnDef::new(Buckets::UpdatedAt).date_time().null()) + .foreign_key( + ForeignKey::create() + .from(Buckets::Table, Buckets::ResourceGroupId) + .to(ResourceGroups::Table, ResourceGroups::Id) + .on_update(ForeignKeyAction::NoAction) + .on_delete(ForeignKeyAction::SetNull), + ) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(BucketAccessKeys::Table) + .if_not_exists() + .col( + ColumnDef::new(BucketAccessKeys::Id) + .uuid() + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(BucketAccessKeys::BucketId) + .uuid() + .not_null(), + ) + .col(ColumnDef::new(BucketAccessKeys::Name).string().not_null()) + .col( + ColumnDef::new(BucketAccessKeys::GarageKeyId) + .string() + .not_null(), + ) + .col( + ColumnDef::new(BucketAccessKeys::Permissions) + .string() + .not_null(), + ) + .col(ColumnDef::new(BucketAccessKeys::ExpiresAt).date_time().null()) + .col( + ColumnDef::new(BucketAccessKeys::LastRotatedAt) + .date_time() + .null(), + ) + .col( + ColumnDef::new(BucketAccessKeys::CreatedAt) + .date_time() + .not_null(), + ) + .foreign_key( + ForeignKey::create() + .from(BucketAccessKeys::Table, BucketAccessKeys::BucketId) + .to(Buckets::Table, Buckets::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(GarageNodes::Table) + .if_not_exists() + .col( + ColumnDef::new(GarageNodes::Id) + .uuid() + .not_null() + .primary_key(), + ) + .col(ColumnDef::new(GarageNodes::AgentId).uuid().not_null()) + .col(ColumnDef::new(GarageNodes::GarageNodeId).string().null()) + .col(ColumnDef::new(GarageNodes::Zone).string().not_null()) + .col(ColumnDef::new(GarageNodes::CapacityBytes).big_integer().null()) + .col( + ColumnDef::new(GarageNodes::Role) + .string() + .not_null() + .default("storage"), + ) + .col( + ColumnDef::new(GarageNodes::Status) + .string() + .not_null() + .default("unknown"), + ) + .col(ColumnDef::new(GarageNodes::LayoutVersion).integer().null()) + .col(ColumnDef::new(GarageNodes::LastSeenAt).date_time().null()) + .col(ColumnDef::new(GarageNodes::CreatedAt).date_time().not_null()) + .col(ColumnDef::new(GarageNodes::UpdatedAt).date_time().null()) + .foreign_key( + ForeignKey::create() + .from(GarageNodes::Table, GarageNodes::AgentId) + .to(Agents::Table, Agents::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(Buckets::Table) + .col(Buckets::ResourceGroupId) + .name("idx_buckets_resource_group_id") + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(Buckets::Table) + .col(Buckets::OrganizationId) + .name("idx_buckets_organization_id") + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(Buckets::Table) + .col(Buckets::GlobalAlias) + .name("idx_buckets_global_alias") + .unique() + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(BucketAccessKeys::Table) + .col(BucketAccessKeys::BucketId) + .name("idx_bucket_access_keys_bucket_id") + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(GarageNodes::Table) + .col(GarageNodes::AgentId) + .name("idx_garage_nodes_agent_id") + .unique() + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("idx_garage_nodes_agent_id").to_owned()) + .await?; + manager + .drop_index( + Index::drop() + .name("idx_bucket_access_keys_bucket_id") + .to_owned(), + ) + .await?; + manager + .drop_index(Index::drop().name("idx_buckets_global_alias").to_owned()) + .await?; + manager + .drop_index( + Index::drop() + .name("idx_buckets_organization_id") + .to_owned(), + ) + .await?; + manager + .drop_index( + Index::drop() + .name("idx_buckets_resource_group_id") + .to_owned(), + ) + .await?; + + manager + .drop_table(Table::drop().table(GarageNodes::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(BucketAccessKeys::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(Buckets::Table).to_owned()) + .await?; + + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Buckets { + Table, + Id, + Name, + GarageBucketId, + GlobalAlias, + Exposure, + QuotaMaxSize, + QuotaMaxObjects, + Status, + OrganizationId, + ResourceGroupId, + CreatedAt, + UpdatedAt, +} + +#[derive(DeriveIden)] +enum BucketAccessKeys { + Table, + Id, + BucketId, + Name, + GarageKeyId, + Permissions, + ExpiresAt, + LastRotatedAt, + CreatedAt, +} + +#[derive(DeriveIden)] +enum GarageNodes { + Table, + Id, + AgentId, + GarageNodeId, + Zone, + CapacityBytes, + Role, + Status, + LayoutVersion, + LastSeenAt, + CreatedAt, + UpdatedAt, +} + +#[derive(DeriveIden)] +enum ResourceGroups { + Table, + Id, +} + +#[derive(DeriveIden)] +enum Agents { + Table, + Id, +} From cd84454b5fe281dd15c76424647d39e42f73d55d Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 16:51:07 +0200 Subject: [PATCH 04/24] feat: add object-storage service with bucket crud --- Cargo.toml | 1 + control-plane/object-storage/Cargo.toml | 37 +++++ .../object-storage/src/db/buckets.rs | 108 +++++++++++++ control-plane/object-storage/src/db/mod.rs | 1 + .../object-storage/src/garage/client.rs | 117 ++++++++++++++ .../object-storage/src/garage/mod.rs | 3 + .../object-storage/src/handlers/buckets.rs | 113 +++++++++++++ .../object-storage/src/handlers/mod.rs | 1 + control-plane/object-storage/src/logger.rs | 152 ++++++++++++++++++ control-plane/object-storage/src/main.rs | 66 ++++++++ control-plane/object-storage/src/metrics.rs | 57 +++++++ control-plane/object-storage/src/models.rs | 34 ++++ control-plane/object-storage/src/server.rs | 37 +++++ .../object-storage/src/services/bucket.rs | 99 ++++++++++++ .../object-storage/src/services/mod.rs | 1 + 15 files changed, 827 insertions(+) create mode 100644 control-plane/object-storage/Cargo.toml create mode 100644 control-plane/object-storage/src/db/buckets.rs create mode 100644 control-plane/object-storage/src/db/mod.rs create mode 100644 control-plane/object-storage/src/garage/client.rs create mode 100644 control-plane/object-storage/src/garage/mod.rs create mode 100644 control-plane/object-storage/src/handlers/buckets.rs create mode 100644 control-plane/object-storage/src/handlers/mod.rs create mode 100644 control-plane/object-storage/src/logger.rs create mode 100644 control-plane/object-storage/src/main.rs create mode 100644 control-plane/object-storage/src/metrics.rs create mode 100644 control-plane/object-storage/src/models.rs create mode 100644 control-plane/object-storage/src/server.rs create mode 100644 control-plane/object-storage/src/services/bucket.rs create mode 100644 control-plane/object-storage/src/services/mod.rs diff --git a/Cargo.toml b/Cargo.toml index c9f8531..33c29d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "control-plane/failover-controller", "control-plane/sdn-controller", "control-plane/volume-manager", + "control-plane/object-storage", "control-plane/registry", "control-plane/shared/entity", "control-plane/shared/migration", diff --git a/control-plane/object-storage/Cargo.toml b/control-plane/object-storage/Cargo.toml new file mode 100644 index 0000000..1ba2f7b --- /dev/null +++ b/control-plane/object-storage/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "object-storage" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "object-storage" +path = "src/main.rs" + +[dependencies] +shared = { path = "../shared/shared" } +entity = { path = "../shared/entity" } + +tokio = { workspace = true, features = ["full"] } +axum = { version = "0.8", features = ["macros"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +serde = { workspace = true } +serde_json = { workspace = true } +dotenvy = { workspace = true } +anyhow = { workspace = true } +uuid = { workspace = true, features = ["v4", "serde"] } +chrono = { workspace = true, features = ["serde"] } +sea-orm = { workspace = true } +etcd-client = { workspace = true } +reqwest = { workspace = true, features = ["json"] } +ring = { workspace = true } +rustls = { workspace = true } +prometheus = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } +tracing-opentelemetry = { workspace = true } diff --git a/control-plane/object-storage/src/db/buckets.rs b/control-plane/object-storage/src/db/buckets.rs new file mode 100644 index 0000000..2364b58 --- /dev/null +++ b/control-plane/object-storage/src/db/buckets.rs @@ -0,0 +1,108 @@ +use anyhow::{Context, Result}; +use chrono::Utc; +use entity::entities::buckets; +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, +}; +use uuid::Uuid; + +use crate::models::{BucketResponse, CreateBucketRequest, UpdateBucketRequest}; + +pub fn into_response(model: buckets::Model) -> BucketResponse { + BucketResponse { + id: model.id, + name: model.name, + global_alias: model.global_alias, + exposure: model.exposure, + quota_max_size: model.quota_max_size, + quota_max_objects: model.quota_max_objects, + status: model.status, + organization_id: model.organization_id, + resource_group_id: model.resource_group_id, + created_at: model.created_at, + updated_at: model.updated_at, + } +} + +pub async fn insert( + db: &DatabaseConnection, + req: &CreateBucketRequest, + global_alias: &str, + garage_bucket_id: &str, +) -> Result { + let model = buckets::ActiveModel { + id: Set(Uuid::new_v4()), + name: Set(req.name.clone()), + garage_bucket_id: Set(Some(garage_bucket_id.to_string())), + global_alias: Set(global_alias.to_string()), + exposure: Set(req.exposure.clone().unwrap_or_else(|| "internal".to_string())), + quota_max_size: Set(req.quota_max_size), + quota_max_objects: Set(req.quota_max_objects), + status: Set("active".to_string()), + organization_id: Set(req.organization_id), + resource_group_id: Set(req.resource_group_id), + created_at: Set(Utc::now().naive_utc()), + updated_at: Set(None), + }; + + model.insert(db).await.context("failed to insert bucket") +} + +pub async fn get_by_id(db: &DatabaseConnection, id: Uuid) -> Result> { + buckets::Entity::find_by_id(id) + .one(db) + .await + .context("failed to get bucket") +} + +pub async fn list( + db: &DatabaseConnection, + resource_group_id: Option, + organization_id: Option, +) -> Result> { + let mut query = buckets::Entity::find(); + + if let Some(rg_id) = resource_group_id { + query = query.filter(buckets::Column::ResourceGroupId.eq(rg_id)); + } + + if let Some(org_id) = organization_id { + query = query.filter(buckets::Column::OrganizationId.eq(org_id)); + } + + query.all(db).await.context("failed to list buckets") +} + +pub async fn update( + db: &DatabaseConnection, + id: Uuid, + req: &UpdateBucketRequest, +) -> Result> { + let Some(existing) = get_by_id(db, id).await? else { + return Ok(None); + }; + + let mut model: buckets::ActiveModel = existing.into(); + + if let Some(exposure) = &req.exposure { + model.exposure = Set(exposure.clone()); + } + if req.quota_max_size.is_some() { + model.quota_max_size = Set(req.quota_max_size); + } + if req.quota_max_objects.is_some() { + model.quota_max_objects = Set(req.quota_max_objects); + } + model.updated_at = Set(Some(Utc::now().naive_utc())); + + let updated = model.update(db).await.context("failed to update bucket")?; + Ok(Some(updated)) +} + +pub async fn delete(db: &DatabaseConnection, id: Uuid) -> Result<()> { + buckets::Entity::delete_by_id(id) + .exec(db) + .await + .context("failed to delete bucket")?; + Ok(()) +} diff --git a/control-plane/object-storage/src/db/mod.rs b/control-plane/object-storage/src/db/mod.rs new file mode 100644 index 0000000..0abce3b --- /dev/null +++ b/control-plane/object-storage/src/db/mod.rs @@ -0,0 +1 @@ +pub mod buckets; diff --git a/control-plane/object-storage/src/garage/client.rs b/control-plane/object-storage/src/garage/client.rs new file mode 100644 index 0000000..7f2625e --- /dev/null +++ b/control-plane/object-storage/src/garage/client.rs @@ -0,0 +1,117 @@ +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone)] +pub struct GarageClient { + http: reqwest::Client, + admin_url: String, + admin_token: String, +} + +#[derive(Debug, Deserialize)] +pub struct GarageBucket { + pub id: String, +} + +#[derive(Debug, Serialize)] +struct CreateBucketBody { + #[serde(rename = "globalAlias")] + global_alias: String, +} + +#[derive(Debug, Serialize)] +struct UpdateBucketBody { + quotas: UpdateBucketQuotas, +} + +#[derive(Debug, Serialize)] +struct UpdateBucketQuotas { + #[serde(rename = "maxSize", skip_serializing_if = "Option::is_none")] + max_size: Option, + #[serde(rename = "maxObjects", skip_serializing_if = "Option::is_none")] + max_objects: Option, +} + +impl GarageClient { + pub fn new(admin_url: String, admin_token: String) -> Self { + Self { + http: reqwest::Client::new(), + admin_url, + admin_token, + } + } + + fn url(&self, path: &str) -> String { + format!("{}{}", self.admin_url, path) + } + + async fn check_status(response: reqwest::Response, context: &str) -> Result { + if response.status().is_success() { + return Ok(response); + } + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + bail!("garage admin api error context={} status={} body={}", context, status, body) + } + + pub async fn create_bucket(&self, global_alias: &str) -> Result { + let response = self + .http + .post(self.url("/v2/CreateBucket")) + .bearer_auth(&self.admin_token) + .json(&CreateBucketBody { + global_alias: global_alias.to_string(), + }) + .send() + .await + .context("create_bucket request failed")?; + + let response = Self::check_status(response, "create_bucket").await?; + response + .json::() + .await + .context("failed to parse create_bucket response") + } + + pub async fn update_bucket_quotas( + &self, + garage_bucket_id: &str, + max_size: Option, + max_objects: Option, + ) -> Result<()> { + let response = self + .http + .post(self.url(&format!("/v2/UpdateBucket?id={}", garage_bucket_id))) + .bearer_auth(&self.admin_token) + .json(&UpdateBucketBody { + quotas: UpdateBucketQuotas { + max_size, + max_objects, + }, + }) + .send() + .await + .context("update_bucket_quotas request failed")?; + + Self::check_status(response, "update_bucket_quotas").await?; + Ok(()) + } + + pub async fn delete_bucket(&self, garage_bucket_id: &str) -> Result<()> { + let response = self + .http + .delete(self.url(&format!("/v2/DeleteBucket?id={}", garage_bucket_id))) + .bearer_auth(&self.admin_token) + .send() + .await + .context("delete_bucket request failed")?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + + Self::check_status(response, "delete_bucket").await?; + Ok(()) + } + +} diff --git a/control-plane/object-storage/src/garage/mod.rs b/control-plane/object-storage/src/garage/mod.rs new file mode 100644 index 0000000..f0449a8 --- /dev/null +++ b/control-plane/object-storage/src/garage/mod.rs @@ -0,0 +1,3 @@ +pub mod client; + +pub use client::GarageClient; diff --git a/control-plane/object-storage/src/handlers/buckets.rs b/control-plane/object-storage/src/handlers/buckets.rs new file mode 100644 index 0000000..680f864 --- /dev/null +++ b/control-plane/object-storage/src/handlers/buckets.rs @@ -0,0 +1,113 @@ +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde::Deserialize; +use serde_json::json; +use uuid::Uuid; + +use crate::{ + models::{CreateBucketRequest, UpdateBucketRequest}, + server::AppState, + services::bucket as service, +}; + +#[derive(Debug, Deserialize)] +pub struct ListBucketsQuery { + pub resource_group_id: Option, + pub organization_id: Option, +} + +pub async fn create_bucket( + State(state): State, + Json(req): Json, +) -> Result)> { + match service::create_bucket(&state.db, &state.garage, req).await { + Ok(bucket) => Ok((StatusCode::CREATED, Json(json!(bucket)))), + Err(e) => { + tracing::error!(error = %e, "failed to create bucket"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn list_buckets( + State(state): State, + Query(query): Query, +) -> Result)> { + match service::list_buckets(&state.db, query.resource_group_id, query.organization_id).await { + Ok(buckets) => Ok(Json(json!(buckets))), + Err(e) => { + tracing::error!(error = %e, "failed to list buckets"); + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn get_bucket( + State(state): State, + Path(id): Path, +) -> Result)> { + match service::get_bucket(&state.db, id).await { + Ok(Some(bucket)) => Ok(Json(json!(bucket))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to get bucket"); + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn update_bucket( + State(state): State, + Path(id): Path, + Json(req): Json, +) -> Result)> { + match service::update_bucket(&state.db, &state.garage, id, req).await { + Ok(Some(bucket)) => Ok(Json(json!(bucket))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to update bucket"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn delete_bucket( + State(state): State, + Path(id): Path, +) -> Result)> { + match service::delete_bucket(&state.db, &state.garage, id).await { + Ok(true) => Ok(StatusCode::NO_CONTENT), + Ok(false) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to delete bucket"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} diff --git a/control-plane/object-storage/src/handlers/mod.rs b/control-plane/object-storage/src/handlers/mod.rs new file mode 100644 index 0000000..0abce3b --- /dev/null +++ b/control-plane/object-storage/src/handlers/mod.rs @@ -0,0 +1 @@ +pub mod buckets; diff --git a/control-plane/object-storage/src/logger.rs b/control-plane/object-storage/src/logger.rs new file mode 100644 index 0000000..7d8b652 --- /dev/null +++ b/control-plane/object-storage/src/logger.rs @@ -0,0 +1,152 @@ +use opentelemetry::trace::TracerProvider as _; +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::trace::SdkTracerProvider; +use tracing::{event, Level}; +use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; + +#[derive(Debug, Clone, Copy)] +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, +} + +impl From for Level { + fn from(level: LogLevel) -> Self { + match level { + LogLevel::Trace => Level::TRACE, + LogLevel::Debug => Level::DEBUG, + LogLevel::Info => Level::INFO, + LogLevel::Warn => Level::WARN, + LogLevel::Error => Level::ERROR, + } + } +} + +fn build_otlp_provider(service_name: &str) -> Option { + let endpoint = std::env::var("OTLP_ENDPOINT").ok()?; + + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .build() + .ok()?; + + let resource = opentelemetry_sdk::Resource::builder_empty() + .with_attribute(KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAME, + service_name.to_string(), + )) + .build(); + + let provider = SdkTracerProvider::builder() + .with_resource(resource) + .with_batch_exporter(exporter) + .build(); + + Some(provider) +} + +pub fn init_logger() -> tokio::sync::mpsc::UnboundedReceiver { + init_logger_with_service(env!("CARGO_PKG_NAME")) +} + +pub fn init_logger_with_service( + service_name: &'static str, +) -> tokio::sync::mpsc::UnboundedReceiver { + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let fmt_layer = fmt::layer().with_target(false).with_thread_ids(true); + let (db_layer, db_receiver) = shared::db_log_layer::DbLogLayer::new(service_name); + + let registry = tracing_subscriber::registry() + .with(filter) + .with(fmt_layer) + .with(db_layer); + + match build_otlp_provider(service_name) { + Some(provider) => { + let tracer = provider.tracer(service_name); + let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer); + registry.with(otel_layer).init(); + tracing::info!(service = service_name, "OpenTelemetry tracing enabled"); + } + None => { + registry.init(); + } + } + + db_receiver +} + +pub fn log_message(level: LogLevel, module: &str, location: &str, description: &str) { + let lvl: Level = level.into(); + + match lvl { + Level::ERROR => { + event!(Level::ERROR, module = %module, location = %location, "{}", description) + } + Level::WARN => { + event!(Level::WARN, module = %module, location = %location, "{}", description) + } + Level::INFO => { + event!(Level::INFO, module = %module, location = %location, "{}", description) + } + Level::DEBUG => { + event!(Level::DEBUG, module = %module, location = %location, "{}", description) + } + Level::TRACE => { + event!(Level::TRACE, module = %module, location = %location, "{}", description) + } + } +} + +#[macro_export] +macro_rules! log_info { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Info, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} + +#[macro_export] +macro_rules! log_warn { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Warn, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} + +#[macro_export] +macro_rules! log_error { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Error, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} + +#[macro_export] +macro_rules! log_debug { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Debug, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} diff --git a/control-plane/object-storage/src/main.rs b/control-plane/object-storage/src/main.rs new file mode 100644 index 0000000..184937c --- /dev/null +++ b/control-plane/object-storage/src/main.rs @@ -0,0 +1,66 @@ +use std::net::SocketAddr; + +mod db; +mod garage; +mod handlers; +mod logger; +mod metrics; +mod models; +mod server; +mod services; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + rustls::crypto::ring::default_provider() + .install_default() + .expect("failed to install ring crypto provider"); + + dotenvy::dotenv().ok(); + + let log_receiver = logger::init_logger(); + + metrics::init(); + log_info!("main", "CSFX Object Storage starting..."); + log_info!("main", &format!("Version: {}", env!("CARGO_PKG_VERSION"))); + + log_info!("main", "Connecting to database..."); + let db = shared::establish_connection() + .await + .expect("Failed to connect to database"); + log_info!("main", "Database connection established"); + shared::spawn_log_writer(log_receiver, db.clone()); + + let admin_url = std::env::var("GARAGE_ADMIN_URL") + .unwrap_or_else(|_| "http://127.0.0.1:3903".to_string()); + let admin_token = std::env::var("GARAGE_ADMIN_TOKEN") + .expect("GARAGE_ADMIN_TOKEN must be set"); + let garage = garage::GarageClient::new(admin_url, admin_token); + + let state = server::AppState::new(db, garage); + let app = server::create_router(state); + + let port = std::env::var("OBJECT_STORAGE_PORT") + .ok() + .and_then(|p| p.parse::().ok()) + .unwrap_or(8006); + + let listen_addr = std::env::var("LISTEN_ADDR").unwrap_or_else(|_| "127.0.0.1".to_string()); + let addr: SocketAddr = format!("{}:{}", listen_addr, port).parse().unwrap(); + log_info!("main", &format!("Object Storage listening port={}", port)); + + let listener = tokio::net::TcpListener::bind(addr).await?; + + tokio::select! { + result = axum::serve(listener, app) => { + if let Err(e) = result { + log_error!("main", &format!("Server error err={}", e)); + } + } + _ = tokio::signal::ctrl_c() => { + log_info!("main", "Shutdown signal received"); + } + } + + log_info!("main", "Object Storage shutting down"); + Ok(()) +} diff --git a/control-plane/object-storage/src/metrics.rs b/control-plane/object-storage/src/metrics.rs new file mode 100644 index 0000000..aa5a979 --- /dev/null +++ b/control-plane/object-storage/src/metrics.rs @@ -0,0 +1,57 @@ +use axum::response::IntoResponse; +use prometheus::{ + register_counter_vec, register_histogram_vec, CounterVec, Encoder, HistogramVec, TextEncoder, +}; +use std::sync::OnceLock; + +static HTTP_REQUESTS_TOTAL: OnceLock = OnceLock::new(); +static HTTP_REQUEST_DURATION_SECONDS: OnceLock = OnceLock::new(); + +pub fn init() { + HTTP_REQUESTS_TOTAL.get_or_init(|| { + register_counter_vec!( + "csfx_http_requests_total", + "Total HTTP requests", + &["method", "path", "status"] + ) + .expect("failed to register csfx_http_requests_total") + }); + + HTTP_REQUEST_DURATION_SECONDS.get_or_init(|| { + register_histogram_vec!( + "csfx_http_request_duration_seconds", + "HTTP request duration in seconds", + &["method", "path"] + ) + .expect("failed to register csfx_http_request_duration_seconds") + }); +} + +pub fn record_request(method: &str, path: &str, status: u16, duration_secs: f64) { + if let Some(counter) = HTTP_REQUESTS_TOTAL.get() { + counter + .with_label_values(&[method, path, &status.to_string()]) + .inc(); + } + if let Some(histogram) = HTTP_REQUEST_DURATION_SECONDS.get() { + histogram + .with_label_values(&[method, path]) + .observe(duration_secs); + } +} + +pub async fn metrics_handler() -> impl IntoResponse { + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buffer = Vec::new(); + encoder + .encode(&metric_families, &mut buffer) + .expect("failed to encode metrics"); + ( + [( + axum::http::header::CONTENT_TYPE, + "text/plain; version=0.0.4", + )], + buffer, + ) +} diff --git a/control-plane/object-storage/src/models.rs b/control-plane/object-storage/src/models.rs new file mode 100644 index 0000000..d8bd25d --- /dev/null +++ b/control-plane/object-storage/src/models.rs @@ -0,0 +1,34 @@ +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateBucketRequest { + pub name: String, + pub exposure: Option, + pub quota_max_size: Option, + pub quota_max_objects: Option, + pub organization_id: Option, + pub resource_group_id: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct UpdateBucketRequest { + pub exposure: Option, + pub quota_max_size: Option, + pub quota_max_objects: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct BucketResponse { + pub id: Uuid, + pub name: String, + pub global_alias: String, + pub exposure: String, + pub quota_max_size: Option, + pub quota_max_objects: Option, + pub status: String, + pub organization_id: Option, + pub resource_group_id: Option, + pub created_at: chrono::NaiveDateTime, + pub updated_at: Option, +} diff --git a/control-plane/object-storage/src/server.rs b/control-plane/object-storage/src/server.rs new file mode 100644 index 0000000..4a7886d --- /dev/null +++ b/control-plane/object-storage/src/server.rs @@ -0,0 +1,37 @@ +use axum::{http::StatusCode, response::IntoResponse, routing::get, Router}; +use sea_orm::DatabaseConnection; + +use crate::{garage::GarageClient, handlers::buckets, metrics}; + +#[derive(Clone)] +pub struct AppState { + pub db: DatabaseConnection, + pub garage: GarageClient, +} + +impl AppState { + pub fn new(db: DatabaseConnection, garage: GarageClient) -> Self { + Self { db, garage } + } +} + +pub async fn health_check() -> impl IntoResponse { + (StatusCode::OK, "Object Storage OK") +} + +pub fn create_router(state: AppState) -> Router { + Router::new() + .route("/health", get(health_check)) + .route("/metrics", get(metrics::metrics_handler)) + .route( + "/buckets", + get(buckets::list_buckets).post(buckets::create_bucket), + ) + .route( + "/buckets/{id}", + get(buckets::get_bucket) + .patch(buckets::update_bucket) + .delete(buckets::delete_bucket), + ) + .with_state(state) +} diff --git a/control-plane/object-storage/src/services/bucket.rs b/control-plane/object-storage/src/services/bucket.rs new file mode 100644 index 0000000..a845a20 --- /dev/null +++ b/control-plane/object-storage/src/services/bucket.rs @@ -0,0 +1,99 @@ +use anyhow::{bail, Result}; +use sea_orm::DatabaseConnection; +use uuid::Uuid; + +use crate::{ + db::buckets as db, + garage::GarageClient, + log_error, log_info, + models::{BucketResponse, CreateBucketRequest, UpdateBucketRequest}, +}; + +pub async fn create_bucket( + db_conn: &DatabaseConnection, + garage: &GarageClient, + req: CreateBucketRequest, +) -> Result { + let global_alias = format!("{}-{}", req.name, Uuid::new_v4().simple()); + + let garage_bucket = match garage.create_bucket(&global_alias).await { + Ok(bucket) => bucket, + Err(e) => { + log_error!("services::bucket", &format!("garage create_bucket failed name={} err={}", req.name, e)); + bail!("failed to create bucket in garage: {}", e); + } + }; + + if req.quota_max_size.is_some() || req.quota_max_objects.is_some() { + if let Err(e) = garage + .update_bucket_quotas(&garage_bucket.id, req.quota_max_size, req.quota_max_objects) + .await + { + log_error!("services::bucket", &format!("garage update_bucket_quotas failed bucket_id={} err={}", garage_bucket.id, e)); + let _ = garage.delete_bucket(&garage_bucket.id).await; + bail!("failed to apply bucket quota: {}", e); + } + } + + let model = db::insert(db_conn, &req, &global_alias, &garage_bucket.id).await?; + log_info!("services::bucket", &format!("bucket created id={} garage_bucket_id={}", model.id, garage_bucket.id)); + + Ok(db::into_response(model)) +} + +pub async fn get_bucket( + db_conn: &DatabaseConnection, + id: Uuid, +) -> Result> { + Ok(db::get_by_id(db_conn, id).await?.map(db::into_response)) +} + +pub async fn list_buckets( + db_conn: &DatabaseConnection, + resource_group_id: Option, + organization_id: Option, +) -> Result> { + let rows = db::list(db_conn, resource_group_id, organization_id).await?; + Ok(rows.into_iter().map(db::into_response).collect()) +} + +pub async fn update_bucket( + db_conn: &DatabaseConnection, + garage: &GarageClient, + id: Uuid, + req: UpdateBucketRequest, +) -> Result> { + let Some(existing) = db::get_by_id(db_conn, id).await? else { + return Ok(None); + }; + + if req.quota_max_size.is_some() || req.quota_max_objects.is_some() { + let Some(garage_bucket_id) = &existing.garage_bucket_id else { + bail!("bucket has no garage_bucket_id, cannot update quota"); + }; + garage + .update_bucket_quotas(garage_bucket_id, req.quota_max_size, req.quota_max_objects) + .await?; + } + + let updated = db::update(db_conn, id, &req).await?.map(db::into_response); + Ok(updated) +} + +pub async fn delete_bucket( + db_conn: &DatabaseConnection, + garage: &GarageClient, + id: Uuid, +) -> Result { + let Some(existing) = db::get_by_id(db_conn, id).await? else { + return Ok(false); + }; + + if let Some(garage_bucket_id) = &existing.garage_bucket_id { + garage.delete_bucket(garage_bucket_id).await?; + } + + db::delete(db_conn, id).await?; + log_info!("services::bucket", &format!("bucket deleted id={}", id)); + Ok(true) +} diff --git a/control-plane/object-storage/src/services/mod.rs b/control-plane/object-storage/src/services/mod.rs new file mode 100644 index 0000000..d25f087 --- /dev/null +++ b/control-plane/object-storage/src/services/mod.rs @@ -0,0 +1 @@ +pub mod bucket; From f1a65b26b1380d02cbeb3a6bf526b19761092766 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 16:53:32 +0200 Subject: [PATCH 05/24] feat: add bucket access key issuance and rotation --- .../object-storage/src/db/access_keys.rs | 73 +++++++++ control-plane/object-storage/src/db/mod.rs | 1 + .../object-storage/src/garage/client.rs | 72 +++++++++ .../object-storage/src/handlers/keys.rs | 86 +++++++++++ .../object-storage/src/handlers/mod.rs | 1 + control-plane/object-storage/src/models.rs | 26 ++++ control-plane/object-storage/src/server.rs | 18 ++- .../object-storage/src/services/access_key.rs | 141 ++++++++++++++++++ .../object-storage/src/services/mod.rs | 1 + 9 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 control-plane/object-storage/src/db/access_keys.rs create mode 100644 control-plane/object-storage/src/handlers/keys.rs create mode 100644 control-plane/object-storage/src/services/access_key.rs diff --git a/control-plane/object-storage/src/db/access_keys.rs b/control-plane/object-storage/src/db/access_keys.rs new file mode 100644 index 0000000..1d1e001 --- /dev/null +++ b/control-plane/object-storage/src/db/access_keys.rs @@ -0,0 +1,73 @@ +use anyhow::{Context, Result}; +use chrono::Utc; +use entity::entities::bucket_access_keys; +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, +}; +use uuid::Uuid; + +pub async fn insert( + db: &DatabaseConnection, + bucket_id: Uuid, + name: &str, + garage_key_id: &str, + permissions: &str, + expires_at: Option, +) -> Result { + let model = bucket_access_keys::ActiveModel { + id: Set(Uuid::new_v4()), + bucket_id: Set(bucket_id), + name: Set(name.to_string()), + garage_key_id: Set(garage_key_id.to_string()), + permissions: Set(permissions.to_string()), + expires_at: Set(expires_at), + last_rotated_at: Set(None), + created_at: Set(Utc::now().naive_utc()), + }; + + model + .insert(db) + .await + .context("failed to insert bucket access key") +} + +pub async fn get_by_id( + db: &DatabaseConnection, + id: Uuid, +) -> Result> { + bucket_access_keys::Entity::find_by_id(id) + .one(db) + .await + .context("failed to get bucket access key") +} + +pub async fn list_for_bucket( + db: &DatabaseConnection, + bucket_id: Uuid, +) -> Result> { + bucket_access_keys::Entity::find() + .filter(bucket_access_keys::Column::BucketId.eq(bucket_id)) + .all(db) + .await + .context("failed to list bucket access keys") +} + +pub async fn touch_rotated(db: &DatabaseConnection, id: Uuid) -> Result<()> { + if let Some(existing) = get_by_id(db, id).await? { + let mut model: bucket_access_keys::ActiveModel = existing.into(); + model.last_rotated_at = Set(Some(Utc::now().naive_utc())); + model + .update(db) + .await + .context("failed to update bucket access key rotation timestamp")?; + } + Ok(()) +} + +pub async fn delete(db: &DatabaseConnection, id: Uuid) -> Result<()> { + bucket_access_keys::Entity::delete_by_id(id) + .exec(db) + .await + .context("failed to delete bucket access key")?; + Ok(()) +} diff --git a/control-plane/object-storage/src/db/mod.rs b/control-plane/object-storage/src/db/mod.rs index 0abce3b..0cc482c 100644 --- a/control-plane/object-storage/src/db/mod.rs +++ b/control-plane/object-storage/src/db/mod.rs @@ -1 +1,2 @@ +pub mod access_keys; pub mod buckets; diff --git a/control-plane/object-storage/src/garage/client.rs b/control-plane/object-storage/src/garage/client.rs index 7f2625e..734fedb 100644 --- a/control-plane/object-storage/src/garage/client.rs +++ b/control-plane/object-storage/src/garage/client.rs @@ -1,5 +1,6 @@ use anyhow::{bail, Context, Result}; use serde::{Deserialize, Serialize}; +use serde_json::json; #[derive(Clone)] pub struct GarageClient { @@ -32,6 +33,14 @@ struct UpdateBucketQuotas { max_objects: Option, } +#[derive(Debug, Deserialize)] +pub struct GarageKey { + #[serde(rename = "accessKeyId")] + pub access_key_id: String, + #[serde(rename = "secretAccessKey")] + pub secret_access_key: String, +} + impl GarageClient { pub fn new(admin_url: String, admin_token: String) -> Self { Self { @@ -114,4 +123,67 @@ impl GarageClient { Ok(()) } + pub async fn create_key(&self, name: &str) -> Result { + let response = self + .http + .post(self.url("/v2/CreateKey")) + .bearer_auth(&self.admin_token) + .json(&json!({ "name": name })) + .send() + .await + .context("create_key request failed")?; + + let response = Self::check_status(response, "create_key").await?; + response + .json::() + .await + .context("failed to parse create_key response") + } + + pub async fn delete_key(&self, garage_key_id: &str) -> Result<()> { + let response = self + .http + .delete(self.url(&format!("/v2/DeleteKey?id={}", garage_key_id))) + .bearer_auth(&self.admin_token) + .send() + .await + .context("delete_key request failed")?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + + Self::check_status(response, "delete_key").await?; + Ok(()) + } + + pub async fn allow_bucket_key( + &self, + garage_bucket_id: &str, + garage_key_id: &str, + permissions: &str, + ) -> Result<()> { + let (read, write, owner) = match permissions { + "read" => (true, false, false), + "readwrite" => (true, true, false), + "owner" => (true, true, true), + other => bail!("unknown bucket key permission permissions={}", other), + }; + + let response = self + .http + .post(self.url("/v2/AllowBucketKey")) + .bearer_auth(&self.admin_token) + .json(&json!({ + "bucketId": garage_bucket_id, + "accessKeyId": garage_key_id, + "permissions": { "read": read, "write": write, "owner": owner }, + })) + .send() + .await + .context("allow_bucket_key request failed")?; + + Self::check_status(response, "allow_bucket_key").await?; + Ok(()) + } } diff --git a/control-plane/object-storage/src/handlers/keys.rs b/control-plane/object-storage/src/handlers/keys.rs new file mode 100644 index 0000000..cb8f1f9 --- /dev/null +++ b/control-plane/object-storage/src/handlers/keys.rs @@ -0,0 +1,86 @@ +use axum::{ + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde_json::json; +use uuid::Uuid; + +use crate::{models::CreateAccessKeyRequest, server::AppState, services::access_key as service}; + +pub async fn create_key( + State(state): State, + Path(bucket_id): Path, + Json(req): Json, +) -> Result)> { + match service::create_key(&state.db, &state.garage, bucket_id, req).await { + Ok(Some(key)) => Ok((StatusCode::CREATED, Json(json!(key)))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to create access key"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn list_keys( + State(state): State, + Path(bucket_id): Path, +) -> Result)> { + match service::list_keys(&state.db, bucket_id).await { + Ok(keys) => Ok(Json(json!(keys))), + Err(e) => { + tracing::error!(error = %e, "failed to list access keys"); + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn rotate_key( + State(state): State, + Path((bucket_id, key_id)): Path<(Uuid, Uuid)>, +) -> Result)> { + match service::rotate_key(&state.db, &state.garage, bucket_id, key_id).await { + Ok(Some(key)) => Ok(Json(json!(key))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Access key not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to rotate access key"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn delete_key( + State(state): State, + Path((bucket_id, key_id)): Path<(Uuid, Uuid)>, +) -> Result)> { + match service::delete_key(&state.db, &state.garage, bucket_id, key_id).await { + Ok(true) => Ok(StatusCode::NO_CONTENT), + Ok(false) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Access key not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to delete access key"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} diff --git a/control-plane/object-storage/src/handlers/mod.rs b/control-plane/object-storage/src/handlers/mod.rs index 0abce3b..78fe757 100644 --- a/control-plane/object-storage/src/handlers/mod.rs +++ b/control-plane/object-storage/src/handlers/mod.rs @@ -1 +1,2 @@ pub mod buckets; +pub mod keys; diff --git a/control-plane/object-storage/src/models.rs b/control-plane/object-storage/src/models.rs index d8bd25d..02525f4 100644 --- a/control-plane/object-storage/src/models.rs +++ b/control-plane/object-storage/src/models.rs @@ -18,6 +18,32 @@ pub struct UpdateBucketRequest { pub quota_max_objects: Option, } +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateAccessKeyRequest { + pub name: String, + pub permissions: Option, + pub expires_at: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct AccessKeyResponse { + pub id: Uuid, + pub bucket_id: Uuid, + pub name: String, + pub garage_key_id: String, + pub permissions: String, + pub expires_at: Option, + pub last_rotated_at: Option, + pub created_at: chrono::NaiveDateTime, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct AccessKeyCreatedResponse { + #[serde(flatten)] + pub key: AccessKeyResponse, + pub secret_access_key: String, +} + #[derive(Debug, Serialize, Deserialize)] pub struct BucketResponse { pub id: Uuid, diff --git a/control-plane/object-storage/src/server.rs b/control-plane/object-storage/src/server.rs index 4a7886d..34f3ff6 100644 --- a/control-plane/object-storage/src/server.rs +++ b/control-plane/object-storage/src/server.rs @@ -1,7 +1,11 @@ use axum::{http::StatusCode, response::IntoResponse, routing::get, Router}; use sea_orm::DatabaseConnection; -use crate::{garage::GarageClient, handlers::buckets, metrics}; +use crate::{ + garage::GarageClient, + handlers::{buckets, keys}, + metrics, +}; #[derive(Clone)] pub struct AppState { @@ -33,5 +37,17 @@ pub fn create_router(state: AppState) -> Router { .patch(buckets::update_bucket) .delete(buckets::delete_bucket), ) + .route( + "/buckets/{id}/keys", + get(keys::list_keys).post(keys::create_key), + ) + .route( + "/buckets/{id}/keys/{key_id}/rotate", + axum::routing::post(keys::rotate_key), + ) + .route( + "/buckets/{id}/keys/{key_id}", + axum::routing::delete(keys::delete_key), + ) .with_state(state) } diff --git a/control-plane/object-storage/src/services/access_key.rs b/control-plane/object-storage/src/services/access_key.rs new file mode 100644 index 0000000..742e0df --- /dev/null +++ b/control-plane/object-storage/src/services/access_key.rs @@ -0,0 +1,141 @@ +use anyhow::{bail, Result}; +use sea_orm::DatabaseConnection; +use uuid::Uuid; + +use crate::{ + db::{access_keys as keys_db, buckets as buckets_db}, + garage::GarageClient, + log_error, log_info, + models::{AccessKeyCreatedResponse, AccessKeyResponse, CreateAccessKeyRequest}, +}; + +fn into_response(model: entity::entities::bucket_access_keys::Model) -> AccessKeyResponse { + AccessKeyResponse { + id: model.id, + bucket_id: model.bucket_id, + name: model.name, + garage_key_id: model.garage_key_id, + permissions: model.permissions, + expires_at: model.expires_at, + last_rotated_at: model.last_rotated_at, + created_at: model.created_at, + } +} + +pub async fn create_key( + db: &DatabaseConnection, + garage: &GarageClient, + bucket_id: Uuid, + req: CreateAccessKeyRequest, +) -> Result> { + let Some(bucket) = buckets_db::get_by_id(db, bucket_id).await? else { + return Ok(None); + }; + let Some(garage_bucket_id) = &bucket.garage_bucket_id else { + bail!("bucket has no garage_bucket_id, cannot create access key"); + }; + + let permissions = req.permissions.unwrap_or_else(|| "readwrite".to_string()); + + let garage_key = match garage.create_key(&req.name).await { + Ok(key) => key, + Err(e) => { + log_error!("services::access_key", &format!("garage create_key failed name={} err={}", req.name, e)); + bail!("failed to create access key in garage: {}", e); + } + }; + + if let Err(e) = garage + .allow_bucket_key(garage_bucket_id, &garage_key.access_key_id, &permissions) + .await + { + log_error!("services::access_key", &format!("garage allow_bucket_key failed key_id={} err={}", garage_key.access_key_id, e)); + let _ = garage.delete_key(&garage_key.access_key_id).await; + bail!("failed to grant bucket access: {}", e); + } + + let model = keys_db::insert( + db, + bucket_id, + &req.name, + &garage_key.access_key_id, + &permissions, + req.expires_at, + ) + .await?; + + log_info!("services::access_key", &format!("access key created id={} bucket_id={}", model.id, bucket_id)); + + Ok(Some(AccessKeyCreatedResponse { + key: into_response(model), + secret_access_key: garage_key.secret_access_key, + })) +} + +pub async fn list_keys( + db: &DatabaseConnection, + bucket_id: Uuid, +) -> Result> { + let rows = keys_db::list_for_bucket(db, bucket_id).await?; + Ok(rows.into_iter().map(into_response).collect()) +} + +pub async fn rotate_key( + db: &DatabaseConnection, + garage: &GarageClient, + bucket_id: Uuid, + key_id: Uuid, +) -> Result> { + let Some(existing) = keys_db::get_by_id(db, key_id).await? else { + return Ok(None); + }; + if existing.bucket_id != bucket_id { + return Ok(None); + } + + let created = create_key( + db, + garage, + bucket_id, + CreateAccessKeyRequest { + name: existing.name.clone(), + permissions: Some(existing.permissions.clone()), + expires_at: existing.expires_at, + }, + ) + .await?; + + let Some(created) = created else { + return Ok(None); + }; + + if let Err(e) = garage.delete_key(&existing.garage_key_id).await { + log_error!("services::access_key", &format!("garage delete_key on rotate failed old_key_id={} err={}", existing.garage_key_id, e)); + } + keys_db::delete(db, key_id).await?; + keys_db::touch_rotated(db, created.key.id).await?; + + log_info!("services::access_key", &format!("access key rotated old_id={} new_id={}", key_id, created.key.id)); + + Ok(Some(created)) +} + +pub async fn delete_key( + db: &DatabaseConnection, + garage: &GarageClient, + bucket_id: Uuid, + key_id: Uuid, +) -> Result { + let Some(existing) = keys_db::get_by_id(db, key_id).await? else { + return Ok(false); + }; + if existing.bucket_id != bucket_id { + return Ok(false); + } + + garage.delete_key(&existing.garage_key_id).await?; + keys_db::delete(db, key_id).await?; + + log_info!("services::access_key", &format!("access key deleted id={}", key_id)); + Ok(true) +} diff --git a/control-plane/object-storage/src/services/mod.rs b/control-plane/object-storage/src/services/mod.rs index d25f087..65cc81d 100644 --- a/control-plane/object-storage/src/services/mod.rs +++ b/control-plane/object-storage/src/services/mod.rs @@ -1 +1,2 @@ +pub mod access_key; pub mod bucket; From c6a7ca2f3954afecc5d83e86c3ca2bdacbe8d08d Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 16:55:57 +0200 Subject: [PATCH 06/24] feat: add garage cluster reconciliation with dynamic replication factor --- .../object-storage/src/db/garage_nodes.rs | 39 ++++++ control-plane/object-storage/src/db/mod.rs | 1 + .../object-storage/src/garage/client.rs | 93 +++++++++++++ .../object-storage/src/garage/layout.rs | 123 ++++++++++++++++++ .../object-storage/src/garage/leader.rs | 97 ++++++++++++++ .../object-storage/src/garage/mod.rs | 2 + .../object-storage/src/handlers/cluster.rs | 31 +++++ .../object-storage/src/handlers/mod.rs | 1 + control-plane/object-storage/src/main.rs | 18 +++ control-plane/object-storage/src/models.rs | 8 ++ control-plane/object-storage/src/server.rs | 3 +- 11 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 control-plane/object-storage/src/db/garage_nodes.rs create mode 100644 control-plane/object-storage/src/garage/layout.rs create mode 100644 control-plane/object-storage/src/garage/leader.rs create mode 100644 control-plane/object-storage/src/handlers/cluster.rs diff --git a/control-plane/object-storage/src/db/garage_nodes.rs b/control-plane/object-storage/src/db/garage_nodes.rs new file mode 100644 index 0000000..4f33424 --- /dev/null +++ b/control-plane/object-storage/src/db/garage_nodes.rs @@ -0,0 +1,39 @@ +use anyhow::{Context, Result}; +use chrono::Utc; +use entity::entities::garage_nodes; +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, +}; +use uuid::Uuid; + +pub async fn list(db: &DatabaseConnection) -> Result> { + garage_nodes::Entity::find() + .all(db) + .await + .context("failed to list garage nodes") +} + +pub async fn get_by_agent_id( + db: &DatabaseConnection, + agent_id: Uuid, +) -> Result> { + garage_nodes::Entity::find() + .filter(garage_nodes::Column::AgentId.eq(agent_id)) + .one(db) + .await + .context("failed to get garage node by agent id") +} + +pub async fn mark_down(db: &DatabaseConnection, agent_id: Uuid) -> Result<()> { + if let Some(existing) = get_by_agent_id(db, agent_id).await? { + let mut model: garage_nodes::ActiveModel = existing.into(); + model.status = Set("down".to_string()); + model.updated_at = Set(Some(Utc::now().naive_utc())); + model + .update(db) + .await + .context("failed to mark garage node down")?; + } + Ok(()) +} + diff --git a/control-plane/object-storage/src/db/mod.rs b/control-plane/object-storage/src/db/mod.rs index 0cc482c..1018a92 100644 --- a/control-plane/object-storage/src/db/mod.rs +++ b/control-plane/object-storage/src/db/mod.rs @@ -1,2 +1,3 @@ pub mod access_keys; pub mod buckets; +pub mod garage_nodes; diff --git a/control-plane/object-storage/src/garage/client.rs b/control-plane/object-storage/src/garage/client.rs index 734fedb..b4fbbfe 100644 --- a/control-plane/object-storage/src/garage/client.rs +++ b/control-plane/object-storage/src/garage/client.rs @@ -41,6 +41,29 @@ pub struct GarageKey { pub secret_access_key: String, } +#[derive(Debug, Deserialize)] +pub struct ClusterStatusNode { + pub id: String, + #[serde(default)] + pub is_up: bool, +} + +#[derive(Debug, Deserialize)] +pub struct ClusterStatus { + #[serde(default)] + pub nodes: Vec, + #[serde(rename = "layoutVersion", default)] + pub layout_version: i64, +} + +#[derive(Debug, Serialize)] +pub struct LayoutRole { + pub id: String, + pub zone: String, + pub capacity: Option, + pub tags: Vec, +} + impl GarageClient { pub fn new(admin_url: String, admin_token: String) -> Self { Self { @@ -186,4 +209,74 @@ impl GarageClient { Self::check_status(response, "allow_bucket_key").await?; Ok(()) } + + pub async fn get_cluster_status(&self) -> Result { + let response = self + .http + .get(self.url("/v2/GetClusterStatus")) + .bearer_auth(&self.admin_token) + .send() + .await + .context("get_cluster_status request failed")?; + + let response = Self::check_status(response, "get_cluster_status").await?; + response + .json::() + .await + .context("failed to parse get_cluster_status response") + } + + pub async fn connect_cluster_nodes(&self, node_addrs: &[String]) -> Result<()> { + if node_addrs.is_empty() { + return Ok(()); + } + + let response = self + .http + .post(self.url("/v2/ConnectClusterNodes")) + .bearer_auth(&self.admin_token) + .json(node_addrs) + .send() + .await + .context("connect_cluster_nodes request failed")?; + + Self::check_status(response, "connect_cluster_nodes").await?; + Ok(()) + } + + pub async fn update_cluster_layout( + &self, + roles: Vec, + parameters_replication_factor: u32, + ) -> Result<()> { + let response = self + .http + .post(self.url("/v2/UpdateClusterLayout")) + .bearer_auth(&self.admin_token) + .json(&json!({ + "roles": roles, + "parameters": { "zone_redundancy": "maximum" }, + "replication_factor": parameters_replication_factor, + })) + .send() + .await + .context("update_cluster_layout request failed")?; + + Self::check_status(response, "update_cluster_layout").await?; + Ok(()) + } + + pub async fn apply_cluster_layout(&self, version: i64) -> Result<()> { + let response = self + .http + .post(self.url("/v2/ApplyClusterLayout")) + .bearer_auth(&self.admin_token) + .json(&json!({ "version": version })) + .send() + .await + .context("apply_cluster_layout request failed")?; + + Self::check_status(response, "apply_cluster_layout").await?; + Ok(()) + } } diff --git a/control-plane/object-storage/src/garage/layout.rs b/control-plane/object-storage/src/garage/layout.rs new file mode 100644 index 0000000..def3406 --- /dev/null +++ b/control-plane/object-storage/src/garage/layout.rs @@ -0,0 +1,123 @@ +use anyhow::Result; +use entity::entities::{agents, garage_nodes}; +use sea_orm::{DatabaseConnection, EntityTrait}; +use tokio::time::{sleep, Duration}; + +use crate::{ + db::garage_nodes as garage_nodes_db, + garage::{client::LayoutRole, leader::LayoutLeader, GarageClient}, + log_error, log_info, log_warn, +}; + +const RECONCILE_INTERVAL_SECONDS: u64 = 30; +const MIN_STORAGE_NODES_FOR_FULL_REPLICATION: usize = 3; + +pub fn replication_factor_for(storage_node_count: usize) -> u32 { + if storage_node_count >= MIN_STORAGE_NODES_FOR_FULL_REPLICATION { + 3 + } else { + 1 + } +} + +async fn peer_addrs(db: &DatabaseConnection, known_nodes: &[garage_nodes::Model]) -> Result> { + let mut addrs = Vec::new(); + + for node in known_nodes { + let Some(garage_node_id) = &node.garage_node_id else { + continue; + }; + let Some(agent) = agents::Entity::find_by_id(node.agent_id).one(db).await? else { + continue; + }; + let Some(wg_ip) = agent.wg_tunnel_ip else { + continue; + }; + addrs.push(format!("{}@{}:3901", garage_node_id, wg_ip)); + } + + Ok(addrs) +} + +async fn reconcile_once(db: &DatabaseConnection, garage: &GarageClient) -> Result<()> { + let known_nodes = garage_nodes_db::list(db).await?; + + let addrs = peer_addrs(db, &known_nodes).await?; + if let Err(e) = garage.connect_cluster_nodes(&addrs).await { + log_warn!("garage::layout", &format!("failed to connect garage cluster nodes err={}", e)); + } + + let status = match garage.get_cluster_status().await { + Ok(status) => status, + Err(e) => { + log_warn!("garage::layout", &format!("failed to read garage cluster status err={}", e)); + return Ok(()); + } + }; + + for node in &known_nodes { + let is_up = status + .nodes + .iter() + .any(|n| Some(n.id.clone()) == node.garage_node_id && n.is_up); + + if !is_up && node.status == "up" { + log_warn!("garage::layout", &format!("garage node reported down agent_id={}", node.agent_id)); + garage_nodes_db::mark_down(db, node.agent_id).await?; + } + } + + let storage_nodes: Vec<&garage_nodes::Model> = known_nodes + .iter() + .filter(|n| n.role == "storage" && n.capacity_bytes.is_some()) + .collect(); + + if storage_nodes.is_empty() { + return Ok(()); + } + + let factor = replication_factor_for(storage_nodes.len()); + + let roles: Vec = known_nodes + .iter() + .filter_map(|n| { + n.garage_node_id.as_ref().map(|garage_id| LayoutRole { + id: garage_id.clone(), + zone: n.zone.clone(), + capacity: n.capacity_bytes, + tags: vec![], + }) + }) + .collect(); + + if roles.is_empty() { + return Ok(()); + } + + if let Err(e) = garage.update_cluster_layout(roles, factor).await { + log_error!("garage::layout", &format!("failed to stage cluster layout err={}", e)); + return Ok(()); + } + + if let Err(e) = garage.apply_cluster_layout(status.layout_version + 1).await { + log_error!("garage::layout", &format!("failed to apply cluster layout err={}", e)); + return Ok(()); + } + + log_info!("garage::layout", &format!("applied cluster layout storage_nodes={} replication_factor={}", storage_nodes.len(), factor)); + Ok(()) +} + +pub async fn run_reconcile_loop(db: DatabaseConnection, garage: GarageClient, leader: LayoutLeader) { + loop { + sleep(Duration::from_secs(RECONCILE_INTERVAL_SECONDS)).await; + + if !leader.is_leader() { + continue; + } + + if let Err(e) = reconcile_once(&db, &garage).await { + log_error!("garage::layout", &format!("reconcile loop iteration failed err={}", e)); + } + } +} diff --git a/control-plane/object-storage/src/garage/leader.rs b/control-plane/object-storage/src/garage/leader.rs new file mode 100644 index 0000000..01f1ac6 --- /dev/null +++ b/control-plane/object-storage/src/garage/leader.rs @@ -0,0 +1,97 @@ +use anyhow::{Context, Result}; +use etcd_client::{Client, Compare, CompareOp, PutOptions, Txn, TxnOp}; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; +use std::sync::Arc; +use tokio::time::{sleep, Duration}; + +use crate::{log_error, log_info, log_warn}; + +const LAYOUT_LOCK_KEY: &str = "/csfx/object-storage/layout-lock"; +const LEASE_TTL_SECONDS: i64 = 10; + +#[derive(Clone)] +pub struct LayoutLeader { + etcd: Client, + node_id: String, + lease_id: Arc, + is_leader: Arc, +} + +impl LayoutLeader { + pub fn new(etcd: Client, node_id: String) -> Self { + Self { + etcd, + node_id, + lease_id: Arc::new(AtomicI64::new(0)), + is_leader: Arc::new(AtomicBool::new(false)), + } + } + + pub fn is_leader(&self) -> bool { + self.is_leader.load(Ordering::SeqCst) + } + + async fn campaign(&mut self) -> Result<()> { + if self.is_leader() { + return Ok(()); + } + + let lease = self + .etcd + .lease_grant(LEASE_TTL_SECONDS, None) + .await + .context("failed to grant etcd lease")?; + let lease_id = lease.id(); + + let txn = Txn::new() + .when(vec![Compare::create_revision(LAYOUT_LOCK_KEY, CompareOp::Equal, 0)]) + .and_then(vec![TxnOp::put( + LAYOUT_LOCK_KEY, + self.node_id.as_bytes(), + Some(PutOptions::new().with_lease(lease_id)), + )]); + + let response = self.etcd.txn(txn).await.context("etcd txn failed")?; + + if response.succeeded() { + self.lease_id.store(lease_id, Ordering::SeqCst); + self.is_leader.store(true, Ordering::SeqCst); + log_info!("garage::leader", &format!("became object-storage layout leader node_id={}", self.node_id)); + self.spawn_lease_renewal(lease_id); + } else { + let _ = self.etcd.lease_revoke(lease_id).await; + } + + Ok(()) + } + + fn spawn_lease_renewal(&self, lease_id: i64) { + let mut etcd = self.etcd.clone(); + let is_leader = Arc::clone(&self.is_leader); + let node_id = self.node_id.clone(); + + tokio::spawn(async move { + loop { + sleep(Duration::from_secs(4)).await; + if !is_leader.load(Ordering::SeqCst) { + break; + } + if let Err(e) = etcd.lease_keep_alive(lease_id).await { + log_error!("garage::leader", &format!("lease renewal failed node_id={} err={}", node_id, e)); + is_leader.store(false, Ordering::SeqCst); + log_warn!("garage::leader", &format!("lost object-storage layout leadership node_id={}", node_id)); + break; + } + } + }); + } + + pub async fn run_campaign_loop(mut self) { + loop { + if let Err(e) = self.campaign().await { + log_error!("garage::leader", &format!("layout leader campaign failed err={}", e)); + } + sleep(Duration::from_secs(5)).await; + } + } +} diff --git a/control-plane/object-storage/src/garage/mod.rs b/control-plane/object-storage/src/garage/mod.rs index f0449a8..307192c 100644 --- a/control-plane/object-storage/src/garage/mod.rs +++ b/control-plane/object-storage/src/garage/mod.rs @@ -1,3 +1,5 @@ pub mod client; +pub mod leader; +pub mod layout; pub use client::GarageClient; diff --git a/control-plane/object-storage/src/handlers/cluster.rs b/control-plane/object-storage/src/handlers/cluster.rs new file mode 100644 index 0000000..e12afd1 --- /dev/null +++ b/control-plane/object-storage/src/handlers/cluster.rs @@ -0,0 +1,31 @@ +use axum::{extract::State, http::StatusCode, response::{IntoResponse, Json}}; +use serde_json::json; + +use crate::{db::garage_nodes, garage::layout::replication_factor_for, models::ClusterStatusResponse, server::AppState}; + +pub async fn get_cluster_status( + State(state): State, +) -> Result)> { + let nodes = garage_nodes::list(&state.db).await.map_err(|e| { + tracing::error!(error = %e, "failed to list garage nodes"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + ) + })?; + + let storage_node_count = nodes + .iter() + .filter(|n| n.role == "storage" && n.status == "up") + .count(); + + let replication_factor = replication_factor_for(storage_node_count); + let degraded = storage_node_count == 0 || (storage_node_count as u32) < replication_factor; + + Ok(Json(json!(ClusterStatusResponse { + storage_node_count: storage_node_count as u32, + replication_factor, + degraded, + nodes, + }))) +} diff --git a/control-plane/object-storage/src/handlers/mod.rs b/control-plane/object-storage/src/handlers/mod.rs index 78fe757..f7f1f8f 100644 --- a/control-plane/object-storage/src/handlers/mod.rs +++ b/control-plane/object-storage/src/handlers/mod.rs @@ -1,2 +1,3 @@ pub mod buckets; +pub mod cluster; pub mod keys; diff --git a/control-plane/object-storage/src/main.rs b/control-plane/object-storage/src/main.rs index 184937c..5dc9f9a 100644 --- a/control-plane/object-storage/src/main.rs +++ b/control-plane/object-storage/src/main.rs @@ -1,4 +1,5 @@ use std::net::SocketAddr; +use uuid::Uuid; mod db; mod garage; @@ -36,6 +37,23 @@ async fn main() -> anyhow::Result<()> { .expect("GARAGE_ADMIN_TOKEN must be set"); let garage = garage::GarageClient::new(admin_url, admin_token); + let etcd_url = + std::env::var("ETCD_URL").unwrap_or_else(|_| "http://localhost:2379".to_string()); + let etcd = etcd_client::Client::connect([etcd_url.as_str()], None) + .await + .expect("Failed to connect to etcd"); + log_info!("main", "etcd connection established"); + + let node_id = Uuid::new_v4().to_string(); + let leader = garage::leader::LayoutLeader::new(etcd, node_id); + + tokio::spawn(leader.clone().run_campaign_loop()); + tokio::spawn(garage::layout::run_reconcile_loop( + db.clone(), + garage.clone(), + leader, + )); + let state = server::AppState::new(db, garage); let app = server::create_router(state); diff --git a/control-plane/object-storage/src/models.rs b/control-plane/object-storage/src/models.rs index 02525f4..cd1db51 100644 --- a/control-plane/object-storage/src/models.rs +++ b/control-plane/object-storage/src/models.rs @@ -44,6 +44,14 @@ pub struct AccessKeyCreatedResponse { pub secret_access_key: String, } +#[derive(Debug, Serialize, Deserialize)] +pub struct ClusterStatusResponse { + pub storage_node_count: u32, + pub replication_factor: u32, + pub degraded: bool, + pub nodes: Vec, +} + #[derive(Debug, Serialize, Deserialize)] pub struct BucketResponse { pub id: Uuid, diff --git a/control-plane/object-storage/src/server.rs b/control-plane/object-storage/src/server.rs index 34f3ff6..e310eed 100644 --- a/control-plane/object-storage/src/server.rs +++ b/control-plane/object-storage/src/server.rs @@ -3,7 +3,7 @@ use sea_orm::DatabaseConnection; use crate::{ garage::GarageClient, - handlers::{buckets, keys}, + handlers::{buckets, cluster, keys}, metrics, }; @@ -49,5 +49,6 @@ pub fn create_router(state: AppState) -> Router { "/buckets/{id}/keys/{key_id}", axum::routing::delete(keys::delete_key), ) + .route("/cluster", get(cluster::get_cluster_status)) .with_state(state) } From c62c7d64fe116f187d81e6a1355560f2d968e6b0 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 16:58:08 +0200 Subject: [PATCH 07/24] feat: wire object-storage into api-gateway with rbac and tenant-scoped bucket listing --- control-plane/api-gateway/src/auth/rbac.rs | 4 + control-plane/api-gateway/src/init.rs | 15 ++ .../api-gateway/src/routes/buckets.rs | 245 ++++++++++++++++++ control-plane/api-gateway/src/routes/mod.rs | 2 + .../api-gateway/src/routes/resource_groups.rs | 48 +++- .../api-gateway/src/service_client.rs | 60 +++++ 6 files changed, 372 insertions(+), 2 deletions(-) create mode 100644 control-plane/api-gateway/src/routes/buckets.rs diff --git a/control-plane/api-gateway/src/auth/rbac.rs b/control-plane/api-gateway/src/auth/rbac.rs index ce94fd9..e61373f 100644 --- a/control-plane/api-gateway/src/auth/rbac.rs +++ b/control-plane/api-gateway/src/auth/rbac.rs @@ -24,6 +24,8 @@ pub struct CanViewResourceGroups(pub Claims); pub struct CanManageResourceGroups(pub Claims); pub struct CanViewLogs(pub Claims); pub struct CanManageLogs(pub Claims); +pub struct CanViewBuckets(pub Claims); +pub struct CanManageBuckets(pub Claims); async fn extract_claims(parts: &mut Parts, state: &AppState) -> Result { let token = parts @@ -121,3 +123,5 @@ impl_extractor!(CanViewResourceGroups, "resource_groups", "view"); impl_extractor!(CanManageResourceGroups, "resource_groups", "manage"); impl_extractor!(CanViewLogs, "logs", "view"); impl_extractor!(CanManageLogs, "logs", "manage"); +impl_extractor!(CanViewBuckets, "buckets", "view"); +impl_extractor!(CanManageBuckets, "buckets", "manage"); diff --git a/control-plane/api-gateway/src/init.rs b/control-plane/api-gateway/src/init.rs index 9fb029e..e5eb447 100644 --- a/control-plane/api-gateway/src/init.rs +++ b/control-plane/api-gateway/src/init.rs @@ -167,6 +167,18 @@ pub async fn initialize_database( "manage", "Manage log retention settings", ), + ( + "buckets.view", + "buckets", + "view", + "View buckets and access keys", + ), + ( + "buckets.manage", + "buckets", + "manage", + "Create, update and delete buckets and access keys", + ), ]; let mut permission_map = std::collections::HashMap::new(); @@ -287,6 +299,8 @@ pub async fn initialize_database( "resource_groups.view", "resource_groups.manage", "logs.view", + "buckets.view", + "buckets.manage", ]; for perm_name in operator_perms { if let Some(perm_id) = permission_map.get(perm_name) { @@ -332,6 +346,7 @@ pub async fn initialize_database( "members.view", "resource_groups.view", "logs.view", + "buckets.view", ]; for perm_name in viewer_perms { if let Some(perm_id) = permission_map.get(perm_name) { diff --git a/control-plane/api-gateway/src/routes/buckets.rs b/control-plane/api-gateway/src/routes/buckets.rs new file mode 100644 index 0000000..8fa6329 --- /dev/null +++ b/control-plane/api-gateway/src/routes/buckets.rs @@ -0,0 +1,245 @@ +use axum::{ + body::Body, + extract::{Path, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Json}, + routing::{get, post}, + Router, +}; +use serde_json::json; + +use crate::{ + auth::rbac::{CanManageBuckets, CanViewBuckets}, + AppState, +}; + +async fn proxy_to_object_storage( + state: &AppState, + method: reqwest::Method, + path: &str, + body: Option, + headers: Option>, +) -> Result)> { + match state + .service_client + .forward_to_object_storage(method, path, body, headers) + .await + { + Ok((status, Some(body))) => { + let axum_status = + StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Ok((axum_status, Json(body)).into_response()) + } + Ok((status, None)) => { + let axum_status = + StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Ok((axum_status, Body::empty()).into_response()) + } + Err(e) => { + tracing::error!("Failed to forward request to object-storage: {}", e); + Err(( + StatusCode::BAD_GATEWAY, + Json( + json!({ "error": "Object Storage service unavailable", "details": e.to_string() }), + ), + )) + } + } +} + +fn header_vec(headers: &HeaderMap) -> Vec<(String, String)> { + headers + .iter() + .filter_map(|(k, v)| v.to_str().ok().map(|val| (k.to_string(), val.to_string()))) + .collect() +} + +pub async fn create_bucket( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + "/buckets", + body_json, + Some(header_map), + ) + .await +} + +pub async fn list_buckets( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + "/buckets", + None, + Some(header_map), + ) + .await +} + +pub async fn get_bucket( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + &format!("/buckets/{}", id), + None, + Some(header_map), + ) + .await +} + +pub async fn update_bucket( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::PATCH, + &format!("/buckets/{}", id), + body_json, + Some(header_map), + ) + .await +} + +pub async fn delete_bucket( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::DELETE, + &format!("/buckets/{}", id), + None, + Some(header_map), + ) + .await +} + +pub async fn list_keys( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + &format!("/buckets/{}/keys", id), + None, + Some(header_map), + ) + .await +} + +pub async fn create_key( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + &format!("/buckets/{}/keys", id), + body_json, + Some(header_map), + ) + .await +} + +pub async fn rotate_key( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path((bucket_id, key_id)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + &format!("/buckets/{}/keys/{}/rotate", bucket_id, key_id), + None, + Some(header_map), + ) + .await +} + +pub async fn delete_key( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path((bucket_id, key_id)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::DELETE, + &format!("/buckets/{}/keys/{}", bucket_id, key_id), + None, + Some(header_map), + ) + .await +} + +pub async fn get_cluster_status( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + "/cluster", + None, + Some(header_map), + ) + .await +} + +pub fn buckets_routes() -> Router { + Router::new() + .route("/buckets", post(create_bucket)) + .route("/buckets", get(list_buckets)) + .route("/buckets/{id}", get(get_bucket)) + .route("/buckets/{id}", axum::routing::patch(update_bucket)) + .route("/buckets/{id}", axum::routing::delete(delete_bucket)) + .route("/buckets/{id}/keys", get(list_keys)) + .route("/buckets/{id}/keys", post(create_key)) + .route("/buckets/{bucket_id}/keys/{key_id}/rotate", post(rotate_key)) + .route( + "/buckets/{bucket_id}/keys/{key_id}", + axum::routing::delete(delete_key), + ) + .route("/object-storage/cluster", get(get_cluster_status)) +} diff --git a/control-plane/api-gateway/src/routes/mod.rs b/control-plane/api-gateway/src/routes/mod.rs index d6b3d96..07527cb 100644 --- a/control-plane/api-gateway/src/routes/mod.rs +++ b/control-plane/api-gateway/src/routes/mod.rs @@ -20,6 +20,7 @@ use tracing::{info_span, Span}; pub mod agent_proxy; pub mod agent_stream; pub mod agents; +pub mod buckets; pub mod events; pub mod logs; pub mod networks; @@ -120,6 +121,7 @@ pub fn create_router() -> Router { let rate_limited_router = Router::new() .merge(agent_proxy::agent_proxy_routes()) .merge(agents::agents_routes()) + .merge(buckets::buckets_routes()) .merge(networks::networks_routes()) .merge(organizations::routes()) .merge(ssh_keys::ssh_keys_routes()) diff --git a/control-plane/api-gateway/src/routes/resource_groups.rs b/control-plane/api-gateway/src/routes/resource_groups.rs index ff9487d..a76f27a 100644 --- a/control-plane/api-gateway/src/routes/resource_groups.rs +++ b/control-plane/api-gateway/src/routes/resource_groups.rs @@ -8,8 +8,8 @@ use axum::{ use base64::{engine::general_purpose::STANDARD as B64, Engine}; use chrono::Utc; use entity::{ - entities::{agents, networks, resource_group_vpn_peers, resource_groups, volumes, workloads}, - Agents, Networks, ResourceGroupVpnPeers, ResourceGroups, Volumes, Workloads, + entities::{agents, buckets, networks, resource_group_vpn_peers, resource_groups, volumes, workloads}, + Agents, Buckets, Networks, ResourceGroupVpnPeers, ResourceGroups, Volumes, Workloads, }; use ring::rand::{SecureRandom, SystemRandom}; use sea_orm::{ @@ -483,6 +483,46 @@ pub async fn list_resource_group_volumes( Ok((StatusCode::OK, Json(json!(vols)))) } +pub async fn list_resource_group_buckets( + CanViewResourceGroups(_claims): CanViewResourceGroups, + State(state): State, + Path(id): Path, +) -> Result)> { + let org_id = get_org_id(&state); + + ResourceGroups::find_by_id(id) + .filter(resource_groups::Column::OrganizationId.eq(org_id)) + .one(&state.db_conn) + .await + .map_err(|e| { + tracing::error!(error = %e, "failed to find resource group"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": "database error" })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::NOT_FOUND, + Json(json!({ "error": "resource group not found" })), + ) + })?; + + let buckets = Buckets::find() + .filter(buckets::Column::ResourceGroupId.eq(id)) + .all(&state.db_conn) + .await + .map_err(|e| { + tracing::error!(error = %e, "failed to list buckets"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": "database error" })), + ) + })?; + + Ok((StatusCode::OK, Json(json!(buckets)))) +} + pub async fn list_resource_group_networks( CanViewResourceGroups(_claims): CanViewResourceGroups, State(state): State, @@ -950,6 +990,10 @@ pub fn resource_groups_routes() -> Router { "/resource-groups/{id}/volumes", get(list_resource_group_volumes), ) + .route( + "/resource-groups/{id}/buckets", + get(list_resource_group_buckets), + ) .route( "/resource-groups/{id}/networks", get(list_resource_group_networks), diff --git a/control-plane/api-gateway/src/service_client.rs b/control-plane/api-gateway/src/service_client.rs index 1d09615..610768f 100644 --- a/control-plane/api-gateway/src/service_client.rs +++ b/control-plane/api-gateway/src/service_client.rs @@ -11,6 +11,7 @@ pub struct ServiceClient { volume_manager_url: String, failover_controller_url: String, sdn_controller_url: String, + object_storage_url: String, } impl ServiceClient { @@ -30,6 +31,9 @@ impl ServiceClient { let sdn_controller_url = std::env::var("SDN_CONTROLLER_URL") .unwrap_or_else(|_| "http://localhost:8005".to_string()); + let object_storage_url = std::env::var("OBJECT_STORAGE_URL") + .unwrap_or_else(|_| "http://localhost:8006".to_string()); + let client = Client::builder() .timeout(Duration::from_secs(30)) .build() @@ -42,6 +46,7 @@ impl ServiceClient { volume_manager_url, failover_controller_url, sdn_controller_url, + object_storage_url, } } @@ -334,4 +339,59 @@ impl ServiceClient { Ok((status, json_body)) } + + pub async fn forward_to_object_storage( + &self, + method: reqwest::Method, + path: &str, + body: Option, + headers: Option>, + ) -> Result<(StatusCode, Option)> { + let url = format!("{}{}", self.object_storage_url, path); + + tracing::debug!("Forwarding {} request to object-storage: {}", method, url); + + let mut request = match method { + reqwest::Method::GET => self.client.get(&url), + reqwest::Method::POST => self.client.post(&url), + reqwest::Method::PATCH => self.client.patch(&url), + reqwest::Method::DELETE => self.client.delete(&url), + _ => return Err(anyhow::anyhow!("Unsupported HTTP method")), + }; + + if let Some(headers) = headers { + for (key, value) in headers { + let key_lower = key.to_lowercase(); + if key_lower == "content-length" + || key_lower == "host" + || key_lower == "content-type" + || key_lower == "transfer-encoding" + { + continue; + } + request = request.header(key, value); + } + } + + if let Some(body) = body { + request = request.json(&body); + } + + let response = request + .send() + .await + .context("Failed to send request to object-storage service")?; + + let status = response.status(); + let body_text = response.text().await.ok(); + let json_body = body_text.and_then(|text| { + if text.is_empty() { + None + } else { + serde_json::from_str(&text).ok() + } + }); + + Ok((status, json_body)) + } } From 630dedcd9a8a127d3ad439e4f3872e10b7fe5a76 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:01:58 +0200 Subject: [PATCH 08/24] feat: add rg-internal s3 dns and dnat wiring in agent --- agent/src/firecracker/runtime.rs | 18 ++++++++++++++---- agent/src/main.rs | 5 +++-- agent/src/nftables.rs | 24 ++++++++++++++++++++++++ agent/src/rg_network.rs | 23 +++++++++++++++++++++-- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/agent/src/firecracker/runtime.rs b/agent/src/firecracker/runtime.rs index 6e81fe0..ff9154c 100644 --- a/agent/src/firecracker/runtime.rs +++ b/agent/src/firecracker/runtime.rs @@ -3,6 +3,7 @@ use serde_json::json; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::UnixStream; use tokio::process::Command; @@ -153,16 +154,21 @@ struct GuestNetwork { pub struct FirecrackerRuntime { wg_private_key_b64: String, dns_supervisor: RgDnsProcessSupervisor, + rg_dns_registry: Arc, handles: Mutex>, next_cid: Mutex, reconciled: AtomicBool, } impl FirecrackerRuntime { - pub fn new(wg_private_key_b64: String) -> Self { + pub fn new( + wg_private_key_b64: String, + rg_dns_registry: Arc, + ) -> Self { Self { wg_private_key_b64, dns_supervisor: RgDnsProcessSupervisor::new(), + rg_dns_registry, handles: Mutex::new(HashMap::new()), next_cid: Mutex::new(next_free_cid_on_host()), reconciled: AtomicBool::new(false), @@ -194,9 +200,13 @@ impl FirecrackerRuntime { resource_group_id: &str, resource_group_cidr: Option<&str>, ) -> Result { - let iface = crate::rg_network::ensure_bridge(resource_group_id, resource_group_cidr) - .await - .context("Failed to ensure resource group bridge")?; + let iface = crate::rg_network::ensure_bridge( + resource_group_id, + resource_group_cidr, + &self.rg_dns_registry, + ) + .await + .context("Failed to ensure resource group bridge")?; if let Some(cidr) = resource_group_cidr { self.dns_supervisor diff --git a/agent/src/main.rs b/agent/src/main.rs index dc701e7..41d504e 100644 --- a/agent/src/main.rs +++ b/agent/src/main.rs @@ -137,8 +137,11 @@ async fn main() -> Result<()> { warn!(error = %e, "Failed to initialize nftables resource group isolation"); } + let rg_dns_registry = Arc::new(rg_dns::RgDnsRegistry::new()); + let firecracker_runtime = Arc::new(firecracker::runtime::FirecrackerRuntime::new( wg_identity.private_key_b64.clone(), + Arc::clone(&rg_dns_registry), )); let running_containers: Arc>> = @@ -153,8 +156,6 @@ async fn main() -> Result<()> { let service_dns_registry: Arc>> = Arc::new(Mutex::new(HashMap::new())); - let rg_dns_registry = rg_dns::RgDnsRegistry::new(); - if let Some(port) = std::env::var("CSFX_AGENT_PORT") .ok() .and_then(|v| v.parse::().ok()) diff --git a/agent/src/nftables.rs b/agent/src/nftables.rs index 097deda..bd57cda 100644 --- a/agent/src/nftables.rs +++ b/agent/src/nftables.rs @@ -130,6 +130,30 @@ pub async fn add_rg_port_dnat( .await } +pub async fn dnat_bridge_port(bridge_name: &str, rg_gateway_ip: &str, port: u16) -> Result<()> { + run_nft(&[ + "add", + "rule", + "ip", + NAT_TABLE_NAME, + NAT_CHAIN_NAME, + "iifname", + bridge_name, + "ip", + "daddr", + rg_gateway_ip, + "tcp", + "dport", + &port.to_string(), + "dnat", + "to", + &format!("127.0.0.1:{}", port), + "comment", + &format!("\"{}-s3\"", bridge_name), + ]) + .await +} + pub async fn remove_node_port_rules(workload_id: &str) -> Result<()> { let output = Command::new("nft") .args(["-a", "list", "chain", "ip", NAT_TABLE_NAME, NAT_CHAIN_NAME]) diff --git a/agent/src/rg_network.rs b/agent/src/rg_network.rs index bc438aa..69cc964 100644 --- a/agent/src/rg_network.rs +++ b/agent/src/rg_network.rs @@ -3,11 +3,18 @@ use std::path::{Path, PathBuf}; use tokio::process::Command; use tracing::info; +use crate::rg_dns::RgDnsRegistry; use crate::spec::{rg_bridge_iface_name, second_host_ip}; const RG_REGISTRY_DIR: &str = "/var/lib/csfx-agent/rg-networks"; - -pub async fn ensure_bridge(resource_group_id: &str, cidr: Option<&str>) -> Result { +const S3_SERVICE_NAME: &str = "s3"; +const S3_DNAT_PORT: u16 = 3900; + +pub async fn ensure_bridge( + resource_group_id: &str, + cidr: Option<&str>, + rg_dns_registry: &RgDnsRegistry, +) -> Result { let iface = rg_bridge_iface_name(resource_group_id); write_registry_entry(resource_group_id).await?; @@ -29,6 +36,18 @@ pub async fn ensure_bridge(resource_group_id: &str, cidr: Option<&str>) -> Resul &iface, ]) .await?; + + if let Err(e) = rg_dns_registry + .upsert(resource_group_id, S3_SERVICE_NAME, &gateway) + .await + { + info!(resource_group_id = %resource_group_id, error = %e, "Failed to register s3 dns record"); + } + + if let Err(e) = crate::nftables::dnat_bridge_port(&iface, &gateway, S3_DNAT_PORT).await + { + info!(resource_group_id = %resource_group_id, error = %e, "Failed to set up s3 dnat rule"); + } } } From 11095dde2fdf8ad2bad540170a7567b921b47690 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:03:16 +0200 Subject: [PATCH 09/24] feat: add sigv4-preserving s3 streaming proxy for external buckets --- control-plane/api-gateway/src/routes/mod.rs | 2 + .../api-gateway/src/routes/s3_proxy.rs | 145 ++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 control-plane/api-gateway/src/routes/s3_proxy.rs diff --git a/control-plane/api-gateway/src/routes/mod.rs b/control-plane/api-gateway/src/routes/mod.rs index 07527cb..b934b9e 100644 --- a/control-plane/api-gateway/src/routes/mod.rs +++ b/control-plane/api-gateway/src/routes/mod.rs @@ -28,6 +28,7 @@ pub mod organizations; pub mod registry; pub mod releases; pub mod resource_groups; +pub mod s3_proxy; pub mod settings; pub mod ssh_keys; pub mod system; @@ -133,6 +134,7 @@ pub fn create_router() -> Router { .merge(resource_groups::resource_groups_routes()) .merge(logs::logs_routes()) .merge(settings::settings_routes()) + .merge(s3_proxy::s3_proxy_routes()) .layer(GovernorLayer::new(governor_config)); let login_rate_limited_router = Router::new() diff --git a/control-plane/api-gateway/src/routes/s3_proxy.rs b/control-plane/api-gateway/src/routes/s3_proxy.rs new file mode 100644 index 0000000..a3ac29b --- /dev/null +++ b/control-plane/api-gateway/src/routes/s3_proxy.rs @@ -0,0 +1,145 @@ +use axum::{ + body::Body, + extract::{Path, State}, + http::{HeaderMap, Method, StatusCode}, + response::{IntoResponse, Json}, + routing::any, + Router, +}; +use entity::entities::{agents, buckets, garage_nodes}; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use serde_json::json; + +use crate::AppState; + +const S3_PORT: u16 = 3900; + +async fn resolve_bucket_target( + state: &AppState, + global_alias: &str, +) -> Result)> { + let bucket = buckets::Entity::find() + .filter(buckets::Column::GlobalAlias.eq(global_alias)) + .one(&state.db_conn) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("database error: {}", e) })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::NOT_FOUND, + Json(json!({ "error": "bucket not found" })), + ) + })?; + + if bucket.exposure != "external" { + return Err(( + StatusCode::NOT_FOUND, + Json(json!({ "error": "bucket not found" })), + )); + } + + let node = garage_nodes::Entity::find() + .filter(garage_nodes::Column::Status.eq("up")) + .one(&state.db_conn) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("database error: {}", e) })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "error": "no garage node available" })), + ) + })?; + + let agent = agents::Entity::find_by_id(node.agent_id) + .one(&state.db_conn) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("database error: {}", e) })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "error": "garage node has no agent record" })), + ) + })?; + + let tunnel_ip = agent.wg_tunnel_ip.ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "error": "garage node has no known tunnel address" })), + ) + })?; + + Ok(tunnel_ip) +} + +pub async fn proxy_s3_request( + State(state): State, + Path((bucket, path)): Path<(String, String)>, + method: Method, + headers: HeaderMap, + body: Body, +) -> Result)> { + let tunnel_ip = resolve_bucket_target(&state, &bucket).await?; + + let url = format!("http://{}:{}/{}/{}", tunnel_ip, S3_PORT, bucket, path); + + let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes()) + .unwrap_or(reqwest::Method::GET); + + let body_bytes = axum::body::to_bytes(body, usize::MAX) + .await + .map_err(|e| { + ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": format!("failed to read request body: {}", e) })), + ) + })?; + + let client = reqwest::Client::new(); + let mut request = client.request(reqwest_method, &url).body(body_bytes); + + for (key, value) in headers.iter() { + let key_lower = key.as_str().to_lowercase(); + if key_lower == "content-length" { + continue; + } + if let Ok(value_str) = value.to_str() { + request = request.header(key.as_str(), value_str); + } + } + + let response = request.send().await.map_err(|e| { + ( + StatusCode::BAD_GATEWAY, + Json(json!({ "error": format!("failed to reach garage node: {}", e) })), + ) + })?; + + let status = + StatusCode::from_u16(response.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let mut response_headers = HeaderMap::new(); + for (key, value) in response.headers().iter() { + response_headers.insert(key.clone(), value.clone()); + } + + let stream = response.bytes_stream(); + + Ok((status, response_headers, Body::from_stream(stream))) +} + +pub fn s3_proxy_routes() -> Router { + Router::new().route("/s3/{bucket}/{*path}", any(proxy_s3_request)) +} From 834144f877790a7f844553e9960efcfacd97bb67 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:05:56 +0200 Subject: [PATCH 10/24] feat: inject bucket credentials into workload env on bucket binding --- Cargo.lock | 29 +++++ .../scheduler/src/models/workload.rs | 9 ++ .../scheduler/src/services/bucket_bindings.rs | 119 ++++++++++++++++++ control-plane/scheduler/src/services/mod.rs | 1 + .../scheduler/src/services/scheduler.rs | 18 ++- 5 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 control-plane/scheduler/src/services/bucket_bindings.rs diff --git a/Cargo.lock b/Cargo.lock index 7d1a48c..beec855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3218,6 +3218,35 @@ dependencies = [ "libm", ] +[[package]] +name = "object-storage" +version = "0.2.2" +dependencies = [ + "anyhow", + "axum", + "chrono", + "dotenvy", + "entity", + "etcd-client", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "prometheus", + "reqwest", + "ring", + "rustls", + "sea-orm", + "serde", + "serde_json", + "shared", + "tokio", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", + "uuid", +] + [[package]] name = "oci-client" version = "0.17.0" diff --git a/control-plane/scheduler/src/models/workload.rs b/control-plane/scheduler/src/models/workload.rs index 1320dce..a6b87d1 100644 --- a/control-plane/scheduler/src/models/workload.rs +++ b/control-plane/scheduler/src/models/workload.rs @@ -50,6 +50,13 @@ pub struct VolumeMount { pub mount_path: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BucketBinding { + pub bucket_id: Uuid, + #[serde(default)] + pub permissions: Option, +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum RestartPolicy { @@ -153,6 +160,8 @@ pub struct CreateWorkloadRequest { pub env_vars: Option>, pub ports: Option>, pub volume_mounts: Option>, + #[serde(default)] + pub bucket_bindings: Option>, pub resource_group_id: Option, #[serde(default)] pub stack_id: Option, diff --git a/control-plane/scheduler/src/services/bucket_bindings.rs b/control-plane/scheduler/src/services/bucket_bindings.rs new file mode 100644 index 0000000..048e6c7 --- /dev/null +++ b/control-plane/scheduler/src/services/bucket_bindings.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; + +use crate::models::workload::BucketBinding; + +const OBJECT_STORAGE_URL_ENV: &str = "OBJECT_STORAGE_URL"; +const DEFAULT_OBJECT_STORAGE_URL: &str = "http://localhost:8006"; + +fn base_url() -> String { + std::env::var(OBJECT_STORAGE_URL_ENV).unwrap_or_else(|_| DEFAULT_OBJECT_STORAGE_URL.to_string()) +} + +pub async fn resolve_env_vars( + workload_id: uuid::Uuid, + resource_group_id: Option, + bindings: &[BucketBinding], +) -> HashMap { + let mut env = HashMap::new(); + let client = reqwest::Client::new(); + let base = base_url(); + + for (index, binding) in bindings.iter().enumerate() { + let bucket_id = binding.bucket_id; + + let bucket = match client + .get(format!("{}/buckets/{}", base, bucket_id)) + .send() + .await + { + Ok(resp) if resp.status().is_success() => { + match resp.json::().await { + Ok(body) => body, + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to parse bucket response"); + continue; + } + } + } + Ok(resp) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, status = %resp.status(), "bucket lookup failed"); + continue; + } + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to reach object-storage"); + continue; + } + }; + + let global_alias = match bucket["global_alias"].as_str() { + Some(alias) => alias.to_string(), + None => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, "bucket response missing global_alias"); + continue; + } + }; + + let key_name = format!("workload-{}", workload_id); + let key_body = serde_json::json!({ + "name": key_name, + "permissions": binding.permissions.clone().unwrap_or_else(|| "readwrite".to_string()), + }); + + let key = match client + .post(format!("{}/buckets/{}/keys", base, bucket_id)) + .json(&key_body) + .send() + .await + { + Ok(resp) if resp.status().is_success() => { + match resp.json::().await { + Ok(body) => body, + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to parse access key response"); + continue; + } + } + } + Ok(resp) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, status = %resp.status(), "access key creation failed"); + continue; + } + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to reach object-storage"); + continue; + } + }; + + let (Some(access_key_id), Some(secret_access_key)) = ( + key["garage_key_id"].as_str(), + key["secret_access_key"].as_str(), + ) else { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, "access key response missing credentials"); + continue; + }; + + let suffix = if index == 0 { + String::new() + } else { + format!("_{}", index) + }; + + let endpoint = match resource_group_id { + Some(rg_id) => format!("http://s3.svc.{}.internal:3900", rg_id), + None => "http://127.0.0.1:3900".to_string(), + }; + + env.insert( + format!("AWS_ACCESS_KEY_ID{}", suffix), + access_key_id.to_string(), + ); + env.insert( + format!("AWS_SECRET_ACCESS_KEY{}", suffix), + secret_access_key.to_string(), + ); + env.insert(format!("AWS_BUCKET{}", suffix), global_alias); + env.insert(format!("AWS_ENDPOINT_URL{}", suffix), endpoint); + } + + env +} diff --git a/control-plane/scheduler/src/services/mod.rs b/control-plane/scheduler/src/services/mod.rs index b34df0d..b4cf3c1 100644 --- a/control-plane/scheduler/src/services/mod.rs +++ b/control-plane/scheduler/src/services/mod.rs @@ -1,3 +1,4 @@ +pub mod bucket_bindings; pub mod compose_parser; pub mod etcd; pub mod gateway_notify; diff --git a/control-plane/scheduler/src/services/scheduler.rs b/control-plane/scheduler/src/services/scheduler.rs index 6868e52..d30524f 100644 --- a/control-plane/scheduler/src/services/scheduler.rs +++ b/control-plane/scheduler/src/services/scheduler.rs @@ -25,8 +25,23 @@ impl SchedulerService { pub async fn schedule( &self, - req: CreateWorkloadRequest, + mut req: CreateWorkloadRequest, ) -> Result { + if let Some(bindings) = req.bucket_bindings.clone() { + if !bindings.is_empty() { + let placeholder_id = Uuid::new_v4(); + let bucket_env = crate::services::bucket_bindings::resolve_env_vars( + placeholder_id, + req.resource_group_id, + &bindings, + ) + .await; + req.env_vars + .get_or_insert_with(std::collections::HashMap::new) + .extend(bucket_env); + } + } + let workload = crate::db::workloads::create(&self.db, &req) .await .map_err(|e| format!("Failed to persist workload: {}", e))?; @@ -303,6 +318,7 @@ impl SchedulerService { env_vars: service.env_vars.clone(), ports: service.ports.clone(), volume_mounts: None, + bucket_bindings: None, resource_group_id: Some(resource_group_id), stack_id: Some(stack_id), service_name: Some(service.service_name.clone()), From 7be44504b8ee460d6e9ad0054216e91f62364500 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:12:14 +0200 Subject: [PATCH 11/24] feat: add bucket tab to resource group dashboard --- app/src/lib/api/resource-groups.ts | 109 +++++++ .../(app)/resource-groups/[id]/+page.svelte | 273 +++++++++++++++++- 2 files changed, 375 insertions(+), 7 deletions(-) diff --git a/app/src/lib/api/resource-groups.ts b/app/src/lib/api/resource-groups.ts index fca551f..f17821c 100644 --- a/app/src/lib/api/resource-groups.ts +++ b/app/src/lib/api/resource-groups.ts @@ -55,6 +55,47 @@ export interface Volume { created_at: string; } +export interface Bucket { + id: string; + name: string; + global_alias: string; + exposure: 'internal' | 'external' | 'node_port'; + quota_max_size: number | null; + quota_max_objects: number | null; + status: string; + resource_group_id: string | null; + created_at: string; + updated_at: string | null; +} + +export interface BucketAccessKey { + id: string; + bucket_id: string; + name: string; + garage_key_id: string; + permissions: string; + expires_at: string | null; + last_rotated_at: string | null; + created_at: string; +} + +export interface BucketAccessKeyCreated extends BucketAccessKey { + secret_access_key: string; +} + +export interface CreateBucketRequest { + name: string; + resource_group_id?: string; + exposure?: 'internal' | 'external'; + quota_max_size?: number; + quota_max_objects?: number; +} + +export interface CreateBucketAccessKeyRequest { + name: string; + permissions?: string; +} + export interface Workload { id: string; name: string; @@ -408,3 +449,71 @@ export async function deleteVolume(token: string, id: string): Promise { }); if (!res.ok) throw new Error(`Failed to delete volume: ${res.status}`); } + +export async function listResourceGroupBuckets(token: string, rgId: string): Promise { + const res = await authedFetch(`${API_BASE}/resource-groups/${rgId}/buckets`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to list buckets: ${res.status}`); + return res.json(); +} + +export async function createBucket(token: string, req: CreateBucketRequest): Promise { + const res = await authedFetch(`${API_BASE}/buckets`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(req), + }); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to create bucket: ${res.status}`); + } + return res.json(); +} + +export async function deleteBucket(token: string, id: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${id}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to delete bucket: ${res.status}`); +} + +export async function listBucketKeys(token: string, bucketId: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/keys`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to list access keys: ${res.status}`); + return res.json(); +} + +export async function createBucketKey( + token: string, + bucketId: string, + req: CreateBucketAccessKeyRequest +): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/keys`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(req), + }); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to create access key: ${res.status}`); + } + return res.json(); +} + +export async function deleteBucketKey(token: string, bucketId: string, keyId: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/keys/${keyId}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to delete access key: ${res.status}`); +} diff --git a/app/src/routes/(app)/resource-groups/[id]/+page.svelte b/app/src/routes/(app)/resource-groups/[id]/+page.svelte index 4dc5945..3a64ee2 100644 --- a/app/src/routes/(app)/resource-groups/[id]/+page.svelte +++ b/app/src/routes/(app)/resource-groups/[id]/+page.svelte @@ -22,11 +22,18 @@ updateWorkload, createVolume, deleteVolume, + listResourceGroupBuckets, + createBucket, + deleteBucket, + listBucketKeys, + createBucketKey, streamWorkloadLogs, openWorkloadExecSocket, type ResourceGroup, type Workload, type Volume, + type Bucket, + type BucketAccessKey, type PortMapping, type VolumeMount, } from "$lib/api/resource-groups"; @@ -49,6 +56,7 @@ let group = $state(null); let workloads = $state([]); let volumes = $state([]); + let buckets = $state([]); let loading = $state(true); let error = $state(null); @@ -64,7 +72,7 @@ let savingRgSettings = $state(false); let rgSettingsError = $state(null); - let activeTab = $state<"all" | "container" | "volume">("all"); + let activeTab = $state<"all" | "container" | "volume" | "bucket">("all"); let filterText = $state(""); let nodeIpCache = $state>({}); @@ -98,13 +106,17 @@ let deployDialog = $state(null); let volumeDialog = $state(null); + let bucketDialog = $state(null); + let bucketDetailDialog = $state(null); let resourcePickerDialog = $state(null); let composeDialog = $state(null); let deploying = $state(false); let creatingVolume = $state(false); + let creatingBucket = $state(false); let deployingStack = $state(false); let deployError = $state(null); let volumeError = $state(null); + let bucketError = $state(null); let composeError = $state(null); let downloadingVpn = $state(false); let expandedStacks = $state>(new Set()); @@ -147,6 +159,16 @@ let volFormName = $state(""); let volFormSize = $state("10"); + let bucketFormName = $state(""); + let bucketFormExposure = $state<"internal" | "external">("internal"); + + let activeBucket = $state(null); + let bucketKeys = $state([]); + let bucketKeysError = $state(null); + let newKeyName = $state(""); + let creatingKey = $state(false); + let createdKeySecret = $state(null); + let containerDialog = $state(null); let containerDialogTab = $state<"logs" | "shell" | "insights" | "network" | "settings">("logs"); let activeContainer = $state(null); @@ -173,10 +195,11 @@ async function load() { if (!auth.token) return; try { - [group, workloads, volumes] = await Promise.all([ + [group, workloads, volumes, buckets] = await Promise.all([ getResourceGroup(auth.token, rgId), listResourceGroupWorkloads(auth.token, rgId), listResourceGroupVolumes(auth.token, rgId), + listResourceGroupBuckets(auth.token, rgId), ]); } catch (e) { error = e instanceof Error ? e.message : "Failed to load"; @@ -389,6 +412,74 @@ } } + async function handleCreateBucket() { + if (!auth.token || !bucketFormName) return; + creatingBucket = true; + bucketError = null; + try { + await createBucket(auth.token, { + name: bucketFormName, + resource_group_id: rgId, + exposure: bucketFormExposure, + }); + bucketDialog?.close(); + bucketFormName = ""; + bucketFormExposure = "internal"; + buckets = await listResourceGroupBuckets(auth.token, rgId); + } catch (e) { + bucketError = e instanceof Error ? e.message : "Failed to create bucket"; + } finally { + creatingBucket = false; + } + } + + async function handleDeleteBucket(id: string) { + if (!auth.token) return; + try { + await deleteBucket(auth.token, id); + buckets = buckets.filter((b) => b.id !== id); + } catch (e) { + error = e instanceof Error ? e.message : "Failed to delete bucket"; + } + } + + async function openBucketDetail(bucket: Bucket) { + if (!auth.token) return; + activeBucket = bucket; + bucketKeysError = null; + createdKeySecret = null; + newKeyName = ""; + bucketDetailDialog?.showModal(); + try { + bucketKeys = await listBucketKeys(auth.token, bucket.id); + } catch (e) { + bucketKeysError = e instanceof Error ? e.message : "Failed to load access keys"; + } + } + + async function handleCreateBucketKey() { + if (!auth.token || !activeBucket || !newKeyName) return; + creatingKey = true; + bucketKeysError = null; + try { + const created = await createBucketKey(auth.token, activeBucket.id, { name: newKeyName }); + createdKeySecret = created.secret_access_key; + newKeyName = ""; + bucketKeys = await listBucketKeys(auth.token, activeBucket.id); + } catch (e) { + bucketKeysError = e instanceof Error ? e.message : "Failed to create access key"; + } finally { + creatingKey = false; + } + } + + function bucketEndpointUrl(bucket: Bucket): string { + if (bucket.exposure === "external") { + return `${window.location.origin}/api/s3/${bucket.global_alias}`; + } + return `http://s3.svc.${rgId}.internal:3900`; + } + function loadSettingsForm(workload: Workload) { settingsImage = workload.image; settingsEnvText = Object.entries(workload.env_vars ?? {}) @@ -681,7 +772,8 @@ type ResourceItem = | { kind: "container"; data: Workload } | { kind: "stack"; data: WorkloadStack } - | { kind: "volume"; data: Volume }; + | { kind: "volume"; data: Volume } + | { kind: "bucket"; data: Bucket }; function groupWorkloadsByStack(items: Workload[]): ResourceItem[] { const standalone: Workload[] = []; @@ -717,12 +809,14 @@ let allResources = $derived([ ...groupWorkloadsByStack(workloads), ...volumes.map((v): ResourceItem => ({ kind: "volume", data: v })), + ...buckets.map((b): ResourceItem => ({ kind: "bucket", data: b })), ]); let filteredResources = $derived( allResources.filter((r) => { - if (activeTab === "container" && r.kind === "volume") return false; + if (activeTab === "container" && (r.kind === "volume" || r.kind === "bucket")) return false; if (activeTab === "volume" && r.kind !== "volume") return false; + if (activeTab === "bucket" && r.kind !== "bucket") return false; if (!filterText) return true; const q = filterText.toLowerCase(); if (r.kind === "container") { @@ -1256,6 +1350,123 @@ + { bucketError = null; }} +> +
+
+

Create Bucket

+ +
+
+
+ + +
+
+ + +
+
+ {#if bucketError} +

{bucketError}

+ {/if} +
+ + +
+
+ + + { activeBucket = null; createdKeySecret = null; }} +> + {#if activeBucket} +
+
+
+

{activeBucket.name}

+

{bucketEndpointUrl(activeBucket)}

+
+ +
+ +
+

Access Keys

+ {#if bucketKeys.length === 0} +

no access keys yet

+ {:else} +
+ {#each bucketKeys as key (key.id)} +
+
+

{key.name}

+

{key.garage_key_id}

+
+ {key.permissions} +
+ {/each} +
+ {/if} +
+ + {#if createdKeySecret} +
+

Secret access key created

+

{createdKeySecret}

+

this will not be shown again, copy it now

+ +
+ {/if} + + {#if bucketKeysError} +

{bucketKeysError}

+ {/if} + +
+ + +
+
+ {/if} + + volumeDialog?.showModal()}> Add Volume + + {:else if item.kind === "bucket"} + {@const b = item.data} + openBucketDetail(b)} + > + +
+
+ +
+
+

{b.name}

+

{b.global_alias}

+
+
+ + + Bucket + + + {b.quota_max_size ? fmtBytes(b.quota_max_size) : "unlimited"} + + + {b.exposure} + + + + + + + + {/if} {/each} {/if} From 2add4b15d06f19a280779db365cd454b9710fd25 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:14:46 +0200 Subject: [PATCH 12/24] fix: add object-storage crate to docker build and compose --- Dockerfile | 3 +++ docker-compose.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/Dockerfile b/Dockerfile index 246cfc1..d53fcef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ COPY control-plane/scheduler/Cargo.toml ./control-plane/scheduler/ COPY control-plane/failover-controller/Cargo.toml ./control-plane/failover-controller/ COPY control-plane/sdn-controller/Cargo.toml ./control-plane/sdn-controller/ COPY control-plane/volume-manager/Cargo.toml ./control-plane/volume-manager/ +COPY control-plane/object-storage/Cargo.toml ./control-plane/object-storage/ COPY control-plane/registry/Cargo.toml ./control-plane/registry/ COPY control-plane/shared/entity/Cargo.toml ./control-plane/shared/entity/ COPY control-plane/shared/migration/Cargo.toml ./control-plane/shared/migration/ @@ -39,6 +40,7 @@ RUN mkdir -p agent/src \ control-plane/failover-controller/src \ control-plane/sdn-controller/src \ control-plane/volume-manager/src \ + control-plane/object-storage/src \ control-plane/registry/src \ control-plane/shared/entity/src \ control-plane/shared/migration/src \ @@ -51,6 +53,7 @@ RUN mkdir -p agent/src \ && echo "fn main() {}" > control-plane/failover-controller/src/main.rs \ && echo "fn main() {}" > control-plane/sdn-controller/src/main.rs \ && echo "fn main() {}" > control-plane/volume-manager/src/main.rs \ + && echo "fn main() {}" > control-plane/object-storage/src/main.rs \ && echo "fn main() {}" > control-plane/registry/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-migrate/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-updater/src/main.rs \ diff --git a/docker-compose.yml b/docker-compose.yml index f397963..7c1d1d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,6 +78,7 @@ services: VOLUME_MANAGER_URL: http://volume-manager:8003 FAILOVER_CONTROLLER_URL: http://failover-controller:8004 SDN_CONTROLLER_URL: http://sdn-controller:8005 + OBJECT_STORAGE_URL: http://object-storage:8006 FRONTEND_URL: http://localhost:5173 CSFX_INCLUDE_CONTAINERS: "true" TLS_ENABLED: "false" @@ -178,6 +179,32 @@ services: - cargo_target_volume_manager:/app/target command: cargo watch -x "run -p volume-manager" + object-storage: + <<: *rust-service + container_name: csfx-object-storage-dev + environment: + DATABASE_URL: postgres://${POSTGRES_USER:-csfx_user}:${POSTGRES_PASSWORD:-csfx_password}@postgres:5432/${POSTGRES_DB:-csfx_core} + ETCD_URL: http://etcd:2379 + OBJECT_STORAGE_PORT: "8006" + LISTEN_ADDR: "0.0.0.0" + GARAGE_ADMIN_URL: http://127.0.0.1:3903 + GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-dev-garage-admin-token} + RUST_LOG: ${RUST_LOG:-debug} + ports: + - "8006:8006" + depends_on: + postgres: + condition: service_healthy + etcd: + condition: service_started + migrate: + condition: service_completed_successfully + volumes: + - ./control-plane/object-storage:/app/control-plane/object-storage + - ./control-plane/shared:/app/control-plane/shared + - cargo_target_object_storage:/app/target + command: cargo watch -x "run -p object-storage" + failover-controller: <<: *rust-service container_name: csfx-failover-controller-dev @@ -320,6 +347,7 @@ volumes: cargo_target_volume_manager: cargo_target_failover_controller: cargo_target_sdn_controller: + cargo_target_object_storage: cargo_target_agent: agent_state_1: agent_state_2: From a8739e2af7b0ec9b877466c073555462b514f570 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:15:48 +0200 Subject: [PATCH 13/24] fix: reload hypervisor version reactively once auth token hydrates --- .../lib/components/sidebar/app-sidebar.svelte | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/src/lib/components/sidebar/app-sidebar.svelte b/app/src/lib/components/sidebar/app-sidebar.svelte index 2959d60..3287dad 100644 --- a/app/src/lib/components/sidebar/app-sidebar.svelte +++ b/app/src/lib/components/sidebar/app-sidebar.svelte @@ -80,7 +80,6 @@ From 19cbb00224c79a86ffde1a36d9f7c7aed458787b Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:17:17 +0200 Subject: [PATCH 14/24] fix: add object-storage crate to control-plane docker build --- control-plane/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/control-plane/Dockerfile b/control-plane/Dockerfile index 8d42b63..2a6f1ff 100644 --- a/control-plane/Dockerfile +++ b/control-plane/Dockerfile @@ -27,6 +27,7 @@ COPY control-plane/scheduler/Cargo.toml ./control-plane/scheduler/ COPY control-plane/failover-controller/Cargo.toml ./control-plane/failover-controller/ COPY control-plane/sdn-controller/Cargo.toml ./control-plane/sdn-controller/ COPY control-plane/volume-manager/Cargo.toml ./control-plane/volume-manager/ +COPY control-plane/object-storage/Cargo.toml ./control-plane/object-storage/ COPY control-plane/registry/Cargo.toml ./control-plane/registry/ COPY control-plane/shared/entity/Cargo.toml ./control-plane/shared/entity/ COPY control-plane/shared/migration/Cargo.toml ./control-plane/shared/migration/ @@ -41,6 +42,7 @@ RUN mkdir -p agent/src \ control-plane/failover-controller/src \ control-plane/sdn-controller/src \ control-plane/volume-manager/src \ + control-plane/object-storage/src \ control-plane/registry/src \ control-plane/shared/entity/src \ control-plane/shared/migration/src \ @@ -54,6 +56,7 @@ RUN mkdir -p agent/src \ && echo "fn main() {}" > control-plane/failover-controller/src/main.rs \ && echo "fn main() {}" > control-plane/sdn-controller/src/main.rs \ && echo "fn main() {}" > control-plane/volume-manager/src/main.rs \ + && echo "fn main() {}" > control-plane/object-storage/src/main.rs \ && echo "fn main() {}" > control-plane/registry/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-migrate/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-updater/src/main.rs \ From ac451d7379cb31bbfdc6580363360dfe28707b6a Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 17:22:49 +0200 Subject: [PATCH 15/24] feat: consolidate add bucket into add resource picker and add buckets overview page --- app/src/lib/api/resource-groups.ts | 8 + .../lib/components/sidebar/app-sidebar.svelte | 2 +- app/src/routes/(app)/buckets/+page.svelte | 154 ++++++++++++++++++ .../(app)/resource-groups/[id]/+page.svelte | 11 +- 4 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 app/src/routes/(app)/buckets/+page.svelte diff --git a/app/src/lib/api/resource-groups.ts b/app/src/lib/api/resource-groups.ts index f17821c..69fe78e 100644 --- a/app/src/lib/api/resource-groups.ts +++ b/app/src/lib/api/resource-groups.ts @@ -450,6 +450,14 @@ export async function deleteVolume(token: string, id: string): Promise { if (!res.ok) throw new Error(`Failed to delete volume: ${res.status}`); } +export async function listBuckets(token: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to list buckets: ${res.status}`); + return res.json(); +} + export async function listResourceGroupBuckets(token: string, rgId: string): Promise { const res = await authedFetch(`${API_BASE}/resource-groups/${rgId}/buckets`, { headers: { Authorization: `Bearer ${token}` }, diff --git a/app/src/lib/components/sidebar/app-sidebar.svelte b/app/src/lib/components/sidebar/app-sidebar.svelte index 3287dad..32fcea2 100644 --- a/app/src/lib/components/sidebar/app-sidebar.svelte +++ b/app/src/lib/components/sidebar/app-sidebar.svelte @@ -57,7 +57,7 @@ projects: [ { name: "S3 Buckets", - url: "#", + url: "/buckets", icon: IconBucket, }, { diff --git a/app/src/routes/(app)/buckets/+page.svelte b/app/src/routes/(app)/buckets/+page.svelte new file mode 100644 index 0000000..7cd524c --- /dev/null +++ b/app/src/routes/(app)/buckets/+page.svelte @@ -0,0 +1,154 @@ + + +
+ + / + S3 Buckets +
+ +
+
+
+

S3 Buckets

+

Object storage buckets across all resource groups

+
+ +
+ +
+ + + + + + + + + + + + + {#if loading} + + + + {:else if error} + + + + {:else if buckets.length === 0} + + + + {:else} + {#each buckets as bucket (bucket.id)} + + + + + + + + + {/each} + {/if} + +
NameGlobal AliasExposureQuotaStatusResource Group
Loading...
{error}
No buckets found
+
+
+ +
+ {bucket.name} +
+ 		{bucket.global_alias}{bucket.exposure} + {bucket.quota_max_size ? fmtBytes(bucket.quota_max_size) : "unlimited"} + + + + {#if bucket.resource_group_id} + + {bucket.resource_group_id.slice(0, 8)} + + {:else} + - + {/if} +
+
+
diff --git a/app/src/routes/(app)/resource-groups/[id]/+page.svelte b/app/src/routes/(app)/resource-groups/[id]/+page.svelte index 3a64ee2..73fadf2 100644 --- a/app/src/routes/(app)/resource-groups/[id]/+page.svelte +++ b/app/src/routes/(app)/resource-groups/[id]/+page.svelte @@ -124,6 +124,8 @@ const RESOURCE_TYPES = [ { key: "docker-container", label: "Docker Container", description: "Deploy a single container", icon: "logos:docker-icon" }, { key: "docker-compose", label: "Docker Compose", description: "Deploy multiple related containers as one stack", icon: "logos:docker-icon" }, + { key: "volume", label: "Volume", description: "Add a block storage volume", icon: "mdi:database-outline" }, + { key: "bucket", label: "S3 Bucket", description: "Add an S3-compatible object storage bucket", icon: "mdi:bucket-outline" }, ] as const; let formImage = $state(""); @@ -376,8 +378,12 @@ resourcePickerDialog?.close(); if (key === "docker-container") { deployDialog?.showModal(); - } else { + } else if (key === "docker-compose") { composeDialog?.showModal(); + } else if (key === "volume") { + volumeDialog?.showModal(); + } else { + bucketDialog?.showModal(); } } @@ -1803,9 +1809,6 @@ - + + + +
+ + {#each breadcrumbParts() as part} + / + + {/each} +
+ + {#if error} +

{error}

+ {/if} + +
+ + + + + + + + + + + {#if folders.length === 0 && objects.length === 0} + + + + {:else} + {#each folders as folder (folder)} + openFolder(folder)} + > + + + + + + {/each} + {#each objects as object (object.key)} + + + + + + + {/each} + {/if} + +
NameSizeLast ModifiedActions
+ This folder is empty +
+
+ + {folderName(folder)} +
+ 		--
+
+ + {objectName(object.key)} +
+ 		{fmtBytes(object.size)}{object.last_modified} +
+ + +
+
+
+ {/if} + diff --git a/app/src/routes/(app)/resource-groups/[id]/+page.svelte b/app/src/routes/(app)/resource-groups/[id]/+page.svelte index 73fadf2..c04dcb0 100644 --- a/app/src/routes/(app)/resource-groups/[id]/+page.svelte +++ b/app/src/routes/(app)/resource-groups/[id]/+page.svelte @@ -2105,14 +2105,24 @@ - +
+ + +
{/if} diff --git a/control-plane/api-gateway/Cargo.toml b/control-plane/api-gateway/Cargo.toml index 7cb88f0..b8d4acb 100644 --- a/control-plane/api-gateway/Cargo.toml +++ b/control-plane/api-gateway/Cargo.toml @@ -9,6 +9,8 @@ name = "api-gateway" path = "src/main.rs" [dependencies] +percent-encoding = "2.3" + # Internal dependencies entity = { path = "../shared/entity" } migration = { path = "../shared/migration" } diff --git a/control-plane/api-gateway/src/routes/buckets.rs b/control-plane/api-gateway/src/routes/buckets.rs index 8fa6329..63148f1 100644 --- a/control-plane/api-gateway/src/routes/buckets.rs +++ b/control-plane/api-gateway/src/routes/buckets.rs @@ -1,11 +1,12 @@ use axum::{ body::Body, - extract::{Path, State}, + extract::{Path, Query, State}, http::{HeaderMap, StatusCode}, response::{IntoResponse, Json}, routing::{get, post}, Router, }; +use serde::Deserialize; use serde_json::json; use crate::{ @@ -227,6 +228,89 @@ pub async fn get_cluster_status( .await } +#[derive(Deserialize)] +pub struct ListObjectsQuery { + #[serde(default)] + prefix: String, + #[serde(default)] + continuation_token: Option, +} + +pub async fn list_objects( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path(id): Path, + Query(query): Query, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + let mut path = format!( + "/buckets/{}/objects?prefix={}", + id, + percent_encoding::utf8_percent_encode(&query.prefix, percent_encoding::NON_ALPHANUMERIC) + ); + if let Some(token) = &query.continuation_token { + path.push_str(&format!( + "&continuation_token={}", + percent_encoding::utf8_percent_encode(token, percent_encoding::NON_ALPHANUMERIC) + )); + } + proxy_to_object_storage(&state, reqwest::Method::GET, &path, None, Some(header_map)).await +} + +pub async fn delete_object( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path((bucket_id, key)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::DELETE, + &format!("/buckets/{}/objects/{}", bucket_id, key), + None, + Some(header_map), + ) + .await +} + +pub async fn presign_upload( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + &format!("/buckets/{}/objects/presign-upload", id), + body_json, + Some(header_map), + ) + .await +} + +pub async fn presign_download( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path((bucket_id, key)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + &format!("/buckets/{}/objects/presign-download/{}", bucket_id, key), + None, + Some(header_map), + ) + .await +} + pub fn buckets_routes() -> Router { Router::new() .route("/buckets", post(create_bucket)) @@ -236,10 +320,23 @@ pub fn buckets_routes() -> Router { .route("/buckets/{id}", axum::routing::delete(delete_bucket)) .route("/buckets/{id}/keys", get(list_keys)) .route("/buckets/{id}/keys", post(create_key)) - .route("/buckets/{bucket_id}/keys/{key_id}/rotate", post(rotate_key)) + .route( + "/buckets/{bucket_id}/keys/{key_id}/rotate", + post(rotate_key), + ) .route( "/buckets/{bucket_id}/keys/{key_id}", axum::routing::delete(delete_key), ) + .route("/buckets/{id}/objects", get(list_objects)) + .route("/buckets/{id}/objects/presign-upload", post(presign_upload)) + .route( + "/buckets/{bucket_id}/objects/presign-download/{*key}", + get(presign_download), + ) + .route( + "/buckets/{bucket_id}/objects/{*key}", + axum::routing::delete(delete_object), + ) .route("/object-storage/cluster", get(get_cluster_status)) } diff --git a/control-plane/object-storage/Cargo.toml b/control-plane/object-storage/Cargo.toml index 1ba2f7b..6bea513 100644 --- a/control-plane/object-storage/Cargo.toml +++ b/control-plane/object-storage/Cargo.toml @@ -16,6 +16,11 @@ entity = { path = "../shared/entity" } tokio = { workspace = true, features = ["full"] } axum = { version = "0.8", features = ["macros"] } +aws-sigv4 = "1.5" +aws-credential-types = "1.2" +aws-smithy-runtime-api = "1.7" +http = "1" +percent-encoding = "2.3" tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } serde = { workspace = true } diff --git a/control-plane/object-storage/dev/bootstrap.sh b/control-plane/object-storage/dev/bootstrap.sh index ba99cbb..425f916 100755 --- a/control-plane/object-storage/dev/bootstrap.sh +++ b/control-plane/object-storage/dev/bootstrap.sh @@ -1,24 +1,38 @@ #!/bin/sh set -eu -GARAGE="garage -c /etc/garage.toml" +ADMIN_URL="http://garage:3903" +TOKEN="dev-garage-admin-token" -until $GARAGE status >/dev/null 2>&1; do - echo "waiting for garage rpc..." +until curl -sf -H "Authorization: Bearer ${TOKEN}" "${ADMIN_URL}/v2/GetClusterStatus" >/tmp/status.json; do + echo "waiting for garage admin api..." sleep 1 done -NODE_ID=$($GARAGE status | awk '/NODE ID/{found=1; next} found && NF {print $1; exit}') -if [ -z "$NODE_ID" ]; then - NODE_ID=$($GARAGE node id -q | cut -d'@' -f1) -fi +NODE_ID=$(jq -r '.nodes[0].id' /tmp/status.json) +LAYOUT_VERSION=$(jq -r '.layoutVersion' /tmp/status.json) + +EXISTING_ROLES=$(curl -sf -H "Authorization: Bearer ${TOKEN}" "${ADMIN_URL}/v2/GetClusterLayout" | jq -r '.roles | length') -if $GARAGE layout show | grep -q "No nodes"; then - echo "assigning layout to node ${NODE_ID}" - $GARAGE layout assign -z dev -c 1G "$NODE_ID" - $GARAGE layout apply --version 1 -else +if [ "$EXISTING_ROLES" -gt 0 ]; then echo "layout already assigned" + exit 0 fi +echo "assigning layout to node ${NODE_ID}" + +curl -sf -X POST \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"roles\":[{\"id\":\"${NODE_ID}\",\"zone\":\"dev\",\"capacity\":1000000000,\"tags\":[]}]}" \ + "${ADMIN_URL}/v2/UpdateClusterLayout" + +APPLY_VERSION=$((LAYOUT_VERSION + 1)) + +curl -sf -X POST \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"version\":${APPLY_VERSION}}" \ + "${ADMIN_URL}/v2/ApplyClusterLayout" + echo "garage bootstrap complete" diff --git a/control-plane/object-storage/src/crypto.rs b/control-plane/object-storage/src/crypto.rs new file mode 100644 index 0000000..24e6ccf --- /dev/null +++ b/control-plane/object-storage/src/crypto.rs @@ -0,0 +1,68 @@ +use anyhow::{bail, Context, Result}; +use ring::aead::{Aad, LessSafeKey, Nonce, UnboundKey, AES_256_GCM, NONCE_LEN}; +use ring::rand::{SecureRandom, SystemRandom}; + +pub struct SecretBox { + key: LessSafeKey, +} + +impl SecretBox { + pub fn from_env() -> Result { + let hex_key = std::env::var("OBJECT_STORAGE_ENCRYPTION_KEY") + .context("OBJECT_STORAGE_ENCRYPTION_KEY must be set")?; + let bytes = hex_decode(&hex_key).context("OBJECT_STORAGE_ENCRYPTION_KEY must be hex")?; + if bytes.len() != 32 { + bail!("OBJECT_STORAGE_ENCRYPTION_KEY must decode to 32 bytes"); + } + let unbound = UnboundKey::new(&AES_256_GCM, &bytes) + .map_err(|_| anyhow::anyhow!("failed to build encryption key"))?; + Ok(Self { + key: LessSafeKey::new(unbound), + }) + } + + pub fn encrypt(&self, plaintext: &str) -> Result> { + let rng = SystemRandom::new(); + let mut nonce_bytes = [0u8; NONCE_LEN]; + rng.fill(&mut nonce_bytes) + .map_err(|_| anyhow::anyhow!("failed to generate nonce"))?; + + let mut in_out = plaintext.as_bytes().to_vec(); + self.key + .seal_in_place_append_tag( + Nonce::assume_unique_for_key(nonce_bytes), + Aad::empty(), + &mut in_out, + ) + .map_err(|_| anyhow::anyhow!("encryption failed"))?; + + let mut output = nonce_bytes.to_vec(); + output.extend_from_slice(&in_out); + Ok(output) + } + + pub fn decrypt(&self, ciphertext: &[u8]) -> Result { + if ciphertext.len() < NONCE_LEN { + bail!("ciphertext too short"); + } + let (nonce_bytes, sealed) = ciphertext.split_at(NONCE_LEN); + let mut buf = sealed.to_vec(); + let nonce = Nonce::try_assume_unique_for_key(nonce_bytes) + .map_err(|_| anyhow::anyhow!("invalid nonce"))?; + let plaintext = self + .key + .open_in_place(nonce, Aad::empty(), &mut buf) + .map_err(|_| anyhow::anyhow!("decryption failed"))?; + String::from_utf8(plaintext.to_vec()).context("decrypted secret is not valid utf-8") + } +} + +fn hex_decode(input: &str) -> Result> { + if input.len() % 2 != 0 { + bail!("hex string must have even length"); + } + (0..input.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&input[i..i + 2], 16).context("invalid hex digit")) + .collect() +} diff --git a/control-plane/object-storage/src/db/buckets.rs b/control-plane/object-storage/src/db/buckets.rs index 2364b58..6d59469 100644 --- a/control-plane/object-storage/src/db/buckets.rs +++ b/control-plane/object-storage/src/db/buckets.rs @@ -29,16 +29,23 @@ pub async fn insert( req: &CreateBucketRequest, global_alias: &str, garage_bucket_id: &str, + master_key_id: &str, + master_key_secret_encrypted: Vec, ) -> Result { let model = buckets::ActiveModel { id: Set(Uuid::new_v4()), name: Set(req.name.clone()), garage_bucket_id: Set(Some(garage_bucket_id.to_string())), global_alias: Set(global_alias.to_string()), - exposure: Set(req.exposure.clone().unwrap_or_else(|| "internal".to_string())), + exposure: Set(req + .exposure + .clone() + .unwrap_or_else(|| "internal".to_string())), quota_max_size: Set(req.quota_max_size), quota_max_objects: Set(req.quota_max_objects), status: Set("active".to_string()), + master_key_id: Set(Some(master_key_id.to_string())), + master_key_secret_encrypted: Set(Some(master_key_secret_encrypted)), organization_id: Set(req.organization_id), resource_group_id: Set(req.resource_group_id), created_at: Set(Utc::now().naive_utc()), diff --git a/control-plane/object-storage/src/db/garage_nodes.rs b/control-plane/object-storage/src/db/garage_nodes.rs index 4f33424..9bcced9 100644 --- a/control-plane/object-storage/src/db/garage_nodes.rs +++ b/control-plane/object-storage/src/db/garage_nodes.rs @@ -36,4 +36,3 @@ pub async fn mark_down(db: &DatabaseConnection, agent_id: Uuid) -> Result<()> { } Ok(()) } - diff --git a/control-plane/object-storage/src/garage/client.rs b/control-plane/object-storage/src/garage/client.rs index b4fbbfe..d57e05b 100644 --- a/control-plane/object-storage/src/garage/client.rs +++ b/control-plane/object-storage/src/garage/client.rs @@ -83,7 +83,12 @@ impl GarageClient { } let status = response.status(); let body = response.text().await.unwrap_or_default(); - bail!("garage admin api error context={} status={} body={}", context, status, body) + bail!( + "garage admin api error context={} status={} body={}", + context, + status, + body + ) } pub async fn create_bucket(&self, global_alias: &str) -> Result { diff --git a/control-plane/object-storage/src/garage/layout.rs b/control-plane/object-storage/src/garage/layout.rs index def3406..236d979 100644 --- a/control-plane/object-storage/src/garage/layout.rs +++ b/control-plane/object-storage/src/garage/layout.rs @@ -20,7 +20,10 @@ pub fn replication_factor_for(storage_node_count: usize) -> u32 { } } -async fn peer_addrs(db: &DatabaseConnection, known_nodes: &[garage_nodes::Model]) -> Result> { +async fn peer_addrs( + db: &DatabaseConnection, + known_nodes: &[garage_nodes::Model], +) -> Result> { let mut addrs = Vec::new(); for node in known_nodes { @@ -44,13 +47,19 @@ async fn reconcile_once(db: &DatabaseConnection, garage: &GarageClient) -> Resul let addrs = peer_addrs(db, &known_nodes).await?; if let Err(e) = garage.connect_cluster_nodes(&addrs).await { - log_warn!("garage::layout", &format!("failed to connect garage cluster nodes err={}", e)); + log_warn!( + "garage::layout", + &format!("failed to connect garage cluster nodes err={}", e) + ); } let status = match garage.get_cluster_status().await { Ok(status) => status, Err(e) => { - log_warn!("garage::layout", &format!("failed to read garage cluster status err={}", e)); + log_warn!( + "garage::layout", + &format!("failed to read garage cluster status err={}", e) + ); return Ok(()); } }; @@ -62,7 +71,10 @@ async fn reconcile_once(db: &DatabaseConnection, garage: &GarageClient) -> Resul .any(|n| Some(n.id.clone()) == node.garage_node_id && n.is_up); if !is_up && node.status == "up" { - log_warn!("garage::layout", &format!("garage node reported down agent_id={}", node.agent_id)); + log_warn!( + "garage::layout", + &format!("garage node reported down agent_id={}", node.agent_id) + ); garage_nodes_db::mark_down(db, node.agent_id).await?; } } @@ -95,20 +107,37 @@ async fn reconcile_once(db: &DatabaseConnection, garage: &GarageClient) -> Resul } if let Err(e) = garage.update_cluster_layout(roles, factor).await { - log_error!("garage::layout", &format!("failed to stage cluster layout err={}", e)); + log_error!( + "garage::layout", + &format!("failed to stage cluster layout err={}", e) + ); return Ok(()); } if let Err(e) = garage.apply_cluster_layout(status.layout_version + 1).await { - log_error!("garage::layout", &format!("failed to apply cluster layout err={}", e)); + log_error!( + "garage::layout", + &format!("failed to apply cluster layout err={}", e) + ); return Ok(()); } - log_info!("garage::layout", &format!("applied cluster layout storage_nodes={} replication_factor={}", storage_nodes.len(), factor)); + log_info!( + "garage::layout", + &format!( + "applied cluster layout storage_nodes={} replication_factor={}", + storage_nodes.len(), + factor + ) + ); Ok(()) } -pub async fn run_reconcile_loop(db: DatabaseConnection, garage: GarageClient, leader: LayoutLeader) { +pub async fn run_reconcile_loop( + db: DatabaseConnection, + garage: GarageClient, + leader: LayoutLeader, +) { loop { sleep(Duration::from_secs(RECONCILE_INTERVAL_SECONDS)).await; @@ -117,7 +146,10 @@ pub async fn run_reconcile_loop(db: DatabaseConnection, garage: GarageClient, le } if let Err(e) = reconcile_once(&db, &garage).await { - log_error!("garage::layout", &format!("reconcile loop iteration failed err={}", e)); + log_error!( + "garage::layout", + &format!("reconcile loop iteration failed err={}", e) + ); } } } diff --git a/control-plane/object-storage/src/garage/leader.rs b/control-plane/object-storage/src/garage/leader.rs index 01f1ac6..d56c228 100644 --- a/control-plane/object-storage/src/garage/leader.rs +++ b/control-plane/object-storage/src/garage/leader.rs @@ -44,7 +44,11 @@ impl LayoutLeader { let lease_id = lease.id(); let txn = Txn::new() - .when(vec![Compare::create_revision(LAYOUT_LOCK_KEY, CompareOp::Equal, 0)]) + .when(vec![Compare::create_revision( + LAYOUT_LOCK_KEY, + CompareOp::Equal, + 0, + )]) .and_then(vec![TxnOp::put( LAYOUT_LOCK_KEY, self.node_id.as_bytes(), @@ -56,7 +60,13 @@ impl LayoutLeader { if response.succeeded() { self.lease_id.store(lease_id, Ordering::SeqCst); self.is_leader.store(true, Ordering::SeqCst); - log_info!("garage::leader", &format!("became object-storage layout leader node_id={}", self.node_id)); + log_info!( + "garage::leader", + &format!( + "became object-storage layout leader node_id={}", + self.node_id + ) + ); self.spawn_lease_renewal(lease_id); } else { let _ = self.etcd.lease_revoke(lease_id).await; @@ -77,9 +87,15 @@ impl LayoutLeader { break; } if let Err(e) = etcd.lease_keep_alive(lease_id).await { - log_error!("garage::leader", &format!("lease renewal failed node_id={} err={}", node_id, e)); + log_error!( + "garage::leader", + &format!("lease renewal failed node_id={} err={}", node_id, e) + ); is_leader.store(false, Ordering::SeqCst); - log_warn!("garage::leader", &format!("lost object-storage layout leadership node_id={}", node_id)); + log_warn!( + "garage::leader", + &format!("lost object-storage layout leadership node_id={}", node_id) + ); break; } } @@ -89,7 +105,10 @@ impl LayoutLeader { pub async fn run_campaign_loop(mut self) { loop { if let Err(e) = self.campaign().await { - log_error!("garage::leader", &format!("layout leader campaign failed err={}", e)); + log_error!( + "garage::leader", + &format!("layout leader campaign failed err={}", e) + ); } sleep(Duration::from_secs(5)).await; } diff --git a/control-plane/object-storage/src/garage/mod.rs b/control-plane/object-storage/src/garage/mod.rs index 307192c..57c8b77 100644 --- a/control-plane/object-storage/src/garage/mod.rs +++ b/control-plane/object-storage/src/garage/mod.rs @@ -1,5 +1,7 @@ pub mod client; -pub mod leader; pub mod layout; +pub mod leader; +pub mod s3_client; pub use client::GarageClient; +pub use s3_client::S3Client; diff --git a/control-plane/object-storage/src/garage/s3_client.rs b/control-plane/object-storage/src/garage/s3_client.rs new file mode 100644 index 0000000..2cc5cbf --- /dev/null +++ b/control-plane/object-storage/src/garage/s3_client.rs @@ -0,0 +1,278 @@ +use anyhow::{Context, Result}; +use aws_credential_types::Credentials; +use aws_sigv4::http_request::{ + sign, PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, SigningSettings, +}; +use aws_sigv4::sign::v4; +use serde::Deserialize; +use std::time::{Duration, SystemTime}; + +const S3_REGION: &str = "csfx"; +const S3_SERVICE: &str = "s3"; + +#[derive(Clone)] +pub struct S3Client { + s3_url: String, + http: reqwest::Client, +} + +#[derive(Debug, Deserialize)] +pub struct S3Object { + pub key: String, + pub size: i64, + pub last_modified: String, +} + +#[derive(Debug, Default)] +pub struct ListObjectsResult { + pub objects: Vec, + pub common_prefixes: Vec, +} + +impl S3Client { + pub fn new(s3_url: String) -> Self { + Self { + s3_url, + http: reqwest::Client::new(), + } + } + + fn identity( + access_key_id: &str, + secret_access_key: &str, + ) -> aws_smithy_runtime_api::client::identity::Identity { + Credentials::new( + access_key_id, + secret_access_key, + None, + None, + "csfx-object-storage", + ) + .into() + } + + fn signing_params<'a>( + identity: &'a aws_smithy_runtime_api::client::identity::Identity, + settings: SigningSettings, + ) -> Result> { + v4::SigningParams::builder() + .identity(identity) + .region(S3_REGION) + .name(S3_SERVICE) + .time(SystemTime::now()) + .settings(settings) + .build() + .context("failed to build sigv4 signing params") + } + + pub async fn list_objects( + &self, + bucket: &str, + access_key_id: &str, + secret_access_key: &str, + prefix: &str, + delimiter: &str, + continuation_token: Option<&str>, + ) -> Result { + let mut url = format!( + "{}/{}?list-type=2&prefix={}&delimiter={}", + self.s3_url, + bucket, + percent_encoding::utf8_percent_encode(prefix, percent_encoding::NON_ALPHANUMERIC), + percent_encoding::utf8_percent_encode(delimiter, percent_encoding::NON_ALPHANUMERIC), + ); + if let Some(token) = continuation_token { + url.push_str(&format!( + "&continuation-token={}", + percent_encoding::utf8_percent_encode(token, percent_encoding::NON_ALPHANUMERIC) + )); + } + + let identity = Self::identity(access_key_id, secret_access_key); + let params = Self::signing_params(&identity, SigningSettings::default())?; + + let signable = + SignableRequest::new("GET", &url, std::iter::empty(), SignableBody::Bytes(&[])) + .context("failed to build signable request")?; + + let signed_headers: Vec<(String, String)> = sign(signable, ¶ms.into()) + .context("failed to sign request")? + .into_parts() + .0 + .headers() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + + let mut request = self + .http + .get(&url) + .build() + .context("failed to build request")?; + for (name, value) in signed_headers { + request.headers_mut().insert( + http::HeaderName::from_bytes(name.as_bytes()) + .context("invalid signed header name")?, + value.parse().context("invalid signed header value")?, + ); + } + + let response = self + .http + .execute(request) + .await + .context("list_objects request failed")?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("list_objects failed status={} body={}", status, body); + } + + let body = response + .text() + .await + .context("failed to read list_objects response")?; + parse_list_objects_xml(&body) + } + + pub async fn delete_object( + &self, + bucket: &str, + access_key_id: &str, + secret_access_key: &str, + key: &str, + ) -> Result<()> { + let url = format!( + "{}/{}/{}", + self.s3_url, + bucket, + percent_encoding::utf8_percent_encode(key, percent_encoding::NON_ALPHANUMERIC) + ); + + let identity = Self::identity(access_key_id, secret_access_key); + let params = Self::signing_params(&identity, SigningSettings::default())?; + + let signable = + SignableRequest::new("DELETE", &url, std::iter::empty(), SignableBody::Bytes(&[])) + .context("failed to build signable request")?; + + let signed_headers: Vec<(String, String)> = sign(signable, ¶ms.into()) + .context("failed to sign request")? + .into_parts() + .0 + .headers() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + + let mut request = self + .http + .delete(&url) + .build() + .context("failed to build request")?; + for (name, value) in signed_headers { + request.headers_mut().insert( + http::HeaderName::from_bytes(name.as_bytes()) + .context("invalid signed header name")?, + value.parse().context("invalid signed header value")?, + ); + } + + let response = self + .http + .execute(request) + .await + .context("delete_object request failed")?; + + if !response.status().is_success() && response.status().as_u16() != 404 { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("delete_object failed status={} body={}", status, body); + } + + Ok(()) + } + + pub fn presign_url( + &self, + method: &str, + bucket: &str, + key: &str, + access_key_id: &str, + secret_access_key: &str, + expires_in: Duration, + ) -> Result { + let url = format!( + "{}/{}/{}", + self.s3_url, + bucket, + percent_encoding::utf8_percent_encode(key, percent_encoding::NON_ALPHANUMERIC) + ); + + let identity = Self::identity(access_key_id, secret_access_key); + let mut settings = SigningSettings::default(); + settings.percent_encoding_mode = PercentEncodingMode::Single; + settings.signature_location = SignatureLocation::QueryParams; + settings.expires_in = Some(expires_in); + let params = Self::signing_params(&identity, settings)?; + + let signable = SignableRequest::new( + method, + &url, + std::iter::empty(), + SignableBody::UnsignedPayload, + ) + .context("failed to build signable request")?; + + let (instructions, _) = sign(signable, ¶ms.into()) + .context("failed to sign presigned url")? + .into_parts(); + + let mut request = http::Request::builder() + .method(method) + .uri(&url) + .body(()) + .context("failed to build presign request")?; + instructions.apply_to_request_http1x(&mut request); + + Ok(request.uri().to_string()) + } +} + +fn parse_list_objects_xml(body: &str) -> Result { + let mut result = ListObjectsResult::default(); + + for segment in body.split("").skip(1) { + let end = segment.find("").unwrap_or(segment.len()); + let entry = &segment[..end]; + let key = extract_xml_tag(entry, "Key").unwrap_or_default(); + let size = extract_xml_tag(entry, "Size") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + let last_modified = extract_xml_tag(entry, "LastModified").unwrap_or_default(); + if !key.is_empty() { + result.objects.push(S3Object { + key, + size, + last_modified, + }); + } + } + + for segment in body.split("").skip(1) { + let end = segment.find("").unwrap_or(segment.len()); + let entry = &segment[..end]; + if let Some(prefix) = extract_xml_tag(entry, "Prefix") { + result.common_prefixes.push(prefix); + } + } + + Ok(result) +} + +fn extract_xml_tag(input: &str, tag: &str) -> Option { + let open = format!("<{}>", tag); + let close = format!("", tag); + let start = input.find(&open)? + open.len(); + let end = input[start..].find(&close)? + start; + Some(input[start..end].to_string()) +} diff --git a/control-plane/object-storage/src/handlers/buckets.rs b/control-plane/object-storage/src/handlers/buckets.rs index 680f864..5522955 100644 --- a/control-plane/object-storage/src/handlers/buckets.rs +++ b/control-plane/object-storage/src/handlers/buckets.rs @@ -23,7 +23,7 @@ pub async fn create_bucket( State(state): State, Json(req): Json, ) -> Result)> { - match service::create_bucket(&state.db, &state.garage, req).await { + match service::create_bucket(&state.db, &state.garage, &state.secret_box, req).await { Ok(bucket) => Ok((StatusCode::CREATED, Json(json!(bucket)))), Err(e) => { tracing::error!(error = %e, "failed to create bucket"); diff --git a/control-plane/object-storage/src/handlers/cluster.rs b/control-plane/object-storage/src/handlers/cluster.rs index e12afd1..2a69b0f 100644 --- a/control-plane/object-storage/src/handlers/cluster.rs +++ b/control-plane/object-storage/src/handlers/cluster.rs @@ -1,7 +1,14 @@ -use axum::{extract::State, http::StatusCode, response::{IntoResponse, Json}}; +use axum::{ + extract::State, + http::StatusCode, + response::{IntoResponse, Json}, +}; use serde_json::json; -use crate::{db::garage_nodes, garage::layout::replication_factor_for, models::ClusterStatusResponse, server::AppState}; +use crate::{ + db::garage_nodes, garage::layout::replication_factor_for, models::ClusterStatusResponse, + server::AppState, +}; pub async fn get_cluster_status( State(state): State, diff --git a/control-plane/object-storage/src/handlers/mod.rs b/control-plane/object-storage/src/handlers/mod.rs index f7f1f8f..f1397a8 100644 --- a/control-plane/object-storage/src/handlers/mod.rs +++ b/control-plane/object-storage/src/handlers/mod.rs @@ -1,3 +1,4 @@ pub mod buckets; pub mod cluster; pub mod keys; +pub mod objects; diff --git a/control-plane/object-storage/src/handlers/objects.rs b/control-plane/object-storage/src/handlers/objects.rs new file mode 100644 index 0000000..4791d2a --- /dev/null +++ b/control-plane/object-storage/src/handlers/objects.rs @@ -0,0 +1,107 @@ +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde_json::json; +use uuid::Uuid; + +use crate::{ + models::{ListObjectsQuery, PresignUploadRequest}, + server::AppState, + services::object as service, +}; + +pub async fn list_objects( + State(state): State, + Path(bucket_id): Path, + Query(query): Query, +) -> Result)> { + match service::list_objects( + &state.db, + &state.s3, + &state.secret_box, + bucket_id, + &query.prefix, + query.continuation_token.as_deref(), + ) + .await + { + Ok(Some(result)) => Ok(Json(json!(result))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to list objects"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn delete_object( + State(state): State, + Path((bucket_id, key)): Path<(Uuid, String)>, +) -> Result)> { + match service::delete_object(&state.db, &state.s3, &state.secret_box, bucket_id, &key).await { + Ok(true) => Ok(StatusCode::NO_CONTENT), + Ok(false) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to delete object"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn presign_upload( + State(state): State, + Path(bucket_id): Path, + Json(req): Json, +) -> Result)> { + match service::presign_upload(&state.db, &state.s3, &state.secret_box, bucket_id, &req.key) + .await + { + Ok(Some(result)) => Ok(Json(json!(result))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to presign upload"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn presign_download( + State(state): State, + Path((bucket_id, key)): Path<(Uuid, String)>, +) -> Result)> { + match service::presign_download(&state.db, &state.s3, &state.secret_box, bucket_id, &key).await + { + Ok(Some(result)) => Ok(Json(json!(result))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to presign download"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} diff --git a/control-plane/object-storage/src/main.rs b/control-plane/object-storage/src/main.rs index 5dc9f9a..bf9a2de 100644 --- a/control-plane/object-storage/src/main.rs +++ b/control-plane/object-storage/src/main.rs @@ -1,6 +1,7 @@ use std::net::SocketAddr; use uuid::Uuid; +mod crypto; mod db; mod garage; mod handlers; @@ -31,12 +32,15 @@ async fn main() -> anyhow::Result<()> { log_info!("main", "Database connection established"); shared::spawn_log_writer(log_receiver, db.clone()); - let admin_url = std::env::var("GARAGE_ADMIN_URL") - .unwrap_or_else(|_| "http://127.0.0.1:3903".to_string()); - let admin_token = std::env::var("GARAGE_ADMIN_TOKEN") - .expect("GARAGE_ADMIN_TOKEN must be set"); + let admin_url = + std::env::var("GARAGE_ADMIN_URL").unwrap_or_else(|_| "http://127.0.0.1:3903".to_string()); + let admin_token = std::env::var("GARAGE_ADMIN_TOKEN").expect("GARAGE_ADMIN_TOKEN must be set"); let garage = garage::GarageClient::new(admin_url, admin_token); + let s3_url = + std::env::var("GARAGE_S3_URL").unwrap_or_else(|_| "http://127.0.0.1:3900".to_string()); + let s3_client = garage::S3Client::new(s3_url); + let etcd_url = std::env::var("ETCD_URL").unwrap_or_else(|_| "http://localhost:2379".to_string()); let etcd = etcd_client::Client::connect([etcd_url.as_str()], None) @@ -54,7 +58,11 @@ async fn main() -> anyhow::Result<()> { leader, )); - let state = server::AppState::new(db, garage); + let secret_box = std::sync::Arc::new( + crypto::SecretBox::from_env().expect("Failed to initialize encryption key"), + ); + + let state = server::AppState::new(db, garage, s3_client, secret_box); let app = server::create_router(state); let port = std::env::var("OBJECT_STORAGE_PORT") diff --git a/control-plane/object-storage/src/models.rs b/control-plane/object-storage/src/models.rs index cd1db51..05939a5 100644 --- a/control-plane/object-storage/src/models.rs +++ b/control-plane/object-storage/src/models.rs @@ -44,6 +44,39 @@ pub struct AccessKeyCreatedResponse { pub secret_access_key: String, } +#[derive(Debug, Serialize, Deserialize)] +pub struct ObjectEntry { + pub key: String, + pub size: i64, + pub last_modified: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ListObjectsResponse { + pub objects: Vec, + pub folders: Vec, + pub next_continuation_token: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ListObjectsQuery { + #[serde(default)] + pub prefix: String, + #[serde(default)] + pub continuation_token: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PresignUploadRequest { + pub key: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PresignResponse { + pub url: String, + pub expires_in_seconds: u64, +} + #[derive(Debug, Serialize, Deserialize)] pub struct ClusterStatusResponse { pub storage_node_count: u32, diff --git a/control-plane/object-storage/src/server.rs b/control-plane/object-storage/src/server.rs index e310eed..5e07ba8 100644 --- a/control-plane/object-storage/src/server.rs +++ b/control-plane/object-storage/src/server.rs @@ -1,9 +1,11 @@ use axum::{http::StatusCode, response::IntoResponse, routing::get, Router}; use sea_orm::DatabaseConnection; +use std::sync::Arc; use crate::{ - garage::GarageClient, - handlers::{buckets, cluster, keys}, + crypto::SecretBox, + garage::{GarageClient, S3Client}, + handlers::{buckets, cluster, keys, objects}, metrics, }; @@ -11,11 +13,23 @@ use crate::{ pub struct AppState { pub db: DatabaseConnection, pub garage: GarageClient, + pub s3: S3Client, + pub secret_box: Arc, } impl AppState { - pub fn new(db: DatabaseConnection, garage: GarageClient) -> Self { - Self { db, garage } + pub fn new( + db: DatabaseConnection, + garage: GarageClient, + s3: S3Client, + secret_box: Arc, + ) -> Self { + Self { + db, + garage, + s3, + secret_box, + } } } @@ -49,6 +63,19 @@ pub fn create_router(state: AppState) -> Router { "/buckets/{id}/keys/{key_id}", axum::routing::delete(keys::delete_key), ) + .route("/buckets/{id}/objects", get(objects::list_objects)) + .route( + "/buckets/{id}/objects/presign-upload", + axum::routing::post(objects::presign_upload), + ) + .route( + "/buckets/{bucket_id}/objects/presign-download/{*key}", + get(objects::presign_download), + ) + .route( + "/buckets/{bucket_id}/objects/{*key}", + axum::routing::delete(objects::delete_object), + ) .route("/cluster", get(cluster::get_cluster_status)) .with_state(state) } diff --git a/control-plane/object-storage/src/services/access_key.rs b/control-plane/object-storage/src/services/access_key.rs index 742e0df..fbfa8c7 100644 --- a/control-plane/object-storage/src/services/access_key.rs +++ b/control-plane/object-storage/src/services/access_key.rs @@ -40,7 +40,10 @@ pub async fn create_key( let garage_key = match garage.create_key(&req.name).await { Ok(key) => key, Err(e) => { - log_error!("services::access_key", &format!("garage create_key failed name={} err={}", req.name, e)); + log_error!( + "services::access_key", + &format!("garage create_key failed name={} err={}", req.name, e) + ); bail!("failed to create access key in garage: {}", e); } }; @@ -49,7 +52,13 @@ pub async fn create_key( .allow_bucket_key(garage_bucket_id, &garage_key.access_key_id, &permissions) .await { - log_error!("services::access_key", &format!("garage allow_bucket_key failed key_id={} err={}", garage_key.access_key_id, e)); + log_error!( + "services::access_key", + &format!( + "garage allow_bucket_key failed key_id={} err={}", + garage_key.access_key_id, e + ) + ); let _ = garage.delete_key(&garage_key.access_key_id).await; bail!("failed to grant bucket access: {}", e); } @@ -64,7 +73,10 @@ pub async fn create_key( ) .await?; - log_info!("services::access_key", &format!("access key created id={} bucket_id={}", model.id, bucket_id)); + log_info!( + "services::access_key", + &format!("access key created id={} bucket_id={}", model.id, bucket_id) + ); Ok(Some(AccessKeyCreatedResponse { key: into_response(model), @@ -72,10 +84,7 @@ pub async fn create_key( })) } -pub async fn list_keys( - db: &DatabaseConnection, - bucket_id: Uuid, -) -> Result> { +pub async fn list_keys(db: &DatabaseConnection, bucket_id: Uuid) -> Result> { let rows = keys_db::list_for_bucket(db, bucket_id).await?; Ok(rows.into_iter().map(into_response).collect()) } @@ -110,12 +119,24 @@ pub async fn rotate_key( }; if let Err(e) = garage.delete_key(&existing.garage_key_id).await { - log_error!("services::access_key", &format!("garage delete_key on rotate failed old_key_id={} err={}", existing.garage_key_id, e)); + log_error!( + "services::access_key", + &format!( + "garage delete_key on rotate failed old_key_id={} err={}", + existing.garage_key_id, e + ) + ); } keys_db::delete(db, key_id).await?; keys_db::touch_rotated(db, created.key.id).await?; - log_info!("services::access_key", &format!("access key rotated old_id={} new_id={}", key_id, created.key.id)); + log_info!( + "services::access_key", + &format!( + "access key rotated old_id={} new_id={}", + key_id, created.key.id + ) + ); Ok(Some(created)) } @@ -136,6 +157,9 @@ pub async fn delete_key( garage.delete_key(&existing.garage_key_id).await?; keys_db::delete(db, key_id).await?; - log_info!("services::access_key", &format!("access key deleted id={}", key_id)); + log_info!( + "services::access_key", + &format!("access key deleted id={}", key_id) + ); Ok(true) } diff --git a/control-plane/object-storage/src/services/bucket.rs b/control-plane/object-storage/src/services/bucket.rs index a845a20..57175d4 100644 --- a/control-plane/object-storage/src/services/bucket.rs +++ b/control-plane/object-storage/src/services/bucket.rs @@ -3,6 +3,7 @@ use sea_orm::DatabaseConnection; use uuid::Uuid; use crate::{ + crypto::SecretBox, db::buckets as db, garage::GarageClient, log_error, log_info, @@ -12,6 +13,7 @@ use crate::{ pub async fn create_bucket( db_conn: &DatabaseConnection, garage: &GarageClient, + secret_box: &SecretBox, req: CreateBucketRequest, ) -> Result { let global_alias = format!("{}-{}", req.name, Uuid::new_v4().simple()); @@ -19,7 +21,10 @@ pub async fn create_bucket( let garage_bucket = match garage.create_bucket(&global_alias).await { Ok(bucket) => bucket, Err(e) => { - log_error!("services::bucket", &format!("garage create_bucket failed name={} err={}", req.name, e)); + log_error!( + "services::bucket", + &format!("garage create_bucket failed name={} err={}", req.name, e) + ); bail!("failed to create bucket in garage: {}", e); } }; @@ -29,22 +34,72 @@ pub async fn create_bucket( .update_bucket_quotas(&garage_bucket.id, req.quota_max_size, req.quota_max_objects) .await { - log_error!("services::bucket", &format!("garage update_bucket_quotas failed bucket_id={} err={}", garage_bucket.id, e)); + log_error!( + "services::bucket", + &format!( + "garage update_bucket_quotas failed bucket_id={} err={}", + garage_bucket.id, e + ) + ); let _ = garage.delete_bucket(&garage_bucket.id).await; bail!("failed to apply bucket quota: {}", e); } } - let model = db::insert(db_conn, &req, &global_alias, &garage_bucket.id).await?; - log_info!("services::bucket", &format!("bucket created id={} garage_bucket_id={}", model.id, garage_bucket.id)); + let master_key = match garage.create_key(&format!("{}-master", global_alias)).await { + Ok(key) => key, + Err(e) => { + log_error!( + "services::bucket", + &format!( + "garage create_key for master key failed bucket_id={} err={}", + garage_bucket.id, e + ) + ); + let _ = garage.delete_bucket(&garage_bucket.id).await; + bail!("failed to create master access key: {}", e); + } + }; + + if let Err(e) = garage + .allow_bucket_key(&garage_bucket.id, &master_key.access_key_id, "owner") + .await + { + log_error!( + "services::bucket", + &format!( + "garage allow_bucket_key for master key failed bucket_id={} err={}", + garage_bucket.id, e + ) + ); + let _ = garage.delete_key(&master_key.access_key_id).await; + let _ = garage.delete_bucket(&garage_bucket.id).await; + bail!("failed to grant master key access: {}", e); + } + + let encrypted_secret = secret_box.encrypt(&master_key.secret_access_key)?; + + let model = db::insert( + db_conn, + &req, + &global_alias, + &garage_bucket.id, + &master_key.access_key_id, + encrypted_secret, + ) + .await?; + log_info!( + "services::bucket", + &format!( + "bucket created id={} garage_bucket_id={}", + model.id, garage_bucket.id + ) + ); Ok(db::into_response(model)) } -pub async fn get_bucket( - db_conn: &DatabaseConnection, - id: Uuid, -) -> Result> { +pub async fn get_bucket(db_conn: &DatabaseConnection, id: Uuid) -> Result> { Ok(db::get_by_id(db_conn, id).await?.map(db::into_response)) } diff --git a/control-plane/object-storage/src/services/mod.rs b/control-plane/object-storage/src/services/mod.rs index 65cc81d..6d8d394 100644 --- a/control-plane/object-storage/src/services/mod.rs +++ b/control-plane/object-storage/src/services/mod.rs @@ -1,2 +1,3 @@ pub mod access_key; pub mod bucket; +pub mod object; diff --git a/control-plane/object-storage/src/services/object.rs b/control-plane/object-storage/src/services/object.rs new file mode 100644 index 0000000..8b0f542 --- /dev/null +++ b/control-plane/object-storage/src/services/object.rs @@ -0,0 +1,145 @@ +use anyhow::{bail, Result}; +use sea_orm::DatabaseConnection; +use std::time::Duration; +use uuid::Uuid; + +use crate::{ + crypto::SecretBox, + db::buckets as buckets_db, + garage::S3Client, + models::{ListObjectsResponse, ObjectEntry, PresignResponse}, +}; + +const PRESIGN_EXPIRY_SECONDS: u64 = 900; + +async fn resolve_master_credentials( + db: &DatabaseConnection, + secret_box: &SecretBox, + bucket_id: Uuid, +) -> Result<(String, String, String)> { + let Some(bucket) = buckets_db::get_by_id(db, bucket_id).await? else { + bail!("bucket not found"); + }; + + let (Some(access_key_id), Some(encrypted_secret)) = + (bucket.master_key_id, bucket.master_key_secret_encrypted) + else { + bail!("bucket has no master key configured"); + }; + + let secret_access_key = secret_box.decrypt(&encrypted_secret)?; + + Ok((bucket.global_alias, access_key_id, secret_access_key)) +} + +pub async fn list_objects( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + prefix: &str, + continuation_token: Option<&str>, +) -> Result> { + let (global_alias, access_key_id, secret_access_key) = + match resolve_master_credentials(db, secret_box, bucket_id).await { + Ok(creds) => creds, + Err(e) if e.to_string() == "bucket not found" => return Ok(None), + Err(e) => return Err(e), + }; + + let result = s3 + .list_objects( + &global_alias, + &access_key_id, + &secret_access_key, + prefix, + "/", + continuation_token, + ) + .await?; + + Ok(Some(ListObjectsResponse { + objects: result + .objects + .into_iter() + .map(|o| ObjectEntry { + key: o.key, + size: o.size, + last_modified: o.last_modified, + }) + .collect(), + folders: result.common_prefixes, + next_continuation_token: None, + })) +} + +pub async fn delete_object( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, +) -> Result { + let (global_alias, access_key_id, secret_access_key) = + match resolve_master_credentials(db, secret_box, bucket_id).await { + Ok(creds) => creds, + Err(e) if e.to_string() == "bucket not found" => return Ok(false), + Err(e) => return Err(e), + }; + + s3.delete_object(&global_alias, &access_key_id, &secret_access_key, key) + .await?; + + Ok(true) +} + +pub async fn presign_upload( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, +) -> Result> { + presign(db, s3, secret_box, bucket_id, key, "PUT").await +} + +pub async fn presign_download( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, +) -> Result> { + presign(db, s3, secret_box, bucket_id, key, "GET").await +} + +async fn presign( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, + method: &str, +) -> Result> { + let (global_alias, access_key_id, secret_access_key) = + match resolve_master_credentials(db, secret_box, bucket_id).await { + Ok(creds) => creds, + Err(e) if e.to_string() == "bucket not found" => return Ok(None), + Err(e) => return Err(e), + }; + + let expires_in = Duration::from_secs(PRESIGN_EXPIRY_SECONDS); + let url = s3.presign_url( + method, + &global_alias, + key, + &access_key_id, + &secret_access_key, + expires_in, + )?; + + Ok(Some(PresignResponse { + url, + expires_in_seconds: PRESIGN_EXPIRY_SECONDS, + })) +} diff --git a/control-plane/shared/entity/src/entities/buckets.rs b/control-plane/shared/entity/src/entities/buckets.rs index 2cb8090..e49313d 100644 --- a/control-plane/shared/entity/src/entities/buckets.rs +++ b/control-plane/shared/entity/src/entities/buckets.rs @@ -13,6 +13,8 @@ pub struct Model { pub quota_max_size: Option, pub quota_max_objects: Option, pub status: String, + pub master_key_id: Option, + pub master_key_secret_encrypted: Option>, pub organization_id: Option, pub resource_group_id: Option, pub created_at: chrono::NaiveDateTime, diff --git a/control-plane/shared/migration/src/lib.rs b/control-plane/shared/migration/src/lib.rs index 961cef2..9acadb3 100644 --- a/control-plane/shared/migration/src/lib.rs +++ b/control-plane/shared/migration/src/lib.rs @@ -33,6 +33,7 @@ mod m20260723_000000_add_resource_group_appearance; mod m20260723_010000_runtime_class_default_firecracker; mod m20260809_000000_add_user_gravatar_email; mod m20260816_000000_add_object_storage; +mod m20260816_010000_add_bucket_master_key; pub struct Migrator; @@ -73,6 +74,7 @@ impl MigratorTrait for Migrator { Box::new(m20260723_010000_runtime_class_default_firecracker::Migration), Box::new(m20260809_000000_add_user_gravatar_email::Migration), Box::new(m20260816_000000_add_object_storage::Migration), + Box::new(m20260816_010000_add_bucket_master_key::Migration), ] } } diff --git a/control-plane/shared/migration/src/m20260816_010000_add_bucket_master_key.rs b/control-plane/shared/migration/src/m20260816_010000_add_bucket_master_key.rs new file mode 100644 index 0000000..2eba933 --- /dev/null +++ b/control-plane/shared/migration/src/m20260816_010000_add_bucket_master_key.rs @@ -0,0 +1,37 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("buckets")) + .add_column_if_not_exists( + ColumnDef::new(Alias::new("master_key_id")).string().null(), + ) + .add_column_if_not_exists( + ColumnDef::new(Alias::new("master_key_secret_encrypted")) + .binary() + .null(), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("buckets")) + .drop_column(Alias::new("master_key_secret_encrypted")) + .drop_column(Alias::new("master_key_id")) + .to_owned(), + ) + .await + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 056da3e..2928a08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -180,7 +180,7 @@ services: command: cargo watch -x "run -p volume-manager" garage: - image: dxflrs/garage:v1.0.1 + image: dxflrs/garage:v2.3.0 container_name: csfx-garage-dev volumes: - ./control-plane/object-storage/dev/garage.toml:/etc/garage.toml:ro @@ -193,18 +193,17 @@ services: - csfx-network restart: unless-stopped healthcheck: - test: ["CMD", "garage", "-c", "/etc/garage.toml", "status"] + test: ["CMD", "/garage", "-c", "/etc/garage.toml", "status"] interval: 5s timeout: 3s retries: 10 start_period: 5s garage-bootstrap: - image: dxflrs/garage:v1.0.1 + image: alpine:3.20 container_name: csfx-garage-bootstrap-dev - entrypoint: ["/bin/sh", "/bootstrap.sh"] + entrypoint: ["/bin/sh", "-c", "apk add --no-cache curl jq >/dev/null && /bootstrap.sh"] volumes: - - ./control-plane/object-storage/dev/garage.toml:/etc/garage.toml:ro - ./control-plane/object-storage/dev/bootstrap.sh:/bootstrap.sh:ro networks: - csfx-network @@ -222,6 +221,8 @@ services: LISTEN_ADDR: "0.0.0.0" GARAGE_ADMIN_URL: http://garage:3903 GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-dev-garage-admin-token} + GARAGE_S3_URL: http://garage:3900 + OBJECT_STORAGE_ENCRYPTION_KEY: ${OBJECT_STORAGE_ENCRYPTION_KEY:-9a29b5d4c92cbfd4abe14c2a645f39eff6b57b1fa13305d3a946a2221e630c29} RUST_LOG: ${RUST_LOG:-debug} ports: - "8006:8006" From e8c70fa4acdfcf1e611a434f6f09b21f6a27689f Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 18:30:41 +0200 Subject: [PATCH 21/24] fix: route presigned s3 uploads through dedicated gateway port --- Dockerfile | 2 +- .../lib/components/sidebar/icon-bucket.svelte | 2 +- app/src/routes/(app)/buckets/+page.svelte | 4 +- .../(app)/resource-groups/[id]/+page.svelte | 6 +- control-plane/Dockerfile | 2 +- control-plane/api-gateway/src/main.rs | 24 +++++- control-plane/api-gateway/src/routes/mod.rs | 31 +++++++ .../api-gateway/src/routes/resource_groups.rs | 4 +- .../api-gateway/src/routes/s3_proxy.rs | 86 +++++++++++++++---- .../object-storage/src/garage/client.rs | 4 +- .../object-storage/src/garage/s3_client.rs | 29 +++++-- control-plane/object-storage/src/main.rs | 3 +- docker-compose.yml | 2 + 13 files changed, 159 insertions(+), 40 deletions(-) diff --git a/Dockerfile b/Dockerfile index d53fcef..25f4710 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.88-slim-bookworm AS base +FROM rust:1.97-slim-bookworm AS base WORKDIR /app diff --git a/app/src/lib/components/sidebar/icon-bucket.svelte b/app/src/lib/components/sidebar/icon-bucket.svelte index 7fc159b..d020daa 100644 --- a/app/src/lib/components/sidebar/icon-bucket.svelte +++ b/app/src/lib/components/sidebar/icon-bucket.svelte @@ -5,4 +5,4 @@ let { class: className, ...restProps }: HTMLAttributes = $props(); - + diff --git a/app/src/routes/(app)/buckets/+page.svelte b/app/src/routes/(app)/buckets/+page.svelte index 0b82bce..5bc8cae 100644 --- a/app/src/routes/(app)/buckets/+page.svelte +++ b/app/src/routes/(app)/buckets/+page.svelte @@ -123,9 +123,7 @@ >
-
- -
+ {bucket.name}
diff --git a/app/src/routes/(app)/resource-groups/[id]/+page.svelte b/app/src/routes/(app)/resource-groups/[id]/+page.svelte index c04dcb0..c07ce8a 100644 --- a/app/src/routes/(app)/resource-groups/[id]/+page.svelte +++ b/app/src/routes/(app)/resource-groups/[id]/+page.svelte @@ -125,7 +125,7 @@ { key: "docker-container", label: "Docker Container", description: "Deploy a single container", icon: "logos:docker-icon" }, { key: "docker-compose", label: "Docker Compose", description: "Deploy multiple related containers as one stack", icon: "logos:docker-icon" }, { key: "volume", label: "Volume", description: "Add a block storage volume", icon: "mdi:database-outline" }, - { key: "bucket", label: "S3 Bucket", description: "Add an S3-compatible object storage bucket", icon: "mdi:bucket-outline" }, + { key: "bucket", label: "S3 Bucket", description: "Add an S3-compatible object storage bucket", icon: "fluent-emoji-high-contrast:bucket" }, ] as const; let formImage = $state(""); @@ -2083,9 +2083,7 @@ >
-
- -
+

{b.name}

{b.global_alias}

diff --git a/control-plane/Dockerfile b/control-plane/Dockerfile index 2a6f1ff..8fd08d7 100644 --- a/control-plane/Dockerfile +++ b/control-plane/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.88-slim-bookworm AS base +FROM rust:1.97-slim-bookworm AS base WORKDIR /app diff --git a/control-plane/api-gateway/src/main.rs b/control-plane/api-gateway/src/main.rs index 0e05f82..9dc77b8 100644 --- a/control-plane/api-gateway/src/main.rs +++ b/control-plane/api-gateway/src/main.rs @@ -177,7 +177,29 @@ async fn main() { let app = routes::create_router() .merge(SwaggerUi::new("/swagger-ui").url("/api-docs/openapi.json", ApiDoc::openapi())) - .with_state(state); + .with_state(state.clone()); + + let object_data_app = routes::create_object_data_router().with_state(state); + + let object_data_port = std::env::var("OBJECT_DATA_PORT") + .ok() + .and_then(|p| p.parse::().ok()) + .unwrap_or(8007); + let object_data_listen_addr = + std::env::var("LISTEN_ADDR").unwrap_or_else(|_| "0.0.0.0".to_string()); + let object_data_addr: SocketAddr = format!("{}:{}", object_data_listen_addr, object_data_port) + .parse() + .unwrap(); + + tokio::spawn(async move { + let listener = tokio::net::TcpListener::bind(object_data_addr) + .await + .expect("failed to bind object data port"); + tracing::info!(addr = %object_data_addr, "object data proxy listening"); + axum::serve(listener, object_data_app.into_make_service()) + .await + .expect("object data proxy server failed"); + }); let port = std::env::var("GATEWAY_PORT") .ok() diff --git a/control-plane/api-gateway/src/routes/mod.rs b/control-plane/api-gateway/src/routes/mod.rs index b934b9e..da09aea 100644 --- a/control-plane/api-gateway/src/routes/mod.rs +++ b/control-plane/api-gateway/src/routes/mod.rs @@ -38,6 +38,37 @@ pub mod volumes; pub mod workloads; /// Creates the main application router and logs all registered routes. +pub fn create_object_data_router() -> Router { + let frontend_url = + std::env::var("FRONTEND_URL").unwrap_or_else(|_| "http://localhost:3000".to_string()); + + let allowed_origins = vec![ + "http://localhost:3000", + "http://localhost:5173", + "http://127.0.0.1:3000", + "http://127.0.0.1:5173", + &frontend_url, + ]; + + let cors = CorsLayer::new() + .allow_origin( + allowed_origins + .into_iter() + .filter_map(|origin| origin.parse::().ok()) + .collect::>(), + ) + .allow_methods(vec![ + Method::GET, + Method::PUT, + Method::DELETE, + Method::OPTIONS, + ]) + .allow_headers(tower_http::cors::Any) + .expose_headers(tower_http::cors::Any); + + s3_proxy::object_data_router().layer(cors) +} + pub fn create_router() -> Router { let rate_limit_per_second: u64 = std::env::var("RATE_LIMIT_PER_SECOND") .ok() diff --git a/control-plane/api-gateway/src/routes/resource_groups.rs b/control-plane/api-gateway/src/routes/resource_groups.rs index a76f27a..e85e8db 100644 --- a/control-plane/api-gateway/src/routes/resource_groups.rs +++ b/control-plane/api-gateway/src/routes/resource_groups.rs @@ -8,7 +8,9 @@ use axum::{ use base64::{engine::general_purpose::STANDARD as B64, Engine}; use chrono::Utc; use entity::{ - entities::{agents, buckets, networks, resource_group_vpn_peers, resource_groups, volumes, workloads}, + entities::{ + agents, buckets, networks, resource_group_vpn_peers, resource_groups, volumes, workloads, + }, Agents, Buckets, Networks, ResourceGroupVpnPeers, ResourceGroups, Volumes, Workloads, }; use ring::rand::{SecureRandom, SystemRandom}; diff --git a/control-plane/api-gateway/src/routes/s3_proxy.rs b/control-plane/api-gateway/src/routes/s3_proxy.rs index a3ac29b..7da4296 100644 --- a/control-plane/api-gateway/src/routes/s3_proxy.rs +++ b/control-plane/api-gateway/src/routes/s3_proxy.rs @@ -1,6 +1,6 @@ use axum::{ body::Body, - extract::{Path, State}, + extract::{OriginalUri, Path, State}, http::{HeaderMap, Method, StatusCode}, response::{IntoResponse, Json}, routing::any, @@ -17,6 +17,7 @@ const S3_PORT: u16 = 3900; async fn resolve_bucket_target( state: &AppState, global_alias: &str, + require_external: bool, ) -> Result)> { let bucket = buckets::Entity::find() .filter(buckets::Column::GlobalAlias.eq(global_alias)) @@ -35,7 +36,7 @@ async fn resolve_bucket_target( ) })?; - if bucket.exposure != "external" { + if require_external && bucket.exposure != "external" { return Err(( StatusCode::NOT_FOUND, Json(json!({ "error": "bucket not found" })), @@ -85,28 +86,33 @@ async fn resolve_bucket_target( Ok(tunnel_ip) } -pub async fn proxy_s3_request( - State(state): State, - Path((bucket, path)): Path<(String, String)>, +async fn proxy( + state: &AppState, + bucket: &str, + path: &str, + query: Option<&str>, + require_external: bool, method: Method, headers: HeaderMap, body: Body, ) -> Result)> { - let tunnel_ip = resolve_bucket_target(&state, &bucket).await?; + let tunnel_ip = resolve_bucket_target(state, bucket, require_external).await?; - let url = format!("http://{}:{}/{}/{}", tunnel_ip, S3_PORT, bucket, path); + let mut url = format!("http://{}:{}/{}/{}", tunnel_ip, S3_PORT, bucket, path); + if let Some(query) = query { + url.push('?'); + url.push_str(query); + } - let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes()) - .unwrap_or(reqwest::Method::GET); + let reqwest_method = + reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap_or(reqwest::Method::GET); - let body_bytes = axum::body::to_bytes(body, usize::MAX) - .await - .map_err(|e| { - ( - StatusCode::BAD_REQUEST, - Json(json!({ "error": format!("failed to read request body: {}", e) })), - ) - })?; + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.map_err(|e| { + ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": format!("failed to read request body: {}", e) })), + ) + })?; let client = reqwest::Client::new(); let mut request = client.request(reqwest_method, &url).body(body_bytes); @@ -140,6 +146,52 @@ pub async fn proxy_s3_request( Ok((status, response_headers, Body::from_stream(stream))) } +pub async fn proxy_s3_request( + State(state): State, + Path((bucket, path)): Path<(String, String)>, + OriginalUri(uri): OriginalUri, + method: Method, + headers: HeaderMap, + body: Body, +) -> Result)> { + proxy( + &state, + &bucket, + &path, + uri.query(), + true, + method, + headers, + body, + ) + .await +} + +pub async fn proxy_object_data( + State(state): State, + Path((bucket, key)): Path<(String, String)>, + OriginalUri(uri): OriginalUri, + method: Method, + headers: HeaderMap, + body: Body, +) -> Result)> { + proxy( + &state, + &bucket, + &key, + uri.query(), + false, + method, + headers, + body, + ) + .await +} + pub fn s3_proxy_routes() -> Router { Router::new().route("/s3/{bucket}/{*path}", any(proxy_s3_request)) } + +pub fn object_data_router() -> Router { + Router::new().route("/{bucket}/{*key}", any(proxy_object_data)) +} diff --git a/control-plane/object-storage/src/garage/client.rs b/control-plane/object-storage/src/garage/client.rs index d57e05b..857c24b 100644 --- a/control-plane/object-storage/src/garage/client.rs +++ b/control-plane/object-storage/src/garage/client.rs @@ -137,7 +137,7 @@ impl GarageClient { pub async fn delete_bucket(&self, garage_bucket_id: &str) -> Result<()> { let response = self .http - .delete(self.url(&format!("/v2/DeleteBucket?id={}", garage_bucket_id))) + .post(self.url(&format!("/v2/DeleteBucket?id={}", garage_bucket_id))) .bearer_auth(&self.admin_token) .send() .await @@ -171,7 +171,7 @@ impl GarageClient { pub async fn delete_key(&self, garage_key_id: &str) -> Result<()> { let response = self .http - .delete(self.url(&format!("/v2/DeleteKey?id={}", garage_key_id))) + .post(self.url(&format!("/v2/DeleteKey?id={}", garage_key_id))) .bearer_auth(&self.admin_token) .send() .await diff --git a/control-plane/object-storage/src/garage/s3_client.rs b/control-plane/object-storage/src/garage/s3_client.rs index 2cc5cbf..1a44251 100644 --- a/control-plane/object-storage/src/garage/s3_client.rs +++ b/control-plane/object-storage/src/garage/s3_client.rs @@ -9,10 +9,12 @@ use std::time::{Duration, SystemTime}; const S3_REGION: &str = "csfx"; const S3_SERVICE: &str = "s3"; +const EMPTY_BODY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; #[derive(Clone)] pub struct S3Client { s3_url: String, + public_s3_url: String, http: reqwest::Client, } @@ -30,9 +32,10 @@ pub struct ListObjectsResult { } impl S3Client { - pub fn new(s3_url: String) -> Self { + pub fn new(s3_url: String, public_s3_url: String) -> Self { Self { s3_url, + public_s3_url, http: reqwest::Client::new(), } } @@ -91,9 +94,13 @@ impl S3Client { let identity = Self::identity(access_key_id, secret_access_key); let params = Self::signing_params(&identity, SigningSettings::default())?; - let signable = - SignableRequest::new("GET", &url, std::iter::empty(), SignableBody::Bytes(&[])) - .context("failed to build signable request")?; + let signable = SignableRequest::new( + "GET", + &url, + std::iter::once(("x-amz-content-sha256", EMPTY_BODY_SHA256)), + SignableBody::Bytes(&[]), + ) + .context("failed to build signable request")?; let signed_headers: Vec<(String, String)> = sign(signable, ¶ms.into()) .context("failed to sign request")? @@ -106,6 +113,7 @@ impl S3Client { let mut request = self .http .get(&url) + .header("x-amz-content-sha256", EMPTY_BODY_SHA256) .build() .context("failed to build request")?; for (name, value) in signed_headers { @@ -152,9 +160,13 @@ impl S3Client { let identity = Self::identity(access_key_id, secret_access_key); let params = Self::signing_params(&identity, SigningSettings::default())?; - let signable = - SignableRequest::new("DELETE", &url, std::iter::empty(), SignableBody::Bytes(&[])) - .context("failed to build signable request")?; + let signable = SignableRequest::new( + "DELETE", + &url, + std::iter::once(("x-amz-content-sha256", EMPTY_BODY_SHA256)), + SignableBody::Bytes(&[]), + ) + .context("failed to build signable request")?; let signed_headers: Vec<(String, String)> = sign(signable, ¶ms.into()) .context("failed to sign request")? @@ -167,6 +179,7 @@ impl S3Client { let mut request = self .http .delete(&url) + .header("x-amz-content-sha256", EMPTY_BODY_SHA256) .build() .context("failed to build request")?; for (name, value) in signed_headers { @@ -203,7 +216,7 @@ impl S3Client { ) -> Result { let url = format!( "{}/{}/{}", - self.s3_url, + self.public_s3_url, bucket, percent_encoding::utf8_percent_encode(key, percent_encoding::NON_ALPHANUMERIC) ); diff --git a/control-plane/object-storage/src/main.rs b/control-plane/object-storage/src/main.rs index bf9a2de..c255035 100644 --- a/control-plane/object-storage/src/main.rs +++ b/control-plane/object-storage/src/main.rs @@ -39,7 +39,8 @@ async fn main() -> anyhow::Result<()> { let s3_url = std::env::var("GARAGE_S3_URL").unwrap_or_else(|_| "http://127.0.0.1:3900".to_string()); - let s3_client = garage::S3Client::new(s3_url); + let public_s3_url = std::env::var("GARAGE_PUBLIC_S3_URL").unwrap_or_else(|_| s3_url.clone()); + let s3_client = garage::S3Client::new(s3_url, public_s3_url); let etcd_url = std::env::var("ETCD_URL").unwrap_or_else(|_| "http://localhost:2379".to_string()); diff --git a/docker-compose.yml b/docker-compose.yml index 2928a08..5cfe4f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,7 @@ services: TLS_ENABLED: "false" ports: - "8000:8000" + - "8007:8007" depends_on: postgres: condition: service_healthy @@ -222,6 +223,7 @@ services: GARAGE_ADMIN_URL: http://garage:3903 GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-dev-garage-admin-token} GARAGE_S3_URL: http://garage:3900 + GARAGE_PUBLIC_S3_URL: ${GARAGE_PUBLIC_S3_URL:-http://localhost:8007} OBJECT_STORAGE_ENCRYPTION_KEY: ${OBJECT_STORAGE_ENCRYPTION_KEY:-9a29b5d4c92cbfd4abe14c2a645f39eff6b57b1fa13305d3a946a2221e630c29} RUST_LOG: ${RUST_LOG:-debug} ports: From 077a2a023b794beffdb31b113e0674ff4ba415de Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 18:42:06 +0200 Subject: [PATCH 22/24] fix: register single-node garage deployments without a csfx agent --- .../api-gateway/src/routes/s3_proxy.rs | 6 +- .../object-storage/src/db/garage_nodes.rs | 65 +++++++++++++++---- .../object-storage/src/garage/layout.rs | 61 ++++++++++++++++- control-plane/object-storage/src/main.rs | 9 +++ .../entity/src/entities/garage_nodes.rs | 2 +- control-plane/shared/migration/src/lib.rs | 2 + ...6_020000_garage_nodes_agent_id_nullable.rs | 29 +++++++++ 7 files changed, 156 insertions(+), 18 deletions(-) create mode 100644 control-plane/shared/migration/src/m20260816_020000_garage_nodes_agent_id_nullable.rs diff --git a/control-plane/api-gateway/src/routes/s3_proxy.rs b/control-plane/api-gateway/src/routes/s3_proxy.rs index 7da4296..d094aba 100644 --- a/control-plane/api-gateway/src/routes/s3_proxy.rs +++ b/control-plane/api-gateway/src/routes/s3_proxy.rs @@ -60,7 +60,11 @@ async fn resolve_bucket_target( ) })?; - let agent = agents::Entity::find_by_id(node.agent_id) + let Some(agent_id) = node.agent_id else { + return Ok(std::env::var("GARAGE_INTERNAL_HOST").unwrap_or_else(|_| "garage".to_string())); + }; + + let agent = agents::Entity::find_by_id(agent_id) .one(&state.db_conn) .await .map_err(|e| { diff --git a/control-plane/object-storage/src/db/garage_nodes.rs b/control-plane/object-storage/src/db/garage_nodes.rs index 9bcced9..f534c17 100644 --- a/control-plane/object-storage/src/db/garage_nodes.rs +++ b/control-plane/object-storage/src/db/garage_nodes.rs @@ -13,19 +13,8 @@ pub async fn list(db: &DatabaseConnection) -> Result> { .context("failed to list garage nodes") } -pub async fn get_by_agent_id( - db: &DatabaseConnection, - agent_id: Uuid, -) -> Result> { - garage_nodes::Entity::find() - .filter(garage_nodes::Column::AgentId.eq(agent_id)) - .one(db) - .await - .context("failed to get garage node by agent id") -} - -pub async fn mark_down(db: &DatabaseConnection, agent_id: Uuid) -> Result<()> { - if let Some(existing) = get_by_agent_id(db, agent_id).await? { +pub async fn mark_down(db: &DatabaseConnection, id: Uuid) -> Result<()> { + if let Some(existing) = garage_nodes::Entity::find_by_id(id).one(db).await? { let mut model: garage_nodes::ActiveModel = existing.into(); model.status = Set("down".to_string()); model.updated_at = Set(Some(Utc::now().naive_utc())); @@ -36,3 +25,53 @@ pub async fn mark_down(db: &DatabaseConnection, agent_id: Uuid) -> Result<()> { } Ok(()) } + +pub async fn upsert_self( + db: &DatabaseConnection, + garage_node_id: &str, + zone: &str, + capacity_bytes: Option, +) -> Result { + let existing = garage_nodes::Entity::find() + .filter(garage_nodes::Column::GarageNodeId.eq(garage_node_id)) + .one(db) + .await + .context("failed to look up existing garage node")?; + + let now = Utc::now().naive_utc(); + + if let Some(existing) = existing { + let mut model: garage_nodes::ActiveModel = existing.into(); + model.status = Set("up".to_string()); + model.capacity_bytes = Set(capacity_bytes); + model.last_seen_at = Set(Some(now)); + model.updated_at = Set(Some(now)); + return model + .update(db) + .await + .context("failed to update self garage node"); + } + + let model = garage_nodes::ActiveModel { + id: Set(Uuid::new_v4()), + agent_id: Set(None), + garage_node_id: Set(Some(garage_node_id.to_string())), + zone: Set(zone.to_string()), + capacity_bytes: Set(capacity_bytes), + role: Set(if capacity_bytes.is_some() { + "storage".to_string() + } else { + "gateway".to_string() + }), + status: Set("up".to_string()), + layout_version: Set(None), + last_seen_at: Set(Some(now)), + created_at: Set(now), + updated_at: Set(None), + }; + + model + .insert(db) + .await + .context("failed to insert self garage node") +} diff --git a/control-plane/object-storage/src/garage/layout.rs b/control-plane/object-storage/src/garage/layout.rs index 236d979..cc256d3 100644 --- a/control-plane/object-storage/src/garage/layout.rs +++ b/control-plane/object-storage/src/garage/layout.rs @@ -30,7 +30,10 @@ async fn peer_addrs( let Some(garage_node_id) = &node.garage_node_id else { continue; }; - let Some(agent) = agents::Entity::find_by_id(node.agent_id).one(db).await? else { + let Some(agent_id) = node.agent_id else { + continue; + }; + let Some(agent) = agents::Entity::find_by_id(agent_id).one(db).await? else { continue; }; let Some(wg_ip) = agent.wg_tunnel_ip else { @@ -73,9 +76,9 @@ async fn reconcile_once(db: &DatabaseConnection, garage: &GarageClient) -> Resul if !is_up && node.status == "up" { log_warn!( "garage::layout", - &format!("garage node reported down agent_id={}", node.agent_id) + &format!("garage node reported down id={}", node.id) ); - garage_nodes_db::mark_down(db, node.agent_id).await?; + garage_nodes_db::mark_down(db, node.id).await?; } } @@ -153,3 +156,55 @@ pub async fn run_reconcile_loop( } } } + +const SELF_REGISTER_RETRY_SECONDS: u64 = 5; +const SELF_REGISTER_MAX_ATTEMPTS: u32 = 60; + +pub async fn register_self_as_node(db: &DatabaseConnection, garage: &GarageClient, zone: &str) { + for attempt in 1..=SELF_REGISTER_MAX_ATTEMPTS { + match garage.get_cluster_status().await { + Ok(status) => { + let Some(self_node) = status.nodes.first() else { + log_warn!( + "garage::layout", + "garage cluster status returned no nodes yet" + ); + sleep(Duration::from_secs(SELF_REGISTER_RETRY_SECONDS)).await; + continue; + }; + + match crate::db::garage_nodes::upsert_self(db, &self_node.id, zone, None).await { + Ok(node) => { + log_info!( + "garage::layout", + &format!("registered self as garage node id={}", node.id) + ); + return; + } + Err(e) => { + log_error!( + "garage::layout", + &format!("failed to persist self garage node registration err={}", e) + ); + return; + } + } + } + Err(e) => { + log_warn!( + "garage::layout", + &format!( + "self-registration attempt {}/{} failed err={}", + attempt, SELF_REGISTER_MAX_ATTEMPTS, e + ) + ); + sleep(Duration::from_secs(SELF_REGISTER_RETRY_SECONDS)).await; + } + } + } + + log_error!( + "garage::layout", + "giving up on self garage node registration after max attempts" + ); +} diff --git a/control-plane/object-storage/src/main.rs b/control-plane/object-storage/src/main.rs index c255035..430fc26 100644 --- a/control-plane/object-storage/src/main.rs +++ b/control-plane/object-storage/src/main.rs @@ -59,6 +59,15 @@ async fn main() -> anyhow::Result<()> { leader, )); + let self_register_zone = std::env::var("GARAGE_ZONE").unwrap_or_else(|_| "dev".to_string()); + tokio::spawn({ + let db = db.clone(); + let garage = garage.clone(); + async move { + garage::layout::register_self_as_node(&db, &garage, &self_register_zone).await; + } + }); + let secret_box = std::sync::Arc::new( crypto::SecretBox::from_env().expect("Failed to initialize encryption key"), ); diff --git a/control-plane/shared/entity/src/entities/garage_nodes.rs b/control-plane/shared/entity/src/entities/garage_nodes.rs index 06da9b7..245034c 100644 --- a/control-plane/shared/entity/src/entities/garage_nodes.rs +++ b/control-plane/shared/entity/src/entities/garage_nodes.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; pub struct Model { #[sea_orm(primary_key, auto_increment = false)] pub id: Uuid, - pub agent_id: Uuid, + pub agent_id: Option, pub garage_node_id: Option, pub zone: String, pub capacity_bytes: Option, diff --git a/control-plane/shared/migration/src/lib.rs b/control-plane/shared/migration/src/lib.rs index 9acadb3..72ffb99 100644 --- a/control-plane/shared/migration/src/lib.rs +++ b/control-plane/shared/migration/src/lib.rs @@ -34,6 +34,7 @@ mod m20260723_010000_runtime_class_default_firecracker; mod m20260809_000000_add_user_gravatar_email; mod m20260816_000000_add_object_storage; mod m20260816_010000_add_bucket_master_key; +mod m20260816_020000_garage_nodes_agent_id_nullable; pub struct Migrator; @@ -75,6 +76,7 @@ impl MigratorTrait for Migrator { Box::new(m20260809_000000_add_user_gravatar_email::Migration), Box::new(m20260816_000000_add_object_storage::Migration), Box::new(m20260816_010000_add_bucket_master_key::Migration), + Box::new(m20260816_020000_garage_nodes_agent_id_nullable::Migration), ] } } diff --git a/control-plane/shared/migration/src/m20260816_020000_garage_nodes_agent_id_nullable.rs b/control-plane/shared/migration/src/m20260816_020000_garage_nodes_agent_id_nullable.rs new file mode 100644 index 0000000..28161ca --- /dev/null +++ b/control-plane/shared/migration/src/m20260816_020000_garage_nodes_agent_id_nullable.rs @@ -0,0 +1,29 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("garage_nodes")) + .modify_column(ColumnDef::new(Alias::new("agent_id")).uuid().null()) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("garage_nodes")) + .modify_column(ColumnDef::new(Alias::new("agent_id")).uuid().not_null()) + .to_owned(), + ) + .await + } +} From d41bda182eee16e14ae8de4b554e2cad07303994 Mon Sep 17 00:00:00 2001 From: CodeMaster4711 Date: Sun, 16 Aug 2026 18:55:47 +0200 Subject: [PATCH 23/24] fix: correct sigv4 encoding and garage node status tracking --- control-plane/api-gateway/src/main.rs | 24 +------------- control-plane/api-gateway/src/routes/mod.rs | 32 +------------------ .../object-storage/src/db/garage_nodes.rs | 5 +++ .../object-storage/src/garage/client.rs | 12 ++++++- .../object-storage/src/garage/layout.rs | 17 +++++++++- .../object-storage/src/garage/s3_client.rs | 17 +++++++--- docker-compose.yml | 3 +- 7 files changed, 47 insertions(+), 63 deletions(-) diff --git a/control-plane/api-gateway/src/main.rs b/control-plane/api-gateway/src/main.rs index 9dc77b8..0e05f82 100644 --- a/control-plane/api-gateway/src/main.rs +++ b/control-plane/api-gateway/src/main.rs @@ -177,29 +177,7 @@ async fn main() { let app = routes::create_router() .merge(SwaggerUi::new("/swagger-ui").url("/api-docs/openapi.json", ApiDoc::openapi())) - .with_state(state.clone()); - - let object_data_app = routes::create_object_data_router().with_state(state); - - let object_data_port = std::env::var("OBJECT_DATA_PORT") - .ok() - .and_then(|p| p.parse::().ok()) - .unwrap_or(8007); - let object_data_listen_addr = - std::env::var("LISTEN_ADDR").unwrap_or_else(|_| "0.0.0.0".to_string()); - let object_data_addr: SocketAddr = format!("{}:{}", object_data_listen_addr, object_data_port) - .parse() - .unwrap(); - - tokio::spawn(async move { - let listener = tokio::net::TcpListener::bind(object_data_addr) - .await - .expect("failed to bind object data port"); - tracing::info!(addr = %object_data_addr, "object data proxy listening"); - axum::serve(listener, object_data_app.into_make_service()) - .await - .expect("object data proxy server failed"); - }); + .with_state(state); let port = std::env::var("GATEWAY_PORT") .ok() diff --git a/control-plane/api-gateway/src/routes/mod.rs b/control-plane/api-gateway/src/routes/mod.rs index da09aea..bd123b0 100644 --- a/control-plane/api-gateway/src/routes/mod.rs +++ b/control-plane/api-gateway/src/routes/mod.rs @@ -38,37 +38,6 @@ pub mod volumes; pub mod workloads; /// Creates the main application router and logs all registered routes. -pub fn create_object_data_router() -> Router { - let frontend_url = - std::env::var("FRONTEND_URL").unwrap_or_else(|_| "http://localhost:3000".to_string()); - - let allowed_origins = vec![ - "http://localhost:3000", - "http://localhost:5173", - "http://127.0.0.1:3000", - "http://127.0.0.1:5173", - &frontend_url, - ]; - - let cors = CorsLayer::new() - .allow_origin( - allowed_origins - .into_iter() - .filter_map(|origin| origin.parse::().ok()) - .collect::>(), - ) - .allow_methods(vec![ - Method::GET, - Method::PUT, - Method::DELETE, - Method::OPTIONS, - ]) - .allow_headers(tower_http::cors::Any) - .expose_headers(tower_http::cors::Any); - - s3_proxy::object_data_router().layer(cors) -} - pub fn create_router() -> Router { let rate_limit_per_second: u64 = std::env::var("RATE_LIMIT_PER_SECOND") .ok() @@ -186,6 +155,7 @@ pub fn create_router() -> Router { Router::new() .route("/metrics", get(metrics::metrics_handler)) + .merge(s3_proxy::object_data_router()) .logged_nest("/api", api_router) .logged_nest("/api", internal_api_router) .fallback_service(serve_dir) diff --git a/control-plane/object-storage/src/db/garage_nodes.rs b/control-plane/object-storage/src/db/garage_nodes.rs index f534c17..ab4f45e 100644 --- a/control-plane/object-storage/src/db/garage_nodes.rs +++ b/control-plane/object-storage/src/db/garage_nodes.rs @@ -44,6 +44,11 @@ pub async fn upsert_self( let mut model: garage_nodes::ActiveModel = existing.into(); model.status = Set("up".to_string()); model.capacity_bytes = Set(capacity_bytes); + model.role = Set(if capacity_bytes.is_some() { + "storage".to_string() + } else { + "gateway".to_string() + }); model.last_seen_at = Set(Some(now)); model.updated_at = Set(Some(now)); return model diff --git a/control-plane/object-storage/src/garage/client.rs b/control-plane/object-storage/src/garage/client.rs index 857c24b..0331492 100644 --- a/control-plane/object-storage/src/garage/client.rs +++ b/control-plane/object-storage/src/garage/client.rs @@ -44,8 +44,18 @@ pub struct GarageKey { #[derive(Debug, Deserialize)] pub struct ClusterStatusNode { pub id: String, - #[serde(default)] + #[serde(rename = "isUp", default)] pub is_up: bool, + #[serde(default)] + pub role: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ClusterStatusNodeRole { + #[serde(default)] + pub zone: String, + #[serde(default)] + pub capacity: Option, } #[derive(Debug, Deserialize)] diff --git a/control-plane/object-storage/src/garage/layout.rs b/control-plane/object-storage/src/garage/layout.rs index cc256d3..0014853 100644 --- a/control-plane/object-storage/src/garage/layout.rs +++ b/control-plane/object-storage/src/garage/layout.rs @@ -173,7 +173,22 @@ pub async fn register_self_as_node(db: &DatabaseConnection, garage: &GarageClien continue; }; - match crate::db::garage_nodes::upsert_self(db, &self_node.id, zone, None).await { + let capacity_bytes = self_node.role.as_ref().and_then(|r| r.capacity); + let node_zone = self_node + .role + .as_ref() + .map(|r| r.zone.as_str()) + .filter(|z| !z.is_empty()) + .unwrap_or(zone); + + match crate::db::garage_nodes::upsert_self( + db, + &self_node.id, + node_zone, + capacity_bytes, + ) + .await + { Ok(node) => { log_info!( "garage::layout", diff --git a/control-plane/object-storage/src/garage/s3_client.rs b/control-plane/object-storage/src/garage/s3_client.rs index 1a44251..119bbe1 100644 --- a/control-plane/object-storage/src/garage/s3_client.rs +++ b/control-plane/object-storage/src/garage/s3_client.rs @@ -4,6 +4,7 @@ use aws_sigv4::http_request::{ sign, PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, SigningSettings, }; use aws_sigv4::sign::v4; +use percent_encoding::AsciiSet; use serde::Deserialize; use std::time::{Duration, SystemTime}; @@ -11,6 +12,12 @@ const S3_REGION: &str = "csfx"; const S3_SERVICE: &str = "s3"; const EMPTY_BODY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +const SIGV4_UNRESERVED: &AsciiSet = &percent_encoding::NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + #[derive(Clone)] pub struct S3Client { s3_url: String, @@ -81,13 +88,13 @@ impl S3Client { "{}/{}?list-type=2&prefix={}&delimiter={}", self.s3_url, bucket, - percent_encoding::utf8_percent_encode(prefix, percent_encoding::NON_ALPHANUMERIC), - percent_encoding::utf8_percent_encode(delimiter, percent_encoding::NON_ALPHANUMERIC), + percent_encoding::utf8_percent_encode(prefix, SIGV4_UNRESERVED), + percent_encoding::utf8_percent_encode(delimiter, SIGV4_UNRESERVED), ); if let Some(token) = continuation_token { url.push_str(&format!( "&continuation-token={}", - percent_encoding::utf8_percent_encode(token, percent_encoding::NON_ALPHANUMERIC) + percent_encoding::utf8_percent_encode(token, SIGV4_UNRESERVED) )); } @@ -154,7 +161,7 @@ impl S3Client { "{}/{}/{}", self.s3_url, bucket, - percent_encoding::utf8_percent_encode(key, percent_encoding::NON_ALPHANUMERIC) + percent_encoding::utf8_percent_encode(key, SIGV4_UNRESERVED) ); let identity = Self::identity(access_key_id, secret_access_key); @@ -218,7 +225,7 @@ impl S3Client { "{}/{}/{}", self.public_s3_url, bucket, - percent_encoding::utf8_percent_encode(key, percent_encoding::NON_ALPHANUMERIC) + percent_encoding::utf8_percent_encode(key, SIGV4_UNRESERVED) ); let identity = Self::identity(access_key_id, secret_access_key); diff --git a/docker-compose.yml b/docker-compose.yml index 5cfe4f9..370baa5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,6 @@ services: TLS_ENABLED: "false" ports: - "8000:8000" - - "8007:8007" depends_on: postgres: condition: service_healthy @@ -223,7 +222,7 @@ services: GARAGE_ADMIN_URL: http://garage:3903 GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-dev-garage-admin-token} GARAGE_S3_URL: http://garage:3900 - GARAGE_PUBLIC_S3_URL: ${GARAGE_PUBLIC_S3_URL:-http://localhost:8007} + GARAGE_PUBLIC_S3_URL: ${GARAGE_PUBLIC_S3_URL:-http://localhost:8000} OBJECT_STORAGE_ENCRYPTION_KEY: ${OBJECT_STORAGE_ENCRYPTION_KEY:-9a29b5d4c92cbfd4abe14c2a645f39eff6b57b1fa13305d3a946a2221e630c29} RUST_LOG: ${RUST_LOG:-debug} ports: From 62000bd91c6577bdae3572bc941440b4273199ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 16 Aug 2026 16:58:58 +0000 Subject: [PATCH 24/24] style: apply cargo fmt and clippy fixes --- control-plane/object-storage/src/crypto.rs | 2 +- .../m20260816_000000_add_object_storage.rs | 36 +++++++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/control-plane/object-storage/src/crypto.rs b/control-plane/object-storage/src/crypto.rs index 24e6ccf..3f4aed5 100644 --- a/control-plane/object-storage/src/crypto.rs +++ b/control-plane/object-storage/src/crypto.rs @@ -58,7 +58,7 @@ impl SecretBox { } fn hex_decode(input: &str) -> Result> { - if input.len() % 2 != 0 { + if !input.len().is_multiple_of(2) { bail!("hex string must have even length"); } (0..input.len()) diff --git a/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs b/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs index 505227c..e5fb693 100644 --- a/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs +++ b/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs @@ -22,7 +22,11 @@ impl MigrationTrait for Migration { .default("internal"), ) .col(ColumnDef::new(Buckets::QuotaMaxSize).big_integer().null()) - .col(ColumnDef::new(Buckets::QuotaMaxObjects).big_integer().null()) + .col( + ColumnDef::new(Buckets::QuotaMaxObjects) + .big_integer() + .null(), + ) .col( ColumnDef::new(Buckets::Status) .string() @@ -55,11 +59,7 @@ impl MigrationTrait for Migration { .not_null() .primary_key(), ) - .col( - ColumnDef::new(BucketAccessKeys::BucketId) - .uuid() - .not_null(), - ) + .col(ColumnDef::new(BucketAccessKeys::BucketId).uuid().not_null()) .col(ColumnDef::new(BucketAccessKeys::Name).string().not_null()) .col( ColumnDef::new(BucketAccessKeys::GarageKeyId) @@ -71,7 +71,11 @@ impl MigrationTrait for Migration { .string() .not_null(), ) - .col(ColumnDef::new(BucketAccessKeys::ExpiresAt).date_time().null()) + .col( + ColumnDef::new(BucketAccessKeys::ExpiresAt) + .date_time() + .null(), + ) .col( ColumnDef::new(BucketAccessKeys::LastRotatedAt) .date_time() @@ -106,7 +110,11 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(GarageNodes::AgentId).uuid().not_null()) .col(ColumnDef::new(GarageNodes::GarageNodeId).string().null()) .col(ColumnDef::new(GarageNodes::Zone).string().not_null()) - .col(ColumnDef::new(GarageNodes::CapacityBytes).big_integer().null()) + .col( + ColumnDef::new(GarageNodes::CapacityBytes) + .big_integer() + .null(), + ) .col( ColumnDef::new(GarageNodes::Role) .string() @@ -121,7 +129,11 @@ impl MigrationTrait for Migration { ) .col(ColumnDef::new(GarageNodes::LayoutVersion).integer().null()) .col(ColumnDef::new(GarageNodes::LastSeenAt).date_time().null()) - .col(ColumnDef::new(GarageNodes::CreatedAt).date_time().not_null()) + .col( + ColumnDef::new(GarageNodes::CreatedAt) + .date_time() + .not_null(), + ) .col(ColumnDef::new(GarageNodes::UpdatedAt).date_time().null()) .foreign_key( ForeignKey::create() @@ -203,11 +215,7 @@ impl MigrationTrait for Migration { .drop_index(Index::drop().name("idx_buckets_global_alias").to_owned()) .await?; manager - .drop_index( - Index::drop() - .name("idx_buckets_organization_id") - .to_owned(), - ) + .drop_index(Index::drop().name("idx_buckets_organization_id").to_owned()) .await?; manager .drop_index(