diff --git a/Cargo.lock b/Cargo.lock index 7d1a48c6..cfab5534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,7 +175,7 @@ dependencies = [ "entity", "etcd-client", "futures-util", - "http", + "http 1.4.2", "hyper", "hyper-util", "image", @@ -185,6 +185,7 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", + "percent-encoding", "prometheus", "qrcode", "rand 0.10.2", @@ -520,6 +521,124 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "aws-credential-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-sigv4" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac 0.13.0", + "http 0.2.12", + "http 1.4.2", + "percent-encoding", + "sha2 0.11.0", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.2", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "aws-smithy-types" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", +] + [[package]] name = "axum" version = "0.8.9" @@ -532,8 +651,8 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-util", @@ -565,8 +684,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -596,8 +715,8 @@ dependencies = [ "arc-swap", "bytes", "fs-err", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "hyper", "hyper-util", "pin-project-lite", @@ -627,6 +746,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -828,6 +957,16 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "caps" version = "0.5.6" @@ -952,6 +1091,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "color_quant" version = "1.1.0" @@ -1252,6 +1397,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -1453,6 +1607,7 @@ dependencies = [ "block-buffer 0.12.1", "const-oid 0.10.2", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1594,7 +1749,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef5da6e9a6ae89f4a91f80ba1caae45a5a924397a19947e18f5121a43285e9bc" dependencies = [ - "http", + "http 1.4.2", "prost", "tokio", "tokio-stream", @@ -2082,7 +2237,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.2", "indexmap", "slab", "tokio", @@ -2183,7 +2338,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -2195,6 +2350,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "home" version = "0.5.12" @@ -2204,6 +2368,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.2" @@ -2223,6 +2398,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -2230,7 +2416,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.2", ] [[package]] @@ -2241,8 +2427,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", ] @@ -2284,8 +2470,8 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2301,7 +2487,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http", + "http 1.4.2", "hyper", "hyper-util", "rustls", @@ -2333,8 +2519,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "hyper", "ipnet", "libc", @@ -2681,7 +2867,7 @@ dependencies = [ "base64", "ed25519-dalek", "getrandom 0.2.17", - "hmac", + "hmac 0.12.1", "js-sys", "p256", "p384", @@ -3218,6 +3404,40 @@ dependencies = [ "libm", ] +[[package]] +name = "object-storage" +version = "0.2.2" +dependencies = [ + "anyhow", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-runtime-api", + "axum", + "chrono", + "dotenvy", + "entity", + "etcd-client", + "http 1.4.2", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "percent-encoding", + "prometheus", + "reqwest", + "ring", + "rustls", + "sea-orm", + "serde", + "serde_json", + "shared", + "tokio", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", + "uuid", +] + [[package]] name = "oci-client" version = "0.17.0" @@ -3228,7 +3448,7 @@ dependencies = [ "chrono", "futures-util", "hex", - "http", + "http 1.4.2", "http-auth", "jsonwebtoken", "lazy_static", @@ -3343,7 +3563,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ - "http", + "http 1.4.2", "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", @@ -3424,6 +3644,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "p256" version = "0.13.2" @@ -4363,8 +4589,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -4400,7 +4626,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -5359,7 +5585,7 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", "md-5", @@ -5402,7 +5628,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -5847,8 +6073,8 @@ dependencies = [ "base64", "bytes", "h2", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-timeout", @@ -5925,7 +6151,7 @@ checksum = "50e69a15e21b2ff22c415446983978bded3244195f17d59cb113551c1e806f91" dependencies = [ "base32", "constant_time_eq", - "hmac", + "hmac 0.12.1", "qrcodegen-image", "sha1 0.10.6", "sha2 0.10.9", @@ -5961,8 +6187,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", "tower", "tower-layer", @@ -5980,8 +6206,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "http-range-header", "httpdate", @@ -6017,7 +6243,7 @@ dependencies = [ "axum", "forwarded-header-value", "governor", - "http", + "http 1.4.2", "pin-project", "thiserror 2.0.18", "tonic", @@ -6117,7 +6343,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.2", "httparse", "log", "rand 0.9.4", @@ -6345,6 +6571,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vsock" version = "0.5.4" diff --git a/Cargo.toml b/Cargo.toml index c9f8531a..33c29d49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "control-plane/failover-controller", "control-plane/sdn-controller", "control-plane/volume-manager", + "control-plane/object-storage", "control-plane/registry", "control-plane/shared/entity", "control-plane/shared/migration", diff --git a/Dockerfile b/Dockerfile index 246cfc1f..25f47108 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.88-slim-bookworm AS base +FROM rust:1.97-slim-bookworm AS base WORKDIR /app @@ -26,6 +26,7 @@ COPY control-plane/scheduler/Cargo.toml ./control-plane/scheduler/ COPY control-plane/failover-controller/Cargo.toml ./control-plane/failover-controller/ COPY control-plane/sdn-controller/Cargo.toml ./control-plane/sdn-controller/ COPY control-plane/volume-manager/Cargo.toml ./control-plane/volume-manager/ +COPY control-plane/object-storage/Cargo.toml ./control-plane/object-storage/ COPY control-plane/registry/Cargo.toml ./control-plane/registry/ COPY control-plane/shared/entity/Cargo.toml ./control-plane/shared/entity/ COPY control-plane/shared/migration/Cargo.toml ./control-plane/shared/migration/ @@ -39,6 +40,7 @@ RUN mkdir -p agent/src \ control-plane/failover-controller/src \ control-plane/sdn-controller/src \ control-plane/volume-manager/src \ + control-plane/object-storage/src \ control-plane/registry/src \ control-plane/shared/entity/src \ control-plane/shared/migration/src \ @@ -51,6 +53,7 @@ RUN mkdir -p agent/src \ && echo "fn main() {}" > control-plane/failover-controller/src/main.rs \ && echo "fn main() {}" > control-plane/sdn-controller/src/main.rs \ && echo "fn main() {}" > control-plane/volume-manager/src/main.rs \ + && echo "fn main() {}" > control-plane/object-storage/src/main.rs \ && echo "fn main() {}" > control-plane/registry/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-migrate/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-updater/src/main.rs \ diff --git a/agent/src/firecracker/rootfs.rs b/agent/src/firecracker/rootfs.rs index 462c4edb..adbc22a8 100644 --- a/agent/src/firecracker/rootfs.rs +++ b/agent/src/firecracker/rootfs.rs @@ -68,10 +68,7 @@ impl RootfsBuilder { (Some(tag), None) => format!(":{tag}"), (None, None) => String::new(), }; - let mirrored = format!( - "{mirror}/{}{tag_or_digest}", - reference.repository() - ); + let mirrored = format!("{mirror}/{}{tag_or_digest}", reference.repository()); mirrored.parse().context("Invalid mirrored image reference") } diff --git a/agent/src/firecracker/runtime.rs b/agent/src/firecracker/runtime.rs index 6e81fe0d..ff9154c1 100644 --- a/agent/src/firecracker/runtime.rs +++ b/agent/src/firecracker/runtime.rs @@ -3,6 +3,7 @@ use serde_json::json; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::UnixStream; use tokio::process::Command; @@ -153,16 +154,21 @@ struct GuestNetwork { pub struct FirecrackerRuntime { wg_private_key_b64: String, dns_supervisor: RgDnsProcessSupervisor, + rg_dns_registry: Arc, handles: Mutex>, next_cid: Mutex, reconciled: AtomicBool, } impl FirecrackerRuntime { - pub fn new(wg_private_key_b64: String) -> Self { + pub fn new( + wg_private_key_b64: String, + rg_dns_registry: Arc, + ) -> Self { Self { wg_private_key_b64, dns_supervisor: RgDnsProcessSupervisor::new(), + rg_dns_registry, handles: Mutex::new(HashMap::new()), next_cid: Mutex::new(next_free_cid_on_host()), reconciled: AtomicBool::new(false), @@ -194,9 +200,13 @@ impl FirecrackerRuntime { resource_group_id: &str, resource_group_cidr: Option<&str>, ) -> Result { - let iface = crate::rg_network::ensure_bridge(resource_group_id, resource_group_cidr) - .await - .context("Failed to ensure resource group bridge")?; + let iface = crate::rg_network::ensure_bridge( + resource_group_id, + resource_group_cidr, + &self.rg_dns_registry, + ) + .await + .context("Failed to ensure resource group bridge")?; if let Some(cidr) = resource_group_cidr { self.dns_supervisor diff --git a/agent/src/main.rs b/agent/src/main.rs index dc701e79..25fc0f90 100644 --- a/agent/src/main.rs +++ b/agent/src/main.rs @@ -137,8 +137,11 @@ async fn main() -> Result<()> { warn!(error = %e, "Failed to initialize nftables resource group isolation"); } + let rg_dns_registry = Arc::new(rg_dns::RgDnsRegistry::new()); + let firecracker_runtime = Arc::new(firecracker::runtime::FirecrackerRuntime::new( wg_identity.private_key_b64.clone(), + Arc::clone(&rg_dns_registry), )); let running_containers: Arc>> = @@ -153,8 +156,6 @@ async fn main() -> Result<()> { let service_dns_registry: Arc>> = Arc::new(Mutex::new(HashMap::new())); - let rg_dns_registry = rg_dns::RgDnsRegistry::new(); - if let Some(port) = std::env::var("CSFX_AGENT_PORT") .ok() .and_then(|v| v.parse::().ok()) @@ -363,7 +364,7 @@ async fn run_heartbeat_loop( mounted_volumes: Arc>>, restart_counts: Arc>>, service_dns_registry: Arc>>, - rg_dns_registry: rg_dns::RgDnsRegistry, + rg_dns_registry: Arc, mut assignment_signal: tokio::sync::mpsc::UnboundedReceiver<()>, ) { let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); diff --git a/agent/src/nftables.rs b/agent/src/nftables.rs index 097dedad..bd57cdaf 100644 --- a/agent/src/nftables.rs +++ b/agent/src/nftables.rs @@ -130,6 +130,30 @@ pub async fn add_rg_port_dnat( .await } +pub async fn dnat_bridge_port(bridge_name: &str, rg_gateway_ip: &str, port: u16) -> Result<()> { + run_nft(&[ + "add", + "rule", + "ip", + NAT_TABLE_NAME, + NAT_CHAIN_NAME, + "iifname", + bridge_name, + "ip", + "daddr", + rg_gateway_ip, + "tcp", + "dport", + &port.to_string(), + "dnat", + "to", + &format!("127.0.0.1:{}", port), + "comment", + &format!("\"{}-s3\"", bridge_name), + ]) + .await +} + pub async fn remove_node_port_rules(workload_id: &str) -> Result<()> { let output = Command::new("nft") .args(["-a", "list", "chain", "ip", NAT_TABLE_NAME, NAT_CHAIN_NAME]) diff --git a/agent/src/rg_network.rs b/agent/src/rg_network.rs index bc438aa7..69cc964a 100644 --- a/agent/src/rg_network.rs +++ b/agent/src/rg_network.rs @@ -3,11 +3,18 @@ use std::path::{Path, PathBuf}; use tokio::process::Command; use tracing::info; +use crate::rg_dns::RgDnsRegistry; use crate::spec::{rg_bridge_iface_name, second_host_ip}; const RG_REGISTRY_DIR: &str = "/var/lib/csfx-agent/rg-networks"; - -pub async fn ensure_bridge(resource_group_id: &str, cidr: Option<&str>) -> Result { +const S3_SERVICE_NAME: &str = "s3"; +const S3_DNAT_PORT: u16 = 3900; + +pub async fn ensure_bridge( + resource_group_id: &str, + cidr: Option<&str>, + rg_dns_registry: &RgDnsRegistry, +) -> Result { let iface = rg_bridge_iface_name(resource_group_id); write_registry_entry(resource_group_id).await?; @@ -29,6 +36,18 @@ pub async fn ensure_bridge(resource_group_id: &str, cidr: Option<&str>) -> Resul &iface, ]) .await?; + + if let Err(e) = rg_dns_registry + .upsert(resource_group_id, S3_SERVICE_NAME, &gateway) + .await + { + info!(resource_group_id = %resource_group_id, error = %e, "Failed to register s3 dns record"); + } + + if let Err(e) = crate::nftables::dnat_bridge_port(&iface, &gateway, S3_DNAT_PORT).await + { + info!(resource_group_id = %resource_group_id, error = %e, "Failed to set up s3 dnat rule"); + } } } diff --git a/agent/src/server.rs b/agent/src/server.rs index 1c962e8d..72df7f22 100644 --- a/agent/src/server.rs +++ b/agent/src/server.rs @@ -187,13 +187,15 @@ async fn logs_handler( })?; info!(workload_id = %workload_id, container_id = %container_id, "opening log stream to guest"); - let stream = futures_util::stream::once(async { - Ok::<_, std::io::Error>(axum::body::Bytes::new()) - }) - .chain(state.firecracker.logs(&container_id)); + let stream = + futures_util::stream::once(async { Ok::<_, std::io::Error>(axum::body::Bytes::new()) }) + .chain(state.firecracker.logs(&container_id)); Ok(( - [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")], + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], axum::body::Body::from_stream(stream), )) } diff --git a/app/src/lib/api/resource-groups.ts b/app/src/lib/api/resource-groups.ts index fca551ff..ccd6d426 100644 --- a/app/src/lib/api/resource-groups.ts +++ b/app/src/lib/api/resource-groups.ts @@ -55,6 +55,47 @@ export interface Volume { created_at: string; } +export interface Bucket { + id: string; + name: string; + global_alias: string; + exposure: 'internal' | 'external' | 'node_port'; + quota_max_size: number | null; + quota_max_objects: number | null; + status: string; + resource_group_id: string | null; + created_at: string; + updated_at: string | null; +} + +export interface BucketAccessKey { + id: string; + bucket_id: string; + name: string; + garage_key_id: string; + permissions: string; + expires_at: string | null; + last_rotated_at: string | null; + created_at: string; +} + +export interface BucketAccessKeyCreated extends BucketAccessKey { + secret_access_key: string; +} + +export interface CreateBucketRequest { + name: string; + resource_group_id?: string; + exposure?: 'internal' | 'external'; + quota_max_size?: number; + quota_max_objects?: number; +} + +export interface CreateBucketAccessKeyRequest { + name: string; + permissions?: string; +} + export interface Workload { id: string; name: string; @@ -408,3 +449,175 @@ export async function deleteVolume(token: string, id: string): Promise { }); if (!res.ok) throw new Error(`Failed to delete volume: ${res.status}`); } + +export async function listBuckets(token: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to list buckets: ${res.status}`); + return res.json(); +} + +export async function getBucket(token: string, id: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${id}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to get bucket: ${res.status}`); + } + return res.json(); +} + +export async function listResourceGroupBuckets(token: string, rgId: string): Promise { + const res = await authedFetch(`${API_BASE}/resource-groups/${rgId}/buckets`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to list buckets: ${res.status}`); + return res.json(); +} + +export async function createBucket(token: string, req: CreateBucketRequest): Promise { + const res = await authedFetch(`${API_BASE}/buckets`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(req), + }); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to create bucket: ${res.status}`); + } + return res.json(); +} + +export async function deleteBucket(token: string, id: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${id}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to delete bucket: ${res.status}`); +} + +export async function listBucketKeys(token: string, bucketId: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/keys`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to list access keys: ${res.status}`); + return res.json(); +} + +export async function createBucketKey( + token: string, + bucketId: string, + req: CreateBucketAccessKeyRequest +): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/keys`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(req), + }); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to create access key: ${res.status}`); + } + return res.json(); +} + +export async function deleteBucketKey(token: string, bucketId: string, keyId: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/keys/${keyId}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to delete access key: ${res.status}`); +} + +export interface ObjectEntry { + key: string; + size: number; + last_modified: string; +} + +export interface ListObjectsResult { + objects: ObjectEntry[]; + folders: string[]; + next_continuation_token: string | null; +} + +export interface PresignResult { + url: string; + expires_in_seconds: number; +} + +export async function listBucketObjects( + token: string, + bucketId: string, + prefix: string +): Promise { + const res = await authedFetch( + `${API_BASE}/buckets/${bucketId}/objects?prefix=${encodeURIComponent(prefix)}`, + { headers: { Authorization: `Bearer ${token}` } } + ); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to list objects: ${res.status}`); + } + return res.json(); +} + +export async function deleteBucketObject(token: string, bucketId: string, key: string): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/objects/${key}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` }, + }); + if (!res.ok) throw new Error(`Failed to delete object: ${res.status}`); +} + +export async function presignObjectUpload( + token: string, + bucketId: string, + key: string +): Promise { + const res = await authedFetch(`${API_BASE}/buckets/${bucketId}/objects/presign-upload`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ key }), + }); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to presign upload: ${res.status}`); + } + return res.json(); +} + +export async function presignObjectDownload( + token: string, + bucketId: string, + key: string +): Promise { + const res = await authedFetch( + `${API_BASE}/buckets/${bucketId}/objects/presign-download/${key}`, + { headers: { Authorization: `Bearer ${token}` } } + ); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.status })); + throw new Error(err.error ?? `Failed to presign download: ${res.status}`); + } + return res.json(); +} + +export async function uploadObjectToPresignedUrl(url: string, file: File): Promise { + const res = await fetch(url, { + method: 'PUT', + body: file, + }); + if (!res.ok) throw new Error(`Upload failed: ${res.status}`); +} diff --git a/app/src/lib/components/sidebar/app-sidebar.svelte b/app/src/lib/components/sidebar/app-sidebar.svelte index 2959d605..32fcea29 100644 --- a/app/src/lib/components/sidebar/app-sidebar.svelte +++ b/app/src/lib/components/sidebar/app-sidebar.svelte @@ -57,7 +57,7 @@ projects: [ { name: "S3 Buckets", - url: "#", + url: "/buckets", icon: IconBucket, }, { @@ -80,7 +80,6 @@ diff --git a/app/src/lib/components/sidebar/icon-bucket.svelte b/app/src/lib/components/sidebar/icon-bucket.svelte index 7fc159be..d020daac 100644 --- a/app/src/lib/components/sidebar/icon-bucket.svelte +++ b/app/src/lib/components/sidebar/icon-bucket.svelte @@ -5,4 +5,4 @@ let { class: className, ...restProps }: HTMLAttributes = $props(); - + diff --git a/app/src/routes/(app)/buckets/+page.svelte b/app/src/routes/(app)/buckets/+page.svelte new file mode 100644 index 00000000..5bc8caea --- /dev/null +++ b/app/src/routes/(app)/buckets/+page.svelte @@ -0,0 +1,157 @@ + + +
+ + / + S3 Buckets +
+ +
+
+
+

S3 Buckets

+

Object storage buckets across all resource groups

+
+ +
+ +
+ + + + + + + + + + + + + {#if loading} + + + + {:else if error} + + + + {:else if buckets.length === 0} + + + + {:else} + {#each buckets as bucket (bucket.id)} + goto(`/buckets/${bucket.id}`)} + > + + + + + + + + {/each} + {/if} + +
NameGlobal AliasExposureQuotaStatusResource Group
Loading...
{error}
No buckets found
+
+ + {bucket.name} +
+ 		{bucket.global_alias}{bucket.exposure} + {bucket.quota_max_size ? fmtBytes(bucket.quota_max_size) : "unlimited"} + + + + {#if bucket.resource_group_id} + e.stopPropagation()} + > + {bucket.resource_group_id.slice(0, 8)} + + {:else} + - + {/if} +
+
+
diff --git a/app/src/routes/(app)/buckets/[id]/+page.svelte b/app/src/routes/(app)/buckets/[id]/+page.svelte new file mode 100644 index 00000000..8e2f2186 --- /dev/null +++ b/app/src/routes/(app)/buckets/[id]/+page.svelte @@ -0,0 +1,261 @@ + + +
+ + / + S3 Buckets + / + {bucket?.name ?? bucketId.slice(0, 8)} +
+ +
+ {#if loading} +

Loading...

+ {:else if error && !bucket} +

{error}

+ {:else if bucket} +
+
+

{bucket.name}

+

{bucket.global_alias}

+
+
+ + +
+
+ +
+ + {#each breadcrumbParts() as part} + / + + {/each} +
+ + {#if error} +

{error}

+ {/if} + +
+ + + + + + + + + + + {#if folders.length === 0 && objects.length === 0} + + + + {:else} + {#each folders as folder (folder)} + openFolder(folder)} + > + + + + + + {/each} + {#each objects as object (object.key)} + + + + + + + {/each} + {/if} + +
NameSizeLast ModifiedActions
+ This folder is empty +
+
+ + {folderName(folder)} +
+ 		--
+
+ + {objectName(object.key)} +
+ 		{fmtBytes(object.size)}{object.last_modified} +
+ + +
+
+
+ {/if} +
diff --git a/app/src/routes/(app)/resource-groups/[id]/+page.svelte b/app/src/routes/(app)/resource-groups/[id]/+page.svelte index ab863fec..c07ce8a7 100644 --- a/app/src/routes/(app)/resource-groups/[id]/+page.svelte +++ b/app/src/routes/(app)/resource-groups/[id]/+page.svelte @@ -22,11 +22,18 @@ updateWorkload, createVolume, deleteVolume, + listResourceGroupBuckets, + createBucket, + deleteBucket, + listBucketKeys, + createBucketKey, streamWorkloadLogs, openWorkloadExecSocket, type ResourceGroup, type Workload, type Volume, + type Bucket, + type BucketAccessKey, type PortMapping, type VolumeMount, } from "$lib/api/resource-groups"; @@ -49,6 +56,7 @@ let group = $state(null); let workloads = $state([]); let volumes = $state([]); + let buckets = $state([]); let loading = $state(true); let error = $state(null); @@ -64,7 +72,7 @@ let savingRgSettings = $state(false); let rgSettingsError = $state(null); - let activeTab = $state<"all" | "container" | "volume">("all"); + let activeTab = $state<"all" | "container" | "volume" | "bucket">("all"); let filterText = $state(""); let nodeIpCache = $state>({}); @@ -98,13 +106,17 @@ let deployDialog = $state(null); let volumeDialog = $state(null); + let bucketDialog = $state(null); + let bucketDetailDialog = $state(null); let resourcePickerDialog = $state(null); let composeDialog = $state(null); let deploying = $state(false); let creatingVolume = $state(false); + let creatingBucket = $state(false); let deployingStack = $state(false); let deployError = $state(null); let volumeError = $state(null); + let bucketError = $state(null); let composeError = $state(null); let downloadingVpn = $state(false); let expandedStacks = $state>(new Set()); @@ -112,6 +124,8 @@ const RESOURCE_TYPES = [ { key: "docker-container", label: "Docker Container", description: "Deploy a single container", icon: "logos:docker-icon" }, { key: "docker-compose", label: "Docker Compose", description: "Deploy multiple related containers as one stack", icon: "logos:docker-icon" }, + { key: "volume", label: "Volume", description: "Add a block storage volume", icon: "mdi:database-outline" }, + { key: "bucket", label: "S3 Bucket", description: "Add an S3-compatible object storage bucket", icon: "fluent-emoji-high-contrast:bucket" }, ] as const; let formImage = $state(""); @@ -147,6 +161,16 @@ let volFormName = $state(""); let volFormSize = $state("10"); + let bucketFormName = $state(""); + let bucketFormExposure = $state<"internal" | "external">("internal"); + + let activeBucket = $state(null); + let bucketKeys = $state([]); + let bucketKeysError = $state(null); + let newKeyName = $state(""); + let creatingKey = $state(false); + let createdKeySecret = $state(null); + let containerDialog = $state(null); let containerDialogTab = $state<"logs" | "shell" | "insights" | "network" | "settings">("logs"); let activeContainer = $state(null); @@ -173,10 +197,11 @@ async function load() { if (!auth.token) return; try { - [group, workloads, volumes] = await Promise.all([ + [group, workloads, volumes, buckets] = await Promise.all([ getResourceGroup(auth.token, rgId), listResourceGroupWorkloads(auth.token, rgId), listResourceGroupVolumes(auth.token, rgId), + listResourceGroupBuckets(auth.token, rgId), ]); } catch (e) { error = e instanceof Error ? e.message : "Failed to load"; @@ -353,8 +378,12 @@ resourcePickerDialog?.close(); if (key === "docker-container") { deployDialog?.showModal(); - } else { + } else if (key === "docker-compose") { composeDialog?.showModal(); + } else if (key === "volume") { + volumeDialog?.showModal(); + } else { + bucketDialog?.showModal(); } } @@ -389,6 +418,74 @@ } } + async function handleCreateBucket() { + if (!auth.token || !bucketFormName) return; + creatingBucket = true; + bucketError = null; + try { + await createBucket(auth.token, { + name: bucketFormName, + resource_group_id: rgId, + exposure: bucketFormExposure, + }); + bucketDialog?.close(); + bucketFormName = ""; + bucketFormExposure = "internal"; + buckets = await listResourceGroupBuckets(auth.token, rgId); + } catch (e) { + bucketError = e instanceof Error ? e.message : "Failed to create bucket"; + } finally { + creatingBucket = false; + } + } + + async function handleDeleteBucket(id: string) { + if (!auth.token) return; + try { + await deleteBucket(auth.token, id); + buckets = buckets.filter((b) => b.id !== id); + } catch (e) { + error = e instanceof Error ? e.message : "Failed to delete bucket"; + } + } + + async function openBucketDetail(bucket: Bucket) { + if (!auth.token) return; + activeBucket = bucket; + bucketKeysError = null; + createdKeySecret = null; + newKeyName = ""; + bucketDetailDialog?.showModal(); + try { + bucketKeys = await listBucketKeys(auth.token, bucket.id); + } catch (e) { + bucketKeysError = e instanceof Error ? e.message : "Failed to load access keys"; + } + } + + async function handleCreateBucketKey() { + if (!auth.token || !activeBucket || !newKeyName) return; + creatingKey = true; + bucketKeysError = null; + try { + const created = await createBucketKey(auth.token, activeBucket.id, { name: newKeyName }); + createdKeySecret = created.secret_access_key; + newKeyName = ""; + bucketKeys = await listBucketKeys(auth.token, activeBucket.id); + } catch (e) { + bucketKeysError = e instanceof Error ? e.message : "Failed to create access key"; + } finally { + creatingKey = false; + } + } + + function bucketEndpointUrl(bucket: Bucket): string { + if (bucket.exposure === "external") { + return `${window.location.origin}/api/s3/${bucket.global_alias}`; + } + return `http://s3.svc.${rgId}.internal:3900`; + } + function loadSettingsForm(workload: Workload) { settingsImage = workload.image; settingsEnvText = Object.entries(workload.env_vars ?? {}) @@ -681,7 +778,8 @@ type ResourceItem = | { kind: "container"; data: Workload } | { kind: "stack"; data: WorkloadStack } - | { kind: "volume"; data: Volume }; + | { kind: "volume"; data: Volume } + | { kind: "bucket"; data: Bucket }; function groupWorkloadsByStack(items: Workload[]): ResourceItem[] { const standalone: Workload[] = []; @@ -717,12 +815,14 @@ let allResources = $derived([ ...groupWorkloadsByStack(workloads), ...volumes.map((v): ResourceItem => ({ kind: "volume", data: v })), + ...buckets.map((b): ResourceItem => ({ kind: "bucket", data: b })), ]); let filteredResources = $derived( allResources.filter((r) => { - if (activeTab === "container" && r.kind === "volume") return false; + if (activeTab === "container" && (r.kind === "volume" || r.kind === "bucket")) return false; if (activeTab === "volume" && r.kind !== "volume") return false; + if (activeTab === "bucket" && r.kind !== "bucket") return false; if (!filterText) return true; const q = filterText.toLowerCase(); if (r.kind === "container") { @@ -741,6 +841,7 @@ function stackStatus(children: Workload[]): string { if (children.every((c) => c.status === "running")) return "running"; if (children.some((c) => c.status === "failed" || c.status === "error")) return "failed"; + if (children.every((c) => c.status === "stopped")) return "stopped"; return "pending"; } @@ -1255,6 +1356,123 @@ + { bucketError = null; }} +> +
+
+

Create Bucket

+ +
+
+
+ + +
+
+ + +
+
+ {#if bucketError} +

{bucketError}

+ {/if} +
+ + +
+
+ + + { activeBucket = null; createdKeySecret = null; }} +> + {#if activeBucket} +
+
+
+

{activeBucket.name}

+

{bucketEndpointUrl(activeBucket)}

+
+ +
+ +
+

Access Keys

+ {#if bucketKeys.length === 0} +

no access keys yet

+ {:else} +
+ {#each bucketKeys as key (key.id)} +
+
+

{key.name}

+

{key.garage_key_id}

+
+ {key.permissions} +
+ {/each} +
+ {/if} +
+ + {#if createdKeySecret} +
+

Secret access key created

+

{createdKeySecret}

+

this will not be shown again, copy it now

+ +
+ {/if} + + {#if bucketKeysError} +

{bucketKeysError}

+ {/if} + +
+ + +
+
+ {/if} + + -
+

Containers

{workloads.length}

@@ -1609,6 +1827,11 @@

{volumes.length}

{totalDisk} GB total

+
+

Buckets

+

{buckets.length}

+

{buckets.filter(b => b.exposure === 'external').length} external

+

CPU Requested

{(totalCpu / 1000).toFixed(1)}

@@ -1628,7 +1851,7 @@
- {#each [["all", `All ${allResources.length}`], ["container", `Container ${workloads.length}`], ["volume", `Volume ${volumes.length}`]] as [tab, label]} + {#each [["all", `All ${allResources.length}`], ["container", `Container ${workloads.length}`], ["volume", `Volume ${volumes.length}`], ["bucket", `Bucket ${buckets.length}`]] as [tab, label]} + {:else if item.kind === "bucket"} + {@const b = item.data} + openBucketDetail(b)} + > + +
+ +
+

{b.name}

+

{b.global_alias}

+
+
+ + + Bucket + + + {b.quota_max_size ? fmtBytes(b.quota_max_size) : "unlimited"} + + + {b.exposure} + + + + + +
+ + +
+ + {/if} {/each} {/if} diff --git a/control-plane/Dockerfile b/control-plane/Dockerfile index 8d42b63b..8fd08d75 100644 --- a/control-plane/Dockerfile +++ b/control-plane/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.88-slim-bookworm AS base +FROM rust:1.97-slim-bookworm AS base WORKDIR /app @@ -27,6 +27,7 @@ COPY control-plane/scheduler/Cargo.toml ./control-plane/scheduler/ COPY control-plane/failover-controller/Cargo.toml ./control-plane/failover-controller/ COPY control-plane/sdn-controller/Cargo.toml ./control-plane/sdn-controller/ COPY control-plane/volume-manager/Cargo.toml ./control-plane/volume-manager/ +COPY control-plane/object-storage/Cargo.toml ./control-plane/object-storage/ COPY control-plane/registry/Cargo.toml ./control-plane/registry/ COPY control-plane/shared/entity/Cargo.toml ./control-plane/shared/entity/ COPY control-plane/shared/migration/Cargo.toml ./control-plane/shared/migration/ @@ -41,6 +42,7 @@ RUN mkdir -p agent/src \ control-plane/failover-controller/src \ control-plane/sdn-controller/src \ control-plane/volume-manager/src \ + control-plane/object-storage/src \ control-plane/registry/src \ control-plane/shared/entity/src \ control-plane/shared/migration/src \ @@ -54,6 +56,7 @@ RUN mkdir -p agent/src \ && echo "fn main() {}" > control-plane/failover-controller/src/main.rs \ && echo "fn main() {}" > control-plane/sdn-controller/src/main.rs \ && echo "fn main() {}" > control-plane/volume-manager/src/main.rs \ + && echo "fn main() {}" > control-plane/object-storage/src/main.rs \ && echo "fn main() {}" > control-plane/registry/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-migrate/src/main.rs \ && echo "fn main() {}" > control-plane/csfx-updater/src/main.rs \ diff --git a/control-plane/api-gateway/Cargo.toml b/control-plane/api-gateway/Cargo.toml index 7cb88f0f..b8d4acba 100644 --- a/control-plane/api-gateway/Cargo.toml +++ b/control-plane/api-gateway/Cargo.toml @@ -9,6 +9,8 @@ name = "api-gateway" path = "src/main.rs" [dependencies] +percent-encoding = "2.3" + # Internal dependencies entity = { path = "../shared/entity" } migration = { path = "../shared/migration" } diff --git a/control-plane/api-gateway/src/auth/rbac.rs b/control-plane/api-gateway/src/auth/rbac.rs index ce94fd9b..e61373f0 100644 --- a/control-plane/api-gateway/src/auth/rbac.rs +++ b/control-plane/api-gateway/src/auth/rbac.rs @@ -24,6 +24,8 @@ pub struct CanViewResourceGroups(pub Claims); pub struct CanManageResourceGroups(pub Claims); pub struct CanViewLogs(pub Claims); pub struct CanManageLogs(pub Claims); +pub struct CanViewBuckets(pub Claims); +pub struct CanManageBuckets(pub Claims); async fn extract_claims(parts: &mut Parts, state: &AppState) -> Result { let token = parts @@ -121,3 +123,5 @@ impl_extractor!(CanViewResourceGroups, "resource_groups", "view"); impl_extractor!(CanManageResourceGroups, "resource_groups", "manage"); impl_extractor!(CanViewLogs, "logs", "view"); impl_extractor!(CanManageLogs, "logs", "manage"); +impl_extractor!(CanViewBuckets, "buckets", "view"); +impl_extractor!(CanManageBuckets, "buckets", "manage"); diff --git a/control-plane/api-gateway/src/init.rs b/control-plane/api-gateway/src/init.rs index 9fb029eb..e5eb447a 100644 --- a/control-plane/api-gateway/src/init.rs +++ b/control-plane/api-gateway/src/init.rs @@ -167,6 +167,18 @@ pub async fn initialize_database( "manage", "Manage log retention settings", ), + ( + "buckets.view", + "buckets", + "view", + "View buckets and access keys", + ), + ( + "buckets.manage", + "buckets", + "manage", + "Create, update and delete buckets and access keys", + ), ]; let mut permission_map = std::collections::HashMap::new(); @@ -287,6 +299,8 @@ pub async fn initialize_database( "resource_groups.view", "resource_groups.manage", "logs.view", + "buckets.view", + "buckets.manage", ]; for perm_name in operator_perms { if let Some(perm_id) = permission_map.get(perm_name) { @@ -332,6 +346,7 @@ pub async fn initialize_database( "members.view", "resource_groups.view", "logs.view", + "buckets.view", ]; for perm_name in viewer_perms { if let Some(perm_id) = permission_map.get(perm_name) { diff --git a/control-plane/api-gateway/src/routes/agent_proxy.rs b/control-plane/api-gateway/src/routes/agent_proxy.rs index 2f167930..405887d9 100644 --- a/control-plane/api-gateway/src/routes/agent_proxy.rs +++ b/control-plane/api-gateway/src/routes/agent_proxy.rs @@ -199,13 +199,15 @@ pub async fn stream_workload_logs( )); } - let stream = futures_util::stream::once(async { - Ok::<_, reqwest::Error>(axum::body::Bytes::new()) - }) - .chain(resp.bytes_stream()); + let stream = + futures_util::stream::once(async { Ok::<_, reqwest::Error>(axum::body::Bytes::new()) }) + .chain(resp.bytes_stream()); Ok(( - [(axum::http::header::CONTENT_TYPE, "text/plain; charset=utf-8")], + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], Body::from_stream(stream), )) } diff --git a/control-plane/api-gateway/src/routes/buckets.rs b/control-plane/api-gateway/src/routes/buckets.rs new file mode 100644 index 00000000..63148f1d --- /dev/null +++ b/control-plane/api-gateway/src/routes/buckets.rs @@ -0,0 +1,342 @@ +use axum::{ + body::Body, + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Json}, + routing::{get, post}, + Router, +}; +use serde::Deserialize; +use serde_json::json; + +use crate::{ + auth::rbac::{CanManageBuckets, CanViewBuckets}, + AppState, +}; + +async fn proxy_to_object_storage( + state: &AppState, + method: reqwest::Method, + path: &str, + body: Option, + headers: Option>, +) -> Result)> { + match state + .service_client + .forward_to_object_storage(method, path, body, headers) + .await + { + Ok((status, Some(body))) => { + let axum_status = + StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Ok((axum_status, Json(body)).into_response()) + } + Ok((status, None)) => { + let axum_status = + StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Ok((axum_status, Body::empty()).into_response()) + } + Err(e) => { + tracing::error!("Failed to forward request to object-storage: {}", e); + Err(( + StatusCode::BAD_GATEWAY, + Json( + json!({ "error": "Object Storage service unavailable", "details": e.to_string() }), + ), + )) + } + } +} + +fn header_vec(headers: &HeaderMap) -> Vec<(String, String)> { + headers + .iter() + .filter_map(|(k, v)| v.to_str().ok().map(|val| (k.to_string(), val.to_string()))) + .collect() +} + +pub async fn create_bucket( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + "/buckets", + body_json, + Some(header_map), + ) + .await +} + +pub async fn list_buckets( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + "/buckets", + None, + Some(header_map), + ) + .await +} + +pub async fn get_bucket( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + &format!("/buckets/{}", id), + None, + Some(header_map), + ) + .await +} + +pub async fn update_bucket( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::PATCH, + &format!("/buckets/{}", id), + body_json, + Some(header_map), + ) + .await +} + +pub async fn delete_bucket( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::DELETE, + &format!("/buckets/{}", id), + None, + Some(header_map), + ) + .await +} + +pub async fn list_keys( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + &format!("/buckets/{}/keys", id), + None, + Some(header_map), + ) + .await +} + +pub async fn create_key( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + &format!("/buckets/{}/keys", id), + body_json, + Some(header_map), + ) + .await +} + +pub async fn rotate_key( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path((bucket_id, key_id)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + &format!("/buckets/{}/keys/{}/rotate", bucket_id, key_id), + None, + Some(header_map), + ) + .await +} + +pub async fn delete_key( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path((bucket_id, key_id)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::DELETE, + &format!("/buckets/{}/keys/{}", bucket_id, key_id), + None, + Some(header_map), + ) + .await +} + +pub async fn get_cluster_status( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + "/cluster", + None, + Some(header_map), + ) + .await +} + +#[derive(Deserialize)] +pub struct ListObjectsQuery { + #[serde(default)] + prefix: String, + #[serde(default)] + continuation_token: Option, +} + +pub async fn list_objects( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path(id): Path, + Query(query): Query, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + let mut path = format!( + "/buckets/{}/objects?prefix={}", + id, + percent_encoding::utf8_percent_encode(&query.prefix, percent_encoding::NON_ALPHANUMERIC) + ); + if let Some(token) = &query.continuation_token { + path.push_str(&format!( + "&continuation_token={}", + percent_encoding::utf8_percent_encode(token, percent_encoding::NON_ALPHANUMERIC) + )); + } + proxy_to_object_storage(&state, reqwest::Method::GET, &path, None, Some(header_map)).await +} + +pub async fn delete_object( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path((bucket_id, key)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::DELETE, + &format!("/buckets/{}/objects/{}", bucket_id, key), + None, + Some(header_map), + ) + .await +} + +pub async fn presign_upload( + CanManageBuckets(_claims): CanManageBuckets, + State(state): State, + Path(id): Path, + headers: HeaderMap, + body: String, +) -> Result)> { + let body_json: Option = serde_json::from_str(&body).ok(); + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::POST, + &format!("/buckets/{}/objects/presign-upload", id), + body_json, + Some(header_map), + ) + .await +} + +pub async fn presign_download( + CanViewBuckets(_claims): CanViewBuckets, + State(state): State, + Path((bucket_id, key)): Path<(String, String)>, + headers: HeaderMap, +) -> Result)> { + let header_map = header_vec(&headers); + proxy_to_object_storage( + &state, + reqwest::Method::GET, + &format!("/buckets/{}/objects/presign-download/{}", bucket_id, key), + None, + Some(header_map), + ) + .await +} + +pub fn buckets_routes() -> Router { + Router::new() + .route("/buckets", post(create_bucket)) + .route("/buckets", get(list_buckets)) + .route("/buckets/{id}", get(get_bucket)) + .route("/buckets/{id}", axum::routing::patch(update_bucket)) + .route("/buckets/{id}", axum::routing::delete(delete_bucket)) + .route("/buckets/{id}/keys", get(list_keys)) + .route("/buckets/{id}/keys", post(create_key)) + .route( + "/buckets/{bucket_id}/keys/{key_id}/rotate", + post(rotate_key), + ) + .route( + "/buckets/{bucket_id}/keys/{key_id}", + axum::routing::delete(delete_key), + ) + .route("/buckets/{id}/objects", get(list_objects)) + .route("/buckets/{id}/objects/presign-upload", post(presign_upload)) + .route( + "/buckets/{bucket_id}/objects/presign-download/{*key}", + get(presign_download), + ) + .route( + "/buckets/{bucket_id}/objects/{*key}", + axum::routing::delete(delete_object), + ) + .route("/object-storage/cluster", get(get_cluster_status)) +} diff --git a/control-plane/api-gateway/src/routes/mod.rs b/control-plane/api-gateway/src/routes/mod.rs index d6b3d96d..bd123b0d 100644 --- a/control-plane/api-gateway/src/routes/mod.rs +++ b/control-plane/api-gateway/src/routes/mod.rs @@ -20,6 +20,7 @@ use tracing::{info_span, Span}; pub mod agent_proxy; pub mod agent_stream; pub mod agents; +pub mod buckets; pub mod events; pub mod logs; pub mod networks; @@ -27,6 +28,7 @@ pub mod organizations; pub mod registry; pub mod releases; pub mod resource_groups; +pub mod s3_proxy; pub mod settings; pub mod ssh_keys; pub mod system; @@ -120,6 +122,7 @@ pub fn create_router() -> Router { let rate_limited_router = Router::new() .merge(agent_proxy::agent_proxy_routes()) .merge(agents::agents_routes()) + .merge(buckets::buckets_routes()) .merge(networks::networks_routes()) .merge(organizations::routes()) .merge(ssh_keys::ssh_keys_routes()) @@ -131,6 +134,7 @@ pub fn create_router() -> Router { .merge(resource_groups::resource_groups_routes()) .merge(logs::logs_routes()) .merge(settings::settings_routes()) + .merge(s3_proxy::s3_proxy_routes()) .layer(GovernorLayer::new(governor_config)); let login_rate_limited_router = Router::new() @@ -151,6 +155,7 @@ pub fn create_router() -> Router { Router::new() .route("/metrics", get(metrics::metrics_handler)) + .merge(s3_proxy::object_data_router()) .logged_nest("/api", api_router) .logged_nest("/api", internal_api_router) .fallback_service(serve_dir) diff --git a/control-plane/api-gateway/src/routes/resource_groups.rs b/control-plane/api-gateway/src/routes/resource_groups.rs index ff9487d3..e85e8db7 100644 --- a/control-plane/api-gateway/src/routes/resource_groups.rs +++ b/control-plane/api-gateway/src/routes/resource_groups.rs @@ -8,8 +8,10 @@ use axum::{ use base64::{engine::general_purpose::STANDARD as B64, Engine}; use chrono::Utc; use entity::{ - entities::{agents, networks, resource_group_vpn_peers, resource_groups, volumes, workloads}, - Agents, Networks, ResourceGroupVpnPeers, ResourceGroups, Volumes, Workloads, + entities::{ + agents, buckets, networks, resource_group_vpn_peers, resource_groups, volumes, workloads, + }, + Agents, Buckets, Networks, ResourceGroupVpnPeers, ResourceGroups, Volumes, Workloads, }; use ring::rand::{SecureRandom, SystemRandom}; use sea_orm::{ @@ -483,6 +485,46 @@ pub async fn list_resource_group_volumes( Ok((StatusCode::OK, Json(json!(vols)))) } +pub async fn list_resource_group_buckets( + CanViewResourceGroups(_claims): CanViewResourceGroups, + State(state): State, + Path(id): Path, +) -> Result)> { + let org_id = get_org_id(&state); + + ResourceGroups::find_by_id(id) + .filter(resource_groups::Column::OrganizationId.eq(org_id)) + .one(&state.db_conn) + .await + .map_err(|e| { + tracing::error!(error = %e, "failed to find resource group"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": "database error" })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::NOT_FOUND, + Json(json!({ "error": "resource group not found" })), + ) + })?; + + let buckets = Buckets::find() + .filter(buckets::Column::ResourceGroupId.eq(id)) + .all(&state.db_conn) + .await + .map_err(|e| { + tracing::error!(error = %e, "failed to list buckets"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": "database error" })), + ) + })?; + + Ok((StatusCode::OK, Json(json!(buckets)))) +} + pub async fn list_resource_group_networks( CanViewResourceGroups(_claims): CanViewResourceGroups, State(state): State, @@ -950,6 +992,10 @@ pub fn resource_groups_routes() -> Router { "/resource-groups/{id}/volumes", get(list_resource_group_volumes), ) + .route( + "/resource-groups/{id}/buckets", + get(list_resource_group_buckets), + ) .route( "/resource-groups/{id}/networks", get(list_resource_group_networks), diff --git a/control-plane/api-gateway/src/routes/s3_proxy.rs b/control-plane/api-gateway/src/routes/s3_proxy.rs new file mode 100644 index 00000000..d094aba3 --- /dev/null +++ b/control-plane/api-gateway/src/routes/s3_proxy.rs @@ -0,0 +1,201 @@ +use axum::{ + body::Body, + extract::{OriginalUri, Path, State}, + http::{HeaderMap, Method, StatusCode}, + response::{IntoResponse, Json}, + routing::any, + Router, +}; +use entity::entities::{agents, buckets, garage_nodes}; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use serde_json::json; + +use crate::AppState; + +const S3_PORT: u16 = 3900; + +async fn resolve_bucket_target( + state: &AppState, + global_alias: &str, + require_external: bool, +) -> Result)> { + let bucket = buckets::Entity::find() + .filter(buckets::Column::GlobalAlias.eq(global_alias)) + .one(&state.db_conn) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("database error: {}", e) })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::NOT_FOUND, + Json(json!({ "error": "bucket not found" })), + ) + })?; + + if require_external && bucket.exposure != "external" { + return Err(( + StatusCode::NOT_FOUND, + Json(json!({ "error": "bucket not found" })), + )); + } + + let node = garage_nodes::Entity::find() + .filter(garage_nodes::Column::Status.eq("up")) + .one(&state.db_conn) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("database error: {}", e) })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "error": "no garage node available" })), + ) + })?; + + let Some(agent_id) = node.agent_id else { + return Ok(std::env::var("GARAGE_INTERNAL_HOST").unwrap_or_else(|_| "garage".to_string())); + }; + + let agent = agents::Entity::find_by_id(agent_id) + .one(&state.db_conn) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ "error": format!("database error: {}", e) })), + ) + })? + .ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "error": "garage node has no agent record" })), + ) + })?; + + let tunnel_ip = agent.wg_tunnel_ip.ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ "error": "garage node has no known tunnel address" })), + ) + })?; + + Ok(tunnel_ip) +} + +async fn proxy( + state: &AppState, + bucket: &str, + path: &str, + query: Option<&str>, + require_external: bool, + method: Method, + headers: HeaderMap, + body: Body, +) -> Result)> { + let tunnel_ip = resolve_bucket_target(state, bucket, require_external).await?; + + let mut url = format!("http://{}:{}/{}/{}", tunnel_ip, S3_PORT, bucket, path); + if let Some(query) = query { + url.push('?'); + url.push_str(query); + } + + let reqwest_method = + reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap_or(reqwest::Method::GET); + + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.map_err(|e| { + ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": format!("failed to read request body: {}", e) })), + ) + })?; + + let client = reqwest::Client::new(); + let mut request = client.request(reqwest_method, &url).body(body_bytes); + + for (key, value) in headers.iter() { + let key_lower = key.as_str().to_lowercase(); + if key_lower == "content-length" { + continue; + } + if let Ok(value_str) = value.to_str() { + request = request.header(key.as_str(), value_str); + } + } + + let response = request.send().await.map_err(|e| { + ( + StatusCode::BAD_GATEWAY, + Json(json!({ "error": format!("failed to reach garage node: {}", e) })), + ) + })?; + + let status = + StatusCode::from_u16(response.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let mut response_headers = HeaderMap::new(); + for (key, value) in response.headers().iter() { + response_headers.insert(key.clone(), value.clone()); + } + + let stream = response.bytes_stream(); + + Ok((status, response_headers, Body::from_stream(stream))) +} + +pub async fn proxy_s3_request( + State(state): State, + Path((bucket, path)): Path<(String, String)>, + OriginalUri(uri): OriginalUri, + method: Method, + headers: HeaderMap, + body: Body, +) -> Result)> { + proxy( + &state, + &bucket, + &path, + uri.query(), + true, + method, + headers, + body, + ) + .await +} + +pub async fn proxy_object_data( + State(state): State, + Path((bucket, key)): Path<(String, String)>, + OriginalUri(uri): OriginalUri, + method: Method, + headers: HeaderMap, + body: Body, +) -> Result)> { + proxy( + &state, + &bucket, + &key, + uri.query(), + false, + method, + headers, + body, + ) + .await +} + +pub fn s3_proxy_routes() -> Router { + Router::new().route("/s3/{bucket}/{*path}", any(proxy_s3_request)) +} + +pub fn object_data_router() -> Router { + Router::new().route("/{bucket}/{*key}", any(proxy_object_data)) +} diff --git a/control-plane/api-gateway/src/routes/users.rs b/control-plane/api-gateway/src/routes/users.rs index 1875aee0..f953e0c1 100644 --- a/control-plane/api-gateway/src/routes/users.rs +++ b/control-plane/api-gateway/src/routes/users.rs @@ -564,7 +564,9 @@ pub async fn change_gravatar_email( .change_gravatar_email(claims.user_id, payload.gravatar_email) .await { - Ok(_) => Ok(Json(json!({ "message": "Gravatar email changed successfully" }))), + Ok(_) => Ok(Json( + json!({ "message": "Gravatar email changed successfully" }), + )), Err(err) => { tracing::error!("Failed to change gravatar email: {}", err); Err(StatusCode::INTERNAL_SERVER_ERROR) diff --git a/control-plane/api-gateway/src/service_client.rs b/control-plane/api-gateway/src/service_client.rs index 1d096154..610768ff 100644 --- a/control-plane/api-gateway/src/service_client.rs +++ b/control-plane/api-gateway/src/service_client.rs @@ -11,6 +11,7 @@ pub struct ServiceClient { volume_manager_url: String, failover_controller_url: String, sdn_controller_url: String, + object_storage_url: String, } impl ServiceClient { @@ -30,6 +31,9 @@ impl ServiceClient { let sdn_controller_url = std::env::var("SDN_CONTROLLER_URL") .unwrap_or_else(|_| "http://localhost:8005".to_string()); + let object_storage_url = std::env::var("OBJECT_STORAGE_URL") + .unwrap_or_else(|_| "http://localhost:8006".to_string()); + let client = Client::builder() .timeout(Duration::from_secs(30)) .build() @@ -42,6 +46,7 @@ impl ServiceClient { volume_manager_url, failover_controller_url, sdn_controller_url, + object_storage_url, } } @@ -334,4 +339,59 @@ impl ServiceClient { Ok((status, json_body)) } + + pub async fn forward_to_object_storage( + &self, + method: reqwest::Method, + path: &str, + body: Option, + headers: Option>, + ) -> Result<(StatusCode, Option)> { + let url = format!("{}{}", self.object_storage_url, path); + + tracing::debug!("Forwarding {} request to object-storage: {}", method, url); + + let mut request = match method { + reqwest::Method::GET => self.client.get(&url), + reqwest::Method::POST => self.client.post(&url), + reqwest::Method::PATCH => self.client.patch(&url), + reqwest::Method::DELETE => self.client.delete(&url), + _ => return Err(anyhow::anyhow!("Unsupported HTTP method")), + }; + + if let Some(headers) = headers { + for (key, value) in headers { + let key_lower = key.to_lowercase(); + if key_lower == "content-length" + || key_lower == "host" + || key_lower == "content-type" + || key_lower == "transfer-encoding" + { + continue; + } + request = request.header(key, value); + } + } + + if let Some(body) = body { + request = request.json(&body); + } + + let response = request + .send() + .await + .context("Failed to send request to object-storage service")?; + + let status = response.status(); + let body_text = response.text().await.ok(); + let json_body = body_text.and_then(|text| { + if text.is_empty() { + None + } else { + serde_json::from_str(&text).ok() + } + }); + + Ok((status, json_body)) + } } diff --git a/control-plane/object-storage/Cargo.toml b/control-plane/object-storage/Cargo.toml new file mode 100644 index 00000000..6bea5132 --- /dev/null +++ b/control-plane/object-storage/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "object-storage" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "object-storage" +path = "src/main.rs" + +[dependencies] +shared = { path = "../shared/shared" } +entity = { path = "../shared/entity" } + +tokio = { workspace = true, features = ["full"] } +axum = { version = "0.8", features = ["macros"] } +aws-sigv4 = "1.5" +aws-credential-types = "1.2" +aws-smithy-runtime-api = "1.7" +http = "1" +percent-encoding = "2.3" +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +serde = { workspace = true } +serde_json = { workspace = true } +dotenvy = { workspace = true } +anyhow = { workspace = true } +uuid = { workspace = true, features = ["v4", "serde"] } +chrono = { workspace = true, features = ["serde"] } +sea-orm = { workspace = true } +etcd-client = { workspace = true } +reqwest = { workspace = true, features = ["json"] } +ring = { workspace = true } +rustls = { workspace = true } +prometheus = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } +tracing-opentelemetry = { workspace = true } diff --git a/control-plane/object-storage/dev/bootstrap.sh b/control-plane/object-storage/dev/bootstrap.sh new file mode 100755 index 00000000..425f916a --- /dev/null +++ b/control-plane/object-storage/dev/bootstrap.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +ADMIN_URL="http://garage:3903" +TOKEN="dev-garage-admin-token" + +until curl -sf -H "Authorization: Bearer ${TOKEN}" "${ADMIN_URL}/v2/GetClusterStatus" >/tmp/status.json; do + echo "waiting for garage admin api..." + sleep 1 +done + +NODE_ID=$(jq -r '.nodes[0].id' /tmp/status.json) +LAYOUT_VERSION=$(jq -r '.layoutVersion' /tmp/status.json) + +EXISTING_ROLES=$(curl -sf -H "Authorization: Bearer ${TOKEN}" "${ADMIN_URL}/v2/GetClusterLayout" | jq -r '.roles | length') + +if [ "$EXISTING_ROLES" -gt 0 ]; then + echo "layout already assigned" + exit 0 +fi + +echo "assigning layout to node ${NODE_ID}" + +curl -sf -X POST \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"roles\":[{\"id\":\"${NODE_ID}\",\"zone\":\"dev\",\"capacity\":1000000000,\"tags\":[]}]}" \ + "${ADMIN_URL}/v2/UpdateClusterLayout" + +APPLY_VERSION=$((LAYOUT_VERSION + 1)) + +curl -sf -X POST \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"version\":${APPLY_VERSION}}" \ + "${ADMIN_URL}/v2/ApplyClusterLayout" + +echo "garage bootstrap complete" diff --git a/control-plane/object-storage/dev/garage.toml b/control-plane/object-storage/dev/garage.toml new file mode 100644 index 00000000..a0300db8 --- /dev/null +++ b/control-plane/object-storage/dev/garage.toml @@ -0,0 +1,19 @@ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "lmdb" + +replication_factor = 1 +compression_level = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "garage:3901" +rpc_secret = "c4717cda58d14d9cd2eee1dce98ded1ff6fa2c0cc5372b769fad641e3d277676" + +[s3_api] +s3_region = "csfx" +api_bind_addr = "[::]:3900" +root_domain = "" + +[admin] +api_bind_addr = "[::]:3903" +admin_token = "dev-garage-admin-token" diff --git a/control-plane/object-storage/src/crypto.rs b/control-plane/object-storage/src/crypto.rs new file mode 100644 index 00000000..3f4aed57 --- /dev/null +++ b/control-plane/object-storage/src/crypto.rs @@ -0,0 +1,68 @@ +use anyhow::{bail, Context, Result}; +use ring::aead::{Aad, LessSafeKey, Nonce, UnboundKey, AES_256_GCM, NONCE_LEN}; +use ring::rand::{SecureRandom, SystemRandom}; + +pub struct SecretBox { + key: LessSafeKey, +} + +impl SecretBox { + pub fn from_env() -> Result { + let hex_key = std::env::var("OBJECT_STORAGE_ENCRYPTION_KEY") + .context("OBJECT_STORAGE_ENCRYPTION_KEY must be set")?; + let bytes = hex_decode(&hex_key).context("OBJECT_STORAGE_ENCRYPTION_KEY must be hex")?; + if bytes.len() != 32 { + bail!("OBJECT_STORAGE_ENCRYPTION_KEY must decode to 32 bytes"); + } + let unbound = UnboundKey::new(&AES_256_GCM, &bytes) + .map_err(|_| anyhow::anyhow!("failed to build encryption key"))?; + Ok(Self { + key: LessSafeKey::new(unbound), + }) + } + + pub fn encrypt(&self, plaintext: &str) -> Result> { + let rng = SystemRandom::new(); + let mut nonce_bytes = [0u8; NONCE_LEN]; + rng.fill(&mut nonce_bytes) + .map_err(|_| anyhow::anyhow!("failed to generate nonce"))?; + + let mut in_out = plaintext.as_bytes().to_vec(); + self.key + .seal_in_place_append_tag( + Nonce::assume_unique_for_key(nonce_bytes), + Aad::empty(), + &mut in_out, + ) + .map_err(|_| anyhow::anyhow!("encryption failed"))?; + + let mut output = nonce_bytes.to_vec(); + output.extend_from_slice(&in_out); + Ok(output) + } + + pub fn decrypt(&self, ciphertext: &[u8]) -> Result { + if ciphertext.len() < NONCE_LEN { + bail!("ciphertext too short"); + } + let (nonce_bytes, sealed) = ciphertext.split_at(NONCE_LEN); + let mut buf = sealed.to_vec(); + let nonce = Nonce::try_assume_unique_for_key(nonce_bytes) + .map_err(|_| anyhow::anyhow!("invalid nonce"))?; + let plaintext = self + .key + .open_in_place(nonce, Aad::empty(), &mut buf) + .map_err(|_| anyhow::anyhow!("decryption failed"))?; + String::from_utf8(plaintext.to_vec()).context("decrypted secret is not valid utf-8") + } +} + +fn hex_decode(input: &str) -> Result> { + if !input.len().is_multiple_of(2) { + bail!("hex string must have even length"); + } + (0..input.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&input[i..i + 2], 16).context("invalid hex digit")) + .collect() +} diff --git a/control-plane/object-storage/src/db/access_keys.rs b/control-plane/object-storage/src/db/access_keys.rs new file mode 100644 index 00000000..1d1e001d --- /dev/null +++ b/control-plane/object-storage/src/db/access_keys.rs @@ -0,0 +1,73 @@ +use anyhow::{Context, Result}; +use chrono::Utc; +use entity::entities::bucket_access_keys; +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, +}; +use uuid::Uuid; + +pub async fn insert( + db: &DatabaseConnection, + bucket_id: Uuid, + name: &str, + garage_key_id: &str, + permissions: &str, + expires_at: Option, +) -> Result { + let model = bucket_access_keys::ActiveModel { + id: Set(Uuid::new_v4()), + bucket_id: Set(bucket_id), + name: Set(name.to_string()), + garage_key_id: Set(garage_key_id.to_string()), + permissions: Set(permissions.to_string()), + expires_at: Set(expires_at), + last_rotated_at: Set(None), + created_at: Set(Utc::now().naive_utc()), + }; + + model + .insert(db) + .await + .context("failed to insert bucket access key") +} + +pub async fn get_by_id( + db: &DatabaseConnection, + id: Uuid, +) -> Result> { + bucket_access_keys::Entity::find_by_id(id) + .one(db) + .await + .context("failed to get bucket access key") +} + +pub async fn list_for_bucket( + db: &DatabaseConnection, + bucket_id: Uuid, +) -> Result> { + bucket_access_keys::Entity::find() + .filter(bucket_access_keys::Column::BucketId.eq(bucket_id)) + .all(db) + .await + .context("failed to list bucket access keys") +} + +pub async fn touch_rotated(db: &DatabaseConnection, id: Uuid) -> Result<()> { + if let Some(existing) = get_by_id(db, id).await? { + let mut model: bucket_access_keys::ActiveModel = existing.into(); + model.last_rotated_at = Set(Some(Utc::now().naive_utc())); + model + .update(db) + .await + .context("failed to update bucket access key rotation timestamp")?; + } + Ok(()) +} + +pub async fn delete(db: &DatabaseConnection, id: Uuid) -> Result<()> { + bucket_access_keys::Entity::delete_by_id(id) + .exec(db) + .await + .context("failed to delete bucket access key")?; + Ok(()) +} diff --git a/control-plane/object-storage/src/db/buckets.rs b/control-plane/object-storage/src/db/buckets.rs new file mode 100644 index 00000000..6d59469d --- /dev/null +++ b/control-plane/object-storage/src/db/buckets.rs @@ -0,0 +1,115 @@ +use anyhow::{Context, Result}; +use chrono::Utc; +use entity::entities::buckets; +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, +}; +use uuid::Uuid; + +use crate::models::{BucketResponse, CreateBucketRequest, UpdateBucketRequest}; + +pub fn into_response(model: buckets::Model) -> BucketResponse { + BucketResponse { + id: model.id, + name: model.name, + global_alias: model.global_alias, + exposure: model.exposure, + quota_max_size: model.quota_max_size, + quota_max_objects: model.quota_max_objects, + status: model.status, + organization_id: model.organization_id, + resource_group_id: model.resource_group_id, + created_at: model.created_at, + updated_at: model.updated_at, + } +} + +pub async fn insert( + db: &DatabaseConnection, + req: &CreateBucketRequest, + global_alias: &str, + garage_bucket_id: &str, + master_key_id: &str, + master_key_secret_encrypted: Vec, +) -> Result { + let model = buckets::ActiveModel { + id: Set(Uuid::new_v4()), + name: Set(req.name.clone()), + garage_bucket_id: Set(Some(garage_bucket_id.to_string())), + global_alias: Set(global_alias.to_string()), + exposure: Set(req + .exposure + .clone() + .unwrap_or_else(|| "internal".to_string())), + quota_max_size: Set(req.quota_max_size), + quota_max_objects: Set(req.quota_max_objects), + status: Set("active".to_string()), + master_key_id: Set(Some(master_key_id.to_string())), + master_key_secret_encrypted: Set(Some(master_key_secret_encrypted)), + organization_id: Set(req.organization_id), + resource_group_id: Set(req.resource_group_id), + created_at: Set(Utc::now().naive_utc()), + updated_at: Set(None), + }; + + model.insert(db).await.context("failed to insert bucket") +} + +pub async fn get_by_id(db: &DatabaseConnection, id: Uuid) -> Result> { + buckets::Entity::find_by_id(id) + .one(db) + .await + .context("failed to get bucket") +} + +pub async fn list( + db: &DatabaseConnection, + resource_group_id: Option, + organization_id: Option, +) -> Result> { + let mut query = buckets::Entity::find(); + + if let Some(rg_id) = resource_group_id { + query = query.filter(buckets::Column::ResourceGroupId.eq(rg_id)); + } + + if let Some(org_id) = organization_id { + query = query.filter(buckets::Column::OrganizationId.eq(org_id)); + } + + query.all(db).await.context("failed to list buckets") +} + +pub async fn update( + db: &DatabaseConnection, + id: Uuid, + req: &UpdateBucketRequest, +) -> Result> { + let Some(existing) = get_by_id(db, id).await? else { + return Ok(None); + }; + + let mut model: buckets::ActiveModel = existing.into(); + + if let Some(exposure) = &req.exposure { + model.exposure = Set(exposure.clone()); + } + if req.quota_max_size.is_some() { + model.quota_max_size = Set(req.quota_max_size); + } + if req.quota_max_objects.is_some() { + model.quota_max_objects = Set(req.quota_max_objects); + } + model.updated_at = Set(Some(Utc::now().naive_utc())); + + let updated = model.update(db).await.context("failed to update bucket")?; + Ok(Some(updated)) +} + +pub async fn delete(db: &DatabaseConnection, id: Uuid) -> Result<()> { + buckets::Entity::delete_by_id(id) + .exec(db) + .await + .context("failed to delete bucket")?; + Ok(()) +} diff --git a/control-plane/object-storage/src/db/garage_nodes.rs b/control-plane/object-storage/src/db/garage_nodes.rs new file mode 100644 index 00000000..ab4f45ec --- /dev/null +++ b/control-plane/object-storage/src/db/garage_nodes.rs @@ -0,0 +1,82 @@ +use anyhow::{Context, Result}; +use chrono::Utc; +use entity::entities::garage_nodes; +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, +}; +use uuid::Uuid; + +pub async fn list(db: &DatabaseConnection) -> Result> { + garage_nodes::Entity::find() + .all(db) + .await + .context("failed to list garage nodes") +} + +pub async fn mark_down(db: &DatabaseConnection, id: Uuid) -> Result<()> { + if let Some(existing) = garage_nodes::Entity::find_by_id(id).one(db).await? { + let mut model: garage_nodes::ActiveModel = existing.into(); + model.status = Set("down".to_string()); + model.updated_at = Set(Some(Utc::now().naive_utc())); + model + .update(db) + .await + .context("failed to mark garage node down")?; + } + Ok(()) +} + +pub async fn upsert_self( + db: &DatabaseConnection, + garage_node_id: &str, + zone: &str, + capacity_bytes: Option, +) -> Result { + let existing = garage_nodes::Entity::find() + .filter(garage_nodes::Column::GarageNodeId.eq(garage_node_id)) + .one(db) + .await + .context("failed to look up existing garage node")?; + + let now = Utc::now().naive_utc(); + + if let Some(existing) = existing { + let mut model: garage_nodes::ActiveModel = existing.into(); + model.status = Set("up".to_string()); + model.capacity_bytes = Set(capacity_bytes); + model.role = Set(if capacity_bytes.is_some() { + "storage".to_string() + } else { + "gateway".to_string() + }); + model.last_seen_at = Set(Some(now)); + model.updated_at = Set(Some(now)); + return model + .update(db) + .await + .context("failed to update self garage node"); + } + + let model = garage_nodes::ActiveModel { + id: Set(Uuid::new_v4()), + agent_id: Set(None), + garage_node_id: Set(Some(garage_node_id.to_string())), + zone: Set(zone.to_string()), + capacity_bytes: Set(capacity_bytes), + role: Set(if capacity_bytes.is_some() { + "storage".to_string() + } else { + "gateway".to_string() + }), + status: Set("up".to_string()), + layout_version: Set(None), + last_seen_at: Set(Some(now)), + created_at: Set(now), + updated_at: Set(None), + }; + + model + .insert(db) + .await + .context("failed to insert self garage node") +} diff --git a/control-plane/object-storage/src/db/mod.rs b/control-plane/object-storage/src/db/mod.rs new file mode 100644 index 00000000..1018a929 --- /dev/null +++ b/control-plane/object-storage/src/db/mod.rs @@ -0,0 +1,3 @@ +pub mod access_keys; +pub mod buckets; +pub mod garage_nodes; diff --git a/control-plane/object-storage/src/garage/client.rs b/control-plane/object-storage/src/garage/client.rs new file mode 100644 index 00000000..0331492d --- /dev/null +++ b/control-plane/object-storage/src/garage/client.rs @@ -0,0 +1,297 @@ +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +#[derive(Clone)] +pub struct GarageClient { + http: reqwest::Client, + admin_url: String, + admin_token: String, +} + +#[derive(Debug, Deserialize)] +pub struct GarageBucket { + pub id: String, +} + +#[derive(Debug, Serialize)] +struct CreateBucketBody { + #[serde(rename = "globalAlias")] + global_alias: String, +} + +#[derive(Debug, Serialize)] +struct UpdateBucketBody { + quotas: UpdateBucketQuotas, +} + +#[derive(Debug, Serialize)] +struct UpdateBucketQuotas { + #[serde(rename = "maxSize", skip_serializing_if = "Option::is_none")] + max_size: Option, + #[serde(rename = "maxObjects", skip_serializing_if = "Option::is_none")] + max_objects: Option, +} + +#[derive(Debug, Deserialize)] +pub struct GarageKey { + #[serde(rename = "accessKeyId")] + pub access_key_id: String, + #[serde(rename = "secretAccessKey")] + pub secret_access_key: String, +} + +#[derive(Debug, Deserialize)] +pub struct ClusterStatusNode { + pub id: String, + #[serde(rename = "isUp", default)] + pub is_up: bool, + #[serde(default)] + pub role: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ClusterStatusNodeRole { + #[serde(default)] + pub zone: String, + #[serde(default)] + pub capacity: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ClusterStatus { + #[serde(default)] + pub nodes: Vec, + #[serde(rename = "layoutVersion", default)] + pub layout_version: i64, +} + +#[derive(Debug, Serialize)] +pub struct LayoutRole { + pub id: String, + pub zone: String, + pub capacity: Option, + pub tags: Vec, +} + +impl GarageClient { + pub fn new(admin_url: String, admin_token: String) -> Self { + Self { + http: reqwest::Client::new(), + admin_url, + admin_token, + } + } + + fn url(&self, path: &str) -> String { + format!("{}{}", self.admin_url, path) + } + + async fn check_status(response: reqwest::Response, context: &str) -> Result { + if response.status().is_success() { + return Ok(response); + } + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + bail!( + "garage admin api error context={} status={} body={}", + context, + status, + body + ) + } + + pub async fn create_bucket(&self, global_alias: &str) -> Result { + let response = self + .http + .post(self.url("/v2/CreateBucket")) + .bearer_auth(&self.admin_token) + .json(&CreateBucketBody { + global_alias: global_alias.to_string(), + }) + .send() + .await + .context("create_bucket request failed")?; + + let response = Self::check_status(response, "create_bucket").await?; + response + .json::() + .await + .context("failed to parse create_bucket response") + } + + pub async fn update_bucket_quotas( + &self, + garage_bucket_id: &str, + max_size: Option, + max_objects: Option, + ) -> Result<()> { + let response = self + .http + .post(self.url(&format!("/v2/UpdateBucket?id={}", garage_bucket_id))) + .bearer_auth(&self.admin_token) + .json(&UpdateBucketBody { + quotas: UpdateBucketQuotas { + max_size, + max_objects, + }, + }) + .send() + .await + .context("update_bucket_quotas request failed")?; + + Self::check_status(response, "update_bucket_quotas").await?; + Ok(()) + } + + pub async fn delete_bucket(&self, garage_bucket_id: &str) -> Result<()> { + let response = self + .http + .post(self.url(&format!("/v2/DeleteBucket?id={}", garage_bucket_id))) + .bearer_auth(&self.admin_token) + .send() + .await + .context("delete_bucket request failed")?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + + Self::check_status(response, "delete_bucket").await?; + Ok(()) + } + + pub async fn create_key(&self, name: &str) -> Result { + let response = self + .http + .post(self.url("/v2/CreateKey")) + .bearer_auth(&self.admin_token) + .json(&json!({ "name": name })) + .send() + .await + .context("create_key request failed")?; + + let response = Self::check_status(response, "create_key").await?; + response + .json::() + .await + .context("failed to parse create_key response") + } + + pub async fn delete_key(&self, garage_key_id: &str) -> Result<()> { + let response = self + .http + .post(self.url(&format!("/v2/DeleteKey?id={}", garage_key_id))) + .bearer_auth(&self.admin_token) + .send() + .await + .context("delete_key request failed")?; + + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + + Self::check_status(response, "delete_key").await?; + Ok(()) + } + + pub async fn allow_bucket_key( + &self, + garage_bucket_id: &str, + garage_key_id: &str, + permissions: &str, + ) -> Result<()> { + let (read, write, owner) = match permissions { + "read" => (true, false, false), + "readwrite" => (true, true, false), + "owner" => (true, true, true), + other => bail!("unknown bucket key permission permissions={}", other), + }; + + let response = self + .http + .post(self.url("/v2/AllowBucketKey")) + .bearer_auth(&self.admin_token) + .json(&json!({ + "bucketId": garage_bucket_id, + "accessKeyId": garage_key_id, + "permissions": { "read": read, "write": write, "owner": owner }, + })) + .send() + .await + .context("allow_bucket_key request failed")?; + + Self::check_status(response, "allow_bucket_key").await?; + Ok(()) + } + + pub async fn get_cluster_status(&self) -> Result { + let response = self + .http + .get(self.url("/v2/GetClusterStatus")) + .bearer_auth(&self.admin_token) + .send() + .await + .context("get_cluster_status request failed")?; + + let response = Self::check_status(response, "get_cluster_status").await?; + response + .json::() + .await + .context("failed to parse get_cluster_status response") + } + + pub async fn connect_cluster_nodes(&self, node_addrs: &[String]) -> Result<()> { + if node_addrs.is_empty() { + return Ok(()); + } + + let response = self + .http + .post(self.url("/v2/ConnectClusterNodes")) + .bearer_auth(&self.admin_token) + .json(node_addrs) + .send() + .await + .context("connect_cluster_nodes request failed")?; + + Self::check_status(response, "connect_cluster_nodes").await?; + Ok(()) + } + + pub async fn update_cluster_layout( + &self, + roles: Vec, + parameters_replication_factor: u32, + ) -> Result<()> { + let response = self + .http + .post(self.url("/v2/UpdateClusterLayout")) + .bearer_auth(&self.admin_token) + .json(&json!({ + "roles": roles, + "parameters": { "zone_redundancy": "maximum" }, + "replication_factor": parameters_replication_factor, + })) + .send() + .await + .context("update_cluster_layout request failed")?; + + Self::check_status(response, "update_cluster_layout").await?; + Ok(()) + } + + pub async fn apply_cluster_layout(&self, version: i64) -> Result<()> { + let response = self + .http + .post(self.url("/v2/ApplyClusterLayout")) + .bearer_auth(&self.admin_token) + .json(&json!({ "version": version })) + .send() + .await + .context("apply_cluster_layout request failed")?; + + Self::check_status(response, "apply_cluster_layout").await?; + Ok(()) + } +} diff --git a/control-plane/object-storage/src/garage/layout.rs b/control-plane/object-storage/src/garage/layout.rs new file mode 100644 index 00000000..00148539 --- /dev/null +++ b/control-plane/object-storage/src/garage/layout.rs @@ -0,0 +1,225 @@ +use anyhow::Result; +use entity::entities::{agents, garage_nodes}; +use sea_orm::{DatabaseConnection, EntityTrait}; +use tokio::time::{sleep, Duration}; + +use crate::{ + db::garage_nodes as garage_nodes_db, + garage::{client::LayoutRole, leader::LayoutLeader, GarageClient}, + log_error, log_info, log_warn, +}; + +const RECONCILE_INTERVAL_SECONDS: u64 = 30; +const MIN_STORAGE_NODES_FOR_FULL_REPLICATION: usize = 3; + +pub fn replication_factor_for(storage_node_count: usize) -> u32 { + if storage_node_count >= MIN_STORAGE_NODES_FOR_FULL_REPLICATION { + 3 + } else { + 1 + } +} + +async fn peer_addrs( + db: &DatabaseConnection, + known_nodes: &[garage_nodes::Model], +) -> Result> { + let mut addrs = Vec::new(); + + for node in known_nodes { + let Some(garage_node_id) = &node.garage_node_id else { + continue; + }; + let Some(agent_id) = node.agent_id else { + continue; + }; + let Some(agent) = agents::Entity::find_by_id(agent_id).one(db).await? else { + continue; + }; + let Some(wg_ip) = agent.wg_tunnel_ip else { + continue; + }; + addrs.push(format!("{}@{}:3901", garage_node_id, wg_ip)); + } + + Ok(addrs) +} + +async fn reconcile_once(db: &DatabaseConnection, garage: &GarageClient) -> Result<()> { + let known_nodes = garage_nodes_db::list(db).await?; + + let addrs = peer_addrs(db, &known_nodes).await?; + if let Err(e) = garage.connect_cluster_nodes(&addrs).await { + log_warn!( + "garage::layout", + &format!("failed to connect garage cluster nodes err={}", e) + ); + } + + let status = match garage.get_cluster_status().await { + Ok(status) => status, + Err(e) => { + log_warn!( + "garage::layout", + &format!("failed to read garage cluster status err={}", e) + ); + return Ok(()); + } + }; + + for node in &known_nodes { + let is_up = status + .nodes + .iter() + .any(|n| Some(n.id.clone()) == node.garage_node_id && n.is_up); + + if !is_up && node.status == "up" { + log_warn!( + "garage::layout", + &format!("garage node reported down id={}", node.id) + ); + garage_nodes_db::mark_down(db, node.id).await?; + } + } + + let storage_nodes: Vec<&garage_nodes::Model> = known_nodes + .iter() + .filter(|n| n.role == "storage" && n.capacity_bytes.is_some()) + .collect(); + + if storage_nodes.is_empty() { + return Ok(()); + } + + let factor = replication_factor_for(storage_nodes.len()); + + let roles: Vec = known_nodes + .iter() + .filter_map(|n| { + n.garage_node_id.as_ref().map(|garage_id| LayoutRole { + id: garage_id.clone(), + zone: n.zone.clone(), + capacity: n.capacity_bytes, + tags: vec![], + }) + }) + .collect(); + + if roles.is_empty() { + return Ok(()); + } + + if let Err(e) = garage.update_cluster_layout(roles, factor).await { + log_error!( + "garage::layout", + &format!("failed to stage cluster layout err={}", e) + ); + return Ok(()); + } + + if let Err(e) = garage.apply_cluster_layout(status.layout_version + 1).await { + log_error!( + "garage::layout", + &format!("failed to apply cluster layout err={}", e) + ); + return Ok(()); + } + + log_info!( + "garage::layout", + &format!( + "applied cluster layout storage_nodes={} replication_factor={}", + storage_nodes.len(), + factor + ) + ); + Ok(()) +} + +pub async fn run_reconcile_loop( + db: DatabaseConnection, + garage: GarageClient, + leader: LayoutLeader, +) { + loop { + sleep(Duration::from_secs(RECONCILE_INTERVAL_SECONDS)).await; + + if !leader.is_leader() { + continue; + } + + if let Err(e) = reconcile_once(&db, &garage).await { + log_error!( + "garage::layout", + &format!("reconcile loop iteration failed err={}", e) + ); + } + } +} + +const SELF_REGISTER_RETRY_SECONDS: u64 = 5; +const SELF_REGISTER_MAX_ATTEMPTS: u32 = 60; + +pub async fn register_self_as_node(db: &DatabaseConnection, garage: &GarageClient, zone: &str) { + for attempt in 1..=SELF_REGISTER_MAX_ATTEMPTS { + match garage.get_cluster_status().await { + Ok(status) => { + let Some(self_node) = status.nodes.first() else { + log_warn!( + "garage::layout", + "garage cluster status returned no nodes yet" + ); + sleep(Duration::from_secs(SELF_REGISTER_RETRY_SECONDS)).await; + continue; + }; + + let capacity_bytes = self_node.role.as_ref().and_then(|r| r.capacity); + let node_zone = self_node + .role + .as_ref() + .map(|r| r.zone.as_str()) + .filter(|z| !z.is_empty()) + .unwrap_or(zone); + + match crate::db::garage_nodes::upsert_self( + db, + &self_node.id, + node_zone, + capacity_bytes, + ) + .await + { + Ok(node) => { + log_info!( + "garage::layout", + &format!("registered self as garage node id={}", node.id) + ); + return; + } + Err(e) => { + log_error!( + "garage::layout", + &format!("failed to persist self garage node registration err={}", e) + ); + return; + } + } + } + Err(e) => { + log_warn!( + "garage::layout", + &format!( + "self-registration attempt {}/{} failed err={}", + attempt, SELF_REGISTER_MAX_ATTEMPTS, e + ) + ); + sleep(Duration::from_secs(SELF_REGISTER_RETRY_SECONDS)).await; + } + } + } + + log_error!( + "garage::layout", + "giving up on self garage node registration after max attempts" + ); +} diff --git a/control-plane/object-storage/src/garage/leader.rs b/control-plane/object-storage/src/garage/leader.rs new file mode 100644 index 00000000..d56c2283 --- /dev/null +++ b/control-plane/object-storage/src/garage/leader.rs @@ -0,0 +1,116 @@ +use anyhow::{Context, Result}; +use etcd_client::{Client, Compare, CompareOp, PutOptions, Txn, TxnOp}; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; +use std::sync::Arc; +use tokio::time::{sleep, Duration}; + +use crate::{log_error, log_info, log_warn}; + +const LAYOUT_LOCK_KEY: &str = "/csfx/object-storage/layout-lock"; +const LEASE_TTL_SECONDS: i64 = 10; + +#[derive(Clone)] +pub struct LayoutLeader { + etcd: Client, + node_id: String, + lease_id: Arc, + is_leader: Arc, +} + +impl LayoutLeader { + pub fn new(etcd: Client, node_id: String) -> Self { + Self { + etcd, + node_id, + lease_id: Arc::new(AtomicI64::new(0)), + is_leader: Arc::new(AtomicBool::new(false)), + } + } + + pub fn is_leader(&self) -> bool { + self.is_leader.load(Ordering::SeqCst) + } + + async fn campaign(&mut self) -> Result<()> { + if self.is_leader() { + return Ok(()); + } + + let lease = self + .etcd + .lease_grant(LEASE_TTL_SECONDS, None) + .await + .context("failed to grant etcd lease")?; + let lease_id = lease.id(); + + let txn = Txn::new() + .when(vec![Compare::create_revision( + LAYOUT_LOCK_KEY, + CompareOp::Equal, + 0, + )]) + .and_then(vec![TxnOp::put( + LAYOUT_LOCK_KEY, + self.node_id.as_bytes(), + Some(PutOptions::new().with_lease(lease_id)), + )]); + + let response = self.etcd.txn(txn).await.context("etcd txn failed")?; + + if response.succeeded() { + self.lease_id.store(lease_id, Ordering::SeqCst); + self.is_leader.store(true, Ordering::SeqCst); + log_info!( + "garage::leader", + &format!( + "became object-storage layout leader node_id={}", + self.node_id + ) + ); + self.spawn_lease_renewal(lease_id); + } else { + let _ = self.etcd.lease_revoke(lease_id).await; + } + + Ok(()) + } + + fn spawn_lease_renewal(&self, lease_id: i64) { + let mut etcd = self.etcd.clone(); + let is_leader = Arc::clone(&self.is_leader); + let node_id = self.node_id.clone(); + + tokio::spawn(async move { + loop { + sleep(Duration::from_secs(4)).await; + if !is_leader.load(Ordering::SeqCst) { + break; + } + if let Err(e) = etcd.lease_keep_alive(lease_id).await { + log_error!( + "garage::leader", + &format!("lease renewal failed node_id={} err={}", node_id, e) + ); + is_leader.store(false, Ordering::SeqCst); + log_warn!( + "garage::leader", + &format!("lost object-storage layout leadership node_id={}", node_id) + ); + break; + } + } + }); + } + + pub async fn run_campaign_loop(mut self) { + loop { + if let Err(e) = self.campaign().await { + log_error!( + "garage::leader", + &format!("layout leader campaign failed err={}", e) + ); + } + sleep(Duration::from_secs(5)).await; + } + } +} diff --git a/control-plane/object-storage/src/garage/mod.rs b/control-plane/object-storage/src/garage/mod.rs new file mode 100644 index 00000000..57c8b775 --- /dev/null +++ b/control-plane/object-storage/src/garage/mod.rs @@ -0,0 +1,7 @@ +pub mod client; +pub mod layout; +pub mod leader; +pub mod s3_client; + +pub use client::GarageClient; +pub use s3_client::S3Client; diff --git a/control-plane/object-storage/src/garage/s3_client.rs b/control-plane/object-storage/src/garage/s3_client.rs new file mode 100644 index 00000000..119bbe1b --- /dev/null +++ b/control-plane/object-storage/src/garage/s3_client.rs @@ -0,0 +1,298 @@ +use anyhow::{Context, Result}; +use aws_credential_types::Credentials; +use aws_sigv4::http_request::{ + sign, PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, SigningSettings, +}; +use aws_sigv4::sign::v4; +use percent_encoding::AsciiSet; +use serde::Deserialize; +use std::time::{Duration, SystemTime}; + +const S3_REGION: &str = "csfx"; +const S3_SERVICE: &str = "s3"; +const EMPTY_BODY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +const SIGV4_UNRESERVED: &AsciiSet = &percent_encoding::NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +#[derive(Clone)] +pub struct S3Client { + s3_url: String, + public_s3_url: String, + http: reqwest::Client, +} + +#[derive(Debug, Deserialize)] +pub struct S3Object { + pub key: String, + pub size: i64, + pub last_modified: String, +} + +#[derive(Debug, Default)] +pub struct ListObjectsResult { + pub objects: Vec, + pub common_prefixes: Vec, +} + +impl S3Client { + pub fn new(s3_url: String, public_s3_url: String) -> Self { + Self { + s3_url, + public_s3_url, + http: reqwest::Client::new(), + } + } + + fn identity( + access_key_id: &str, + secret_access_key: &str, + ) -> aws_smithy_runtime_api::client::identity::Identity { + Credentials::new( + access_key_id, + secret_access_key, + None, + None, + "csfx-object-storage", + ) + .into() + } + + fn signing_params<'a>( + identity: &'a aws_smithy_runtime_api::client::identity::Identity, + settings: SigningSettings, + ) -> Result> { + v4::SigningParams::builder() + .identity(identity) + .region(S3_REGION) + .name(S3_SERVICE) + .time(SystemTime::now()) + .settings(settings) + .build() + .context("failed to build sigv4 signing params") + } + + pub async fn list_objects( + &self, + bucket: &str, + access_key_id: &str, + secret_access_key: &str, + prefix: &str, + delimiter: &str, + continuation_token: Option<&str>, + ) -> Result { + let mut url = format!( + "{}/{}?list-type=2&prefix={}&delimiter={}", + self.s3_url, + bucket, + percent_encoding::utf8_percent_encode(prefix, SIGV4_UNRESERVED), + percent_encoding::utf8_percent_encode(delimiter, SIGV4_UNRESERVED), + ); + if let Some(token) = continuation_token { + url.push_str(&format!( + "&continuation-token={}", + percent_encoding::utf8_percent_encode(token, SIGV4_UNRESERVED) + )); + } + + let identity = Self::identity(access_key_id, secret_access_key); + let params = Self::signing_params(&identity, SigningSettings::default())?; + + let signable = SignableRequest::new( + "GET", + &url, + std::iter::once(("x-amz-content-sha256", EMPTY_BODY_SHA256)), + SignableBody::Bytes(&[]), + ) + .context("failed to build signable request")?; + + let signed_headers: Vec<(String, String)> = sign(signable, ¶ms.into()) + .context("failed to sign request")? + .into_parts() + .0 + .headers() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + + let mut request = self + .http + .get(&url) + .header("x-amz-content-sha256", EMPTY_BODY_SHA256) + .build() + .context("failed to build request")?; + for (name, value) in signed_headers { + request.headers_mut().insert( + http::HeaderName::from_bytes(name.as_bytes()) + .context("invalid signed header name")?, + value.parse().context("invalid signed header value")?, + ); + } + + let response = self + .http + .execute(request) + .await + .context("list_objects request failed")?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("list_objects failed status={} body={}", status, body); + } + + let body = response + .text() + .await + .context("failed to read list_objects response")?; + parse_list_objects_xml(&body) + } + + pub async fn delete_object( + &self, + bucket: &str, + access_key_id: &str, + secret_access_key: &str, + key: &str, + ) -> Result<()> { + let url = format!( + "{}/{}/{}", + self.s3_url, + bucket, + percent_encoding::utf8_percent_encode(key, SIGV4_UNRESERVED) + ); + + let identity = Self::identity(access_key_id, secret_access_key); + let params = Self::signing_params(&identity, SigningSettings::default())?; + + let signable = SignableRequest::new( + "DELETE", + &url, + std::iter::once(("x-amz-content-sha256", EMPTY_BODY_SHA256)), + SignableBody::Bytes(&[]), + ) + .context("failed to build signable request")?; + + let signed_headers: Vec<(String, String)> = sign(signable, ¶ms.into()) + .context("failed to sign request")? + .into_parts() + .0 + .headers() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect(); + + let mut request = self + .http + .delete(&url) + .header("x-amz-content-sha256", EMPTY_BODY_SHA256) + .build() + .context("failed to build request")?; + for (name, value) in signed_headers { + request.headers_mut().insert( + http::HeaderName::from_bytes(name.as_bytes()) + .context("invalid signed header name")?, + value.parse().context("invalid signed header value")?, + ); + } + + let response = self + .http + .execute(request) + .await + .context("delete_object request failed")?; + + if !response.status().is_success() && response.status().as_u16() != 404 { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("delete_object failed status={} body={}", status, body); + } + + Ok(()) + } + + pub fn presign_url( + &self, + method: &str, + bucket: &str, + key: &str, + access_key_id: &str, + secret_access_key: &str, + expires_in: Duration, + ) -> Result { + let url = format!( + "{}/{}/{}", + self.public_s3_url, + bucket, + percent_encoding::utf8_percent_encode(key, SIGV4_UNRESERVED) + ); + + let identity = Self::identity(access_key_id, secret_access_key); + let mut settings = SigningSettings::default(); + settings.percent_encoding_mode = PercentEncodingMode::Single; + settings.signature_location = SignatureLocation::QueryParams; + settings.expires_in = Some(expires_in); + let params = Self::signing_params(&identity, settings)?; + + let signable = SignableRequest::new( + method, + &url, + std::iter::empty(), + SignableBody::UnsignedPayload, + ) + .context("failed to build signable request")?; + + let (instructions, _) = sign(signable, ¶ms.into()) + .context("failed to sign presigned url")? + .into_parts(); + + let mut request = http::Request::builder() + .method(method) + .uri(&url) + .body(()) + .context("failed to build presign request")?; + instructions.apply_to_request_http1x(&mut request); + + Ok(request.uri().to_string()) + } +} + +fn parse_list_objects_xml(body: &str) -> Result { + let mut result = ListObjectsResult::default(); + + for segment in body.split("").skip(1) { + let end = segment.find("").unwrap_or(segment.len()); + let entry = &segment[..end]; + let key = extract_xml_tag(entry, "Key").unwrap_or_default(); + let size = extract_xml_tag(entry, "Size") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + let last_modified = extract_xml_tag(entry, "LastModified").unwrap_or_default(); + if !key.is_empty() { + result.objects.push(S3Object { + key, + size, + last_modified, + }); + } + } + + for segment in body.split("").skip(1) { + let end = segment.find("").unwrap_or(segment.len()); + let entry = &segment[..end]; + if let Some(prefix) = extract_xml_tag(entry, "Prefix") { + result.common_prefixes.push(prefix); + } + } + + Ok(result) +} + +fn extract_xml_tag(input: &str, tag: &str) -> Option { + let open = format!("<{}>", tag); + let close = format!("", tag); + let start = input.find(&open)? + open.len(); + let end = input[start..].find(&close)? + start; + Some(input[start..end].to_string()) +} diff --git a/control-plane/object-storage/src/handlers/buckets.rs b/control-plane/object-storage/src/handlers/buckets.rs new file mode 100644 index 00000000..55229556 --- /dev/null +++ b/control-plane/object-storage/src/handlers/buckets.rs @@ -0,0 +1,113 @@ +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde::Deserialize; +use serde_json::json; +use uuid::Uuid; + +use crate::{ + models::{CreateBucketRequest, UpdateBucketRequest}, + server::AppState, + services::bucket as service, +}; + +#[derive(Debug, Deserialize)] +pub struct ListBucketsQuery { + pub resource_group_id: Option, + pub organization_id: Option, +} + +pub async fn create_bucket( + State(state): State, + Json(req): Json, +) -> Result)> { + match service::create_bucket(&state.db, &state.garage, &state.secret_box, req).await { + Ok(bucket) => Ok((StatusCode::CREATED, Json(json!(bucket)))), + Err(e) => { + tracing::error!(error = %e, "failed to create bucket"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn list_buckets( + State(state): State, + Query(query): Query, +) -> Result)> { + match service::list_buckets(&state.db, query.resource_group_id, query.organization_id).await { + Ok(buckets) => Ok(Json(json!(buckets))), + Err(e) => { + tracing::error!(error = %e, "failed to list buckets"); + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn get_bucket( + State(state): State, + Path(id): Path, +) -> Result)> { + match service::get_bucket(&state.db, id).await { + Ok(Some(bucket)) => Ok(Json(json!(bucket))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to get bucket"); + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn update_bucket( + State(state): State, + Path(id): Path, + Json(req): Json, +) -> Result)> { + match service::update_bucket(&state.db, &state.garage, id, req).await { + Ok(Some(bucket)) => Ok(Json(json!(bucket))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to update bucket"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn delete_bucket( + State(state): State, + Path(id): Path, +) -> Result)> { + match service::delete_bucket(&state.db, &state.garage, id).await { + Ok(true) => Ok(StatusCode::NO_CONTENT), + Ok(false) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to delete bucket"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} diff --git a/control-plane/object-storage/src/handlers/cluster.rs b/control-plane/object-storage/src/handlers/cluster.rs new file mode 100644 index 00000000..2a69b0ff --- /dev/null +++ b/control-plane/object-storage/src/handlers/cluster.rs @@ -0,0 +1,38 @@ +use axum::{ + extract::State, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde_json::json; + +use crate::{ + db::garage_nodes, garage::layout::replication_factor_for, models::ClusterStatusResponse, + server::AppState, +}; + +pub async fn get_cluster_status( + State(state): State, +) -> Result)> { + let nodes = garage_nodes::list(&state.db).await.map_err(|e| { + tracing::error!(error = %e, "failed to list garage nodes"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + ) + })?; + + let storage_node_count = nodes + .iter() + .filter(|n| n.role == "storage" && n.status == "up") + .count(); + + let replication_factor = replication_factor_for(storage_node_count); + let degraded = storage_node_count == 0 || (storage_node_count as u32) < replication_factor; + + Ok(Json(json!(ClusterStatusResponse { + storage_node_count: storage_node_count as u32, + replication_factor, + degraded, + nodes, + }))) +} diff --git a/control-plane/object-storage/src/handlers/keys.rs b/control-plane/object-storage/src/handlers/keys.rs new file mode 100644 index 00000000..cb8f1f92 --- /dev/null +++ b/control-plane/object-storage/src/handlers/keys.rs @@ -0,0 +1,86 @@ +use axum::{ + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde_json::json; +use uuid::Uuid; + +use crate::{models::CreateAccessKeyRequest, server::AppState, services::access_key as service}; + +pub async fn create_key( + State(state): State, + Path(bucket_id): Path, + Json(req): Json, +) -> Result)> { + match service::create_key(&state.db, &state.garage, bucket_id, req).await { + Ok(Some(key)) => Ok((StatusCode::CREATED, Json(json!(key)))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to create access key"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn list_keys( + State(state): State, + Path(bucket_id): Path, +) -> Result)> { + match service::list_keys(&state.db, bucket_id).await { + Ok(keys) => Ok(Json(json!(keys))), + Err(e) => { + tracing::error!(error = %e, "failed to list access keys"); + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn rotate_key( + State(state): State, + Path((bucket_id, key_id)): Path<(Uuid, Uuid)>, +) -> Result)> { + match service::rotate_key(&state.db, &state.garage, bucket_id, key_id).await { + Ok(Some(key)) => Ok(Json(json!(key))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Access key not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to rotate access key"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn delete_key( + State(state): State, + Path((bucket_id, key_id)): Path<(Uuid, Uuid)>, +) -> Result)> { + match service::delete_key(&state.db, &state.garage, bucket_id, key_id).await { + Ok(true) => Ok(StatusCode::NO_CONTENT), + Ok(false) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Access key not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to delete access key"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} diff --git a/control-plane/object-storage/src/handlers/mod.rs b/control-plane/object-storage/src/handlers/mod.rs new file mode 100644 index 00000000..f1397a84 --- /dev/null +++ b/control-plane/object-storage/src/handlers/mod.rs @@ -0,0 +1,4 @@ +pub mod buckets; +pub mod cluster; +pub mod keys; +pub mod objects; diff --git a/control-plane/object-storage/src/handlers/objects.rs b/control-plane/object-storage/src/handlers/objects.rs new file mode 100644 index 00000000..4791d2af --- /dev/null +++ b/control-plane/object-storage/src/handlers/objects.rs @@ -0,0 +1,107 @@ +use axum::{ + extract::{Path, Query, State}, + http::StatusCode, + response::{IntoResponse, Json}, +}; +use serde_json::json; +use uuid::Uuid; + +use crate::{ + models::{ListObjectsQuery, PresignUploadRequest}, + server::AppState, + services::object as service, +}; + +pub async fn list_objects( + State(state): State, + Path(bucket_id): Path, + Query(query): Query, +) -> Result)> { + match service::list_objects( + &state.db, + &state.s3, + &state.secret_box, + bucket_id, + &query.prefix, + query.continuation_token.as_deref(), + ) + .await + { + Ok(Some(result)) => Ok(Json(json!(result))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to list objects"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn delete_object( + State(state): State, + Path((bucket_id, key)): Path<(Uuid, String)>, +) -> Result)> { + match service::delete_object(&state.db, &state.s3, &state.secret_box, bucket_id, &key).await { + Ok(true) => Ok(StatusCode::NO_CONTENT), + Ok(false) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to delete object"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn presign_upload( + State(state): State, + Path(bucket_id): Path, + Json(req): Json, +) -> Result)> { + match service::presign_upload(&state.db, &state.s3, &state.secret_box, bucket_id, &req.key) + .await + { + Ok(Some(result)) => Ok(Json(json!(result))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to presign upload"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} + +pub async fn presign_download( + State(state): State, + Path((bucket_id, key)): Path<(Uuid, String)>, +) -> Result)> { + match service::presign_download(&state.db, &state.s3, &state.secret_box, bucket_id, &key).await + { + Ok(Some(result)) => Ok(Json(json!(result))), + Ok(None) => Err(( + StatusCode::NOT_FOUND, + Json(json!({"error": "Bucket not found"})), + )), + Err(e) => { + tracing::error!(error = %e, "failed to presign download"); + Err(( + StatusCode::BAD_GATEWAY, + Json(json!({"error": e.to_string()})), + )) + } + } +} diff --git a/control-plane/object-storage/src/logger.rs b/control-plane/object-storage/src/logger.rs new file mode 100644 index 00000000..7d8b6529 --- /dev/null +++ b/control-plane/object-storage/src/logger.rs @@ -0,0 +1,152 @@ +use opentelemetry::trace::TracerProvider as _; +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::trace::SdkTracerProvider; +use tracing::{event, Level}; +use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; + +#[derive(Debug, Clone, Copy)] +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, +} + +impl From for Level { + fn from(level: LogLevel) -> Self { + match level { + LogLevel::Trace => Level::TRACE, + LogLevel::Debug => Level::DEBUG, + LogLevel::Info => Level::INFO, + LogLevel::Warn => Level::WARN, + LogLevel::Error => Level::ERROR, + } + } +} + +fn build_otlp_provider(service_name: &str) -> Option { + let endpoint = std::env::var("OTLP_ENDPOINT").ok()?; + + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .build() + .ok()?; + + let resource = opentelemetry_sdk::Resource::builder_empty() + .with_attribute(KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAME, + service_name.to_string(), + )) + .build(); + + let provider = SdkTracerProvider::builder() + .with_resource(resource) + .with_batch_exporter(exporter) + .build(); + + Some(provider) +} + +pub fn init_logger() -> tokio::sync::mpsc::UnboundedReceiver { + init_logger_with_service(env!("CARGO_PKG_NAME")) +} + +pub fn init_logger_with_service( + service_name: &'static str, +) -> tokio::sync::mpsc::UnboundedReceiver { + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let fmt_layer = fmt::layer().with_target(false).with_thread_ids(true); + let (db_layer, db_receiver) = shared::db_log_layer::DbLogLayer::new(service_name); + + let registry = tracing_subscriber::registry() + .with(filter) + .with(fmt_layer) + .with(db_layer); + + match build_otlp_provider(service_name) { + Some(provider) => { + let tracer = provider.tracer(service_name); + let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer); + registry.with(otel_layer).init(); + tracing::info!(service = service_name, "OpenTelemetry tracing enabled"); + } + None => { + registry.init(); + } + } + + db_receiver +} + +pub fn log_message(level: LogLevel, module: &str, location: &str, description: &str) { + let lvl: Level = level.into(); + + match lvl { + Level::ERROR => { + event!(Level::ERROR, module = %module, location = %location, "{}", description) + } + Level::WARN => { + event!(Level::WARN, module = %module, location = %location, "{}", description) + } + Level::INFO => { + event!(Level::INFO, module = %module, location = %location, "{}", description) + } + Level::DEBUG => { + event!(Level::DEBUG, module = %module, location = %location, "{}", description) + } + Level::TRACE => { + event!(Level::TRACE, module = %module, location = %location, "{}", description) + } + } +} + +#[macro_export] +macro_rules! log_info { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Info, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} + +#[macro_export] +macro_rules! log_warn { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Warn, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} + +#[macro_export] +macro_rules! log_error { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Error, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} + +#[macro_export] +macro_rules! log_debug { + ($module:expr, $desc:expr) => { + $crate::logger::log_message( + $crate::logger::LogLevel::Debug, + $module, + concat!(file!(), ":", line!()), + $desc, + ) + }; +} diff --git a/control-plane/object-storage/src/main.rs b/control-plane/object-storage/src/main.rs new file mode 100644 index 00000000..430fc26b --- /dev/null +++ b/control-plane/object-storage/src/main.rs @@ -0,0 +1,102 @@ +use std::net::SocketAddr; +use uuid::Uuid; + +mod crypto; +mod db; +mod garage; +mod handlers; +mod logger; +mod metrics; +mod models; +mod server; +mod services; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + rustls::crypto::ring::default_provider() + .install_default() + .expect("failed to install ring crypto provider"); + + dotenvy::dotenv().ok(); + + let log_receiver = logger::init_logger(); + + metrics::init(); + log_info!("main", "CSFX Object Storage starting..."); + log_info!("main", &format!("Version: {}", env!("CARGO_PKG_VERSION"))); + + log_info!("main", "Connecting to database..."); + let db = shared::establish_connection() + .await + .expect("Failed to connect to database"); + log_info!("main", "Database connection established"); + shared::spawn_log_writer(log_receiver, db.clone()); + + let admin_url = + std::env::var("GARAGE_ADMIN_URL").unwrap_or_else(|_| "http://127.0.0.1:3903".to_string()); + let admin_token = std::env::var("GARAGE_ADMIN_TOKEN").expect("GARAGE_ADMIN_TOKEN must be set"); + let garage = garage::GarageClient::new(admin_url, admin_token); + + let s3_url = + std::env::var("GARAGE_S3_URL").unwrap_or_else(|_| "http://127.0.0.1:3900".to_string()); + let public_s3_url = std::env::var("GARAGE_PUBLIC_S3_URL").unwrap_or_else(|_| s3_url.clone()); + let s3_client = garage::S3Client::new(s3_url, public_s3_url); + + let etcd_url = + std::env::var("ETCD_URL").unwrap_or_else(|_| "http://localhost:2379".to_string()); + let etcd = etcd_client::Client::connect([etcd_url.as_str()], None) + .await + .expect("Failed to connect to etcd"); + log_info!("main", "etcd connection established"); + + let node_id = Uuid::new_v4().to_string(); + let leader = garage::leader::LayoutLeader::new(etcd, node_id); + + tokio::spawn(leader.clone().run_campaign_loop()); + tokio::spawn(garage::layout::run_reconcile_loop( + db.clone(), + garage.clone(), + leader, + )); + + let self_register_zone = std::env::var("GARAGE_ZONE").unwrap_or_else(|_| "dev".to_string()); + tokio::spawn({ + let db = db.clone(); + let garage = garage.clone(); + async move { + garage::layout::register_self_as_node(&db, &garage, &self_register_zone).await; + } + }); + + let secret_box = std::sync::Arc::new( + crypto::SecretBox::from_env().expect("Failed to initialize encryption key"), + ); + + let state = server::AppState::new(db, garage, s3_client, secret_box); + let app = server::create_router(state); + + let port = std::env::var("OBJECT_STORAGE_PORT") + .ok() + .and_then(|p| p.parse::().ok()) + .unwrap_or(8006); + + let listen_addr = std::env::var("LISTEN_ADDR").unwrap_or_else(|_| "127.0.0.1".to_string()); + let addr: SocketAddr = format!("{}:{}", listen_addr, port).parse().unwrap(); + log_info!("main", &format!("Object Storage listening port={}", port)); + + let listener = tokio::net::TcpListener::bind(addr).await?; + + tokio::select! { + result = axum::serve(listener, app) => { + if let Err(e) = result { + log_error!("main", &format!("Server error err={}", e)); + } + } + _ = tokio::signal::ctrl_c() => { + log_info!("main", "Shutdown signal received"); + } + } + + log_info!("main", "Object Storage shutting down"); + Ok(()) +} diff --git a/control-plane/object-storage/src/metrics.rs b/control-plane/object-storage/src/metrics.rs new file mode 100644 index 00000000..aa5a9792 --- /dev/null +++ b/control-plane/object-storage/src/metrics.rs @@ -0,0 +1,57 @@ +use axum::response::IntoResponse; +use prometheus::{ + register_counter_vec, register_histogram_vec, CounterVec, Encoder, HistogramVec, TextEncoder, +}; +use std::sync::OnceLock; + +static HTTP_REQUESTS_TOTAL: OnceLock = OnceLock::new(); +static HTTP_REQUEST_DURATION_SECONDS: OnceLock = OnceLock::new(); + +pub fn init() { + HTTP_REQUESTS_TOTAL.get_or_init(|| { + register_counter_vec!( + "csfx_http_requests_total", + "Total HTTP requests", + &["method", "path", "status"] + ) + .expect("failed to register csfx_http_requests_total") + }); + + HTTP_REQUEST_DURATION_SECONDS.get_or_init(|| { + register_histogram_vec!( + "csfx_http_request_duration_seconds", + "HTTP request duration in seconds", + &["method", "path"] + ) + .expect("failed to register csfx_http_request_duration_seconds") + }); +} + +pub fn record_request(method: &str, path: &str, status: u16, duration_secs: f64) { + if let Some(counter) = HTTP_REQUESTS_TOTAL.get() { + counter + .with_label_values(&[method, path, &status.to_string()]) + .inc(); + } + if let Some(histogram) = HTTP_REQUEST_DURATION_SECONDS.get() { + histogram + .with_label_values(&[method, path]) + .observe(duration_secs); + } +} + +pub async fn metrics_handler() -> impl IntoResponse { + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buffer = Vec::new(); + encoder + .encode(&metric_families, &mut buffer) + .expect("failed to encode metrics"); + ( + [( + axum::http::header::CONTENT_TYPE, + "text/plain; version=0.0.4", + )], + buffer, + ) +} diff --git a/control-plane/object-storage/src/models.rs b/control-plane/object-storage/src/models.rs new file mode 100644 index 00000000..05939a53 --- /dev/null +++ b/control-plane/object-storage/src/models.rs @@ -0,0 +1,101 @@ +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateBucketRequest { + pub name: String, + pub exposure: Option, + pub quota_max_size: Option, + pub quota_max_objects: Option, + pub organization_id: Option, + pub resource_group_id: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct UpdateBucketRequest { + pub exposure: Option, + pub quota_max_size: Option, + pub quota_max_objects: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct CreateAccessKeyRequest { + pub name: String, + pub permissions: Option, + pub expires_at: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct AccessKeyResponse { + pub id: Uuid, + pub bucket_id: Uuid, + pub name: String, + pub garage_key_id: String, + pub permissions: String, + pub expires_at: Option, + pub last_rotated_at: Option, + pub created_at: chrono::NaiveDateTime, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct AccessKeyCreatedResponse { + #[serde(flatten)] + pub key: AccessKeyResponse, + pub secret_access_key: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ObjectEntry { + pub key: String, + pub size: i64, + pub last_modified: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ListObjectsResponse { + pub objects: Vec, + pub folders: Vec, + pub next_continuation_token: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ListObjectsQuery { + #[serde(default)] + pub prefix: String, + #[serde(default)] + pub continuation_token: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PresignUploadRequest { + pub key: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct PresignResponse { + pub url: String, + pub expires_in_seconds: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ClusterStatusResponse { + pub storage_node_count: u32, + pub replication_factor: u32, + pub degraded: bool, + pub nodes: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct BucketResponse { + pub id: Uuid, + pub name: String, + pub global_alias: String, + pub exposure: String, + pub quota_max_size: Option, + pub quota_max_objects: Option, + pub status: String, + pub organization_id: Option, + pub resource_group_id: Option, + pub created_at: chrono::NaiveDateTime, + pub updated_at: Option, +} diff --git a/control-plane/object-storage/src/server.rs b/control-plane/object-storage/src/server.rs new file mode 100644 index 00000000..5e07ba8f --- /dev/null +++ b/control-plane/object-storage/src/server.rs @@ -0,0 +1,81 @@ +use axum::{http::StatusCode, response::IntoResponse, routing::get, Router}; +use sea_orm::DatabaseConnection; +use std::sync::Arc; + +use crate::{ + crypto::SecretBox, + garage::{GarageClient, S3Client}, + handlers::{buckets, cluster, keys, objects}, + metrics, +}; + +#[derive(Clone)] +pub struct AppState { + pub db: DatabaseConnection, + pub garage: GarageClient, + pub s3: S3Client, + pub secret_box: Arc, +} + +impl AppState { + pub fn new( + db: DatabaseConnection, + garage: GarageClient, + s3: S3Client, + secret_box: Arc, + ) -> Self { + Self { + db, + garage, + s3, + secret_box, + } + } +} + +pub async fn health_check() -> impl IntoResponse { + (StatusCode::OK, "Object Storage OK") +} + +pub fn create_router(state: AppState) -> Router { + Router::new() + .route("/health", get(health_check)) + .route("/metrics", get(metrics::metrics_handler)) + .route( + "/buckets", + get(buckets::list_buckets).post(buckets::create_bucket), + ) + .route( + "/buckets/{id}", + get(buckets::get_bucket) + .patch(buckets::update_bucket) + .delete(buckets::delete_bucket), + ) + .route( + "/buckets/{id}/keys", + get(keys::list_keys).post(keys::create_key), + ) + .route( + "/buckets/{id}/keys/{key_id}/rotate", + axum::routing::post(keys::rotate_key), + ) + .route( + "/buckets/{id}/keys/{key_id}", + axum::routing::delete(keys::delete_key), + ) + .route("/buckets/{id}/objects", get(objects::list_objects)) + .route( + "/buckets/{id}/objects/presign-upload", + axum::routing::post(objects::presign_upload), + ) + .route( + "/buckets/{bucket_id}/objects/presign-download/{*key}", + get(objects::presign_download), + ) + .route( + "/buckets/{bucket_id}/objects/{*key}", + axum::routing::delete(objects::delete_object), + ) + .route("/cluster", get(cluster::get_cluster_status)) + .with_state(state) +} diff --git a/control-plane/object-storage/src/services/access_key.rs b/control-plane/object-storage/src/services/access_key.rs new file mode 100644 index 00000000..fbfa8c76 --- /dev/null +++ b/control-plane/object-storage/src/services/access_key.rs @@ -0,0 +1,165 @@ +use anyhow::{bail, Result}; +use sea_orm::DatabaseConnection; +use uuid::Uuid; + +use crate::{ + db::{access_keys as keys_db, buckets as buckets_db}, + garage::GarageClient, + log_error, log_info, + models::{AccessKeyCreatedResponse, AccessKeyResponse, CreateAccessKeyRequest}, +}; + +fn into_response(model: entity::entities::bucket_access_keys::Model) -> AccessKeyResponse { + AccessKeyResponse { + id: model.id, + bucket_id: model.bucket_id, + name: model.name, + garage_key_id: model.garage_key_id, + permissions: model.permissions, + expires_at: model.expires_at, + last_rotated_at: model.last_rotated_at, + created_at: model.created_at, + } +} + +pub async fn create_key( + db: &DatabaseConnection, + garage: &GarageClient, + bucket_id: Uuid, + req: CreateAccessKeyRequest, +) -> Result> { + let Some(bucket) = buckets_db::get_by_id(db, bucket_id).await? else { + return Ok(None); + }; + let Some(garage_bucket_id) = &bucket.garage_bucket_id else { + bail!("bucket has no garage_bucket_id, cannot create access key"); + }; + + let permissions = req.permissions.unwrap_or_else(|| "readwrite".to_string()); + + let garage_key = match garage.create_key(&req.name).await { + Ok(key) => key, + Err(e) => { + log_error!( + "services::access_key", + &format!("garage create_key failed name={} err={}", req.name, e) + ); + bail!("failed to create access key in garage: {}", e); + } + }; + + if let Err(e) = garage + .allow_bucket_key(garage_bucket_id, &garage_key.access_key_id, &permissions) + .await + { + log_error!( + "services::access_key", + &format!( + "garage allow_bucket_key failed key_id={} err={}", + garage_key.access_key_id, e + ) + ); + let _ = garage.delete_key(&garage_key.access_key_id).await; + bail!("failed to grant bucket access: {}", e); + } + + let model = keys_db::insert( + db, + bucket_id, + &req.name, + &garage_key.access_key_id, + &permissions, + req.expires_at, + ) + .await?; + + log_info!( + "services::access_key", + &format!("access key created id={} bucket_id={}", model.id, bucket_id) + ); + + Ok(Some(AccessKeyCreatedResponse { + key: into_response(model), + secret_access_key: garage_key.secret_access_key, + })) +} + +pub async fn list_keys(db: &DatabaseConnection, bucket_id: Uuid) -> Result> { + let rows = keys_db::list_for_bucket(db, bucket_id).await?; + Ok(rows.into_iter().map(into_response).collect()) +} + +pub async fn rotate_key( + db: &DatabaseConnection, + garage: &GarageClient, + bucket_id: Uuid, + key_id: Uuid, +) -> Result> { + let Some(existing) = keys_db::get_by_id(db, key_id).await? else { + return Ok(None); + }; + if existing.bucket_id != bucket_id { + return Ok(None); + } + + let created = create_key( + db, + garage, + bucket_id, + CreateAccessKeyRequest { + name: existing.name.clone(), + permissions: Some(existing.permissions.clone()), + expires_at: existing.expires_at, + }, + ) + .await?; + + let Some(created) = created else { + return Ok(None); + }; + + if let Err(e) = garage.delete_key(&existing.garage_key_id).await { + log_error!( + "services::access_key", + &format!( + "garage delete_key on rotate failed old_key_id={} err={}", + existing.garage_key_id, e + ) + ); + } + keys_db::delete(db, key_id).await?; + keys_db::touch_rotated(db, created.key.id).await?; + + log_info!( + "services::access_key", + &format!( + "access key rotated old_id={} new_id={}", + key_id, created.key.id + ) + ); + + Ok(Some(created)) +} + +pub async fn delete_key( + db: &DatabaseConnection, + garage: &GarageClient, + bucket_id: Uuid, + key_id: Uuid, +) -> Result { + let Some(existing) = keys_db::get_by_id(db, key_id).await? else { + return Ok(false); + }; + if existing.bucket_id != bucket_id { + return Ok(false); + } + + garage.delete_key(&existing.garage_key_id).await?; + keys_db::delete(db, key_id).await?; + + log_info!( + "services::access_key", + &format!("access key deleted id={}", key_id) + ); + Ok(true) +} diff --git a/control-plane/object-storage/src/services/bucket.rs b/control-plane/object-storage/src/services/bucket.rs new file mode 100644 index 00000000..57175d4e --- /dev/null +++ b/control-plane/object-storage/src/services/bucket.rs @@ -0,0 +1,154 @@ +use anyhow::{bail, Result}; +use sea_orm::DatabaseConnection; +use uuid::Uuid; + +use crate::{ + crypto::SecretBox, + db::buckets as db, + garage::GarageClient, + log_error, log_info, + models::{BucketResponse, CreateBucketRequest, UpdateBucketRequest}, +}; + +pub async fn create_bucket( + db_conn: &DatabaseConnection, + garage: &GarageClient, + secret_box: &SecretBox, + req: CreateBucketRequest, +) -> Result { + let global_alias = format!("{}-{}", req.name, Uuid::new_v4().simple()); + + let garage_bucket = match garage.create_bucket(&global_alias).await { + Ok(bucket) => bucket, + Err(e) => { + log_error!( + "services::bucket", + &format!("garage create_bucket failed name={} err={}", req.name, e) + ); + bail!("failed to create bucket in garage: {}", e); + } + }; + + if req.quota_max_size.is_some() || req.quota_max_objects.is_some() { + if let Err(e) = garage + .update_bucket_quotas(&garage_bucket.id, req.quota_max_size, req.quota_max_objects) + .await + { + log_error!( + "services::bucket", + &format!( + "garage update_bucket_quotas failed bucket_id={} err={}", + garage_bucket.id, e + ) + ); + let _ = garage.delete_bucket(&garage_bucket.id).await; + bail!("failed to apply bucket quota: {}", e); + } + } + + let master_key = match garage.create_key(&format!("{}-master", global_alias)).await { + Ok(key) => key, + Err(e) => { + log_error!( + "services::bucket", + &format!( + "garage create_key for master key failed bucket_id={} err={}", + garage_bucket.id, e + ) + ); + let _ = garage.delete_bucket(&garage_bucket.id).await; + bail!("failed to create master access key: {}", e); + } + }; + + if let Err(e) = garage + .allow_bucket_key(&garage_bucket.id, &master_key.access_key_id, "owner") + .await + { + log_error!( + "services::bucket", + &format!( + "garage allow_bucket_key for master key failed bucket_id={} err={}", + garage_bucket.id, e + ) + ); + let _ = garage.delete_key(&master_key.access_key_id).await; + let _ = garage.delete_bucket(&garage_bucket.id).await; + bail!("failed to grant master key access: {}", e); + } + + let encrypted_secret = secret_box.encrypt(&master_key.secret_access_key)?; + + let model = db::insert( + db_conn, + &req, + &global_alias, + &garage_bucket.id, + &master_key.access_key_id, + encrypted_secret, + ) + .await?; + log_info!( + "services::bucket", + &format!( + "bucket created id={} garage_bucket_id={}", + model.id, garage_bucket.id + ) + ); + + Ok(db::into_response(model)) +} + +pub async fn get_bucket(db_conn: &DatabaseConnection, id: Uuid) -> Result> { + Ok(db::get_by_id(db_conn, id).await?.map(db::into_response)) +} + +pub async fn list_buckets( + db_conn: &DatabaseConnection, + resource_group_id: Option, + organization_id: Option, +) -> Result> { + let rows = db::list(db_conn, resource_group_id, organization_id).await?; + Ok(rows.into_iter().map(db::into_response).collect()) +} + +pub async fn update_bucket( + db_conn: &DatabaseConnection, + garage: &GarageClient, + id: Uuid, + req: UpdateBucketRequest, +) -> Result> { + let Some(existing) = db::get_by_id(db_conn, id).await? else { + return Ok(None); + }; + + if req.quota_max_size.is_some() || req.quota_max_objects.is_some() { + let Some(garage_bucket_id) = &existing.garage_bucket_id else { + bail!("bucket has no garage_bucket_id, cannot update quota"); + }; + garage + .update_bucket_quotas(garage_bucket_id, req.quota_max_size, req.quota_max_objects) + .await?; + } + + let updated = db::update(db_conn, id, &req).await?.map(db::into_response); + Ok(updated) +} + +pub async fn delete_bucket( + db_conn: &DatabaseConnection, + garage: &GarageClient, + id: Uuid, +) -> Result { + let Some(existing) = db::get_by_id(db_conn, id).await? else { + return Ok(false); + }; + + if let Some(garage_bucket_id) = &existing.garage_bucket_id { + garage.delete_bucket(garage_bucket_id).await?; + } + + db::delete(db_conn, id).await?; + log_info!("services::bucket", &format!("bucket deleted id={}", id)); + Ok(true) +} diff --git a/control-plane/object-storage/src/services/mod.rs b/control-plane/object-storage/src/services/mod.rs new file mode 100644 index 00000000..6d8d394a --- /dev/null +++ b/control-plane/object-storage/src/services/mod.rs @@ -0,0 +1,3 @@ +pub mod access_key; +pub mod bucket; +pub mod object; diff --git a/control-plane/object-storage/src/services/object.rs b/control-plane/object-storage/src/services/object.rs new file mode 100644 index 00000000..8b0f5420 --- /dev/null +++ b/control-plane/object-storage/src/services/object.rs @@ -0,0 +1,145 @@ +use anyhow::{bail, Result}; +use sea_orm::DatabaseConnection; +use std::time::Duration; +use uuid::Uuid; + +use crate::{ + crypto::SecretBox, + db::buckets as buckets_db, + garage::S3Client, + models::{ListObjectsResponse, ObjectEntry, PresignResponse}, +}; + +const PRESIGN_EXPIRY_SECONDS: u64 = 900; + +async fn resolve_master_credentials( + db: &DatabaseConnection, + secret_box: &SecretBox, + bucket_id: Uuid, +) -> Result<(String, String, String)> { + let Some(bucket) = buckets_db::get_by_id(db, bucket_id).await? else { + bail!("bucket not found"); + }; + + let (Some(access_key_id), Some(encrypted_secret)) = + (bucket.master_key_id, bucket.master_key_secret_encrypted) + else { + bail!("bucket has no master key configured"); + }; + + let secret_access_key = secret_box.decrypt(&encrypted_secret)?; + + Ok((bucket.global_alias, access_key_id, secret_access_key)) +} + +pub async fn list_objects( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + prefix: &str, + continuation_token: Option<&str>, +) -> Result> { + let (global_alias, access_key_id, secret_access_key) = + match resolve_master_credentials(db, secret_box, bucket_id).await { + Ok(creds) => creds, + Err(e) if e.to_string() == "bucket not found" => return Ok(None), + Err(e) => return Err(e), + }; + + let result = s3 + .list_objects( + &global_alias, + &access_key_id, + &secret_access_key, + prefix, + "/", + continuation_token, + ) + .await?; + + Ok(Some(ListObjectsResponse { + objects: result + .objects + .into_iter() + .map(|o| ObjectEntry { + key: o.key, + size: o.size, + last_modified: o.last_modified, + }) + .collect(), + folders: result.common_prefixes, + next_continuation_token: None, + })) +} + +pub async fn delete_object( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, +) -> Result { + let (global_alias, access_key_id, secret_access_key) = + match resolve_master_credentials(db, secret_box, bucket_id).await { + Ok(creds) => creds, + Err(e) if e.to_string() == "bucket not found" => return Ok(false), + Err(e) => return Err(e), + }; + + s3.delete_object(&global_alias, &access_key_id, &secret_access_key, key) + .await?; + + Ok(true) +} + +pub async fn presign_upload( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, +) -> Result> { + presign(db, s3, secret_box, bucket_id, key, "PUT").await +} + +pub async fn presign_download( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, +) -> Result> { + presign(db, s3, secret_box, bucket_id, key, "GET").await +} + +async fn presign( + db: &DatabaseConnection, + s3: &S3Client, + secret_box: &SecretBox, + bucket_id: Uuid, + key: &str, + method: &str, +) -> Result> { + let (global_alias, access_key_id, secret_access_key) = + match resolve_master_credentials(db, secret_box, bucket_id).await { + Ok(creds) => creds, + Err(e) if e.to_string() == "bucket not found" => return Ok(None), + Err(e) => return Err(e), + }; + + let expires_in = Duration::from_secs(PRESIGN_EXPIRY_SECONDS); + let url = s3.presign_url( + method, + &global_alias, + key, + &access_key_id, + &secret_access_key, + expires_in, + )?; + + Ok(Some(PresignResponse { + url, + expires_in_seconds: PRESIGN_EXPIRY_SECONDS, + })) +} diff --git a/control-plane/scheduler/src/models/workload.rs b/control-plane/scheduler/src/models/workload.rs index 1320dce1..a6b87d1b 100644 --- a/control-plane/scheduler/src/models/workload.rs +++ b/control-plane/scheduler/src/models/workload.rs @@ -50,6 +50,13 @@ pub struct VolumeMount { pub mount_path: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BucketBinding { + pub bucket_id: Uuid, + #[serde(default)] + pub permissions: Option, +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum RestartPolicy { @@ -153,6 +160,8 @@ pub struct CreateWorkloadRequest { pub env_vars: Option>, pub ports: Option>, pub volume_mounts: Option>, + #[serde(default)] + pub bucket_bindings: Option>, pub resource_group_id: Option, #[serde(default)] pub stack_id: Option, diff --git a/control-plane/scheduler/src/services/bucket_bindings.rs b/control-plane/scheduler/src/services/bucket_bindings.rs new file mode 100644 index 00000000..048e6c7e --- /dev/null +++ b/control-plane/scheduler/src/services/bucket_bindings.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; + +use crate::models::workload::BucketBinding; + +const OBJECT_STORAGE_URL_ENV: &str = "OBJECT_STORAGE_URL"; +const DEFAULT_OBJECT_STORAGE_URL: &str = "http://localhost:8006"; + +fn base_url() -> String { + std::env::var(OBJECT_STORAGE_URL_ENV).unwrap_or_else(|_| DEFAULT_OBJECT_STORAGE_URL.to_string()) +} + +pub async fn resolve_env_vars( + workload_id: uuid::Uuid, + resource_group_id: Option, + bindings: &[BucketBinding], +) -> HashMap { + let mut env = HashMap::new(); + let client = reqwest::Client::new(); + let base = base_url(); + + for (index, binding) in bindings.iter().enumerate() { + let bucket_id = binding.bucket_id; + + let bucket = match client + .get(format!("{}/buckets/{}", base, bucket_id)) + .send() + .await + { + Ok(resp) if resp.status().is_success() => { + match resp.json::().await { + Ok(body) => body, + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to parse bucket response"); + continue; + } + } + } + Ok(resp) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, status = %resp.status(), "bucket lookup failed"); + continue; + } + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to reach object-storage"); + continue; + } + }; + + let global_alias = match bucket["global_alias"].as_str() { + Some(alias) => alias.to_string(), + None => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, "bucket response missing global_alias"); + continue; + } + }; + + let key_name = format!("workload-{}", workload_id); + let key_body = serde_json::json!({ + "name": key_name, + "permissions": binding.permissions.clone().unwrap_or_else(|| "readwrite".to_string()), + }); + + let key = match client + .post(format!("{}/buckets/{}/keys", base, bucket_id)) + .json(&key_body) + .send() + .await + { + Ok(resp) if resp.status().is_success() => { + match resp.json::().await { + Ok(body) => body, + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to parse access key response"); + continue; + } + } + } + Ok(resp) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, status = %resp.status(), "access key creation failed"); + continue; + } + Err(e) => { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, error = %e, "failed to reach object-storage"); + continue; + } + }; + + let (Some(access_key_id), Some(secret_access_key)) = ( + key["garage_key_id"].as_str(), + key["secret_access_key"].as_str(), + ) else { + tracing::warn!(workload_id = %workload_id, bucket_id = %bucket_id, "access key response missing credentials"); + continue; + }; + + let suffix = if index == 0 { + String::new() + } else { + format!("_{}", index) + }; + + let endpoint = match resource_group_id { + Some(rg_id) => format!("http://s3.svc.{}.internal:3900", rg_id), + None => "http://127.0.0.1:3900".to_string(), + }; + + env.insert( + format!("AWS_ACCESS_KEY_ID{}", suffix), + access_key_id.to_string(), + ); + env.insert( + format!("AWS_SECRET_ACCESS_KEY{}", suffix), + secret_access_key.to_string(), + ); + env.insert(format!("AWS_BUCKET{}", suffix), global_alias); + env.insert(format!("AWS_ENDPOINT_URL{}", suffix), endpoint); + } + + env +} diff --git a/control-plane/scheduler/src/services/mod.rs b/control-plane/scheduler/src/services/mod.rs index b34df0dc..b4cf3c17 100644 --- a/control-plane/scheduler/src/services/mod.rs +++ b/control-plane/scheduler/src/services/mod.rs @@ -1,3 +1,4 @@ +pub mod bucket_bindings; pub mod compose_parser; pub mod etcd; pub mod gateway_notify; diff --git a/control-plane/scheduler/src/services/scheduler.rs b/control-plane/scheduler/src/services/scheduler.rs index 6868e52c..d30524fa 100644 --- a/control-plane/scheduler/src/services/scheduler.rs +++ b/control-plane/scheduler/src/services/scheduler.rs @@ -25,8 +25,23 @@ impl SchedulerService { pub async fn schedule( &self, - req: CreateWorkloadRequest, + mut req: CreateWorkloadRequest, ) -> Result { + if let Some(bindings) = req.bucket_bindings.clone() { + if !bindings.is_empty() { + let placeholder_id = Uuid::new_v4(); + let bucket_env = crate::services::bucket_bindings::resolve_env_vars( + placeholder_id, + req.resource_group_id, + &bindings, + ) + .await; + req.env_vars + .get_or_insert_with(std::collections::HashMap::new) + .extend(bucket_env); + } + } + let workload = crate::db::workloads::create(&self.db, &req) .await .map_err(|e| format!("Failed to persist workload: {}", e))?; @@ -303,6 +318,7 @@ impl SchedulerService { env_vars: service.env_vars.clone(), ports: service.ports.clone(), volume_mounts: None, + bucket_bindings: None, resource_group_id: Some(resource_group_id), stack_id: Some(stack_id), service_name: Some(service.service_name.clone()), diff --git a/control-plane/shared/entity/src/entities/bucket_access_keys.rs b/control-plane/shared/entity/src/entities/bucket_access_keys.rs new file mode 100644 index 00000000..3b4f75c4 --- /dev/null +++ b/control-plane/shared/entity/src/entities/bucket_access_keys.rs @@ -0,0 +1,35 @@ +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "bucket_access_keys")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + pub bucket_id: Uuid, + pub name: String, + pub garage_key_id: String, + pub permissions: String, + pub expires_at: Option, + pub last_rotated_at: Option, + pub created_at: chrono::NaiveDateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::buckets::Entity", + from = "Column::BucketId", + to = "super::buckets::Column::Id", + on_delete = "Cascade" + )] + Bucket, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Bucket.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/entity/src/entities/buckets.rs b/control-plane/shared/entity/src/entities/buckets.rs new file mode 100644 index 00000000..e49313d2 --- /dev/null +++ b/control-plane/shared/entity/src/entities/buckets.rs @@ -0,0 +1,50 @@ +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "buckets")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + pub name: String, + pub garage_bucket_id: Option, + pub global_alias: String, + pub exposure: String, + pub quota_max_size: Option, + pub quota_max_objects: Option, + pub status: String, + pub master_key_id: Option, + pub master_key_secret_encrypted: Option>, + pub organization_id: Option, + pub resource_group_id: Option, + pub created_at: chrono::NaiveDateTime, + pub updated_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm(has_many = "super::bucket_access_keys::Entity")] + AccessKeys, + #[sea_orm( + belongs_to = "super::resource_groups::Entity", + from = "Column::ResourceGroupId", + to = "super::resource_groups::Column::Id", + on_update = "NoAction", + on_delete = "SetNull" + )] + ResourceGroup, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::AccessKeys.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::ResourceGroup.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/entity/src/entities/garage_nodes.rs b/control-plane/shared/entity/src/entities/garage_nodes.rs new file mode 100644 index 00000000..245034ce --- /dev/null +++ b/control-plane/shared/entity/src/entities/garage_nodes.rs @@ -0,0 +1,38 @@ +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "garage_nodes")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + pub agent_id: Option, + pub garage_node_id: Option, + pub zone: String, + pub capacity_bytes: Option, + pub role: String, + pub status: String, + pub layout_version: Option, + pub last_seen_at: Option, + pub created_at: chrono::NaiveDateTime, + pub updated_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::agents::Entity", + from = "Column::AgentId", + to = "super::agents::Column::Id", + on_delete = "Cascade" + )] + Agent, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Agent.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/entity/src/entities/mod.rs b/control-plane/shared/entity/src/entities/mod.rs index 343c9f68..f5decf3d 100644 --- a/control-plane/shared/entity/src/entities/mod.rs +++ b/control-plane/shared/entity/src/entities/mod.rs @@ -3,8 +3,11 @@ pub mod agent_certificates; pub mod agent_metrics; pub mod agents; pub mod bootstrap_tokens; +pub mod bucket_access_keys; +pub mod buckets; pub mod certificate_revocations; pub mod failover_events; +pub mod garage_nodes; pub mod invalid_jwt; pub mod key; pub mod logs; @@ -32,8 +35,11 @@ pub use agent_certificates::Entity as AgentCertificates; pub use agent_metrics::Entity as AgentMetrics; pub use agents::Entity as Agents; pub use bootstrap_tokens::Entity as BootstrapTokens; +pub use bucket_access_keys::Entity as BucketAccessKeys; +pub use buckets::Entity as Buckets; pub use certificate_revocations::Entity as CertificateRevocations; pub use failover_events::Entity as FailoverEvents; +pub use garage_nodes::Entity as GarageNodes; pub use invalid_jwt::Entity as InvalidJwt; pub use key::Entity as Key; pub use logs::Entity as Logs; diff --git a/control-plane/shared/entity/src/entities/resource_groups.rs b/control-plane/shared/entity/src/entities/resource_groups.rs index 3994ea42..d3e0c157 100644 --- a/control-plane/shared/entity/src/entities/resource_groups.rs +++ b/control-plane/shared/entity/src/entities/resource_groups.rs @@ -37,6 +37,8 @@ pub enum Relation { WorkloadStacks, #[sea_orm(has_many = "super::resource_group_vpn_peers::Entity")] ResourceGroupVpnPeers, + #[sea_orm(has_many = "super::buckets::Entity")] + Buckets, } impl Related for Entity { @@ -75,4 +77,10 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Buckets.def() + } +} + impl ActiveModelBehavior for ActiveModel {} diff --git a/control-plane/shared/migration/src/lib.rs b/control-plane/shared/migration/src/lib.rs index aeed8750..72ffb996 100644 --- a/control-plane/shared/migration/src/lib.rs +++ b/control-plane/shared/migration/src/lib.rs @@ -32,6 +32,9 @@ mod m20260712_010000_add_resource_group_vpn_peers; mod m20260723_000000_add_resource_group_appearance; mod m20260723_010000_runtime_class_default_firecracker; mod m20260809_000000_add_user_gravatar_email; +mod m20260816_000000_add_object_storage; +mod m20260816_010000_add_bucket_master_key; +mod m20260816_020000_garage_nodes_agent_id_nullable; pub struct Migrator; @@ -71,6 +74,9 @@ impl MigratorTrait for Migrator { Box::new(m20260723_000000_add_resource_group_appearance::Migration), Box::new(m20260723_010000_runtime_class_default_firecracker::Migration), Box::new(m20260809_000000_add_user_gravatar_email::Migration), + Box::new(m20260816_000000_add_object_storage::Migration), + Box::new(m20260816_010000_add_bucket_master_key::Migration), + Box::new(m20260816_020000_garage_nodes_agent_id_nullable::Migration), ] } } diff --git a/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs b/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs new file mode 100644 index 00000000..e5fb693c --- /dev/null +++ b/control-plane/shared/migration/src/m20260816_000000_add_object_storage.rs @@ -0,0 +1,298 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Buckets::Table) + .if_not_exists() + .col(ColumnDef::new(Buckets::Id).uuid().not_null().primary_key()) + .col(ColumnDef::new(Buckets::Name).string().not_null()) + .col(ColumnDef::new(Buckets::GarageBucketId).string().null()) + .col(ColumnDef::new(Buckets::GlobalAlias).string().not_null()) + .col( + ColumnDef::new(Buckets::Exposure) + .string() + .not_null() + .default("internal"), + ) + .col(ColumnDef::new(Buckets::QuotaMaxSize).big_integer().null()) + .col( + ColumnDef::new(Buckets::QuotaMaxObjects) + .big_integer() + .null(), + ) + .col( + ColumnDef::new(Buckets::Status) + .string() + .not_null() + .default("provisioning"), + ) + .col(ColumnDef::new(Buckets::OrganizationId).uuid().null()) + .col(ColumnDef::new(Buckets::ResourceGroupId).uuid().null()) + .col(ColumnDef::new(Buckets::CreatedAt).date_time().not_null()) + .col(ColumnDef::new(Buckets::UpdatedAt).date_time().null()) + .foreign_key( + ForeignKey::create() + .from(Buckets::Table, Buckets::ResourceGroupId) + .to(ResourceGroups::Table, ResourceGroups::Id) + .on_update(ForeignKeyAction::NoAction) + .on_delete(ForeignKeyAction::SetNull), + ) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(BucketAccessKeys::Table) + .if_not_exists() + .col( + ColumnDef::new(BucketAccessKeys::Id) + .uuid() + .not_null() + .primary_key(), + ) + .col(ColumnDef::new(BucketAccessKeys::BucketId).uuid().not_null()) + .col(ColumnDef::new(BucketAccessKeys::Name).string().not_null()) + .col( + ColumnDef::new(BucketAccessKeys::GarageKeyId) + .string() + .not_null(), + ) + .col( + ColumnDef::new(BucketAccessKeys::Permissions) + .string() + .not_null(), + ) + .col( + ColumnDef::new(BucketAccessKeys::ExpiresAt) + .date_time() + .null(), + ) + .col( + ColumnDef::new(BucketAccessKeys::LastRotatedAt) + .date_time() + .null(), + ) + .col( + ColumnDef::new(BucketAccessKeys::CreatedAt) + .date_time() + .not_null(), + ) + .foreign_key( + ForeignKey::create() + .from(BucketAccessKeys::Table, BucketAccessKeys::BucketId) + .to(Buckets::Table, Buckets::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(GarageNodes::Table) + .if_not_exists() + .col( + ColumnDef::new(GarageNodes::Id) + .uuid() + .not_null() + .primary_key(), + ) + .col(ColumnDef::new(GarageNodes::AgentId).uuid().not_null()) + .col(ColumnDef::new(GarageNodes::GarageNodeId).string().null()) + .col(ColumnDef::new(GarageNodes::Zone).string().not_null()) + .col( + ColumnDef::new(GarageNodes::CapacityBytes) + .big_integer() + .null(), + ) + .col( + ColumnDef::new(GarageNodes::Role) + .string() + .not_null() + .default("storage"), + ) + .col( + ColumnDef::new(GarageNodes::Status) + .string() + .not_null() + .default("unknown"), + ) + .col(ColumnDef::new(GarageNodes::LayoutVersion).integer().null()) + .col(ColumnDef::new(GarageNodes::LastSeenAt).date_time().null()) + .col( + ColumnDef::new(GarageNodes::CreatedAt) + .date_time() + .not_null(), + ) + .col(ColumnDef::new(GarageNodes::UpdatedAt).date_time().null()) + .foreign_key( + ForeignKey::create() + .from(GarageNodes::Table, GarageNodes::AgentId) + .to(Agents::Table, Agents::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(Buckets::Table) + .col(Buckets::ResourceGroupId) + .name("idx_buckets_resource_group_id") + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(Buckets::Table) + .col(Buckets::OrganizationId) + .name("idx_buckets_organization_id") + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(Buckets::Table) + .col(Buckets::GlobalAlias) + .name("idx_buckets_global_alias") + .unique() + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(BucketAccessKeys::Table) + .col(BucketAccessKeys::BucketId) + .name("idx_bucket_access_keys_bucket_id") + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .table(GarageNodes::Table) + .col(GarageNodes::AgentId) + .name("idx_garage_nodes_agent_id") + .unique() + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("idx_garage_nodes_agent_id").to_owned()) + .await?; + manager + .drop_index( + Index::drop() + .name("idx_bucket_access_keys_bucket_id") + .to_owned(), + ) + .await?; + manager + .drop_index(Index::drop().name("idx_buckets_global_alias").to_owned()) + .await?; + manager + .drop_index(Index::drop().name("idx_buckets_organization_id").to_owned()) + .await?; + manager + .drop_index( + Index::drop() + .name("idx_buckets_resource_group_id") + .to_owned(), + ) + .await?; + + manager + .drop_table(Table::drop().table(GarageNodes::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(BucketAccessKeys::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(Buckets::Table).to_owned()) + .await?; + + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Buckets { + Table, + Id, + Name, + GarageBucketId, + GlobalAlias, + Exposure, + QuotaMaxSize, + QuotaMaxObjects, + Status, + OrganizationId, + ResourceGroupId, + CreatedAt, + UpdatedAt, +} + +#[derive(DeriveIden)] +enum BucketAccessKeys { + Table, + Id, + BucketId, + Name, + GarageKeyId, + Permissions, + ExpiresAt, + LastRotatedAt, + CreatedAt, +} + +#[derive(DeriveIden)] +enum GarageNodes { + Table, + Id, + AgentId, + GarageNodeId, + Zone, + CapacityBytes, + Role, + Status, + LayoutVersion, + LastSeenAt, + CreatedAt, + UpdatedAt, +} + +#[derive(DeriveIden)] +enum ResourceGroups { + Table, + Id, +} + +#[derive(DeriveIden)] +enum Agents { + Table, + Id, +} diff --git a/control-plane/shared/migration/src/m20260816_010000_add_bucket_master_key.rs b/control-plane/shared/migration/src/m20260816_010000_add_bucket_master_key.rs new file mode 100644 index 00000000..2eba933e --- /dev/null +++ b/control-plane/shared/migration/src/m20260816_010000_add_bucket_master_key.rs @@ -0,0 +1,37 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("buckets")) + .add_column_if_not_exists( + ColumnDef::new(Alias::new("master_key_id")).string().null(), + ) + .add_column_if_not_exists( + ColumnDef::new(Alias::new("master_key_secret_encrypted")) + .binary() + .null(), + ) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("buckets")) + .drop_column(Alias::new("master_key_secret_encrypted")) + .drop_column(Alias::new("master_key_id")) + .to_owned(), + ) + .await + } +} diff --git a/control-plane/shared/migration/src/m20260816_020000_garage_nodes_agent_id_nullable.rs b/control-plane/shared/migration/src/m20260816_020000_garage_nodes_agent_id_nullable.rs new file mode 100644 index 00000000..28161ca7 --- /dev/null +++ b/control-plane/shared/migration/src/m20260816_020000_garage_nodes_agent_id_nullable.rs @@ -0,0 +1,29 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("garage_nodes")) + .modify_column(ColumnDef::new(Alias::new("agent_id")).uuid().null()) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Alias::new("garage_nodes")) + .modify_column(ColumnDef::new(Alias::new("agent_id")).uuid().not_null()) + .to_owned(), + ) + .await + } +} diff --git a/docker-compose.yml b/docker-compose.yml index f397963a..370baa57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,6 +78,7 @@ services: VOLUME_MANAGER_URL: http://volume-manager:8003 FAILOVER_CONTROLLER_URL: http://failover-controller:8004 SDN_CONTROLLER_URL: http://sdn-controller:8005 + OBJECT_STORAGE_URL: http://object-storage:8006 FRONTEND_URL: http://localhost:5173 CSFX_INCLUDE_CONTAINERS: "true" TLS_ENABLED: "false" @@ -178,6 +179,69 @@ services: - cargo_target_volume_manager:/app/target command: cargo watch -x "run -p volume-manager" + garage: + image: dxflrs/garage:v2.3.0 + container_name: csfx-garage-dev + volumes: + - ./control-plane/object-storage/dev/garage.toml:/etc/garage.toml:ro + - garage_meta:/var/lib/garage/meta + - garage_data:/var/lib/garage/data + ports: + - "3900:3900" + - "3903:3903" + networks: + - csfx-network + restart: unless-stopped + healthcheck: + test: ["CMD", "/garage", "-c", "/etc/garage.toml", "status"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + + garage-bootstrap: + image: alpine:3.20 + container_name: csfx-garage-bootstrap-dev + entrypoint: ["/bin/sh", "-c", "apk add --no-cache curl jq >/dev/null && /bootstrap.sh"] + volumes: + - ./control-plane/object-storage/dev/bootstrap.sh:/bootstrap.sh:ro + networks: + - csfx-network + depends_on: + garage: + condition: service_healthy + + object-storage: + <<: *rust-service + container_name: csfx-object-storage-dev + environment: + DATABASE_URL: postgres://${POSTGRES_USER:-csfx_user}:${POSTGRES_PASSWORD:-csfx_password}@postgres:5432/${POSTGRES_DB:-csfx_core} + ETCD_URL: http://etcd:2379 + OBJECT_STORAGE_PORT: "8006" + LISTEN_ADDR: "0.0.0.0" + GARAGE_ADMIN_URL: http://garage:3903 + GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-dev-garage-admin-token} + GARAGE_S3_URL: http://garage:3900 + GARAGE_PUBLIC_S3_URL: ${GARAGE_PUBLIC_S3_URL:-http://localhost:8000} + OBJECT_STORAGE_ENCRYPTION_KEY: ${OBJECT_STORAGE_ENCRYPTION_KEY:-9a29b5d4c92cbfd4abe14c2a645f39eff6b57b1fa13305d3a946a2221e630c29} + RUST_LOG: ${RUST_LOG:-debug} + ports: + - "8006:8006" + depends_on: + postgres: + condition: service_healthy + etcd: + condition: service_started + migrate: + condition: service_completed_successfully + garage: + condition: service_healthy + volumes: + - ./control-plane/object-storage:/app/control-plane/object-storage + - ./control-plane/shared:/app/control-plane/shared + - cargo_target_object_storage:/app/target + command: cargo watch -x "run -p object-storage" + failover-controller: <<: *rust-service container_name: csfx-failover-controller-dev @@ -320,6 +384,9 @@ volumes: cargo_target_volume_manager: cargo_target_failover_controller: cargo_target_sdn_controller: + cargo_target_object_storage: + garage_meta: + garage_data: cargo_target_agent: agent_state_1: agent_state_2: