diff --git a/.dockerignore b/.dockerignore index f36f99f23..4cf143a67 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ Dockerfile doc .git/ .gitignore +.claude/ .github/ tools/ skaffold.yaml diff --git a/AGENTS.md b/AGENTS.md index 53469045c..159c9f7db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,6 +106,66 @@ Backend traits in `crates/core/src/backend.rs` follow CRUD naming: with a mock built the same wrong way won't catch it. Only a live-server request through the real password-auth path surfaces it. +## Running loadtests locally (`tests/loadtest`) + +- `tools/run-loadtest-local.sh [goose args...]` sets up postgres (docker), + SPIRE, an embedded-OPA rust `keystone`, bootstraps the admin user, seeds + data, builds `tests/loadtest`, and runs it against `http://localhost:8080`. + Extra args are forwarded to `load_test`, e.g.: + `tools/run-loadtest-local.sh --users 20 --hatch-rate 4 --run-time 30s --report-file reports/run.md`. + `tools/teardown-loadtest.sh` tears everything down; the runner calls it + itself at the start of every run, so a stale environment from an + interrupted run is cleaned up automatically. +- Requires a working `docker` CLI (postgres container). On a podman-only + host, alias `docker` to `podman` and use fully-qualified image refs + (`docker.io/postgres:17`) — an unqualified `postgres:17` fails to resolve + without configured unqualified-search registries. +- `tests/loadtest` is excluded from the main workspace (`exclude` in root + `Cargo.toml`) — it's a standalone crate. Build it with + `cd tests/loadtest && cargo build`, never `cargo build -p load_test` from + the repo root. +- **Never pipe the runner through `tail`** — same root cause as the + `test_api` warning above: the script backgrounds SPIRE/OPA/keystone + daemons that inherit the pipe's write fd and never let it close, so the + whole thing looks hung even after the actual test finished. Run it via + `nohup ... > /tmp/out.log 2>&1 &` and read the file, or poll with + `kill -0 ` in a loop. +- `keystone-manage db up` alone is **not** enough against a fresh postgres — + it only applies versioned migrations for the drivers that ship them + (credential, federation, k8s-auth, token-restriction, webauthn). Identity, + role, assignment, resource, catalog etc. use entity-based schema creation + via `keystone-manage db sync` instead. Run both, `sync` before `up`. +- `openstack_sdk`'s auth-cache (`$HOME/.osc/`, enabled by default, keyed by + cloud-config hash) survives across server restarts. Since each run + bootstraps a brand-new admin user, a stale cache entry would make the SDK + replay a token for a user id that no longer exists in the fresh DB — + surfaces as widespread, seemingly random 401/500s that have nothing to do + with the actual server. The runner works around this by pointing + `load_test` at an isolated `$HOME` (under `STATE_DIR`, wiped every run) + via `HOME=`/`OS_CLIENT_CONFIG_PATH=` rather than touching your real + `~/.osc` or `~/.config/openstack`. If you invoke the SDK some other way + and hit "user cannot be found: ``" against a freshly bootstrapped + keystone, check `~/.osc` for a stale entry before assuming a real bug. +- `AsyncOpenStack`/`CloudConfig` accepts either a `clouds.yaml` entry + (`OS_CLOUD=`) or plain `OS_*` auth env vars + (`OS_AUTH_URL`/`OS_USERNAME`/`OS_PASSWORD`/...) via + `CloudConfig::from_env()` — `tests/loadtest/src/main.rs`'s + `load_cloud_config()` prefers the env-var form whenever `OS_AUTH_URL` is + set, falling back to `OS_CLOUD` + clouds.yaml otherwise, so a clouds.yaml + file isn't a hard requirement to run the suite. +- Goose logs any non-2xx response into its `ERRORS`/status-code tables at + the HTTP layer regardless of the transaction's own pass/fail logic. + Negative-test transactions (wrong password, revoke-then-validate, rescope + to a nonexistent project) must call `user.set_success(&mut goose.request)` + once they've confirmed the rejection was the *expected* outcome, or the + top-line failure percentage reads misleadingly high even though nothing + is actually broken. +- Loadtest also runs against a keystone deployed via skaffold to local k3s + (see `doc/src/contributor/development.md` for that setup) — skip + `tools/run-loadtest-local.sh` entirely and point `load_test` straight at + the cluster's exposed URL/port with `--host`, using `OS_CLOUD`/`OS_*` env + vars matching that deployment's admin credentials. + ## Commit Message Rules - **Format**: Conventional Commits (`type: subject body`) enforced by diff --git a/crates/api-types/src/error_conv.rs b/crates/api-types/src/error_conv.rs index 864b19855..0e8e54449 100644 --- a/crates/api-types/src/error_conv.rs +++ b/crates/api-types/src/error_conv.rs @@ -286,6 +286,11 @@ impl From for KeystoneApiError { resource: "group".into(), identifier: x, }, + // Delegate to the dedicated `ResourceProviderError` conversion + // below instead of falling through to the generic `InternalError` + // catch-all - e.g. `DomainNotFound` from resolving a + // password-auth `domain.name` must surface as 404, not 500. + IdentityProviderError::ResourceProvider { source } => source.into(), ref err @ IdentityProviderError::Conflict(..) => Self::Conflict(err.to_string()), ref err @ IdentityProviderError::SecurityCompliance(..) => { Self::BadRequest(err.to_string()) diff --git a/crates/keystone/src/bin/keystone.rs b/crates/keystone/src/bin/keystone.rs index c6b6296c3..849779268 100644 --- a/crates/keystone/src/bin/keystone.rs +++ b/crates/keystone/src/bin/keystone.rs @@ -187,7 +187,7 @@ async fn main() -> Result<(), Report> { // Guard must stay alive for the process lifetime to flush buffered // file-appender logs; binding to `_guard` (rather than dropping the // `Option`) keeps that lifetime tied to `main`'s scope. - let _guard = init_tracing(args.verbose, &cfg); + let _guard = init_tracing(args.verbose, &cfg)?; color_eyre::install()?; info!("Starting Keystone..."); @@ -464,7 +464,10 @@ async fn main() -> Result<(), Report> { /// Returns the file-appender's `WorkerGuard`, if file logging is enabled. /// The caller must keep this alive for the process lifetime — dropping it /// stops buffered log lines from being flushed. -fn init_tracing(verbose: u8, cfg: &Config) -> Option { +fn init_tracing( + verbose: u8, + cfg: &Config, +) -> Result> { let external_deps_log_level = match verbose { 0 => LevelFilter::ERROR, 1 => LevelFilter::WARN, @@ -586,7 +589,7 @@ fn init_tracing(verbose: u8, cfg: &Config) -> Option, /// ID of the project created in on_start for ProjectCRUD scenario. pub project_id: Option, + /// ID of the role created in on_start for RoleCRUD scenario. + pub role_id: Option, } #[tokio::main] @@ -54,11 +72,23 @@ async fn main() -> Result<(), GooseError> { v3::user::set_seeded_ids(seed_state.user_ids.clone()); v3::project::set_seeded_ids(seed_state.project_ids.clone()); - // Default to 30 users so all weighted scenarios get at least 1 user - // (total weight = 20; 30 ensures proportional coverage). - // Can be overridden by passing --users on the CLI. + // Share seeded user credentials and the shared auth project with the + // password-auth scenarios. + v3::user::set_seeded_creds(seed_state.user_creds.clone()); + if let Some(auth_project_id) = &seed_state.auth_project_id { + v3::user::set_auth_project_id(auth_project_id.clone()); + } + if let Some(auth_role_id) = &seed_state.auth_role_id { + v3::role_assignment::set_auth_role_id(auth_role_id.clone()); + } + + // Default to 45 users so all weighted scenarios get at least 1 user + // (total weight = 36; 45 ensures proportional coverage). + // Can be overridden by passing --users on the CLI. To actually run this + // as a stress test, scale further with goose's own CLI flags, e.g.: + // --users 500 --hatch-rate 20 --run-time 10m let attack = GooseAttack::initialize()? - .set_default(GooseDefault::Users, 30usize)? + .set_default(GooseDefault::Users, 45usize)? // Read-heavy workload: list endpoints hit the most common production path. .register_scenario( scenario!("ReadHeavy") @@ -100,6 +130,36 @@ async fn main() -> Result<(), GooseError> { .register_transaction(transaction!(project_show)) .register_transaction(transaction!(project_delete).set_on_stop()), ) + // Role CRUD: each virtual user owns one role resource for the test duration. + .register_scenario( + scenario!("RoleCRUD") + .set_weight(1)? + .register_transaction(transaction!(openstack_login).set_on_start()) + .register_transaction(transaction!(role_create).set_on_start()) + .register_transaction(transaction!(role_show)) + .register_transaction(transaction!(role_delete).set_on_stop()), + ) + // Dedicated role listing: exercises GET /v3/roles against the roles + // bootstrap/CRUD scenarios keep populated (admin, manager, member, + // reader, plus the auth-project's member role and any RoleCRUD churn). + .register_scenario( + scenario!("RoleList") + .set_weight(2)? + .register_transaction(transaction!(openstack_login).set_on_start()) + .register_transaction(transaction!(role_list)), + ) + // Role assignment listing: unfiltered plus the user.id/scope.domain.id/ + // role.id filter variants, all against the auth-project's seeded + // (100 users x member role) assignments. + .register_scenario( + scenario!("RoleAssignmentList") + .set_weight(3)? + .register_transaction(transaction!(openstack_login).set_on_start()) + .register_transaction(transaction!(role_assignment_list)) + .register_transaction(transaction!(role_assignment_list_by_user)) + .register_transaction(transaction!(role_assignment_list_by_domain)) + .register_transaction(transaction!(role_assignment_list_by_role)), + ) // Catalog read: list all users then fetch a randomly chosen one from the // pre-seeded pool. Exercises the list + point-read path under realistic // data volumes (100 seeded users). @@ -119,6 +179,40 @@ async fn main() -> Result<(), GooseError> { .register_transaction(transaction!(openstack_login).set_on_start()) .register_transaction(transaction!(project_list)) .register_transaction(transaction!(project_show_random)), + ) + // Password auth: exercises the hot password-login path against the + // pre-provisioned seeded users, both unscoped and project-scoped. + .register_scenario( + scenario!("PasswordAuth") + .set_weight(4)? + .register_transaction(transaction!(password_auth_unscoped)) + .register_transaction(transaction!(password_auth_scoped)), + ) + // Password auth negative path: wrong password must be rejected (401). + .register_scenario( + scenario!("PasswordAuthNegative") + .set_weight(1)? + .register_transaction(transaction!(password_auth_invalid)), + ) + // Rescope the OS_CLOUD-authenticated token to system scope. + .register_scenario( + scenario!("Rescope") + .set_weight(2)? + .register_transaction(transaction!(openstack_login).set_on_start()) + .register_transaction(transaction!(rescope_to_system)), + ) + // Rescope negative path: rescoping to a nonexistent project must fail. + .register_scenario( + scenario!("RescopeNegative") + .set_weight(1)? + .register_transaction(transaction!(openstack_login).set_on_start()) + .register_transaction(transaction!(rescope_invalid_project)), + ) + // System-scoped auth using the initial bootstrap admin credentials. + .register_scenario( + scenario!("SystemScopeAuth") + .set_weight(2)? + .register_transaction(transaction!(system_scope_auth)), ); attack.execute().await?; @@ -128,11 +222,26 @@ async fn main() -> Result<(), GooseError> { Ok(()) } -/// Authenticate via the configured OS_CLOUD and store the token in session data. -pub async fn openstack_login(user: &mut GooseUser) -> TransactionResult { - let cfg = ConfigFile::new().unwrap(); +/// Build the admin cloud config, preferring plain `OS_*` auth env vars +/// (`OS_AUTH_URL`, `OS_USERNAME`, `OS_PASSWORD`, ...) over a `clouds.yaml` +/// entry when `OS_AUTH_URL` is set, so CI/local runs don't need a clouds.yaml +/// at all. Falls back to `OS_CLOUD` (default `devstack`) looked up via +/// clouds.yaml otherwise. +fn load_cloud_config() -> CloudConfig { + if env::var("OS_AUTH_URL").is_ok() { + return CloudConfig::from_env().expect("cannot build cloud config from OS_* env vars"); + } + let cfg = ConfigFile::new().expect("cannot read clouds.yaml"); let cloud_name = env::var("OS_CLOUD").unwrap_or("devstack".to_string()); - let profile = cfg.get_cloud_config(cloud_name).unwrap().unwrap(); + cfg.get_cloud_config(&cloud_name) + .expect("cannot get cloud config") + .unwrap_or_else(|| panic!("cloud '{cloud_name}' not found in clouds.yaml")) +} + +/// Authenticate via the configured OS_CLOUD/OS_* env vars and store the +/// token in session data. +pub async fn openstack_login(user: &mut GooseUser) -> TransactionResult { + let profile = load_cloud_config(); let session = AsyncOpenStack::new(&profile) .await .expect("cannot connect to the cloud"); @@ -141,6 +250,7 @@ pub async fn openstack_login(user: &mut GooseUser) -> TransactionResult { token, user_id: None, project_id: None, + role_id: None, }); Ok(()) } @@ -156,14 +266,9 @@ fn get_host() -> String { "http://localhost:8080".to_string() } -/// Obtain an admin token using the configured OS_CLOUD credentials. +/// Obtain an admin token using the configured OS_CLOUD/OS_* env vars. async fn get_admin_token() -> String { - let cfg = ConfigFile::new().expect("cannot read clouds.yaml"); - let cloud_name = env::var("OS_CLOUD").unwrap_or("devstack".to_string()); - let profile = cfg - .get_cloud_config(cloud_name) - .expect("cannot get cloud config") - .expect("cloud not found"); + let profile = load_cloud_config(); let session = AsyncOpenStack::new(&profile) .await .expect("cannot authenticate"); diff --git a/tests/loadtest/src/seed.rs b/tests/loadtest/src/seed.rs index 0296fc4d3..d2c97d802 100644 --- a/tests/loadtest/src/seed.rs +++ b/tests/loadtest/src/seed.rs @@ -23,10 +23,33 @@ use uuid::Uuid; const DEFAULT_DOMAIN_ID: &str = "default"; const SEED_USERS: usize = 100; const SEED_PROJECTS: usize = 100; +/// Shared password for every seeded user, so password-auth scenarios can log +/// in as any user from the seeded pool without tracking per-user secrets. +pub const SEED_USER_PASSWORD: &str = "LoadTestSeedPassw0rd!"; +/// Dedicated project all seeded users are granted a role on, used by the +/// project-scoped password auth scenarios. +const AUTH_PROJECT_NAME: &str = "loadtest-authproject"; +const AUTH_ROLE_NAME: &str = "member"; + +/// A seeded user's credentials, usable for password authentication. +#[derive(Clone)] +pub struct SeededUserCred { + pub id: String, + pub password: String, +} pub struct SeedState { pub user_ids: Vec, pub project_ids: Vec, + /// Credentials for the seeded users, each holding a `member` role on + /// `auth_project_id`. + pub user_creds: Vec, + /// Project every seeded user has a role assignment on; used to exercise + /// project-scoped password auth under load. + pub auth_project_id: Option, + /// Role every seeded user is granted on `auth_project_id`; used to + /// exercise the `role.id` filter on `GET /v3/role_assignments`. + pub auth_role_id: Option, } /// Create background resources so list endpoints return non-trivial result sets. @@ -35,12 +58,21 @@ pub async fn seed(host: &str, token: &str) -> SeedState { let mut state = SeedState { user_ids: Vec::new(), project_ids: Vec::new(), + user_creds: Vec::new(), + auth_project_id: None, + auth_role_id: None, }; for i in 0..SEED_USERS { let name = format!("loadtest-seed-user-{}-{}", i, Uuid::new_v4().as_simple()); match create_user(&client, host, token, &name, DEFAULT_DOMAIN_ID).await { - Some(id) => state.user_ids.push(id), + Some(id) => { + state.user_creds.push(SeededUserCred { + id: id.clone(), + password: SEED_USER_PASSWORD.to_string(), + }); + state.user_ids.push(id); + } None => eprintln!("seed: failed to create user {name}"), } } @@ -59,9 +91,101 @@ pub async fn seed(host: &str, token: &str) -> SeedState { state.project_ids.len() ); + match seed_auth_project(&client, host, token, &state.user_ids).await { + Some((project_id, role_id)) => { + eprintln!( + "seed: created auth project {project_id} with {} user role assignments (role {role_id})", + state.user_ids.len() + ); + state.auth_project_id = Some(project_id); + state.auth_role_id = Some(role_id); + } + None => eprintln!("seed: failed to set up auth project for password-auth scenarios"), + } + state } +/// Create a dedicated project and grant every seeded user a `member` role on +/// it, so seeded users have a real scope to authenticate against. Returns +/// the project ID and the role ID granted to every user. +async fn seed_auth_project( + client: &Client, + host: &str, + token: &str, + user_ids: &[String], +) -> Option<(String, String)> { + let project_id = + create_project(client, host, token, AUTH_PROJECT_NAME, DEFAULT_DOMAIN_ID).await?; + let role_id = get_or_create_role(client, host, token, AUTH_ROLE_NAME).await?; + + for user_id in user_ids { + if let Err(e) = + assign_project_role(client, host, token, &project_id, user_id, &role_id).await + { + eprintln!("seed: failed to assign role to user {user_id}: {e}"); + } + } + + Some((project_id, role_id)) +} + +/// Find a role by name, creating it if it doesn't exist. +async fn get_or_create_role( + client: &Client, + host: &str, + token: &str, + name: &str, +) -> Option { + let resp = client + .get(format!("{host}/v3/roles")) + .header("x-auth-token", token) + .query(&[("name", name)]) + .send() + .await + .ok()?; + if resp.status().is_success() { + let val: serde_json::Value = resp.json().await.ok()?; + if let Some(id) = val["roles"][0]["id"].as_str() { + return Some(id.to_owned()); + } + } + + let body = json!({ "role": { "name": name } }); + let resp = client + .post(format!("{host}/v3/roles")) + .header("x-auth-token", token) + .json(&body) + .send() + .await + .ok()?; + if !resp.status().is_success() { + eprintln!("seed: create_role HTTP {}", resp.status()); + return None; + } + let val: serde_json::Value = resp.json().await.ok()?; + val["role"]["id"].as_str().map(str::to_owned) +} + +/// Grant a user a role on a project. +async fn assign_project_role( + client: &Client, + host: &str, + token: &str, + project_id: &str, + user_id: &str, + role_id: &str, +) -> Result<(), reqwest::Error> { + client + .put(format!( + "{host}/v3/projects/{project_id}/users/{user_id}/roles/{role_id}" + )) + .header("x-auth-token", token) + .send() + .await?; + Ok(()) +} + /// Delete all resources created during seeding. pub async fn cleanup(host: &str, token: &str, state: &SeedState) { let client = Client::new(); @@ -78,6 +202,18 @@ pub async fn cleanup(host: &str, token: &str, state: &SeedState) { } } + if let Some(auth_project_id) = &state.auth_project_id + && let Err(e) = delete( + &client, + host, + token, + &format!("/v3/projects/{auth_project_id}"), + ) + .await + { + eprintln!("seed cleanup: failed to delete auth project {auth_project_id}: {e}"); + } + eprintln!( "seed cleanup: removed {} users, {} projects", state.user_ids.len(), @@ -97,7 +233,7 @@ async fn create_user( "name": name, "domain_id": domain_id, "enabled": true, - "password": Uuid::new_v4().to_string() + "password": SEED_USER_PASSWORD } }); let resp = client diff --git a/tests/loadtest/src/v3.rs b/tests/loadtest/src/v3.rs index efdfbfc1c..7895c73cb 100644 --- a/tests/loadtest/src/v3.rs +++ b/tests/loadtest/src/v3.rs @@ -15,4 +15,6 @@ pub(crate) mod auth; pub(crate) mod domain; pub(crate) mod project; +pub(crate) mod role; +pub(crate) mod role_assignment; pub(crate) mod user; diff --git a/tests/loadtest/src/v3/auth.rs b/tests/loadtest/src/v3/auth/lifecycle.rs similarity index 78% rename from tests/loadtest/src/v3/auth.rs rename to tests/loadtest/src/v3/auth/lifecycle.rs index cc3774f98..344bcab72 100644 --- a/tests/loadtest/src/v3/auth.rs +++ b/tests/loadtest/src/v3/auth/lifecycle.rs @@ -121,5 +121,32 @@ pub async fn token_lifecycle(user: &mut GooseUser) -> TransactionResult { user.request(goose_request).await?; + // Validate the revoked token: a revoked token must not still validate. + let revalidate_req = user + .get_request_builder(&GooseMethod::Get, "/v3/auth/tokens")? + .header("x-auth-token", &existing_token) + .header("x-subject-token", &new_token); + + let goose_request = GooseRequest::builder() + .name("GET /v3/auth/tokens (validate revoked)") + .set_request_builder(revalidate_req) + .build(); + + let mut goose = user.request(goose_request).await?; + if let Ok(response) = &goose.response { + if response.status().is_success() { + return user.set_failure( + "revoked token still validated successfully", + &mut goose.request, + None, + None, + ); + } + // A revoked token is expected to be rejected (typically 404); mark + // this non-2xx response as a success so it doesn't pollute Goose's + // error stats with an expected negative-test outcome. + return user.set_success(&mut goose.request); + } + Ok(()) } diff --git a/tests/loadtest/src/v3/auth/mod.rs b/tests/loadtest/src/v3/auth/mod.rs new file mode 100644 index 000000000..2211eca46 --- /dev/null +++ b/tests/loadtest/src/v3/auth/mod.rs @@ -0,0 +1,18 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) mod lifecycle; +pub(crate) mod password; +pub(crate) mod rescope; +pub(crate) mod system; diff --git a/tests/loadtest/src/v3/auth/password.rs b/tests/loadtest/src/v3/auth/password.rs new file mode 100644 index 000000000..7252d7794 --- /dev/null +++ b/tests/loadtest/src/v3/auth/password.rs @@ -0,0 +1,173 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//! Password authentication scenarios exercised against the pre-provisioned +//! seeded users (see `crate::seed`), each of which has a role assignment on +//! a shared "auth project" and a known shared password. + +use goose::prelude::*; +use serde_json::json; + +use crate::v3::user::{auth_project_id, random_seeded_cred}; + +/// Password auth with no scope requested. +pub async fn unscoped(user: &mut GooseUser) -> TransactionResult { + let cred = match random_seeded_cred() { + Some(c) => c, + None => return Ok(()), + }; + + let body = json!({ + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "id": cred.id, + "password": cred.password + } + } + } + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v3/auth/tokens")? + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v3/auth/tokens (password, unscoped)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + if let Ok(response) = &goose.response + && !response.status().is_success() + { + let status = response.status(); + return user.set_failure( + &format!("password auth (unscoped) returned {status}"), + &mut goose.request, + None, + None, + ); + } + + Ok(()) +} + +/// Password auth scoped to the shared auth project every seeded user has a +/// role assignment on. +pub async fn scoped(user: &mut GooseUser) -> TransactionResult { + let cred = match random_seeded_cred() { + Some(c) => c, + None => return Ok(()), + }; + let project_id = match auth_project_id() { + Some(id) => id, + None => return Ok(()), + }; + + let body = json!({ + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "id": cred.id, + "password": cred.password + } + } + }, + "scope": { + "project": { "id": project_id } + } + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v3/auth/tokens")? + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v3/auth/tokens (password, project-scoped)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + if let Ok(response) = &goose.response + && !response.status().is_success() + { + let status = response.status(); + return user.set_failure( + &format!("password auth (project-scoped) returned {status}"), + &mut goose.request, + None, + None, + ); + } + + Ok(()) +} + +/// Password auth with a deliberately wrong password; must be rejected with +/// 401. Any other outcome (including success) is a Goose failure. +pub async fn invalid(user: &mut GooseUser) -> TransactionResult { + let cred = match random_seeded_cred() { + Some(c) => c, + None => return Ok(()), + }; + + let body = json!({ + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "id": cred.id, + "password": format!("wrong-{}", cred.password) + } + } + } + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v3/auth/tokens")? + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v3/auth/tokens (password, invalid)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + if let Ok(response) = &goose.response { + if response.status() != reqwest::StatusCode::UNAUTHORIZED { + let status = response.status(); + return user.set_failure( + &format!("invalid password auth returned {status}, expected 401"), + &mut goose.request, + None, + None, + ); + } + // A wrong password is expected to be rejected with 401; mark this + // non-2xx response as a success so it doesn't pollute Goose's error + // stats with an expected negative-test outcome. + return user.set_success(&mut goose.request); + } + + Ok(()) +} diff --git a/tests/loadtest/src/v3/auth/rescope.rs b/tests/loadtest/src/v3/auth/rescope.rs new file mode 100644 index 000000000..57a5a6c86 --- /dev/null +++ b/tests/loadtest/src/v3/auth/rescope.rs @@ -0,0 +1,119 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//! Token rescoping: re-authenticate with method "token" and a new `scope`. +//! +//! These scenarios reuse the `OS_CLOUD`-authenticated session token +//! (`Session.token`), which in the standard devstack/CI setup belongs to the +//! bootstrap admin user and therefore holds the system `admin` role — that +//! is what makes `rescope_to_system` expected to succeed. Running this +//! scenario against a cloud whose `OS_CLOUD` identity lacks a system role +//! assignment will surface as load-test failures. + +use goose::prelude::*; +use serde_json::json; +use uuid::Uuid; + +use crate::Session; + +/// Rescope the current session token to system scope (`{"all": true}`). +pub async fn rescope_to_system(user: &mut GooseUser) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + + let body = json!({ + "auth": { + "identity": { + "methods": ["token"], + "token": { "id": token } + }, + "scope": { + "system": { "all": true } + } + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v3/auth/tokens")? + .header("x-auth-token", &token) + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v3/auth/tokens (rescope to system)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + if let Ok(response) = &goose.response + && !response.status().is_success() + { + let status = response.status(); + return user.set_failure( + &format!("rescope to system scope returned {status}"), + &mut goose.request, + None, + None, + ); + } + + Ok(()) +} + +/// Attempt to rescope the current session token to a project that does not +/// exist; must be rejected. A successful response is a Goose failure. +pub async fn rescope_invalid_project(user: &mut GooseUser) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + let bogus_project_id = Uuid::new_v4().as_simple().to_string(); + + let body = json!({ + "auth": { + "identity": { + "methods": ["token"], + "token": { "id": token } + }, + "scope": { + "project": { "id": bogus_project_id } + } + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v3/auth/tokens")? + .header("x-auth-token", &token) + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v3/auth/tokens (rescope, invalid project)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + if let Ok(response) = &goose.response { + if response.status().is_success() { + return user.set_failure( + "rescope to a nonexistent project unexpectedly succeeded", + &mut goose.request, + None, + None, + ); + } + // Rescoping to a nonexistent project is expected to be rejected; + // mark this non-2xx response as a success so it doesn't pollute + // Goose's error stats with an expected negative-test outcome. + return user.set_success(&mut goose.request); + } + + Ok(()) +} diff --git a/tests/loadtest/src/v3/auth/system.rs b/tests/loadtest/src/v3/auth/system.rs new file mode 100644 index 000000000..91c31ff7a --- /dev/null +++ b/tests/loadtest/src/v3/auth/system.rs @@ -0,0 +1,133 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//! System-scoped auth using the initial bootstrap admin credentials, i.e. +//! the same identity `keystone-manage bootstrap` (see +//! `crates/cli-manage/src/bootstrap.rs`) provisions with a system `admin` +//! role. Defaults fall back to `OS_USERNAME`/`OS_PASSWORD`/ +//! `OS_USER_DOMAIN_NAME` (the same vars `openstack_login`/`get_admin_token` +//! in `main.rs` use to reach the admin cloud), so this scenario's notion of +//! "the admin" can't silently drift from the rest of the suite's; only fall +//! further back to the `tools/start-api.sh`/`tools/start_keystone.sh` +//! convention (`admin`/`password`/`default`) when neither is set. Override +//! with `LOADTEST_ADMIN_USERNAME`/`_PASSWORD`/`_DOMAIN` if the target +//! deployment's admin identity differs from both. + +use goose::prelude::*; +use serde_json::json; +use std::env; + +fn admin_username() -> String { + env::var("LOADTEST_ADMIN_USERNAME") + .or_else(|_| env::var("OS_USERNAME")) + .unwrap_or_else(|_| "admin".to_string()) +} + +fn admin_password() -> String { + env::var("LOADTEST_ADMIN_PASSWORD") + .or_else(|_| env::var("OS_PASSWORD")) + .unwrap_or_else(|_| "password".to_string()) +} + +/// The admin's user domain, keeping track of whether the value is a domain +/// name or a domain id so it lands in the right JSON field — the two are +/// not interchangeable, and conflating them only happens to work when a +/// domain's id and name coincide (e.g. `default` in this dev convention). +enum AdminDomain { + Name(String), + Id(String), +} + +impl AdminDomain { + fn as_json(&self) -> serde_json::Value { + match self { + AdminDomain::Name(name) => json!({ "name": name }), + AdminDomain::Id(id) => json!({ "id": id }), + } + } +} + +fn admin_domain() -> AdminDomain { + if let Ok(name) = env::var("LOADTEST_ADMIN_DOMAIN") { + return AdminDomain::Name(name); + } + if let Ok(name) = env::var("OS_USER_DOMAIN_NAME") { + return AdminDomain::Name(name); + } + if let Ok(id) = env::var("OS_USER_DOMAIN_ID") { + return AdminDomain::Id(id); + } + // The id is the stable, always-lowercase identifier; the display name + // (bootstrapped as "Default", capitalized) is a convention that can + // vary by deployment, so falling back to it by name risks an exact-match + // domain lookup failing even though the id would have resolved fine. + AdminDomain::Id("default".to_string()) +} + +/// Authenticate as the bootstrap admin with system scope +/// (`{"system": {"all": true}}`), exercising the same path an operator uses +/// to get cloud-wide privileges. +pub async fn system_scope_auth(user: &mut GooseUser) -> TransactionResult { + let body = json!({ + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "name": admin_username(), + "domain": admin_domain().as_json(), + "password": admin_password() + } + } + }, + "scope": { + "system": { "all": true } + } + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v3/auth/tokens")? + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v3/auth/tokens (system-scoped, bootstrap admin)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + let response = match goose.response { + Ok(r) => r, + Err(e) => { + return user.set_failure( + &format!("system scope auth failed: {e}"), + &mut goose.request, + None, + None, + ); + } + }; + + if !response.status().is_success() { + let status = response.status(); + return user.set_failure( + &format!("system scope auth returned {status}"), + &mut goose.request, + None, + None, + ); + } + + Ok(()) +} diff --git a/tests/loadtest/src/v3/role.rs b/tests/loadtest/src/v3/role.rs new file mode 100644 index 000000000..5f8e03401 --- /dev/null +++ b/tests/loadtest/src/v3/role.rs @@ -0,0 +1,140 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +use goose::prelude::*; +use serde_json::json; +use uuid::Uuid; + +use crate::Session; + +/// List all roles (dedicated read scenario transaction). +pub async fn list(user: &mut GooseUser) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + + let req = user + .get_request_builder(&GooseMethod::Get, "/v3/roles")? + .header("x-auth-token", &token); + + let goose_request = GooseRequest::builder() + .name("GET /v3/roles") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// Create a role owned by this virtual user (on_start for RoleCRUD scenario). +pub async fn create(user: &mut GooseUser) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + + let name = format!("loadtest-role-{}", Uuid::new_v4().as_simple()); + let body = json!({ + "role": { + "name": name + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v3/roles")? + .header("x-auth-token", &token) + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v3/roles (setup)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + let response = match goose.response { + Ok(r) => r, + Err(e) => { + return user.set_failure( + &format!("role create failed: {e}"), + &mut goose.request, + None, + None, + ); + } + }; + + if !response.status().is_success() { + return user.set_failure( + &format!("role create returned {}", response.status()), + &mut goose.request, + None, + None, + ); + } + + let val: serde_json::Value = response.json().await.unwrap_or_default(); + let role_id = val["role"]["id"].as_str().map(str::to_owned); + + let session = user.get_session_data_unchecked_mut::(); + session.role_id = role_id; + + Ok(()) +} + +/// Show the role owned by this virtual user. +pub async fn show(user: &mut GooseUser) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + let role_id = match &session.role_id { + Some(id) => id.clone(), + None => return Ok(()), + }; + + let path = format!("/v3/roles/{role_id}"); + let req = user + .get_request_builder(&GooseMethod::Get, &path)? + .header("x-auth-token", &token); + + let goose_request = GooseRequest::builder() + .name("GET /v3/roles/:id") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// Delete the role owned by this virtual user (on_stop for RoleCRUD scenario). +pub async fn delete(user: &mut GooseUser) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + let role_id = match &session.role_id { + Some(id) => id.clone(), + None => return Ok(()), + }; + + let path = format!("/v3/roles/{role_id}"); + let req = user + .get_request_builder(&GooseMethod::Delete, &path)? + .header("x-auth-token", &token); + + let goose_request = GooseRequest::builder() + .name("DELETE /v3/roles/:id (teardown)") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + + let session = user.get_session_data_unchecked_mut::(); + session.role_id = None; + + Ok(()) +} diff --git a/tests/loadtest/src/v3/role_assignment.rs b/tests/loadtest/src/v3/role_assignment.rs new file mode 100644 index 000000000..6f1375f16 --- /dev/null +++ b/tests/loadtest/src/v3/role_assignment.rs @@ -0,0 +1,99 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//! `GET /v3/role_assignments` in its unfiltered form and each of the +//! `user.id` / `scope.domain.id` / `role.id` filter variants, exercising the +//! same seeded data (100 users granted the auth-project's `member` role) the +//! password-auth scenarios already rely on. + +use goose::prelude::*; +use std::sync::OnceLock; + +use crate::Session; + +const DEFAULT_DOMAIN_ID: &str = "default"; + +static AUTH_ROLE_ID: OnceLock = OnceLock::new(); + +/// Call once before `GooseAttack::execute()` to share the role every seeded +/// user is granted, for the `role.id` filter. +pub fn set_auth_role_id(id: String) { + AUTH_ROLE_ID.set(id).ok(); +} + +async fn list_with_query( + user: &mut GooseUser, + name: &str, + query: &[(&str, &str)], +) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + + let req = user + .get_request_builder(&GooseMethod::Get, "/v3/role_assignments")? + .header("x-auth-token", &token) + .query(query); + + let goose_request = GooseRequest::builder() + .name(name) + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// List all role assignments, unfiltered. +pub async fn list(user: &mut GooseUser) -> TransactionResult { + list_with_query(user, "GET /v3/role_assignments", &[]).await +} + +/// List role assignments filtered by a randomly chosen seeded user +/// (`user.id`). No-op if the seeded credential pool is empty. +pub async fn list_by_user(user: &mut GooseUser) -> TransactionResult { + let Some(cred) = crate::v3::user::random_seeded_cred() else { + return Ok(()); + }; + let user_id = cred.id.clone(); + list_with_query( + user, + "GET /v3/role_assignments?user.id", + &[("user.id", &user_id)], + ) + .await +} + +/// List role assignments filtered by the default domain (`scope.domain.id`). +pub async fn list_by_domain(user: &mut GooseUser) -> TransactionResult { + list_with_query( + user, + "GET /v3/role_assignments?scope.domain.id", + &[("scope.domain.id", DEFAULT_DOMAIN_ID)], + ) + .await +} + +/// List role assignments filtered by the auth-project's `member` role +/// (`role.id`). No-op if seeding the auth role failed. +pub async fn list_by_role(user: &mut GooseUser) -> TransactionResult { + let Some(role_id) = AUTH_ROLE_ID.get().cloned() else { + return Ok(()); + }; + list_with_query( + user, + "GET /v3/role_assignments?role.id", + &[("role.id", &role_id)], + ) + .await +} diff --git a/tests/loadtest/src/v3/user.rs b/tests/loadtest/src/v3/user.rs index 506adc1d6..51d9e8890 100644 --- a/tests/loadtest/src/v3/user.rs +++ b/tests/loadtest/src/v3/user.rs @@ -22,6 +22,8 @@ use crate::Session; const DEFAULT_DOMAIN_ID: &str = "default"; static SEEDED_USER_IDS: OnceLock> = OnceLock::new(); +static SEEDED_USER_CREDS: OnceLock> = OnceLock::new(); +static AUTH_PROJECT_ID: OnceLock = OnceLock::new(); /// Call once before `GooseAttack::execute()` to share the seeded user ID pool /// with all virtual users. @@ -29,6 +31,34 @@ pub fn set_seeded_ids(ids: Vec) { SEEDED_USER_IDS.set(ids).ok(); } +/// Call once before `GooseAttack::execute()` to share the seeded user +/// credential pool (id + password) with the password-auth scenarios. +pub fn set_seeded_creds(creds: Vec) { + SEEDED_USER_CREDS.set(creds).ok(); +} + +/// Return a randomly chosen seeded user credential, or `None` if the pool is +/// empty (seed failed entirely). +pub fn random_seeded_cred() -> Option<&'static crate::seed::SeededUserCred> { + let creds = SEEDED_USER_CREDS.get()?; + if creds.is_empty() { + return None; + } + Some(&creds[fastrand::usize(..creds.len())]) +} + +/// Call once before `GooseAttack::execute()` to share the project every +/// seeded user has a role assignment on, for project-scoped password auth. +pub fn set_auth_project_id(id: String) { + AUTH_PROJECT_ID.set(id).ok(); +} + +/// Return the project every seeded user has a role assignment on, if seeding +/// succeeded. +pub fn auth_project_id() -> Option<&'static str> { + AUTH_PROJECT_ID.get().map(String::as_str) +} + /// List all users (read-heavy scenario transaction). pub async fn list(user: &mut GooseUser) -> TransactionResult { let session = user.get_session_data_unchecked::(); diff --git a/tools/k8s/keystone/base/bootstrap-job.yaml b/tools/k8s/keystone/base/bootstrap-job.yaml index 93720e1e3..9d09d97c1 100644 --- a/tools/k8s/keystone/base/bootstrap-job.yaml +++ b/tools/k8s/keystone/base/bootstrap-job.yaml @@ -24,6 +24,7 @@ spec: key: fqdn-uri name: keystone-db-app image: py-keystone + imagePullPolicy: IfNotPresent name: keystone-db-py command: [keystone-manage, db_sync] volumeMounts: @@ -39,6 +40,7 @@ spec: key: fqdn-uri name: keystone-db-app image: keystone-rs + imagePullPolicy: IfNotPresent name: keystone-db-rs command: [keystone-manage, db, up] volumeMounts: @@ -56,6 +58,7 @@ spec: key: fqdn-uri name: keystone-db-app image: py-keystone + imagePullPolicy: IfNotPresent name: keystone-bootstrap command: [keystone-manage, "bootstrap", "--bootstrap-username", "admin", "--bootstrap-password", "password", "--bootstrap-public-url", "http://keystone-py.local", "--bootstrap-internal-url", "http://keystone-py.default.svc.cluster.local:5000", "--bootstrap-region-id", "RegionOne"] volumeMounts: diff --git a/tools/k8s/keystone/base/conf/keystone.conf b/tools/k8s/keystone/base/conf/keystone.conf index af9013490..73776dbe4 100644 --- a/tools/k8s/keystone/base/conf/keystone.conf +++ b/tools/k8s/keystone/base/conf/keystone.conf @@ -38,3 +38,8 @@ admin_svid = spiffe://example.org/ns/default/sa/keystone [mapping] cluster_salt = "fbb27433d07ab307cc1fc899d0e174cf197fd398fbcff7285a63fe2f94eec2fe" + +[cache] +enabled = true +backend = dogpile.cache.memcached +memcache_servers = memcached:11211 diff --git a/tools/k8s/keystone/base/deployment-py.yaml b/tools/k8s/keystone/base/deployment-py.yaml index f3e527cf3..df9bb69f8 100644 --- a/tools/k8s/keystone/base/deployment-py.yaml +++ b/tools/k8s/keystone/base/deployment-py.yaml @@ -5,7 +5,7 @@ metadata: app.kubernetes.io/name: keystone-py name: keystone-py spec: - replicas: 1 + replicas: 2 revisionHistoryLimit: 10 progressDeadlineSeconds: 300 selector: @@ -41,8 +41,16 @@ spec: secretKeyRef: key: fqdn-uri name: keystone-db-app - command: ["uwsgi", "--http-socket", ":5000", "-b", "65535", "-w", "keystone.server.wsgi:initialize_public_application()", "-p", "4"] + command: ["uwsgi", "--http-socket", ":5000", "-b", "65535", "--listen", "1024", "-w", "keystone.server.wsgi:initialize_public_application()", "--master", "--lazy-apps", "-p", "4"] image: py-keystone + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "4" + memory: 4Gi livenessProbe: failureThreshold: 3 httpGet: diff --git a/tools/k8s/keystone/base/kustomization.yaml b/tools/k8s/keystone/base/kustomization.yaml index 760f61a2f..7d2d48f17 100644 --- a/tools/k8s/keystone/base/kustomization.yaml +++ b/tools/k8s/keystone/base/kustomization.yaml @@ -26,3 +26,4 @@ resources: - sa.yaml - service.yaml - ingress.yaml + - memcached.yaml diff --git a/tools/k8s/keystone/base/memcached.yaml b/tools/k8s/keystone/base/memcached.yaml new file mode 100644 index 000000000..708eb8e85 --- /dev/null +++ b/tools/k8s/keystone/base/memcached.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: memcached + name: memcached +spec: + replicas: 1 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 300 + selector: + matchLabels: + app.kubernetes.io/name: memcached + template: + metadata: + labels: + app.kubernetes.io/name: memcached + spec: + containers: + - name: memcached + image: memcached:1.6-alpine + args: ["-m", "64"] + ports: + - containerPort: 11211 + name: memcache + protocol: TCP + livenessProbe: + tcpSocket: + port: 11211 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: 11211 + periodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached +spec: + selector: + app.kubernetes.io/name: memcached + ports: + - port: 11211 + targetPort: 11211 + protocol: TCP + name: memcache diff --git a/tools/k8s/keystone/overlays/skaffold/conf/keystone.conf b/tools/k8s/keystone/overlays/skaffold/conf/keystone.conf index af9013490..728b6ae16 100644 --- a/tools/k8s/keystone/overlays/skaffold/conf/keystone.conf +++ b/tools/k8s/keystone/overlays/skaffold/conf/keystone.conf @@ -1,5 +1,5 @@ [DEFAULT] -debug = true +debug = false use_stderr = true log_dir = /var/log/keystone @@ -38,3 +38,9 @@ admin_svid = spiffe://example.org/ns/default/sa/keystone [mapping] cluster_salt = "fbb27433d07ab307cc1fc899d0e174cf197fd398fbcff7285a63fe2f94eec2fe" + +# Only consumed by keystone-py (oslo.cache) - rust ignores this section. +[cache] +enabled = true +backend = dogpile.cache.memcached +memcache_servers = memcached:11211 diff --git a/tools/k8s/keystone/overlays/skaffold/statefulset-dev-patch.yaml b/tools/k8s/keystone/overlays/skaffold/statefulset-dev-patch.yaml index b15d0d251..1bb7ac6a8 100644 --- a/tools/k8s/keystone/overlays/skaffold/statefulset-dev-patch.yaml +++ b/tools/k8s/keystone/overlays/skaffold/statefulset-dev-patch.yaml @@ -77,4 +77,4 @@ spec: value: "4242424242424242424242424242424242424242424242424242424242424242" - name: KEYSTONE_ALLOW_ENV_KEK value: "1" - command: ["keystone", "-vv"] + command: ["keystone", "-v"] diff --git a/tools/run-loadtest-local.sh b/tools/run-loadtest-local.sh new file mode 100755 index 000000000..3fecdfa20 --- /dev/null +++ b/tools/run-loadtest-local.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +STATE_DIR="/tmp/loadtest/keystone" +CONFIG_FILE="${STATE_DIR}/etc/keystone.conf" +REAL_HOME="$HOME" +DATABASE_URL="postgres://postgres:password@127.0.0.1:15432/postgres" +SPIRE_SOCKET="/tmp/spire-ci-test-harness/agent.sock" +SPIFFE_ENDPOINT_SOCKET="unix:///${SPIRE_SOCKET}" +KEYSTONE_PID="" + +cleanup() { + echo "Tearing down loadtest environment..." >&2 + if [ -n "$KEYSTONE_PID" ]; then kill "$KEYSTONE_PID" 2>/dev/null || true; fi + pkill -f "opa run -s .* --addr unix://${STATE_DIR}/opa.sock" 2>/dev/null || true +} +trap cleanup ERR INT TERM + +tools/teardown-loadtest.sh || true +rm -rf "$STATE_DIR" +mkdir -p "${STATE_DIR}/etc/fernet-keys" + +# openstack_sdk caches auth tokens per cloud-config hash under +# $HOME/.osc, surviving across server restarts. Since we bootstrap a +# brand new admin user/token every run, a stale cache entry would make the +# SDK replay a token for a user id that no longer exists in the fresh DB +# (500s downstream). Point load_test at an isolated HOME (reset alongside +# the rest of STATE_DIR above) instead of touching the real ~/.osc. +LOADTEST_HOME="${STATE_DIR}/home" +mkdir -p "$LOADTEST_HOME" + +tools/start-spire.sh + +docker run -d --name loadtest_postgres \ + -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=password \ + -p 15432:5432 docker.io/postgres:17 + +for i in {1..30}; do + if docker exec loadtest_postgres pg_isready -U postgres > /dev/null 2>&1; then + echo "✅ Postgres is ready!" + break + fi + sleep 1 +done + +cat < "$CONFIG_FILE" +[api_policy] +opa_base_url = unix://${STATE_DIR}/opa.sock +enable = true +opa_policies_path = policy + +[auth] +methods = password,token,openid,application_credential,x509 + +[DEFAULT] +debug = true +use_stderr = false +log_dir = ${STATE_DIR} + +[database] +connection = ${DATABASE_URL} + +[fernet_receipts] +key_repository = ${STATE_DIR}/etc/fernet-keys +[fernet_tokens] +key_repository = ${STATE_DIR}/etc/fernet-keys + +[interface_admin] +socket_path = ${STATE_DIR}/keystone.sock +trust_domains = example.org +admin_svid = spiffe://example.org/keystone + +[audit] +spool_dir = ${STATE_DIR}/audit +EOF + +echo "2Rlc-npWYOGqqG1zM-bmfBj2apLacLXhIbBsdyqQ0zg=" > "${STATE_DIR}/etc/fernet-keys/0" + +cargo build --release --bins + +./target/release/keystone-manage -c "$CONFIG_FILE" db sync +./target/release/keystone-manage -c "$CONFIG_FILE" db up + +KEYSTONE_DEV_KEK=4242424242424242424242424242424242424242424242424242424242424242 \ +KEYSTONE_ALLOW_ENV_KEK=1 \ +SPIFFE_ENDPOINT_SOCKET=$SPIFFE_ENDPOINT_SOCKET \ +./target/release/keystone -c "$CONFIG_FILE" -vv > "${STATE_DIR}/rust.log" 2>&1 & +KEYSTONE_PID=$! +echo "$KEYSTONE_PID" > "${STATE_DIR}/keystone.pid" + +URL_METRICS="http://127.0.0.1:8099" +for i in {1..30}; do + if curl -s "${URL_METRICS}/health" > /dev/null; then + echo "✅ Keystone health url started responding!" + break + fi + if [ "$i" -eq 30 ]; then + echo "Server failed to start." >&2 + cat "${STATE_DIR}/rust.log" >&2 + exit 1 + fi + sleep 0.5 +done + +until [ -S "${STATE_DIR}/keystone.sock" ]; do + sleep 0.5 +done +echo "✅ Keystone admin socket appeared!" + +KEYSTONE_DEV_KEK=4242424242424242424242424242424242424242424242424242424242424242 \ +KEYSTONE_ALLOW_ENV_KEK=1 \ +SPIFFE_ENDPOINT_SOCKET=$SPIFFE_ENDPOINT_SOCKET \ +./target/release/keystone-manage -c "$CONFIG_FILE" bootstrap --bootstrap-password password + +echo "✅ Keystone bootstrap completed!" + +(cd tests/loadtest && cargo build --release) + +mkdir -p tests/loadtest/reports +(cd tests/loadtest && \ + HOME="$LOADTEST_HOME" \ + OS_CLIENT_CONFIG_PATH="${REAL_HOME}/.config/openstack" \ + OS_CLOUD=admin \ + ./target/release/load_test \ + --host http://localhost:8080 \ + "$@") diff --git a/tools/teardown-loadtest.sh b/tools/teardown-loadtest.sh new file mode 100755 index 000000000..addd65064 --- /dev/null +++ b/tools/teardown-loadtest.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +STATE_DIR="/tmp/loadtest/keystone" +PID_FILE="${STATE_DIR}/keystone.pid" +SPIRE_PID_DIR="/tmp/spire-ci-test-harness" + +echo "Cleaning up loadtest server process..." +if [ -f "$PID_FILE" ]; then + echo "Killing Keystone process $(cat "${PID_FILE}")" + kill -9 "$(cat "$PID_FILE")" 2>/dev/null || true +fi + +if [ -f "$SPIRE_PID_DIR/agent.pid" ]; then kill -9 "$(cat "$SPIRE_PID_DIR/agent.pid")" 2>/dev/null || true; fi +if [ -f "$SPIRE_PID_DIR/server.pid" ]; then kill -9 "$(cat "$SPIRE_PID_DIR/server.pid")" 2>/dev/null || true; fi + +# `keystone`'s embedded OPA subprocess is never signaled to stop on shutdown, +# see tools/teardown-api.sh -- kill it directly by its socket path. +pkill -9 -f "opa run -s .* --addr unix://${STATE_DIR}/opa.sock" 2>/dev/null || true + +docker rm -f loadtest_postgres 2>/dev/null || true + +echo "Cleanup complete. Logs preserved in $STATE_DIR."