feat(loadtest): Add more auth tests

.dockerignore

@@ -2,6 +2,7 @@ Dockerfile
 doc
 .git/
 .gitignore
+.claude/
 .github/
 tools/
 skaffold.yaml

AGENTS.md

@@ -106,6 +106,66 @@ Backend traits in `crates/core/src/backend.rs` follow CRUD naming:
   with a mock built the same wrong way won't catch it. Only a live-server
   request through the real password-auth path surfaces it.
 
+## Running loadtests locally (`tests/loadtest`)
+
+- `tools/run-loadtest-local.sh [goose args...]` sets up postgres (docker),
+  SPIRE, an embedded-OPA rust `keystone`, bootstraps the admin user, seeds
+  data, builds `tests/loadtest`, and runs it against `http://localhost:8080`.
+  Extra args are forwarded to `load_test`, e.g.:
+  `tools/run-loadtest-local.sh --users 20 --hatch-rate 4 --run-time 30s --report-file reports/run.md`.
+  `tools/teardown-loadtest.sh` tears everything down; the runner calls it
+  itself at the start of every run, so a stale environment from an
+  interrupted run is cleaned up automatically.
+- Requires a working `docker` CLI (postgres container). On a podman-only
+  host, alias `docker` to `podman` and use fully-qualified image refs
+  (`docker.io/postgres:17`) — an unqualified `postgres:17` fails to resolve
+  without configured unqualified-search registries.
+- `tests/loadtest` is excluded from the main workspace (`exclude` in root
+  `Cargo.toml`) — it's a standalone crate. Build it with
+  `cd tests/loadtest && cargo build`, never `cargo build -p load_test` from
+  the repo root.
+- **Never pipe the runner through `tail`** — same root cause as the
+  `test_api` warning above: the script backgrounds SPIRE/OPA/keystone
+  daemons that inherit the pipe's write fd and never let it close, so the
+  whole thing looks hung even after the actual test finished. Run it via
+  `nohup ... > /tmp/out.log 2>&1 &` and read the file, or poll with
+  `kill -0 <pid>` in a loop.
+- `keystone-manage db up` alone is **not** enough against a fresh postgres —
+  it only applies versioned migrations for the drivers that ship them
+  (credential, federation, k8s-auth, token-restriction, webauthn). Identity,
+  role, assignment, resource, catalog etc. use entity-based schema creation
+  via `keystone-manage db sync` instead. Run both, `sync` before `up`.
+- `openstack_sdk`'s auth-cache (`$HOME/.osc/`, enabled by default, keyed by
+  cloud-config hash) survives across server restarts. Since each run
+  bootstraps a brand-new admin user, a stale cache entry would make the SDK
+  replay a token for a user id that no longer exists in the fresh DB —
+  surfaces as widespread, seemingly random 401/500s that have nothing to do
+  with the actual server. The runner works around this by pointing
+  `load_test` at an isolated `$HOME` (under `STATE_DIR`, wiped every run)
+  via `HOME=`/`OS_CLIENT_CONFIG_PATH=` rather than touching your real
+  `~/.osc` or `~/.config/openstack`. If you invoke the SDK some other way
+  and hit "user cannot be found: `<id>`" against a freshly bootstrapped
+  keystone, check `~/.osc` for a stale entry before assuming a real bug.
+- `AsyncOpenStack`/`CloudConfig` accepts either a `clouds.yaml` entry
+  (`OS_CLOUD=<name>`) or plain `OS_*` auth env vars
+  (`OS_AUTH_URL`/`OS_USERNAME`/`OS_PASSWORD`/...) via
+  `CloudConfig::from_env()` — `tests/loadtest/src/main.rs`'s
+  `load_cloud_config()` prefers the env-var form whenever `OS_AUTH_URL` is
+  set, falling back to `OS_CLOUD` + clouds.yaml otherwise, so a clouds.yaml
+  file isn't a hard requirement to run the suite.
+- Goose logs any non-2xx response into its `ERRORS`/status-code tables at
+  the HTTP layer regardless of the transaction's own pass/fail logic.
+  Negative-test transactions (wrong password, revoke-then-validate, rescope
+  to a nonexistent project) must call `user.set_success(&mut goose.request)`
+  once they've confirmed the rejection was the *expected* outcome, or the
+  top-line failure percentage reads misleadingly high even though nothing
+  is actually broken.
+- Loadtest also runs against a keystone deployed via skaffold to local k3s
+  (see `doc/src/contributor/development.md` for that setup) — skip
+  `tools/run-loadtest-local.sh` entirely and point `load_test` straight at
+  the cluster's exposed URL/port with `--host`, using `OS_CLOUD`/`OS_*` env
+  vars matching that deployment's admin credentials.
+
 ## Commit Message Rules
 
 - **Format**: Conventional Commits (`type: subject body`) enforced by

crates/api-types/src/error_conv.rs

@@ -286,6 +286,11 @@ impl From<IdentityProviderError> for KeystoneApiError {
                 resource: "group".into(),
                 identifier: x,
             },
+            // Delegate to the dedicated `ResourceProviderError` conversion
+            // below instead of falling through to the generic `InternalError`
+            // catch-all - e.g. `DomainNotFound` from resolving a
+            // password-auth `domain.name` must surface as 404, not 500.
+            IdentityProviderError::ResourceProvider { source } => source.into(),
             ref err @ IdentityProviderError::Conflict(..) => Self::Conflict(err.to_string()),
             ref err @ IdentityProviderError::SecurityCompliance(..) => {
                 Self::BadRequest(err.to_string())

crates/keystone/src/bin/keystone.rs

@@ -187,7 +187,7 @@ async fn main() -> Result<(), Report> {
     // Guard must stay alive for the process lifetime to flush buffered
     // file-appender logs; binding to `_guard` (rather than dropping the
     // `Option`) keeps that lifetime tied to `main`'s scope.
-    let _guard = init_tracing(args.verbose, &cfg);
+    let _guard = init_tracing(args.verbose, &cfg)?;
     color_eyre::install()?;
 
     info!("Starting Keystone...");
@@ -464,7 +464,10 @@ async fn main() -> Result<(), Report> {
 /// Returns the file-appender's `WorkerGuard`, if file logging is enabled.
 /// The caller must keep this alive for the process lifetime — dropping it
 /// stops buffered log lines from being flushed.
-fn init_tracing(verbose: u8, cfg: &Config) -> Option<tracing_appender::non_blocking::WorkerGuard> {
+fn init_tracing(
+    verbose: u8,
+    cfg: &Config,
+) -> Result<Option<tracing_appender::non_blocking::WorkerGuard>> {
     let external_deps_log_level = match verbose {
         0 => LevelFilter::ERROR,
         1 => LevelFilter::WARN,
@@ -586,7 +589,7 @@ fn init_tracing(verbose: u8, cfg: &Config) -> Option<tracing_appender::non_block
         .with(log_layers)
         .init();
 
-    guard
+    Ok(guard)
 }
 
 /// Build the file-log appender for `[DEFAULT] log_dir`, honoring the

tests/loadtest/.gitignore

@@ -0,0 +1 @@
+/reports/

tests/loadtest/src/main.rs

@@ -14,18 +14,34 @@
 
 use goose::config::GooseDefault;
 use goose::prelude::*;
-use openstack_sdk::{AsyncOpenStack, config::ConfigFile};
+use openstack_sdk::{
+    AsyncOpenStack,
+    config::{CloudConfig, ConfigFile},
+};
 use std::env;
 
 mod seed;
 mod v3;
 
-use crate::v3::auth::{token_lifecycle, validate as validate_token};
+use crate::v3::auth::lifecycle::{token_lifecycle, validate as validate_token};
+use crate::v3::auth::password::{
+    invalid as password_auth_invalid, scoped as password_auth_scoped,
+    unscoped as password_auth_unscoped,
+};
+use crate::v3::auth::rescope::{rescope_invalid_project, rescope_to_system};
+use crate::v3::auth::system::system_scope_auth;
 use crate::v3::domain::list as domain_list;
 use crate::v3::project::{
     create as project_create, delete as project_delete, list as project_list, show as project_show,
     show_random as project_show_random,
 };
+use crate::v3::role::{
+    create as role_create, delete as role_delete, list as role_list, show as role_show,
+};
+use crate::v3::role_assignment::{
+    list as role_assignment_list, list_by_domain as role_assignment_list_by_domain,
+    list_by_role as role_assignment_list_by_role, list_by_user as role_assignment_list_by_user,
+};
 use crate::v3::user::{
     create as user_create, delete as user_delete, list as user_list, show as user_show,
     show_random as user_show_random,
@@ -39,6 +55,8 @@ pub struct Session {
     pub user_id: Option<String>,
     /// ID of the project created in on_start for ProjectCRUD scenario.
     pub project_id: Option<String>,
+    /// ID of the role created in on_start for RoleCRUD scenario.
+    pub role_id: Option<String>,
 }
 
 #[tokio::main]
@@ -54,11 +72,23 @@ async fn main() -> Result<(), GooseError> {
     v3::user::set_seeded_ids(seed_state.user_ids.clone());
     v3::project::set_seeded_ids(seed_state.project_ids.clone());
 
-    // Default to 30 users so all weighted scenarios get at least 1 user
-    // (total weight = 20; 30 ensures proportional coverage).
-    // Can be overridden by passing --users on the CLI.
+    // Share seeded user credentials and the shared auth project with the
+    // password-auth scenarios.
+    v3::user::set_seeded_creds(seed_state.user_creds.clone());
+    if let Some(auth_project_id) = &seed_state.auth_project_id {
+        v3::user::set_auth_project_id(auth_project_id.clone());
+    }
+    if let Some(auth_role_id) = &seed_state.auth_role_id {
+        v3::role_assignment::set_auth_role_id(auth_role_id.clone());
+    }
+
+    // Default to 45 users so all weighted scenarios get at least 1 user
+    // (total weight = 36; 45 ensures proportional coverage).
+    // Can be overridden by passing --users on the CLI. To actually run this
+    // as a stress test, scale further with goose's own CLI flags, e.g.:
+    //   --users 500 --hatch-rate 20 --run-time 10m
     let attack = GooseAttack::initialize()?
-        .set_default(GooseDefault::Users, 30usize)?
+        .set_default(GooseDefault::Users, 45usize)?
         // Read-heavy workload: list endpoints hit the most common production path.
         .register_scenario(
             scenario!("ReadHeavy")
@@ -100,6 +130,36 @@ async fn main() -> Result<(), GooseError> {
                 .register_transaction(transaction!(project_show))
                 .register_transaction(transaction!(project_delete).set_on_stop()),
         )
+        // Role CRUD: each virtual user owns one role resource for the test duration.
+        .register_scenario(
+            scenario!("RoleCRUD")
+                .set_weight(1)?
+                .register_transaction(transaction!(openstack_login).set_on_start())
+                .register_transaction(transaction!(role_create).set_on_start())
+                .register_transaction(transaction!(role_show))
+                .register_transaction(transaction!(role_delete).set_on_stop()),
+        )
+        // Dedicated role listing: exercises GET /v3/roles against the roles
+        // bootstrap/CRUD scenarios keep populated (admin, manager, member,
+        // reader, plus the auth-project's member role and any RoleCRUD churn).
+        .register_scenario(
+            scenario!("RoleList")
+                .set_weight(2)?
+                .register_transaction(transaction!(openstack_login).set_on_start())
+                .register_transaction(transaction!(role_list)),
+        )
+        // Role assignment listing: unfiltered plus the user.id/scope.domain.id/
+        // role.id filter variants, all against the auth-project's seeded
+        // (100 users x member role) assignments.
+        .register_scenario(
+            scenario!("RoleAssignmentList")
+                .set_weight(3)?
+                .register_transaction(transaction!(openstack_login).set_on_start())
+                .register_transaction(transaction!(role_assignment_list))
+                .register_transaction(transaction!(role_assignment_list_by_user))
+                .register_transaction(transaction!(role_assignment_list_by_domain))
+                .register_transaction(transaction!(role_assignment_list_by_role)),
+        )
         // Catalog read: list all users then fetch a randomly chosen one from the
         // pre-seeded pool.  Exercises the list + point-read path under realistic
         // data volumes (100 seeded users).
@@ -119,6 +179,40 @@ async fn main() -> Result<(), GooseError> {
                 .register_transaction(transaction!(openstack_login).set_on_start())
                 .register_transaction(transaction!(project_list))
                 .register_transaction(transaction!(project_show_random)),
+        )
+        // Password auth: exercises the hot password-login path against the
+        // pre-provisioned seeded users, both unscoped and project-scoped.
+        .register_scenario(
+            scenario!("PasswordAuth")
+                .set_weight(4)?
+                .register_transaction(transaction!(password_auth_unscoped))
+                .register_transaction(transaction!(password_auth_scoped)),
+        )
+        // Password auth negative path: wrong password must be rejected (401).
+        .register_scenario(
+            scenario!("PasswordAuthNegative")
+                .set_weight(1)?
+                .register_transaction(transaction!(password_auth_invalid)),
+        )
+        // Rescope the OS_CLOUD-authenticated token to system scope.
+        .register_scenario(
+            scenario!("Rescope")
+                .set_weight(2)?
+                .register_transaction(transaction!(openstack_login).set_on_start())
+                .register_transaction(transaction!(rescope_to_system)),
+        )
+        // Rescope negative path: rescoping to a nonexistent project must fail.
+        .register_scenario(
+            scenario!("RescopeNegative")
+                .set_weight(1)?
+                .register_transaction(transaction!(openstack_login).set_on_start())
+                .register_transaction(transaction!(rescope_invalid_project)),
+        )
+        // System-scoped auth using the initial bootstrap admin credentials.
+        .register_scenario(
+            scenario!("SystemScopeAuth")
+                .set_weight(2)?
+                .register_transaction(transaction!(system_scope_auth)),
         );
 
     attack.execute().await?;
@@ -128,11 +222,26 @@ async fn main() -> Result<(), GooseError> {
     Ok(())
 }
 
-/// Authenticate via the configured OS_CLOUD and store the token in session data.
-pub async fn openstack_login(user: &mut GooseUser) -> TransactionResult {
-    let cfg = ConfigFile::new().unwrap();
+/// Build the admin cloud config, preferring plain `OS_*` auth env vars
+/// (`OS_AUTH_URL`, `OS_USERNAME`, `OS_PASSWORD`, ...) over a `clouds.yaml`
+/// entry when `OS_AUTH_URL` is set, so CI/local runs don't need a clouds.yaml
+/// at all. Falls back to `OS_CLOUD` (default `devstack`) looked up via
+/// clouds.yaml otherwise.
+fn load_cloud_config() -> CloudConfig {
+    if env::var("OS_AUTH_URL").is_ok() {
+        return CloudConfig::from_env().expect("cannot build cloud config from OS_* env vars");
+    }
+    let cfg = ConfigFile::new().expect("cannot read clouds.yaml");
     let cloud_name = env::var("OS_CLOUD").unwrap_or("devstack".to_string());
-    let profile = cfg.get_cloud_config(cloud_name).unwrap().unwrap();
+    cfg.get_cloud_config(&cloud_name)
+        .expect("cannot get cloud config")
+        .unwrap_or_else(|| panic!("cloud '{cloud_name}' not found in clouds.yaml"))
+}
+
+/// Authenticate via the configured OS_CLOUD/OS_* env vars and store the
+/// token in session data.
+pub async fn openstack_login(user: &mut GooseUser) -> TransactionResult {
+    let profile = load_cloud_config();
     let session = AsyncOpenStack::new(&profile)
         .await
         .expect("cannot connect to the cloud");
@@ -141,6 +250,7 @@ pub async fn openstack_login(user: &mut GooseUser) -> TransactionResult {
         token,
         user_id: None,
         project_id: None,
+        role_id: None,
     });
     Ok(())
 }
@@ -156,14 +266,9 @@ fn get_host() -> String {
     "http://localhost:8080".to_string()
 }
 
-/// Obtain an admin token using the configured OS_CLOUD credentials.
+/// Obtain an admin token using the configured OS_CLOUD/OS_* env vars.
 async fn get_admin_token() -> String {
-    let cfg = ConfigFile::new().expect("cannot read clouds.yaml");
-    let cloud_name = env::var("OS_CLOUD").unwrap_or("devstack".to_string());
-    let profile = cfg
-        .get_cloud_config(cloud_name)
-        .expect("cannot get cloud config")
-        .expect("cloud not found");
+    let profile = load_cloud_config();
     let session = AsyncOpenStack::new(&profile)
         .await
         .expect("cannot authenticate");