diff --git a/crates/core/src/api/api_key_auth.rs b/crates/core/src/api/api_key_auth.rs index 927ca088c..05e831fa2 100644 --- a/crates/core/src/api/api_key_auth.rs +++ b/crates/core/src/api/api_key_auth.rs @@ -25,7 +25,7 @@ use axum::extract::{FromRef, FromRequestParts, Path}; use axum::http::request::Parts; use governor::clock::Clock as _; use ipnet::IpNet; -use tracing::warn; +use tracing::{debug, warn}; use openstack_keystone_core_types::api_key::ApiClientResource; use openstack_keystone_core_types::auth::AuthenticationError; @@ -241,12 +241,25 @@ async fn resolve_verified_api_client( // Dummy hash: burn the same Argon2id cost as a real verification // to prevent timing-based enumeration of valid lookup hashes // (ADR 0021 Invariant 7). + debug!( + domain_id, + lookup_hash = %parsed.lookup_hash, + "api_key lookup returned no resource" + ); let _ = crypto::generate_dummy_hash(&cfg).await; return Err(AuthenticationError::Unauthorized.into()); }; let now = chrono::Utc::now().timestamp(); if !resource.is_active(now) { + debug!( + domain_id, + lookup_hash = %parsed.lookup_hash, + enabled = resource.enabled, + expires_at = resource.expires_at, + now, + "api_key resource found but inactive" + ); let _ = crypto::generate_dummy_hash(&cfg).await; return Err(AuthenticationError::Unauthorized.into()); } @@ -260,6 +273,20 @@ async fn resolve_verified_api_client( cfg.trusted_header, ); if !ip_allowed(effective_ip, &resource.allowed_ips) { + // Burn the same Argon2id cost as a real verification here too -- + // otherwise this branch returns near-instantly while the + // not-found/inactive/wrong-secret branches all pay the hashing + // cost, letting an attacker distinguish "this lookup_hash exists + // but my IP is disallowed" from "this lookup_hash doesn't exist" + // by response latency alone (ADR 0021 Invariant 7). + debug!( + domain_id, + client_id = %resource.client_id, + ?effective_ip, + allowed_ips = ?resource.allowed_ips, + "api_key resource found but IP not allowed" + ); + let _ = crypto::generate_dummy_hash(&cfg).await; return Err(AuthenticationError::Unauthorized.into()); } @@ -271,6 +298,12 @@ async fn resolve_verified_api_client( AuthenticationError::Unauthorized })?; if !verified { + debug!( + domain_id, + client_id = %resource.client_id, + lookup_hash = %parsed.lookup_hash, + "api_key resource found, IP allowed, but secret verification failed" + ); return Err(AuthenticationError::Unauthorized.into()); } diff --git a/crates/storage/src/app.rs b/crates/storage/src/app.rs index fd3665c5f..4e3ee87a1 100644 --- a/crates/storage/src/app.rs +++ b/crates/storage/src/app.rs @@ -832,18 +832,20 @@ impl StorageApi for Storage { forwarding get_by_key to leader" ); - if let Ok(forwarded) = self + // A failed forward means we can no longer prove the + // leader we found is still current (or reachable) -- + // falling back to a local read here would silently + // serve possibly-stale data from a follower instead of + // surfacing the inconsistency (this bit us as + // intermittent false-negative auth lookups under load). + // Fail the read; the caller retries. + return self .forwarded_get_by_key(lead_id, lead_node.rpc_addr, key, keyspace) - .await - { - return Ok(forwarded); - } - debug!("forwarded get_by_key failed; falling back to local read"); + .await; } else { - debug!( - "ensure_linearizable (ReadIndex) returned ForwardToLeader \ - without leader info; falling back to local read" - ); + return Err(ApiStoreError::Other(Box::new(StoreError::Other(eyre::eyre!( + "ReadIndex returned ForwardToLeader without leader info" + ))))); } } Err(e) => { @@ -927,19 +929,15 @@ impl StorageApi for Storage { forwarding prefix to leader" ); - if let Ok(forwarded) = self + // See `get_by_key`'s equivalent branch: a failed forward + // must not fall back to an unguarded local read. + return self .forwarded_prefix_read(lead_id, lead_node.rpc_addr, prefix, keyspace) - .await - { - return Ok(forwarded); - } - debug!("forwarded prefix failed; falling back to local read"); + .await; } else { - // No leader info — fall back to local read (e.g., removed node). - debug!( - "ensure_linearizable (ReadIndex) returned ForwardToLeader \ - without leader info; falling back to local read" - ); + return Err(ApiStoreError::Other(Box::new(StoreError::Other(eyre::eyre!( + "ReadIndex returned ForwardToLeader without leader info" + ))))); } } Err(e) => { @@ -1021,18 +1019,21 @@ impl StorageApi for Storage { for prefix_index; forwarding to leader" ); - if let Ok(forwarded) = self + // See `get_by_key`'s equivalent branch: a failed forward + // must not fall back to an unguarded local read. This + // path backs `get_ruleset_by_source`'s index scan -- + // relevant to the intermittent false-negative "mapping + // ruleset not found" 401s observed under concurrent + // load, whose confirmed root cause was the leader-side + // forwarded handler skipping its own leadership/read- + // index re-check (fixed in `storage_service.rs`). + return self .forwarded_prefix_index(lead_id, lead_node.rpc_addr, prefix) - .await - { - return Ok(forwarded); - } - debug!("forwarded prefix_index failed; falling back to local read"); + .await; } else { - debug!( - "ensure_linearizable (ReadIndex) returned ForwardToLeader \ - without leader info; falling back to local read" - ); + return Err(ApiStoreError::Other(Box::new(StoreError::Other(eyre::eyre!( + "ReadIndex returned ForwardToLeader without leader info" + ))))); } } Err(e) => { diff --git a/crates/storage/src/grpc/storage_service.rs b/crates/storage/src/grpc/storage_service.rs index b1774573c..2cf33da4f 100644 --- a/crates/storage/src/grpc/storage_service.rs +++ b/crates/storage/src/grpc/storage_service.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use openraft::ReadPolicy; use tonic::{Request, Response, Status}; use crate::DataTier; @@ -58,6 +59,23 @@ impl StorageServiceImpl { state_machine_store, } } + + /// Re-confirm this node is still leader with an up-to-date read index + /// before serving a forwarded read. + /// + /// A follower that observed `ForwardToLeader` forwards the read here, + /// but leadership can change between that observation and this RPC + /// landing -- without re-checking, a former leader would serve a local + /// read that is no longer guaranteed linearizable (e.g. a competing + /// leader committed writes this node hasn't seen), silently breaking + /// read-your-writes for the forwarded caller. + async fn ensure_leader_linearizable(&self) -> Result<(), Status> { + self.raft_node + .ensure_linearizable(ReadPolicy::ReadIndex) + .await + .map(|_| ()) + .map_err(|e| Status::unavailable(format!("not linearizable leader: {e}"))) + } } #[tonic::async_trait] @@ -98,6 +116,7 @@ impl StorageService for StorageServiceImpl { &self, request: Request, ) -> Result, Status> { + self.ensure_leader_linearizable().await?; let req = request.into_inner(); let key = match std::str::from_utf8(&req.key) { Ok(s) => s.to_string(), @@ -177,6 +196,7 @@ impl StorageService for StorageServiceImpl { &self, request: Request, ) -> Result, Status> { + self.ensure_leader_linearizable().await?; let req = request.into_inner(); let ks = match &req.keyspace { @@ -256,6 +276,7 @@ impl StorageService for StorageServiceImpl { &self, request: Request, ) -> Result, Status> { + self.ensure_leader_linearizable().await?; let req = request.into_inner(); let items: Vec<_> = self diff --git a/tests/loadtest/src/main.rs b/tests/loadtest/src/main.rs index 3ed46235b..146ca06c3 100644 --- a/tests/loadtest/src/main.rs +++ b/tests/loadtest/src/main.rs @@ -60,6 +60,14 @@ use crate::v4::oauth2::{ jwks as oauth2_jwks, list_clients as oauth2_client_list, show_client as oauth2_client_show, well_known as oauth2_well_known, }; +use crate::v4::scim_realm::{ + list as scim_realm_list, show as scim_realm_show, update as scim_realm_update, +}; +use crate::v4::scim_sync::{ + provision_api_key as scim_sync_provision_api_key, sync_create as scim_sync_create, + sync_delete as scim_sync_delete, sync_list as scim_sync_list, sync_patch as scim_sync_patch, + sync_show as scim_sync_show, +}; /// Per-GooseUser session state shared across transactions. pub struct Session { @@ -104,6 +112,10 @@ async fn main() -> Result<(), GooseError> { if let Some(auth_role_id) = &seed_state.auth_role_id { v3::role_assignment::set_auth_role_id(auth_role_id.clone()); } + if let Some(scim_realm_provider_id) = &seed_state.scim_realm_provider_id { + v4::scim_realm::set_realm_provider_id(scim_realm_provider_id.clone()); + v4::scim_sync::set_realm_provider_id(scim_realm_provider_id.clone()); + } // Default to 45 users so all weighted scenarios get at least 1 user // (total weight = 36; 45 ensures proportional coverage). @@ -278,6 +290,33 @@ async fn main() -> Result<(), GooseError> { .set_weight(3)? .register_transaction(transaction!(oauth2_jwks)) .register_transaction(transaction!(oauth2_well_known)), + ) + // Raft-backed SCIM realm reads (ADR 0024): scim-realm-driver-raft. + // No per-VU create/delete -- realm is seeded once (no delete endpoint + // exists, only per-resource purge), shared read-mostly across VUs. + .register_scenario( + scenario!("ScimRealmRead") + .set_weight(1)? + .register_transaction(transaction!(openstack_login).set_on_start()) + .register_transaction(transaction!(scim_realm_show)) + .register_transaction(transaction!(scim_realm_list)) + .register_transaction(transaction!(scim_realm_update)), + ) + // Real SCIM sync simulation (ADR 0021 + 0024): unlike ScimRealmRead + // (admin x-auth-token reads), this authenticates via a per-VU + // `kscim_...` API Key bearer against `/SCIM/v2`, the actual ingress + // path an IdP-side SCIM connector uses -- provision, reconcile + // (list/show), attribute drift (PATCH), offboard (DELETE). + .register_scenario( + scenario!("ScimSync") + .set_weight(1)? + .register_transaction(transaction!(openstack_login).set_on_start()) + .register_transaction(transaction!(scim_sync_provision_api_key).set_on_start()) + .register_transaction(transaction!(scim_sync_create).set_on_start()) + .register_transaction(transaction!(scim_sync_list)) + .register_transaction(transaction!(scim_sync_show)) + .register_transaction(transaction!(scim_sync_patch)) + .register_transaction(transaction!(scim_sync_delete).set_on_stop()), ); attack.execute().await?; diff --git a/tests/loadtest/src/seed.rs b/tests/loadtest/src/seed.rs index d2c97d802..6f272e7de 100644 --- a/tests/loadtest/src/seed.rs +++ b/tests/loadtest/src/seed.rs @@ -30,6 +30,8 @@ pub const SEED_USER_PASSWORD: &str = "LoadTestSeedPassw0rd!"; /// project-scoped password auth scenarios. const AUTH_PROJECT_NAME: &str = "loadtest-authproject"; const AUTH_ROLE_NAME: &str = "member"; +const SCIM_IDP_NAME: &str = "loadtest-scim-idp"; +const SCIM_REALM_PROVIDER_ID: &str = "loadtest-scim-realm"; /// A seeded user's credentials, usable for password authentication. #[derive(Clone)] @@ -50,6 +52,14 @@ pub struct SeedState { /// Role every seeded user is granted on `auth_project_id`; used to /// exercise the `role.id` filter on `GET /v3/role_assignments`. pub auth_role_id: Option, + /// Federation identity provider backing `scim_realm_provider_id`, used by + /// the SCIM realm loadtest scenario (ADR 0024). SCIM realms cannot be + /// created without a real, existing IdP (`get_identity_provider` check). + pub scim_idp_id: Option, + /// `provider_id` of the SCIM realm seeded for `ScimRealmRead`. There is + /// no realm-delete endpoint (only per-resource purge), so this realm is + /// not torn down -- only the backing IdP is, best-effort, in `cleanup`. + pub scim_realm_provider_id: Option, } /// Create background resources so list endpoints return non-trivial result sets. @@ -61,6 +71,8 @@ pub async fn seed(host: &str, token: &str) -> SeedState { user_creds: Vec::new(), auth_project_id: None, auth_role_id: None, + scim_idp_id: None, + scim_realm_provider_id: None, }; for i in 0..SEED_USERS { @@ -103,9 +115,112 @@ pub async fn seed(host: &str, token: &str) -> SeedState { None => eprintln!("seed: failed to set up auth project for password-auth scenarios"), } + match seed_scim_realm(&client, host, token).await { + Some((idp_id, provider_id)) => { + eprintln!("seed: created SCIM idp {idp_id} + realm {provider_id}"); + state.scim_idp_id = Some(idp_id); + state.scim_realm_provider_id = Some(provider_id); + } + None => eprintln!("seed: failed to set up SCIM idp/realm for ScimRealmRead scenario"), + } + state } +/// Create a global federation IdP and a SCIM realm bound to it, so the SCIM +/// realm loadtest scenario (raft-backed, ADR 0024) has a real realm to read. +/// `create_realm` 404s unless the referenced `idp_id` already resolves via +/// `get_identity_provider`, so the IdP must exist first. +async fn seed_scim_realm(client: &Client, host: &str, token: &str) -> Option<(String, String)> { + let idp_body = json!({ + "identity_provider": { + "name": SCIM_IDP_NAME, + "domain_id": DEFAULT_DOMAIN_ID + } + }); + let resp = client + .post(format!("{host}/v4/federation/identity_providers")) + .header("x-auth-token", token) + .json(&idp_body) + .send() + .await + .ok()?; + let idp_id = if resp.status() == reqwest::StatusCode::CONFLICT { + // A prior interrupted run's IdP survived cleanup -- reuse it by name + // rather than aborting seed setup. + let resp = client + .get(format!( + "{host}/v4/federation/identity_providers?domain_id={DEFAULT_DOMAIN_ID}" + )) + .header("x-auth-token", token) + .send() + .await + .ok()?; + if !resp.status().is_success() { + eprintln!("seed: list identity_providers HTTP {}", resp.status()); + return None; + } + let val: serde_json::Value = resp.json().await.ok()?; + val["identity_providers"] + .as_array()? + .iter() + .find(|idp| idp["name"].as_str() == Some(SCIM_IDP_NAME))?["id"] + .as_str()? + .to_owned() + } else { + if !resp.status().is_success() { + eprintln!("seed: create identity_provider HTTP {}", resp.status()); + return None; + } + let val: serde_json::Value = resp.json().await.ok()?; + val["identity_provider"]["id"].as_str()?.to_owned() + }; + + let realm_body = json!({ + "scim_realm": { + "domain_id": DEFAULT_DOMAIN_ID, + "provider_id": SCIM_REALM_PROVIDER_ID, + "idp_id": idp_id, + "display_name": "Loadtest SCIM realm" + } + }); + let resp = client + .post(format!("{host}/v4/scim_realms")) + .header("x-auth-token", token) + .json(&realm_body) + .send() + .await + .ok()?; + if resp.status() == reqwest::StatusCode::CONFLICT { + // A prior run's realm survived (no delete endpoint exists for it, + // ADR 0024) -- reuse it rather than aborting seed setup. + let resp = client + .get(format!( + "{host}/v4/scim_realms/{DEFAULT_DOMAIN_ID}/{SCIM_REALM_PROVIDER_ID}" + )) + .header("x-auth-token", token) + .send() + .await + .ok()?; + if !resp.status().is_success() { + eprintln!("seed: fetch existing scim_realm HTTP {}", resp.status()); + return None; + } + let val: serde_json::Value = resp.json().await.ok()?; + let existing_idp_id = val["scim_realm"]["idp_id"].as_str()?.to_owned(); + let provider_id = val["scim_realm"]["provider_id"].as_str()?.to_owned(); + return Some((existing_idp_id, provider_id)); + } + if !resp.status().is_success() { + eprintln!("seed: create scim_realm HTTP {}", resp.status()); + return None; + } + let val: serde_json::Value = resp.json().await.ok()?; + let provider_id = val["scim_realm"]["provider_id"].as_str()?.to_owned(); + + Some((idp_id, provider_id)) +} + /// Create a dedicated project and grant every seeded user a `member` role on /// it, so seeded users have a real scope to authenticate against. Returns /// the project ID and the role ID granted to every user. @@ -214,6 +329,18 @@ pub async fn cleanup(host: &str, token: &str, state: &SeedState) { eprintln!("seed cleanup: failed to delete auth project {auth_project_id}: {e}"); } + if let Some(idp_id) = &state.scim_idp_id + && let Err(e) = delete( + &client, + host, + token, + &format!("/v4/federation/identity_providers/{idp_id}"), + ) + .await + { + eprintln!("seed cleanup: failed to delete scim idp {idp_id}: {e}"); + } + eprintln!( "seed cleanup: removed {} users, {} projects", state.user_ids.len(), diff --git a/tests/loadtest/src/v4.rs b/tests/loadtest/src/v4.rs index 21120e7eb..d45edd32e 100644 --- a/tests/loadtest/src/v4.rs +++ b/tests/loadtest/src/v4.rs @@ -15,3 +15,5 @@ pub(crate) mod api_key; pub(crate) mod mapping; pub(crate) mod oauth2; +pub(crate) mod scim_realm; +pub(crate) mod scim_sync; diff --git a/tests/loadtest/src/v4/scim_realm.rs b/tests/loadtest/src/v4/scim_realm.rs new file mode 100644 index 000000000..4c41f17a0 --- /dev/null +++ b/tests/loadtest/src/v4/scim_realm.rs @@ -0,0 +1,103 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//! Raft-backed SCIM realm reads (ADR 0024). `create_realm` requires an +//! existing federation IdP (checked live against `get_identity_provider`), +//! so the realm exercised here is seeded once in `seed.rs`, not per-VU -- +//! there is also no realm-delete endpoint (only per-resource purge), so a +//! per-VU create/delete cycle isn't possible for this domain. + +use goose::prelude::*; +use serde_json::json; +use std::sync::OnceLock; + +use crate::Session; + +const DEFAULT_DOMAIN_ID: &str = "default"; + +static REALM_PROVIDER_ID: OnceLock = OnceLock::new(); + +/// Call once before `GooseAttack::execute()` to share the seeded SCIM realm +/// with all virtual users. +pub fn set_realm_provider_id(id: String) { + REALM_PROVIDER_ID.set(id).ok(); +} + +/// List SCIM realms in the default domain. +pub async fn list(user: &mut GooseUser) -> TransactionResult { + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + + let path = format!("/v4/scim_realms?domain_id={DEFAULT_DOMAIN_ID}"); + let req = user + .get_request_builder(&GooseMethod::Get, &path)? + .header("x-auth-token", &token); + + let goose_request = GooseRequest::builder() + .name("GET /v4/scim_realms") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// Show the seeded SCIM realm. +pub async fn show(user: &mut GooseUser) -> TransactionResult { + let Some(provider_id) = REALM_PROVIDER_ID.get() else { + return Ok(()); + }; + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + + let path = format!("/v4/scim_realms/{DEFAULT_DOMAIN_ID}/{provider_id}"); + let req = user + .get_request_builder(&GooseMethod::Get, &path)? + .header("x-auth-token", &token); + + let goose_request = GooseRequest::builder() + .name("GET /v4/scim_realms/:domain_id/:provider_id") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// Idempotently rewrite the seeded realm's `display_name` (raft write path). +pub async fn update(user: &mut GooseUser) -> TransactionResult { + let Some(provider_id) = REALM_PROVIDER_ID.get() else { + return Ok(()); + }; + let session = user.get_session_data_unchecked::(); + let token = session.token.clone(); + + let body = json!({ + "scim_realm": { + "display_name": "Loadtest SCIM realm" + } + }); + let path = format!("/v4/scim_realms/{DEFAULT_DOMAIN_ID}/{provider_id}"); + let req = user + .get_request_builder(&GooseMethod::Put, &path)? + .header("x-auth-token", &token) + .json(&body); + + let goose_request = GooseRequest::builder() + .name("PUT /v4/scim_realms/:domain_id/:provider_id") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} diff --git a/tests/loadtest/src/v4/scim_sync.rs b/tests/loadtest/src/v4/scim_sync.rs new file mode 100644 index 000000000..46a9d76a6 --- /dev/null +++ b/tests/loadtest/src/v4/scim_sync.rs @@ -0,0 +1,296 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//! Simulates a real IdP-driven SCIM sync cycle against `/SCIM/v2` (ADR +//! 0024): unlike `v4::scim_realm` (which reads the realm as an admin, via +//! `x-auth-token`), this module authenticates as the machine identity an +//! IdP-side SCIM connector actually uses -- a `kscim_...` API Key bearer +//! token (ADR 0021) -- and drives the RFC 7644 lifecycle a sync connector +//! generates each cycle: provision (`POST`), reconcile (`GET` list/show), +//! attribute drift (`PATCH`), full-replace (`PUT`), offboarding (`DELETE`). +//! +//! Requires its own per-VU API Key (raft-backed, `provider_id` bound to the +//! seeded SCIM realm so `ScimRealmAuth`'s Realm Activation Gate passes) -- +//! the shared admin token used everywhere else in this suite is rejected +//! outright by `/SCIM/v2` (ADR 0021 §4 Sub-Router Isolation only accepts API +//! Key bearer tokens there). + +use chrono::{Duration, Utc}; +use goose::prelude::*; +use serde_json::json; +use std::sync::OnceLock; +use uuid::Uuid; + +const DEFAULT_DOMAIN_ID: &str = "default"; +const USER_SCHEMA: &str = "urn:ietf:params:scim:schemas:core:2.0:User"; +const PATCH_OP_SCHEMA: &str = "urn:ietf:params:scim:api:messages:2.0:PatchOp"; + +static REALM_PROVIDER_ID: OnceLock = OnceLock::new(); + +/// Call once before `GooseAttack::execute()` to share the seeded SCIM realm +/// this scenario's API Keys must bind to. +pub fn set_realm_provider_id(id: String) { + REALM_PROVIDER_ID.set(id).ok(); +} + +/// Per-VU SCIM sync session state: the API Key bearer token minted in +/// `provision_api_key` and the SCIM user id created in `sync_create`. +#[derive(Default)] +pub struct ScimSession { + pub bearer: Option, + pub user_id: Option, + pub external_id: Option, +} + +/// Mint a per-VU API Key bound to the seeded realm's `provider_id` (on_start, +/// admin-token setup step -- not itself part of the simulated sync traffic). +/// Mirrors `v4::api_key::create` but keeps the full bearer `token`, which +/// that module never needs since it never calls `/SCIM/v2`. +pub async fn provision_api_key(user: &mut GooseUser) -> TransactionResult { + let Some(provider_id) = REALM_PROVIDER_ID.get() else { + return Ok(()); + }; + let session = user.get_session_data_unchecked::(); + let admin_token = session.token.clone(); + + let expires_at = Utc::now() + Duration::hours(1); + let body = json!({ + "api_key": { + "domain_id": DEFAULT_DOMAIN_ID, + "provider_id": provider_id, + "expires_at": expires_at.to_rfc3339(), + "description": "loadtest SCIM sync connector" + } + }); + + let req = user + .get_request_builder(&GooseMethod::Post, "/v4/api-keys")? + .header("x-auth-token", &admin_token) + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /v4/api-keys (SCIM sync setup)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + let response = match goose.response { + Ok(r) => r, + Err(e) => { + return user.set_failure( + &format!("scim_sync api_key provision failed: {e}"), + &mut goose.request, + None, + None, + ); + } + }; + if !response.status().is_success() { + return user.set_failure( + &format!("scim_sync api_key provision returned {}", response.status()), + &mut goose.request, + None, + None, + ); + } + + let val: serde_json::Value = response.json().await.unwrap_or_default(); + let bearer = val["token"].as_str().map(str::to_owned); + + user.set_session_data(ScimSession { + bearer, + user_id: None, + external_id: None, + }); + Ok(()) +} + +fn bearer_header(user: &GooseUser) -> Option { + user.get_session_data::() + .and_then(|s| s.bearer.clone()) + .map(|b| format!("Bearer {b}")) +} + +/// `POST /SCIM/v2/{domain_id}/Users` -- an IdP connector provisioning a new +/// user (ADR 0024 §3.C/§3.D). `externalId` is the IdP-side identifier a real +/// sync connector uses to converge repeat syncs onto the same shadow user. +pub async fn sync_create(user: &mut GooseUser) -> TransactionResult { + let Some(auth) = bearer_header(user) else { + return Ok(()); + }; + + let external_id = format!("loadtest-ext-{}", Uuid::new_v4().as_simple()); + let user_name = format!("loadtest-scim-{}", Uuid::new_v4().as_simple()); + let body = json!({ + "schemas": [USER_SCHEMA], + "externalId": external_id, + "userName": user_name, + "name": {"givenName": "Load", "familyName": "Test"}, + "emails": [{"value": format!("{user_name}@example.org"), "primary": true}], + "active": true + }); + + let path = format!("/SCIM/v2/{DEFAULT_DOMAIN_ID}/Users"); + let req = user + .get_request_builder(&GooseMethod::Post, &path)? + .header("authorization", &auth) + .json(&body); + + let goose_request = GooseRequest::builder() + .name("POST /SCIM/v2/:domain_id/Users (sync provision)") + .set_request_builder(req) + .build(); + + let mut goose = user.request(goose_request).await?; + let response = match goose.response { + Ok(r) => r, + Err(e) => { + return user.set_failure( + &format!("scim sync_create failed: {e}"), + &mut goose.request, + None, + None, + ); + } + }; + if !response.status().is_success() { + return user.set_failure( + &format!("scim sync_create returned {}", response.status()), + &mut goose.request, + None, + None, + ); + } + let val: serde_json::Value = response.json().await.unwrap_or_default(); + let user_id = val["id"].as_str().map(str::to_owned); + + let session = user.get_session_data_unchecked_mut::(); + session.user_id = user_id; + session.external_id = Some(external_id); + Ok(()) +} + +/// `GET /SCIM/v2/{domain_id}/Users?filter=userName eq "..."` -- the +/// reconciliation read a sync connector issues each cycle to check for +/// drift before deciding whether to `PATCH`/`PUT`. +pub async fn sync_list(user: &mut GooseUser) -> TransactionResult { + let Some(auth) = bearer_header(user) else { + return Ok(()); + }; + + let path = format!("/SCIM/v2/{DEFAULT_DOMAIN_ID}/Users?count=20"); + let req = user + .get_request_builder(&GooseMethod::Get, &path)? + .header("authorization", &auth); + + let goose_request = GooseRequest::builder() + .name("GET /SCIM/v2/:domain_id/Users (sync reconcile list)") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// `GET /SCIM/v2/{domain_id}/Users/{id}` -- per-record reconciliation read. +pub async fn sync_show(user: &mut GooseUser) -> TransactionResult { + let Some(auth) = bearer_header(user) else { + return Ok(()); + }; + let Some(user_id) = user + .get_session_data::() + .and_then(|s| s.user_id.clone()) + else { + return Ok(()); + }; + + let path = format!("/SCIM/v2/{DEFAULT_DOMAIN_ID}/Users/{user_id}"); + let req = user + .get_request_builder(&GooseMethod::Get, &path)? + .header("authorization", &auth); + + let goose_request = GooseRequest::builder() + .name("GET /SCIM/v2/:domain_id/Users/:id (sync reconcile show)") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// `PATCH /SCIM/v2/{domain_id}/Users/{id}` -- an IdP-side attribute change +/// (e.g. `displayName` edited upstream) landing as a partial update, the +/// dominant write shape in steady-state sync (RFC 7644 §3.5.2, ADR 0024 +/// §5.C). +pub async fn sync_patch(user: &mut GooseUser) -> TransactionResult { + let Some(auth) = bearer_header(user) else { + return Ok(()); + }; + let Some(user_id) = user + .get_session_data::() + .and_then(|s| s.user_id.clone()) + else { + return Ok(()); + }; + + let body = json!({ + "schemas": [PATCH_OP_SCHEMA], + "Operations": [ + {"op": "replace", "path": "displayName", "value": "Load Test (synced)"} + ] + }); + + let path = format!("/SCIM/v2/{DEFAULT_DOMAIN_ID}/Users/{user_id}"); + let req = user + .get_request_builder(&GooseMethod::Patch, &path)? + .header("authorization", &auth) + .json(&body); + + let goose_request = GooseRequest::builder() + .name("PATCH /SCIM/v2/:domain_id/Users/:id (sync attribute drift)") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + Ok(()) +} + +/// `DELETE /SCIM/v2/{domain_id}/Users/{id}` -- offboarding, the terminal +/// event of a sync cycle when the IdP-side record is removed (on_stop). +pub async fn sync_delete(user: &mut GooseUser) -> TransactionResult { + let Some(auth) = bearer_header(user) else { + return Ok(()); + }; + let Some(user_id) = user + .get_session_data::() + .and_then(|s| s.user_id.clone()) + else { + return Ok(()); + }; + + let path = format!("/SCIM/v2/{DEFAULT_DOMAIN_ID}/Users/{user_id}"); + let req = user + .get_request_builder(&GooseMethod::Delete, &path)? + .header("authorization", &auth); + + let goose_request = GooseRequest::builder() + .name("DELETE /SCIM/v2/:domain_id/Users/:id (sync offboard)") + .set_request_builder(req) + .build(); + + user.request(goose_request).await?; + + let session = user.get_session_data_unchecked_mut::(); + session.user_id = None; + Ok(()) +}