//! `test(store)` suite: atomic Lua, tombstones, grant index, durability. //! //! Every test here spins its own single-node Redis 7.2 container with AOF //! enabled (mirroring `state::test_support`), so persistence, barriers, and //! concurrent compare-and-set run against a real server. use std::{ sync::atomic::{AtomicU64, Ordering}, time::Duration, }; use testcontainers::{ GenericImage, ImageExt, core::{IntoContainerPort, WaitFor}, runners::AsyncRunner, }; use super::{ ErrorClass, OpClass, Store, StoreError, classify_redis, codes, dcr, device, durability_barrier, grants, hget_record, hset_record, may_retry_op, par, pending, ratelimit, retry_op, sessions, sha256_hex, tokens, }; use crate::{ config::{ConfigError, DurabilityMode, RedisConfig, TtlConfig, redis::RawRedisConfig}, state::RedisConn, test_guard::ContainerGuard, }; /// Test body result (no `unwrap`/`expect` under the deny lints). type TestResult = Result<(), TestError>; /// Errors raised by the store test scaffolding itself. #[derive(Debug, thiserror::Error)] enum TestError { /// A testcontainers operation failed. #[error("container error: {0}")] Containers(#[from] testcontainers::TestcontainersError), /// The Redis configuration was rejected. #[error("config error: {0}")] Config(#[from] ConfigError), /// A Redis command failed. #[error("redis error: {0}")] Redis(#[from] redis::RedisError), /// A store operation failed. #[error("store error: {0}")] Store(#[from] StoreError), /// A spawned task failed to join. #[error("task join failed")] Join, /// A test assertion observed an unexpected value. #[error("unexpected test value: {0}")] Unexpected(String), } static TEST_NONCE: AtomicU64 = AtomicU64::new(10_000); fn unique(label: &str) -> String { let n = TEST_NONCE.fetch_add(1, Ordering::Relaxed); format!("store-test-{n}-{label}") } /// Start a single-node Redis with AOF on plus extra server flags. /// /// The guard removes the container synchronously when the test drops it. async fn redis_container(extra: Vec<&str>) -> Result<(ContainerGuard, String), TestError> { let mut args = vec!["redis-server", "--appendonly", "yes"]; args.extend(extra); let container = GenericImage::new("redis", "7.2-alpine") .with_exposed_port(6379.tcp()) .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections")) .with_cmd(args) .start() .await?; let guarded = ContainerGuard::new(container); let port = guarded.get_host_port_ipv4(6379).await?; Ok((guarded, format!("redis://127.0.0.1:{port}"))) } /// Connect a [`RedisConn`] plus [`Store`] directly (startup assertions live in /// task 4; here the store contract is what is under test). /// /// Timeouts are generous on purpose: the blocking `WAITAOF` barrier may wait /// out a full `everysec` fsync cycle, so the client response timeout must sit /// well above the server-side barrier wait. async fn direct_store( url: &str, durability: DurabilityMode, ) -> Result<(RedisConn, Store), TestError> { let client = redis::Client::open(url)?; let manager_config = redis::aio::ConnectionManagerConfig::new() .set_response_timeout(Some(Duration::from_secs(10))) .set_connection_timeout(Some(Duration::from_secs(5))); let manager = redis::aio::ConnectionManager::new_with_config(client, manager_config).await?; let raw = RawRedisConfig { url: Some(url.to_owned()), durability: Some(durability), response_timeout_secs: 10, connect_timeout_secs: 5, ..Default::default() }; let config = RedisConfig::from_raw(raw)?; Ok((RedisConn::Single(manager), Store::from_config(&config))) } fn code_record(client: &str, grant: &str) -> codes::CodeRecord { codes::CodeRecord { client_id: client.to_owned(), redirect_uri: "https://client.example.test/cb".to_owned(), scope: "orders:read".to_owned(), code_challenge: "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM".to_owned(), subject: "user-1".to_owned(), grant_id: grant.to_owned(), cnf: String::new(), } } fn refresh_record(client: &str, grant: &str) -> tokens::RefreshRecord { tokens::RefreshRecord { client_id: client.to_owned(), grant_id: grant.to_owned(), scope: "orders:read".to_owned(), subject: "user-1".to_owned(), cnf: String::new(), } } fn access_record(client: &str, grant: &str) -> tokens::AccessRecord { tokens::AccessRecord { client_id: client.to_owned(), grant_id: grant.to_owned(), scope: "orders:read".to_owned(), subject: "user-1".to_owned(), cnf: String::new(), } } fn rotation_input( old: &str, client: &str, new_refresh: &str, new_access: &str, grant: &str, request_id: Option, ) -> tokens::RotationInput { tokens::RotationInput { old_refresh_hash: old.to_owned(), expected_client_id: client.to_owned(), new_refresh_hash: new_refresh.to_owned(), new_access_hash: new_access.to_owned(), new_refresh: refresh_record(client, grant), new_access: access_record(client, grant), new_refresh_ttl: Duration::from_secs(3600), new_access_ttl: Duration::from_secs(300), request_id, } } /// Remaining TTL of a key in milliseconds (`-1` persistent, `-2` absent). async fn pttl_ms(conn: &mut RedisConn, key: &str) -> Result { let pttl: i64 = redis::cmd("PTTL").arg(key).query_async(conn).await?; Ok(pttl) } /// Assert a key carries a live TTL inside the design window `(0, cap]`. /// /// `cap` comes from the configured/design lifetime, never from the observed /// value. A non-positive TTL means the key is persistent or absent (a defect: /// the artifact would outlive its intended window); the upper bound can only /// drift downward while the test runs, so a slow run cannot flake it. fn assert_ttl_within(pttl: i64, cap: Duration) -> TestResult { if pttl <= 0 || pttl > cap.as_millis() as i64 { return Err(TestError::Unexpected(format!( "PTTL {pttl} ms must be within (0, {} ms]", cap.as_millis() ))); } Ok(()) } #[tokio::test] async fn store_code_consume_exactly_once() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let hash = unique("code"); let grant = unique("grant"); let client = unique("client"); codes::issue( &mut conn, &store, &hash, &code_record(&client, &grant), Duration::from_secs(60), ) .await?; let first = codes::consume(&mut conn, &store, &hash).await?; let second = codes::consume(&mut conn, &store, &hash).await?; if first .as_ref() .is_none_or(|record| record.client_id != client) { return Err(TestError::Unexpected( "first consume must return the record".to_owned(), )); } if second.is_some() { return Err(TestError::Unexpected( "second consume must return None".to_owned(), )); } Ok(()) } #[tokio::test] async fn store_refresh_rotate_idempotent_replay() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let new_refresh = unique("new-refresh"); let new_access = unique("new-access"); let pepper = b"store-test-pepper"; grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; // Same old token presented twice (deterministic request id): the SAME new // tokens come back, never a double rotation. let first = tokens::rotate( &mut conn, &store, pepper, &rotation_input(&old, &client, &new_refresh, &new_access, &grant, None), ) .await?; let second = tokens::rotate( &mut conn, &store, pepper, &rotation_input(&old, &client, &new_refresh, &new_access, &grant, None), ) .await?; match (first, second) { ( tokens::RotateOutcome::Rotated { new_refresh_hash: first_r, new_access_hash: first_a, }, tokens::RotateOutcome::Replay { new_refresh_hash: second_r, new_access_hash: second_a, }, ) if first_r == new_refresh && first_a == new_access && second_r == new_refresh && second_a == new_access => { Ok(()) } _ => Err(TestError::Unexpected( "expected Rotated then Replay with identical tokens".to_owned(), )), } } #[tokio::test] async fn store_refresh_concurrent_cas_one_rotated_one_reuse() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let pepper = b"store-test-pepper"; grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; // Two concurrent rotations with DISTINCT request ids (two spend attempts): // exactly one rotates, the other observes reuse. let first_tag = unique("race-refresh"); let second_tag = unique("race-refresh"); let first = tokio::spawn({ let conn = conn.clone(); let store = store.clone(); let grant = grant.clone(); let client = client.clone(); let old = old.clone(); async move { let mut conn = conn; let input = rotation_input( &old, &client, &first_tag, &format!("{first_tag}-access"), &grant, Some("req-1".to_owned()), ); tokens::rotate(&mut conn, &store, pepper, &input).await } }); let second = tokio::spawn({ let conn = conn.clone(); let store = store.clone(); async move { let mut conn = conn; let input = rotation_input( &old, &client, &second_tag, &format!("{second_tag}-access"), &grant, Some("req-2".to_owned()), ); tokens::rotate(&mut conn, &store, pepper, &input).await } }); let first = first.await.map_err(|_| TestError::Join)??; let second = second.await.map_err(|_| TestError::Join)??; let rotated = u8::from(matches!(first, tokens::RotateOutcome::Rotated { .. })) + u8::from(matches!(second, tokens::RotateOutcome::Rotated { .. })); let reused = u8::from(matches!(first, tokens::RotateOutcome::Reused { .. })) + u8::from(matches!(second, tokens::RotateOutcome::Reused { .. })); if rotated == 1 && reused == 1 { Ok(()) } else { Err(TestError::Unexpected( "expected exactly one Rotated and one Reused".to_owned(), )) } } #[tokio::test] async fn store_refresh_reuse_without_mapping_cascades() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let new_refresh = unique("new-refresh"); let new_access = unique("new-access"); let pepper = b"store-test-pepper"; grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; let first = tokens::rotate( &mut conn, &store, pepper, &rotation_input(&old, &client, &new_refresh, &new_access, &grant, None), ) .await?; if !matches!(first, tokens::RotateOutcome::Rotated { .. }) { return Err(TestError::Unexpected( "first rotation must apply".to_owned(), )); } // Expire the grace mapping (grace window elapsed), then replay the old // token: reuse, and the whole family dies. let request_id = tokens::derive_rotation_id(pepper, &old)?; let map_key = store.rotation_map_key(&request_id); let _: () = redis::cmd("DEL") .arg(&map_key) .query_async(&mut conn) .await?; let replay = tokens::rotate( &mut conn, &store, pepper, &rotation_input( &old, &client, &unique("other"), &unique("other-a"), &grant, None, ), ) .await?; match replay { tokens::RotateOutcome::Reused { rotated_to } if rotated_to == new_refresh => {} _ => { return Err(TestError::Unexpected( "expected Reused pointing at the rotated token".to_owned(), )); } } if tokens::get_refresh(&mut conn, &store, &new_refresh) .await? .is_some() { return Err(TestError::Unexpected( "cascade must delete the rotated refresh token".to_owned(), )); } if tokens::get_access(&mut conn, &store, &new_access) .await? .is_some() { return Err(TestError::Unexpected( "cascade must delete the rotated access token".to_owned(), )); } if grants::get(&mut conn, &store, &grant).await?.is_some() { return Err(TestError::Unexpected( "cascade must delete the grant record".to_owned(), )); } if !grants::members(&mut conn, &store, &grant).await?.is_empty() { return Err(TestError::Unexpected( "cascade must empty the grant set".to_owned(), )); } Ok(()) } #[tokio::test] async fn store_grant_revoke_cascade() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let refresh = unique("refresh"); let access = unique("access"); grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &refresh, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; tokens::put_access( &mut conn, &store, &access, &access_record(&client, &grant), Duration::from_secs(300), ) .await?; if grants::members(&mut conn, &store, &grant).await?.len() != 2 { return Err(TestError::Unexpected( "grant set must index both tokens".to_owned(), )); } let deleted = grants::revoke(&mut conn, &store, &grant).await?; if deleted < 2 { return Err(TestError::Unexpected( "revoke must delete both token keys".to_owned(), )); } if tokens::get_refresh(&mut conn, &store, &refresh) .await? .is_some() || tokens::get_access(&mut conn, &store, &access) .await? .is_some() { return Err(TestError::Unexpected( "revoke must delete member records".to_owned(), )); } // Unknown grants revoke to zero (idempotent). if grants::revoke(&mut conn, &store, &unique("no-such-grant")).await? != 0 { return Err(TestError::Unexpected( "unknown grant must revoke to zero".to_owned(), )); } Ok(()) } #[tokio::test] async fn store_tombstone_detectable_after_rotation() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let new_refresh = unique("new-refresh"); let new_access = unique("new-access"); let pepper = b"store-test-pepper"; grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; // Live token: no tombstone; unknown hash: no tombstone. if tokens::tombstone_of(&mut conn, &store, &old) .await? .is_some() { return Err(TestError::Unexpected( "live token must not look tombstoned".to_owned(), )); } if tokens::tombstone_of(&mut conn, &store, &unique("absent")) .await? .is_some() { return Err(TestError::Unexpected( "absent token must not look tombstoned".to_owned(), )); } let outcome = tokens::rotate( &mut conn, &store, pepper, &rotation_input(&old, &client, &new_refresh, &new_access, &grant, None), ) .await?; if !matches!(outcome, tokens::RotateOutcome::Rotated { .. }) { return Err(TestError::Unexpected("rotation must apply".to_owned())); } match tokens::tombstone_of(&mut conn, &store, &old).await? { Some(rotated_to) if rotated_to == new_refresh => Ok(()), _ => Err(TestError::Unexpected( "tombstone must point at the new token".to_owned(), )), } } #[tokio::test] async fn store_persistence_survives_restart() -> TestResult { let (container, url) = redis_container(vec!["--appendfsync", "always"]).await?; let grant = unique("grant"); let client = unique("client"); let code_hash = unique("code"); let refresh_hash = unique("refresh"); let jti = unique("jti"); { let (mut conn, store) = direct_store(&url, DurabilityMode::AppendFsyncAlways).await?; codes::issue( &mut conn, &store, &code_hash, &code_record(&client, &grant), Duration::from_secs(600), ) .await?; tokens::put_refresh( &mut conn, &store, &refresh_hash, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; if !dcr::claim_dpop_jti(&mut conn, &store, &jti, Duration::from_secs(600)).await? { return Err(TestError::Unexpected("marker claim must win".to_owned())); } } container.stop_with_timeout(Some(10)).await?; container.start().await?; let port = container.get_host_port_ipv4(6379).await?; let (mut conn, store) = direct_store( &format!("redis://127.0.0.1:{port}"), DurabilityMode::AppendFsyncAlways, ) .await?; let code = codes::consume(&mut conn, &store, &code_hash).await?; if code .as_ref() .is_none_or(|record| record.client_id != client) { return Err(TestError::Unexpected( "code must survive the restart".to_owned(), )); } let refresh = tokens::get_refresh(&mut conn, &store, &refresh_hash).await?; if refresh .as_ref() .is_none_or(|record| record.grant_id != grant) { return Err(TestError::Unexpected( "refresh record must survive the restart".to_owned(), )); } if dcr::claim_dpop_jti(&mut conn, &store, &jti, Duration::from_secs(600)).await? { return Err(TestError::Unexpected( "replay marker must survive the restart".to_owned(), )); } Ok(()) } #[tokio::test] async fn store_durability_barrier_modes() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; // Single-node: WAITAOF barrier succeeds after a write. let (mut conn, waitaof) = direct_store(&url, DurabilityMode::WaitAof).await?; durability_barrier(&mut conn, &waitaof).await?; // `WAIT 1` on a replica-less node acknowledges zero: fail closed. let wait = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{rl}:".to_owned(), DurabilityMode::Wait, ); if durability_barrier(&mut conn, &wait).await.is_ok() { return Err(TestError::Unexpected( "WAIT without replicas must fail closed".to_owned(), )); } // `appendfsync always` assertion fails on an everysec node: fail closed. let always = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{rl}:".to_owned(), DurabilityMode::AppendFsyncAlways, ); if durability_barrier(&mut conn, &always).await.is_ok() { return Err(TestError::Unexpected( "everysec node must fail the always barrier".to_owned(), )); } // ...and succeeds where the server actually fsyncs always. let (_always_container, always_url) = redis_container(vec!["--appendfsync", "always"]).await?; let (mut always_conn, _) = direct_store(&always_url, DurabilityMode::AppendFsyncAlways).await?; durability_barrier(&mut always_conn, &always).await?; Ok(()) } #[tokio::test] async fn store_ratelimit_fixed_window() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let bucket = unique("bucket"); let window = Duration::from_secs(60); for _ in 0..3 { let decision = ratelimit::check_and_incr(&mut conn, &store, &bucket, window, 3).await?; if !decision.allowed { return Err(TestError::Unexpected( "first three hits must be allowed".to_owned(), )); } } let limited = ratelimit::check_and_incr(&mut conn, &store, &bucket, window, 3).await?; if limited.allowed || limited.count != 4 || limited.retry_after_secs == 0 { return Err(TestError::Unexpected( "fourth hit must be limited with Retry-After".to_owned(), )); } // Buckets are hashed under the `{rl}` keyspace, never raw labels. let key = store.ratelimit_key(&ratelimit::hash_bucket(&bucket)); if !key.contains("{rl}") || key.contains(&bucket) { return Err(TestError::Unexpected( "counter key must be hashed and {rl}-tagged".to_owned(), )); } if ratelimit::hash_bucket(&bucket) != ratelimit::hash_bucket(&bucket) { return Err(TestError::Unexpected( "bucket hashing must be deterministic".to_owned(), )); } Ok(()) } #[tokio::test] async fn store_replay_markers_first_claim_wins() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let jti = unique("jti"); let assertion = unique("assertion"); if !dcr::claim_dpop_jti(&mut conn, &store, &jti, Duration::from_secs(300)).await? { return Err(TestError::Unexpected( "first DPoP claim must win".to_owned(), )); } if dcr::claim_dpop_jti(&mut conn, &store, &jti, Duration::from_secs(300)).await? { return Err(TestError::Unexpected( "second DPoP claim must lose".to_owned(), )); } if !dcr::claim_client_assertion_jti(&mut conn, &store, &assertion, Duration::from_secs(300)) .await? { return Err(TestError::Unexpected( "first assertion claim must win".to_owned(), )); } if dcr::claim_client_assertion_jti(&mut conn, &store, &assertion, Duration::from_secs(300)) .await? { return Err(TestError::Unexpected( "second assertion claim must lose".to_owned(), )); } Ok(()) } #[tokio::test] async fn store_sessions_and_login_counters() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let old_session = unique("session"); let new_session = unique("session"); let record = sessions::SessionRecord { subject: "user-1".to_owned(), mfa_verified: true, restricted: false, }; sessions::put( &mut conn, &store, &old_session, &record, Duration::from_secs(3600), ) .await?; sessions::rotate( &mut conn, &store, &old_session, &new_session, &record, Duration::from_secs(3600), ) .await?; if sessions::get(&mut conn, &store, &old_session) .await? .is_some() { return Err(TestError::Unexpected( "old session must be gone after rotate".to_owned(), )); } let current = sessions::get(&mut conn, &store, &new_session).await?; if current.as_ref().is_none_or(|session| !session.mfa_verified) { return Err(TestError::Unexpected( "new session must verify MFA".to_owned(), )); } sessions::destroy(&mut conn, &store, &new_session).await?; if sessions::get(&mut conn, &store, &new_session) .await? .is_some() { return Err(TestError::Unexpected( "destroyed session must be gone".to_owned(), )); } let account = unique("alice@example.test"); let first = sessions::record_failed_login(&mut conn, &store, &account, Duration::from_secs(300)).await?; let second = sessions::record_failed_login(&mut conn, &store, &account, Duration::from_secs(300)).await?; if (first, second) != (1, 2) { return Err(TestError::Unexpected( "failed-login counter must increment".to_owned(), )); } if sessions::failed_login_count(&mut conn, &store, &account).await? != 2 { return Err(TestError::Unexpected( "failed-login count must read back".to_owned(), )); } sessions::clear_failed_logins(&mut conn, &store, &account).await?; if sessions::failed_login_count(&mut conn, &store, &account).await? != 0 { return Err(TestError::Unexpected( "cleared counter must read zero".to_owned(), )); } Ok(()) } #[tokio::test] async fn store_device_par_and_dcr_round_trip() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let client = unique("client"); // Device flow: put, poll, approve, consume exactly once. let device_hash = unique("device"); let user_code = unique("user-code"); device::put( &mut conn, &store, &device_hash, &user_code, &device::DeviceRecord { client_id: client.clone(), scope: "orders:read".to_owned(), status: device::DeviceStatus::Pending, subject: String::new(), grant_id: String::new(), }, Duration::from_secs(600), ) .await?; if device::resolve_user_code(&mut conn, &store, &user_code) .await? .as_deref() != Some(device_hash.as_str()) { return Err(TestError::Unexpected( "user code must resolve to the device".to_owned(), )); } let grant = unique("grant"); if !device::decide( &mut conn, &store, &device_hash, device::DeviceStatus::Approved, "user-1", &grant, ) .await? { return Err(TestError::Unexpected("approve must apply".to_owned())); } let approved = device::poll(&mut conn, &store, &device_hash).await?; if approved .as_ref() .is_none_or(|record| record.status != device::DeviceStatus::Approved) { return Err(TestError::Unexpected( "poll must observe approval".to_owned(), )); } if device::consume(&mut conn, &store, &device_hash) .await? .is_none() { return Err(TestError::Unexpected( "first device consume must return the record".to_owned(), )); } if device::consume(&mut conn, &store, &device_hash) .await? .is_some() { return Err(TestError::Unexpected( "second device consume must return None".to_owned(), )); } // PAR: put, consume exactly once. let request_id = unique("par"); par::put( &mut conn, &store, &request_id, &par::ParRecord { client_id: client.clone(), request: "{\"response_type\":\"code\"}".to_owned(), }, Duration::from_secs(90), ) .await?; if par::consume(&mut conn, &store, &request_id) .await? .is_none() { return Err(TestError::Unexpected( "first PAR consume must return the record".to_owned(), )); } if par::consume(&mut conn, &store, &request_id) .await? .is_some() { return Err(TestError::Unexpected( "second PAR consume must return None".to_owned(), )); } // DCR initial-access token: two uses, then exhausted. let iat = unique("iat"); dcr::issue( &mut conn, &store, &iat, &dcr::InitialAccessRecord { uses_remaining: 2, label: "ops-token".to_owned(), }, Duration::from_secs(3600), ) .await?; let first_use = dcr::consume_one_use(&mut conn, &store, &iat).await?; if first_use .as_ref() .is_none_or(|record| record.uses_remaining != 2) { return Err(TestError::Unexpected( "first use must report the pre-decrement budget".to_owned(), )); } if dcr::consume_one_use(&mut conn, &store, &iat) .await? .is_none() { return Err(TestError::Unexpected( "second (final) use must succeed".to_owned(), )); } if dcr::consume_one_use(&mut conn, &store, &iat) .await? .is_some() { return Err(TestError::Unexpected( "exhausted token must return None".to_owned(), )); } Ok(()) } /// docs/REDIS.md: `oauth:{state}:code:` carries the authorization-code /// lifetime, capped at [`crate::protocol::code::CODE_TTL_CAP`] (60 s). #[tokio::test] async fn store_code_ttl_within_configured_window() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let ttl = TtlConfig::default(); let hash = unique("code"); codes::issue( &mut conn, &store, &hash, &code_record(&unique("client"), &unique("grant")), ttl.code, ) .await?; let cap = ttl.code.min(crate::protocol::code::CODE_TTL_CAP); assert_ttl_within(pttl_ms(&mut conn, &store.code_key(&hash)).await?, cap) } /// docs/REDIS.md: the PAR handle lives under `ttl.par` (90 s) and both the /// device-code record and its user-code mapping under `ttl.device` (600 s). #[tokio::test] async fn store_par_device_and_user_code_ttl_within_configured_windows() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let ttl = TtlConfig::default(); let client = unique("client"); let request_id = unique("par"); par::put( &mut conn, &store, &request_id, &par::ParRecord { client_id: client.clone(), request: "{\"response_type\":\"code\"}".to_owned(), }, ttl.par, ) .await?; assert_ttl_within( pttl_ms(&mut conn, &store.par_key(&request_id)).await?, ttl.par, )?; let device_hash = unique("device"); let user_code = unique("user-code"); device::put( &mut conn, &store, &device_hash, &user_code, &device::DeviceRecord { client_id: client, scope: "orders:read".to_owned(), status: device::DeviceStatus::Pending, subject: String::new(), grant_id: String::new(), }, ttl.device, ) .await?; assert_ttl_within( pttl_ms(&mut conn, &store.device_key(&device_hash)).await?, ttl.device, )?; assert_ttl_within( pttl_ms(&mut conn, &store.user_code_key(&user_code)).await?, ttl.device, ) } /// docs/REDIS.md: the DCR initial-access-token record lives under the CLI /// `--ttl`, defaulting to the configured `ttl.dcr_initial_access` (24 h). #[tokio::test] async fn store_dcr_initial_access_ttl_within_configured_window() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let ttl = TtlConfig::default(); let iat = unique("iat"); dcr::issue( &mut conn, &store, &iat, &dcr::InitialAccessRecord { uses_remaining: 2, label: "ops-token".to_owned(), }, ttl.dcr_initial_access, ) .await?; assert_ttl_within( pttl_ms(&mut conn, &store.dcr_iat_key(&iat)).await?, ttl.dcr_initial_access, ) } /// docs/REDIS.md: the rotation idempotency mapping is bounded by /// `min(900 s, refresh TTL)`, the old refresh record becomes a tombstone with /// the refresh lifetime, and the per-grant token index lives with the grant. #[tokio::test] async fn store_rotation_grace_tombstone_and_grant_index_ttl() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let new_refresh = unique("new-refresh"); let new_access = unique("new-access"); let pepper = b"store-test-pepper"; let refresh_ttl = Duration::from_secs(3600); let access_ttl = Duration::from_secs(300); let rotation_grace = Duration::from_secs(900); grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, refresh_ttl, ) .await?; assert_ttl_within( pttl_ms(&mut conn, &store.grant_key(&grant)).await?, refresh_ttl, )?; // The index is born with the access member, then extended by the refresh // member: it must outlive the short access TTL and stay within the grant's. tokens::put_access( &mut conn, &store, &unique("access-seed"), &access_record(&client, &grant), access_ttl, ) .await?; tokens::put_refresh( &mut conn, &store, &old, &refresh_record(&client, &grant), refresh_ttl, ) .await?; let index_key = store.grant_tokens_key(&grant); let index_ttl = pttl_ms(&mut conn, &index_key).await?; if index_ttl <= access_ttl.as_millis() as i64 { return Err(TestError::Unexpected( "grant index must outlive the access-token TTL".to_owned(), )); } assert_ttl_within(index_ttl, refresh_ttl)?; // Simulate the index having expired: the rotation script must re-arm it // rather than recreate it persistent. let _: () = redis::cmd("DEL") .arg(&index_key) .query_async(&mut conn) .await?; let outcome = tokens::rotate( &mut conn, &store, pepper, &rotation_input(&old, &client, &new_refresh, &new_access, &grant, None), ) .await?; if !matches!(outcome, tokens::RotateOutcome::Rotated { .. }) { return Err(TestError::Unexpected("rotation must apply".to_owned())); } assert_ttl_within(pttl_ms(&mut conn, &index_key).await?, refresh_ttl)?; let request_id = tokens::derive_rotation_id(pepper, &old)?; assert_ttl_within( pttl_ms(&mut conn, &store.rotation_map_key(&request_id)).await?, rotation_grace.min(refresh_ttl), )?; assert_ttl_within( pttl_ms(&mut conn, &store.refresh_key(&old)).await?, refresh_ttl, ) } /// docs/REDIS.md: `oauth:{state}:session:` lives under `ttl.session` /// (12 h); a presession is the same record before login binds a subject. #[tokio::test] async fn store_session_and_presession_ttl_within_configured_window() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let session_ttl = TtlConfig::default().session; let session_id = unique("session"); sessions::put( &mut conn, &store, &session_id, &sessions::SessionRecord { subject: "user-1".to_owned(), mfa_verified: true, restricted: false, }, session_ttl, ) .await?; assert_ttl_within( pttl_ms(&mut conn, &store.session_key(&session_id)).await?, session_ttl, )?; let presession_id = unique("presession"); sessions::put( &mut conn, &store, &presession_id, &sessions::SessionRecord { subject: String::new(), mfa_verified: false, restricted: false, }, session_ttl, ) .await?; assert_ttl_within( pttl_ms(&mut conn, &store.session_key(&presession_id)).await?, session_ttl, ) } /// docs/REDIS.md: `oauth:{state}:pending:` is short, request-scoped /// (`routes::PENDING_TTL`, 300 s). #[tokio::test] async fn store_pending_auth_ttl_within_request_window() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let request_id = unique("pending"); pending::put( &mut conn, &store, &request_id, &pending::PendingAuth { client_id: unique("client"), redirect_uri: "https://client.example.test/cb".to_owned(), scope: "orders:read".to_owned(), code_challenge: "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM".to_owned(), state: Some("state-xyz".to_owned()), dpop_jkt: None, iss: "https://as.example.test".to_owned(), post_logout_uri: None, }, crate::routes::PENDING_TTL, ) .await?; assert_ttl_within( pttl_ms(&mut conn, &store.token_key("pending", &request_id)).await?, crate::routes::PENDING_TTL, ) } /// docs/REDIS.md: the failed-login counter lives under the lockout window /// (`ui::login::LOCKOUT_WINDOW`, 15 min) and keys a hash of the account label. #[tokio::test] async fn store_login_failure_counter_ttl_within_lockout_window() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let window = crate::ui::login::LOCKOUT_WINDOW; let account = unique("alice@example.test"); sessions::record_failed_login(&mut conn, &store, &account, window).await?; let key = store.login_fail_key(&sha256_hex(account.as_bytes())); assert_ttl_within(pttl_ms(&mut conn, &key).await?, window) } /// docs/REDIS.md: the DPoP replay marker is bounded by the proof window /// ([`crate::jose::dpop::DPOP_JTI_TTL`]); the `private_key_jwt` marker is /// bounded by the assertion's remaining validity (at most 360 s). #[tokio::test] async fn store_replay_marker_ttl_within_proof_window() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let dpop_ttl = crate::jose::dpop::DPOP_JTI_TTL; let jti = unique("jti"); if !dcr::claim_dpop_jti(&mut conn, &store, &jti, dpop_ttl).await? { return Err(TestError::Unexpected( "first DPoP claim must win".to_owned(), )); } assert_ttl_within( pttl_ms(&mut conn, &store.dpop_jti_key(&jti)).await?, dpop_ttl, )?; let assertion_ttl = Duration::from_secs(300); let assertion = unique("assertion"); if !dcr::claim_client_assertion_jti(&mut conn, &store, &assertion, assertion_ttl).await? { return Err(TestError::Unexpected( "first assertion claim must win".to_owned(), )); } assert_ttl_within( pttl_ms(&mut conn, &store.client_assertion_jti_key(&assertion)).await?, assertion_ttl, ) } #[test] fn store_keys_share_hash_tag_and_keyspaces_split() -> TestResult { let store = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{rl}:".to_owned(), DurabilityMode::WaitAof, ); store.assert_keyspace_separation()?; for key in [ store.code_key("h"), store.access_key("h"), store.refresh_key("h"), store.grant_key("g"), store.grant_tokens_key("g"), store.session_key("s"), store.par_key("p"), store.device_key("d"), store.user_code_key("u"), store.dcr_iat_key("i"), store.dpop_jti_key("j"), store.client_assertion_jti_key("j"), store.rotation_map_key("r"), store.login_fail_key("l"), ] { if !key.contains("{state}") { return Err(TestError::Unexpected( "token key must carry the {state} tag".to_owned(), )); } } if !store.ratelimit_key("b").contains("{rl}") { return Err(TestError::Unexpected( "counter key must carry the {rl} tag".to_owned(), )); } // Colliding prefixes are rejected so counters can never evict tokens. let colliding = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{state}:".to_owned(), DurabilityMode::WaitAof, ); if colliding.assert_keyspace_separation().is_ok() { return Err(TestError::Unexpected( "colliding keyspaces must be rejected".to_owned(), )); } Ok(()) } #[test] fn store_durability_selected_from_topology() -> TestResult { // Single-node topologies must barrier on WAITAOF, never WAIT. let single = RawRedisConfig { url: Some("redis://127.0.0.1:6379".to_owned()), durability: Some(DurabilityMode::Wait), ..Default::default() }; if RedisConfig::from_raw(single).is_ok() { return Err(TestError::Unexpected( "single-node WAIT must be rejected".to_owned(), )); } let single = RawRedisConfig { url: Some("redis://127.0.0.1:6379".to_owned()), durability: Some(DurabilityMode::WaitAof), ..Default::default() }; let config = RedisConfig::from_raw(single)?; if Store::from_config(&config).durability() != DurabilityMode::WaitAof { return Err(TestError::Unexpected( "single-node store must barrier on WAITAOF".to_owned(), )); } Ok(()) } #[test] fn store_rotation_id_deterministic_per_old_token() -> TestResult { let pepper = b"store-test-pepper"; let first = tokens::derive_rotation_id(pepper, "old-hash")?; let second = tokens::derive_rotation_id(pepper, "old-hash")?; if first != second || first.is_empty() { return Err(TestError::Unexpected( "rotation id must be deterministic".to_owned(), )); } if tokens::derive_rotation_id(pepper, "other-hash")? == first { return Err(TestError::Unexpected( "rotation id must differ per token".to_owned(), )); } if tokens::derive_rotation_id(b"other-pepper", "old-hash")? == first { return Err(TestError::Unexpected( "rotation id must differ per pepper".to_owned(), )); } Ok(()) } #[test] fn store_error_classification_and_retry_scope() -> TestResult { let transient_io = redis::RedisError::from(std::io::Error::new( std::io::ErrorKind::TimedOut, "timed out", )); let loading = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::BusyLoading), "busy", )); let try_again = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::TryAgain), "retry", )); let master_down = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::MasterDown), "down", )); let cluster_down = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::ClusterDown), "down", )); for error in [ &transient_io, &loading, &try_again, &master_down, &cluster_down, ] { if classify_redis(error) != ErrorClass::Transient { return Err(TestError::Unexpected( "expected a transient class".to_owned(), )); } if !may_retry_op(OpClass::IdempotentRead, error) || !may_retry_op(OpClass::ConditionalWrite, error) || may_retry_op(OpClass::NonIdempotent, error) { return Err(TestError::Unexpected( "retry scope must exclude non-idempotent ops".to_owned(), )); } } let cross_slot = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::CrossSlot), "slot", )); let readonly = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly), "ro", )); let type_error = redis::RedisError::from((redis::ErrorKind::UnexpectedReturnType, "type")); let auth = redis::RedisError::from((redis::ErrorKind::AuthenticationFailed, "auth")); for error in [&cross_slot, &readonly, &type_error, &auth] { if classify_redis(error) != ErrorClass::Permanent { return Err(TestError::Unexpected( "expected a permanent class".to_owned(), )); } if may_retry_op(OpClass::IdempotentRead, error) { return Err(TestError::Unexpected( "permanent errors must never retry".to_owned(), )); } } // Server-extension codes (`OOM`, ...) classify by code: fail closed. let oom_value = redis::parse_redis_value(b"-OOM command not allowed\r\n"); match oom_value { Ok(redis::Value::ServerError(server)) => { let error = redis::RedisError::from(server); if classify_redis(&error) != ErrorClass::Permanent { return Err(TestError::Unexpected("OOM must fail closed".to_owned())); } } _ => { return Err(TestError::Unexpected( "expected an OOM server error value".to_owned(), )); } } Ok(()) } #[test] fn store_permanent_infra_errors_map_to_503() -> TestResult { use crate::error::{AppError, OAuthErrorCode}; use axum::http::StatusCode; let cross_slot = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::CrossSlot), "slot", )); let readonly = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly), "ro", )); let auth = redis::RedisError::from((redis::ErrorKind::AuthenticationFailed, "auth")); let oom_value = redis::parse_redis_value(b"-OOM command not allowed\r\n"); let oom = match oom_value { Ok(redis::Value::ServerError(server)) => redis::RedisError::from(server), _ => { return Err(TestError::Unexpected( "expected an OOM server error value".to_owned(), )); } }; for error in [cross_slot, readonly, auth, oom] { let mapped = AppError::from(StoreError::Redis(error)); if mapped.code() != OAuthErrorCode::TemporarilyUnavailable { return Err(TestError::Unexpected( "permanent Redis infra failure must map to temporarily_unavailable".to_owned(), )); } if mapped.status() != StatusCode::SERVICE_UNAVAILABLE { return Err(TestError::Unexpected( "permanent Redis infra failure must render 503".to_owned(), )); } } // Genuinely internal bugs stay masked as server_error (500). for error in [StoreError::InvalidData, StoreError::Mismatch] { let mapped = AppError::from(error); if mapped.code() != OAuthErrorCode::ServerError { return Err(TestError::Unexpected( "internal store bug must stay server_error".to_owned(), )); } if mapped.status() != StatusCode::INTERNAL_SERVER_ERROR { return Err(TestError::Unexpected( "internal store bug must render 500".to_owned(), )); } } Ok(()) } #[test] fn store_error_classification() -> TestResult { use crate::error::{AppError, OAuthErrorCode}; use axum::http::StatusCode; let parse = redis::RedisError::from((redis::ErrorKind::Parse, "parse")); let unexpected = redis::RedisError::from((redis::ErrorKind::UnexpectedReturnType, "type")); let client = redis::RedisError::from((redis::ErrorKind::Client, "client")); let wrongtype_value = redis::parse_redis_value( b"-WRONGTYPE Operation against a key holding the wrong kind of value\r\n", ); let wrongtype = match wrongtype_value { Ok(redis::Value::ServerError(server)) => redis::RedisError::from(server), _ => { return Err(TestError::Unexpected( "expected a WRONGTYPE server error value".to_owned(), )); } }; for error in [parse, unexpected, client, wrongtype] { let mapped = AppError::from(StoreError::Redis(error)); if mapped.code() != OAuthErrorCode::ServerError { return Err(TestError::Unexpected( "programming Redis error must map to server_error".to_owned(), )); } if mapped.status() != StatusCode::INTERNAL_SERVER_ERROR { return Err(TestError::Unexpected( "programming Redis error must render 500".to_owned(), )); } } let cross_slot = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::CrossSlot), "slot", )); let readonly = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly), "ro", )); let auth = redis::RedisError::from((redis::ErrorKind::AuthenticationFailed, "auth")); let transient = redis::RedisError::from(std::io::Error::new( std::io::ErrorKind::TimedOut, "timed out", )); let oom_value = redis::parse_redis_value(b"-OOM command not allowed\r\n"); let oom = match oom_value { Ok(redis::Value::ServerError(server)) => redis::RedisError::from(server), _ => { return Err(TestError::Unexpected( "expected an OOM server error value".to_owned(), )); } }; for error in [cross_slot, readonly, auth, transient, oom] { let mapped = AppError::from(StoreError::Redis(error)); if mapped.code() != OAuthErrorCode::TemporarilyUnavailable { return Err(TestError::Unexpected( "infra Redis failure must map to temporarily_unavailable".to_owned(), )); } if mapped.status() != StatusCode::SERVICE_UNAVAILABLE { return Err(TestError::Unexpected( "infra Redis failure must render 503".to_owned(), )); } } Ok(()) } #[tokio::test] async fn store_retry_budget_bounds_idempotent_reads() -> TestResult { use std::sync::atomic::{AtomicU32, Ordering as AtomicOrdering}; static ATTEMPTS: AtomicU32 = AtomicU32::new(0); ATTEMPTS.store(0, AtomicOrdering::Relaxed); let (_container, url) = redis_container(Vec::new()).await?; let (mut _conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let outcome: Result<(), redis::RedisError> = retry_op!(&mut _conn, store.retry_budget(), async { ATTEMPTS.fetch_add(1, AtomicOrdering::Relaxed); Err::<(), redis::RedisError>(redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::CrossSlot), "slot", ))) }); match outcome { Err(error) if classify_redis(&error) == ErrorClass::Permanent && ATTEMPTS.load(AtomicOrdering::Relaxed) == 1 => {} _ => { return Err(TestError::Unexpected( "permanent errors must not retry".to_owned(), )); } } Ok(()) } /// The record write is a single atomic `HSET`+`PEXPIRE` script: a record /// observed after [`hset_record`] always carries both its fields AND a live /// TTL, and a rewrite re-arms the expiry (never a persistent key). #[tokio::test] async fn store_hset_record_writes_fields_and_ttl() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, _store) = direct_store(&url, DurabilityMode::WaitAof).await?; let key = unique("hset-record"); let ttl = Duration::from_secs(300); let pairs = vec![ ("client_id".to_owned(), "client-1".to_owned()), ("scope".to_owned(), "orders:read".to_owned()), ]; hset_record(&mut conn, &key, &pairs, ttl).await?; let record = hget_record(&mut conn, &key).await?; if record .as_ref() .is_none_or(|fields| fields.get("client_id").map(String::as_str) != Some("client-1")) { return Err(TestError::Unexpected( "written record must carry its fields".to_owned(), )); } assert_ttl_within(pttl_ms(&mut conn, &key).await?, ttl)?; // Rewrite: a shorter lifetime must be observable, proving the write // re-arms the TTL and does not leave the previous expiry in place. let rewrite = Duration::from_secs(120); let updated = vec![ ("client_id".to_owned(), "client-2".to_owned()), ("scope".to_owned(), "orders:read".to_owned()), ]; hset_record(&mut conn, &key, &updated, rewrite).await?; let record = hget_record(&mut conn, &key).await?; if record .as_ref() .is_none_or(|fields| fields.get("client_id").map(String::as_str) != Some("client-2")) { return Err(TestError::Unexpected( "rewrite must replace the field value".to_owned(), )); } assert_ttl_within(pttl_ms(&mut conn, &key).await?, rewrite)?; // A key whose TTL was stripped must regain a live expiry when rewritten // through the atomic write, so no record can remain persistent. let _: () = redis::cmd("PERSIST") .arg(&key) .query_async(&mut conn) .await?; if pttl_ms(&mut conn, &key).await? != -1 { return Err(TestError::Unexpected( "PERSIST must strip the TTL for the simulation".to_owned(), )); } hset_record(&mut conn, &key, &pairs, ttl).await?; assert_ttl_within(pttl_ms(&mut conn, &key).await?, ttl) } /// Revoke drain exhaustion must fail closed, never report silent success. /// /// A zero-pass budget models the exhausted drain (a mint landing after every /// snapshot, so the set never reads empty): members still indexed must /// surface [`StoreError::InvalidData`], never a silent `Ok` count. The full /// production budget on the same grant then still drains to zero with no live /// member records behind it. #[tokio::test] async fn store_grant_revoke_exhaustion_is_not_silent() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let seed = unique("seed-refresh"); grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &seed, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; match grants::revoke_with_passes(&mut conn, &store, &grant, 0).await { Err(StoreError::InvalidData) => {} Ok(count) => { return Err(TestError::Unexpected(format!( "exhausted drain must fail closed, got Ok({count})" ))); } Err(error) => { return Err(TestError::Unexpected(format!( "exhausted drain must be InvalidData, got {error:?}" ))); } } // The production budget on the same grant drains fully: no live members. let drained = grants::revoke(&mut conn, &store, &grant).await?; if drained == 0 { return Err(TestError::Unexpected( "production revoke must count the seed member".to_owned(), )); } if tokens::get_refresh(&mut conn, &store, &seed) .await? .is_some() { return Err(TestError::Unexpected( "revoke must leave no live member record".to_owned(), )); } if !grants::members(&mut conn, &store, &grant).await?.is_empty() { return Err(TestError::Unexpected( "revoke must leave the grant set empty".to_owned(), )); } Ok(()) } /// A corrupt poll-interval marker must fail closed, never fail open. /// /// A non-numeric sidecar value trips the script's `-1` path, which /// `claim_poll_slot` surfaces as `InvalidData` (500-masked, never an /// admission). #[tokio::test] async fn claim_poll_slot_corrupt_marker_fails_closed() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let device_hash = unique("device"); let key = device::poll_key(&store, &device_hash); let _: () = redis::cmd("SET") .arg(&key) .arg("not-a-number") .query_async(&mut conn) .await?; match device::claim_poll_slot( &mut conn, &store, &device_hash, 1_700_000_000, Duration::from_secs(5), Duration::from_secs(600), ) .await { Err(StoreError::InvalidData) => Ok(()), Ok(slot) => Err(TestError::Unexpected(format!( "corrupt poll marker must fail closed, admitted {slot:?}" ))), Err(error) => Err(TestError::Unexpected(format!( "corrupt poll marker must be InvalidData, got {error:?}" ))), } } /// Generic Lua-runtime failures (`ERR`/`EXECABORT` server errors that are not /// `WRONGTYPE`) are programming errors: they must map to masked /// `server_error` (500) so 500-alerts fire, while genuine infrastructure /// (`OOM`, `READONLY`, `CROSSSLOT`, auth, transient/connection) stays 503. #[test] fn store_error_classification_lua_runtime_is_500() -> TestResult { use crate::error::{AppError, OAuthErrorCode}; use axum::http::StatusCode; let lua_err = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::ResponseError), "ERR Error running script: @user_script:1: bad argument", )); let exec_abort = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::ExecAbort), "EXECABORT Script aborted", )); for error in [lua_err, exec_abort] { let mapped = AppError::from(StoreError::Redis(error)); if mapped.code() != OAuthErrorCode::ServerError { return Err(TestError::Unexpected( "generic Lua-runtime failure must map to server_error".to_owned(), )); } if mapped.status() != StatusCode::INTERNAL_SERVER_ERROR { return Err(TestError::Unexpected( "generic Lua-runtime failure must render 500".to_owned(), )); } } // Genuine infrastructure stays fail-closed 503. let oom_value = redis::parse_redis_value(b"-OOM command not allowed\r\n"); let oom = match oom_value { Ok(redis::Value::ServerError(server)) => redis::RedisError::from(server), _ => { return Err(TestError::Unexpected( "expected an OOM server error value".to_owned(), )); } }; let readonly = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::ReadOnly), "ro", )); let cross_slot = redis::RedisError::from(( redis::ErrorKind::Server(redis::ServerErrorKind::CrossSlot), "slot", )); let auth = redis::RedisError::from((redis::ErrorKind::AuthenticationFailed, "auth")); let transient = redis::RedisError::from(std::io::Error::new( std::io::ErrorKind::TimedOut, "timed out", )); for error in [oom, readonly, cross_slot, auth, transient] { let mapped = AppError::from(StoreError::Redis(error)); if mapped.code() != OAuthErrorCode::TemporarilyUnavailable { return Err(TestError::Unexpected( "infra failure must map to temporarily_unavailable".to_owned(), )); } if mapped.status() != StatusCode::SERVICE_UNAVAILABLE { return Err(TestError::Unexpected( "infra failure must render 503".to_owned(), )); } } Ok(()) } /// T3-B1: `decide` (EXISTS then HSET) racing `consume` (atomic DEL) must /// never resurrect a zombie record. Every round puts a fresh record and runs /// both ops concurrently; after both settle the key must be gone in every /// case (decide won and consume reaped the verdict, or consume won and /// decide saw the miss) — a surviving record is a verdict recreated after /// the delete, i.e. a lost update. #[tokio::test] async fn store_device_decide_consume_leaves_no_zombie() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let client = unique("client"); for round in 0..100u32 { let device_hash = format!("{}-{round:03}", unique("device-race")); let user_code = format!("{}-{round:03}", unique("user-race")); let grant = format!("{}-{round:03}", unique("grant-race")); device::put( &mut conn, &store, &device_hash, &user_code, &device::DeviceRecord { client_id: client.clone(), scope: "orders:read".to_owned(), status: device::DeviceStatus::Pending, subject: String::new(), grant_id: String::new(), }, Duration::from_secs(600), ) .await?; let decide_task = tokio::spawn({ let conn = conn.clone(); let store = store.clone(); let device_hash = device_hash.clone(); let grant = grant.clone(); async move { let mut conn = conn; device::decide( &mut conn, &store, &device_hash, device::DeviceStatus::Approved, "user-1", &grant, ) .await } }); let consume_task = tokio::spawn({ let conn = conn.clone(); let store = store.clone(); let device_hash = device_hash.clone(); async move { let mut conn = conn; device::consume(&mut conn, &store, &device_hash).await } }); let decided = decide_task .await .map_err(|_| TestError::Join)? .map_err(|error| TestError::Unexpected(format!("round {round} decide: {error:?}")))?; let consumed_present = consume_task .await .map_err(|_| TestError::Join)? .map_err(|error| TestError::Unexpected(format!("round {round} consume: {error:?}")))? .is_some(); let observed = device::poll(&mut conn, &store, &device_hash) .await .map_err(|error| TestError::Unexpected(format!("round {round} poll: {error:?}")))?; if observed.is_some() { return Err(TestError::Unexpected(format!( "round {round}: zombie (decided={decided}, consumed={consumed_present})" ))); } } Ok(()) } /// T3-B4: a rotation into a revoked grant must be fenced, never minted. /// /// A token minted mid-cascade (live record, grant record already gone) must /// not rotate into fresh tokens: the fence rejects with `Absent`, writes no /// new records, indexes no new members, and leaves the presented record live /// (fail closed, no cascade on a fenced write). #[tokio::test] async fn store_grant_revoke_fence_rejects_rotation_into_revoked_grant() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let pepper = b"store-test-pepper"; grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; grants::revoke(&mut conn, &store, &grant).await?; // The mid-cascade mint: a live record re-indexed after the grant record // died (the rotation SADD that slipped through the revoke snapshot). let late_old = unique("late-refresh"); tokens::put_refresh( &mut conn, &store, &late_old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; let new_refresh = unique("fenced-refresh"); let new_access = unique("fenced-access"); let outcome = tokens::rotate( &mut conn, &store, pepper, &rotation_input(&late_old, &client, &new_refresh, &new_access, &grant, None), ) .await?; if !matches!(outcome, tokens::RotateOutcome::Absent) { return Err(TestError::Unexpected( "rotation into a revoked grant must be fenced to Absent".to_owned(), )); } if tokens::get_refresh(&mut conn, &store, &new_refresh) .await? .is_some() || tokens::get_access(&mut conn, &store, &new_access) .await? .is_some() { return Err(TestError::Unexpected( "fenced rotation must leave no minted records behind".to_owned(), )); } if grants::members(&mut conn, &store, &grant) .await? .iter() .any(|member| member == &new_refresh || member == &new_access) { return Err(TestError::Unexpected( "fenced rotation must index no new members".to_owned(), )); } if tokens::get_refresh(&mut conn, &store, &late_old) .await? .is_none() { return Err(TestError::Unexpected( "fenced rotation must not cascade on the presented record".to_owned(), )); } Ok(()) } /// T3-B4: one `revoke` call drains members indexed mid-cascade. /// /// A single-shot `SMEMBERS`-snapshot revoke leaves a member `SADD`ed after /// its snapshot alive; `revoke` must loop until the grant set reads empty so /// a mint that lands mid-cascade is deleted by a later pass. #[tokio::test] async fn store_grant_revoke_drains_mid_cascade_members() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, store) = direct_store(&url, DurabilityMode::WaitAof).await?; let grant = unique("grant"); let client = unique("client"); grants::create( &mut conn, &store, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &store, &unique("seed-refresh"), &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; tokens::put_access( &mut conn, &store, &unique("seed-access"), &access_record(&client, &grant), Duration::from_secs(300), ) .await?; // First pass: the cascade snapshot. A mint lands after it (the SADD the // single-shot script can never see). let set_key = store.grant_tokens_key(&grant); let grant_key = store.grant_key(&grant); let mut first = super::SCRIPT_GRANT_REVOKE.prepare_invoke(); first.key(set_key.as_str()).key(grant_key.as_str()); first .arg(store.refresh_key_prefix().as_str()) .arg(store.access_key_prefix().as_str()); let _: u64 = first.invoke_async(&mut conn).await?; let late_refresh = unique("late-refresh"); let late_access = unique("late-access"); tokens::put_refresh( &mut conn, &store, &late_refresh, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; tokens::put_access( &mut conn, &store, &late_access, &access_record(&client, &grant), Duration::from_secs(300), ) .await?; // One `revoke` call must drain the stragglers: no survivor records, an // empty index, and a count that includes the late pair. let drained = grants::revoke(&mut conn, &store, &grant).await?; if drained < 2 { return Err(TestError::Unexpected( "revoke must count the mid-cascade pair".to_owned(), )); } if tokens::get_refresh(&mut conn, &store, &late_refresh) .await? .is_some() || tokens::get_access(&mut conn, &store, &late_access) .await? .is_some() { return Err(TestError::Unexpected( "revoke must delete tokens minted mid-cascade".to_owned(), )); } if !grants::members(&mut conn, &store, &grant).await?.is_empty() { return Err(TestError::Unexpected( "revoke must leave the grant set empty".to_owned(), )); } Ok(()) } /// T3-B5: a barrier failure after the rotation write returns the originating /// request's tokens instead of a whole-grant lockout. /// /// On a replica-less `WAIT` node the barrier always fails closed; the /// originating rotation (this request id) must still resolve to its /// already-minted pair (`RotatedUnbarriered`), while a non-originating request id on the /// tombstoned old token still cascades (fail closed, no tokens). #[tokio::test] async fn store_rotation_barrier_failure_returns_replay_for_originating_id() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, durable) = direct_store(&url, DurabilityMode::WaitAof).await?; let fragile = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{rl}:".to_owned(), DurabilityMode::Wait, ); let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let new_refresh = unique("new-refresh"); let new_access = unique("new-access"); let pepper = b"store-test-pepper"; grants::create( &mut conn, &durable, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &durable, &old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; let outcome = tokens::rotate( &mut conn, &fragile, pepper, &rotation_input(&old, &client, &new_refresh, &new_access, &grant, None), ) .await?; match outcome { tokens::RotateOutcome::RotatedUnbarriered { new_refresh_hash, new_access_hash, } if new_refresh_hash == new_refresh && new_access_hash == new_access => {} _ => { return Err(TestError::Unexpected( "barrier failure must return the originating pair as RotatedUnbarriered".to_owned(), )); } } if tokens::get_refresh(&mut conn, &durable, &new_refresh) .await? .is_none() || tokens::get_access(&mut conn, &durable, &new_access) .await? .is_none() { return Err(TestError::Unexpected( "originating pair must be readable after barrier failure".to_owned(), )); } // A non-originating request id on the tombstoned old token gets no tokens: // reuse, plus the cascade. let request_id = tokens::derive_rotation_id(pepper, &old)?; let map_key = durable.rotation_map_key(&request_id); let _: () = redis::cmd("DEL") .arg(&map_key) .query_async(&mut conn) .await?; let foreign = tokens::rotate( &mut conn, &durable, pepper, &rotation_input( &old, &client, &unique("foreign"), &unique("foreign-a"), &grant, Some("req-foreign".to_owned()), ), ) .await?; if !matches!(foreign, tokens::RotateOutcome::Reused { .. }) { return Err(TestError::Unexpected( "non-originating retry must stay fail-closed reuse".to_owned(), )); } Ok(()) } /// T3-B5: a barrier failure after a winning replay-marker claim must not burn /// the proof. /// /// The marker is removed before the `Durability` error surfaces, so the /// originating `jti` can be claimed again; a concurrent non-originating /// claimant still loses while the marker stands. #[tokio::test] async fn store_replay_marker_barrier_failure_does_not_burn_proof() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, durable) = direct_store(&url, DurabilityMode::WaitAof).await?; let fragile = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{rl}:".to_owned(), DurabilityMode::Wait, ); let jti = unique("jti"); let ttl = Duration::from_secs(300); match dcr::claim_dpop_jti(&mut conn, &fragile, &jti, ttl).await { Err(StoreError::Durability) => {} _ => { return Err(TestError::Unexpected( "failing barrier must surface Durability".to_owned(), )); } } let marker: Option = redis::cmd("GET") .arg(durable.dpop_jti_key(&jti)) .query_async(&mut conn) .await?; if marker.is_some() { return Err(TestError::Unexpected( "failed claim must not leave the marker behind".to_owned(), )); } if !dcr::claim_dpop_jti(&mut conn, &durable, &jti, ttl).await? { return Err(TestError::Unexpected( "originating jti must be claimable after the failure".to_owned(), )); } Ok(()) } /// A field-less `HSET` is a Redis arity error; the empty-pairs guard falls /// back to the TTL touch alone, so an existing record keeps its fields and a /// live expiry instead of becoming persistent (or erroring out). #[tokio::test] async fn store_hset_record_empty_pairs_keeps_ttl() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, _store) = direct_store(&url, DurabilityMode::WaitAof).await?; let key = unique("hset-empty"); let ttl = Duration::from_secs(300); let pairs = vec![("client_id".to_owned(), "client-1".to_owned())]; hset_record(&mut conn, &key, &pairs, ttl).await?; hset_record(&mut conn, &key, &[], ttl).await?; let record = hget_record(&mut conn, &key).await?; if record .as_ref() .is_none_or(|fields| fields.get("client_id").map(String::as_str) != Some("client-1")) { return Err(TestError::Unexpected( "empty rewrite must preserve the fields".to_owned(), )); } assert_ttl_within(pttl_ms(&mut conn, &key).await?, ttl) } /// Wait-mode writes without a replica always fail the barrier; the pending /// pre-mint must skip it (`Ok`) while the barriered write fails closed. #[tokio::test] async fn store_put_access_pending_skips_barrier() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, _) = direct_store(&url, DurabilityMode::WaitAof).await?; let fragile = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{rl}:".to_owned(), DurabilityMode::Wait, ); let grant = unique("grant"); let client = unique("client"); let pending_hash = unique("pending-access"); let barriered_hash = unique("barriered-access"); tokens::put_access_pending( &mut conn, &fragile, &pending_hash, &access_record(&client, &grant), Duration::from_secs(300), ) .await?; if tokens::get_access(&mut conn, &fragile, &pending_hash) .await? .is_none() { return Err(TestError::Unexpected( "pending write must be readable without a barrier".to_owned(), )); } match tokens::put_access( &mut conn, &fragile, &barriered_hash, &access_record(&client, &grant), Duration::from_secs(300), ) .await { Err(StoreError::Durability) => Ok(()), Ok(()) => Err(TestError::Unexpected( "barriered write without replicas must fail closed".to_owned(), )), Err(error) => Err(TestError::Unexpected(format!( "barriered write must be Durability, got {error:?}" ))), } } /// A barrier failure still mints a readable pair; revoking the grant must /// delete that recovered pair, so no unbarriered token survives revocation. #[tokio::test] async fn store_recovered_pair_revocable_after_unbarriered_rotation() -> TestResult { let (_container, url) = redis_container(Vec::new()).await?; let (mut conn, durable) = direct_store(&url, DurabilityMode::WaitAof).await?; let fragile = Store::from_parts( "oauth:{state}:".to_owned(), "oauth:{rl}:".to_owned(), DurabilityMode::Wait, ); let grant = unique("grant"); let client = unique("client"); let old = unique("old-refresh"); let new_refresh = unique("new-refresh"); let new_access = unique("new-access"); let pepper = b"store-test-pepper"; grants::create( &mut conn, &durable, &grants::GrantRecord { grant_id: grant.clone(), client_id: client.clone(), subject: "user-1".to_owned(), scope: "orders:read".to_owned(), }, Duration::from_secs(3600), ) .await?; tokens::put_refresh( &mut conn, &durable, &old, &refresh_record(&client, &grant), Duration::from_secs(3600), ) .await?; match tokens::rotate( &mut conn, &fragile, pepper, &rotation_input(&old, &client, &new_refresh, &new_access, &grant, None), ) .await? { tokens::RotateOutcome::RotatedUnbarriered { new_refresh_hash, new_access_hash, } if new_refresh_hash == new_refresh && new_access_hash == new_access => {} _ => { return Err(TestError::Unexpected( "barrier failure must force RotatedUnbarriered".to_owned(), )); } } grants::revoke(&mut conn, &durable, &grant).await?; if tokens::get_refresh(&mut conn, &durable, &new_refresh) .await? .is_some() { return Err(TestError::Unexpected( "revoke must delete the recovered refresh token".to_owned(), )); } if tokens::get_access(&mut conn, &durable, &new_access) .await? .is_some() { return Err(TestError::Unexpected( "revoke must delete the recovered access token".to_owned(), )); } Ok(()) }