From 5f51bdc1adb465428b6a4330c44a63f7e9877237 Mon Sep 17 00:00:00 2001 From: Rufet Arzumanov Date: Fri, 14 Aug 2026 04:59:15 +0200 Subject: [PATCH 1/3] Add explicit slate diversity selector --- .../phoenix_candidate_pipeline.rs | 10 +- home-mixer/params/param.rs | 25 +++ home-mixer/selectors/mod.rs | 2 +- home-mixer/selectors/top_k_score_selector.rs | 200 +++++++++++++++++- 4 files changed, 230 insertions(+), 7 deletions(-) diff --git a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs index dd38792c..ff0d9e97 100644 --- a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs +++ b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs @@ -78,7 +78,7 @@ use crate::query_hydrators::user_installed_apps_query_hydrator::UserInstalledApp use crate::scorers::phoenix_scorer::PhoenixScorer; use crate::scorers::ranking_scorer::RankingScorer; use crate::scorers::vm_ranker::VMRanker; -use crate::selectors::TopKScoreSelector; +use crate::selectors::SlateDiversitySelector; use crate::side_effects::author_served_metrics_side_effect::AuthorServedMetricsSideEffect; use crate::side_effects::debug_side_effect::DebugSideEffect; use crate::side_effects::mutual_follow_stats_side_effect::MutualFollowStatsSideEffect; @@ -106,8 +106,8 @@ use xai_candidate_pipeline::component_library::clients::gender_prediction_client GenderPredictionGrpcClient, MockGenderPredictionGrpcClient, ProdGenderPredictionGrpcClient, }; use xai_candidate_pipeline::component_library::clients::kafka_publisher_client::{ - KafkaCluster, KafkaPublisherClient, MockKafkaPublisherClient, ProdKafkaPublisherClient, - PHOENIX_SCORES_TOPIC, RERANKING_TOPIC, + KafkaCluster, KafkaPublisherClient, MockKafkaPublisherClient, PHOENIX_SCORES_TOPIC, + ProdKafkaPublisherClient, RERANKING_TOPIC, }; use xai_candidate_pipeline::component_library::clients::media_info_cache_client::{ MediaInfoCacheClient, MockMediaInfoCacheClient, ProdMediaInfoCacheClient, @@ -176,7 +176,7 @@ pub struct PhoenixCandidatePipeline { hydrators: Vec>>, filters: Vec>>, scorers: Vec>>, - selector: TopKScoreSelector, + selector: SlateDiversitySelector, post_selection_hydrators: Vec>>, post_selection_filters: Vec>>, side_effects: Arc>>>, @@ -395,7 +395,7 @@ impl PhoenixCandidatePipeline { let scorers: Vec>> = vec![phoenix_scorer, ranking_scorer, vm_ranker]; - let selector = TopKScoreSelector; + let selector = SlateDiversitySelector; let post_selection_hydrators: Vec>> = vec![ Box::new( diff --git a/home-mixer/params/param.rs b/home-mixer/params/param.rs index dc3a12e4..14f8024c 100644 --- a/home-mixer/params/param.rs +++ b/home-mixer/params/param.rs @@ -276,6 +276,31 @@ param!( 0 ); +param!( + EnableSlateDiversity, + bool, + "rust_home_mixer_enable_slate_diversity", + true +); +param!( + SlateDiversityAuthorWindowSize, + u32, + "rust_home_mixer_slate_diversity_author_window_size", + 20 +); +param!( + SlateDiversityMaxPostsPerAuthor, + u32, + "rust_home_mixer_slate_diversity_max_posts_per_author", + 2 +); +param!( + EnableSlateSemanticDiversity, + bool, + "rust_home_mixer_enable_slate_semantic_diversity", + true +); + // These weights reflect a combination of how much an action is // valued in ranking and typical propensities of these actions // across the X network (e.g. negative feedback is overall rare). diff --git a/home-mixer/selectors/mod.rs b/home-mixer/selectors/mod.rs index f3ebd995..dec1b918 100644 --- a/home-mixer/selectors/mod.rs +++ b/home-mixer/selectors/mod.rs @@ -6,4 +6,4 @@ mod top_k_score_selector; pub use blender_selector::BlenderSelector; pub use following_blender_selector::FollowingBlenderSelector; pub use passthrough_selector::PassthroughSelector; -pub use top_k_score_selector::TopKScoreSelector; +pub use top_k_score_selector::{SlateDiversitySelector, TopKScoreSelector}; diff --git a/home-mixer/selectors/top_k_score_selector.rs b/home-mixer/selectors/top_k_score_selector.rs index 6f0ba37b..83066ca2 100644 --- a/home-mixer/selectors/top_k_score_selector.rs +++ b/home-mixer/selectors/top_k_score_selector.rs @@ -1,7 +1,9 @@ +use crate::models::candidate::CandidateHelpers; use crate::models::candidate::PostCandidate; use crate::models::query::ScoredPostsQuery; use crate::params; -use xai_candidate_pipeline::selector::Selector; +use rustc_hash::FxHashMap; +use xai_candidate_pipeline::selector::{SelectResult, Selector}; pub struct TopKScoreSelector; @@ -13,3 +15,199 @@ impl Selector for TopKScoreSelector { Some(params::TOP_K_CANDIDATES_TO_SELECT) } } + +/// Selects the highest-scoring candidates while improving slate diversity. +/// The author cap applies near the top; semantic adjacency applies throughout. +/// +/// Constraints are applied greedily in score order. When the remaining pool +/// cannot satisfy every constraint, semantic adjacency is relaxed first and +/// the author cap second. This keeps the response full while making any +/// constraint violation an explicit best-effort fallback. +pub struct SlateDiversitySelector; + +impl Selector for SlateDiversitySelector { + fn score(&self, candidate: &PostCandidate) -> f64 { + candidate.score.unwrap_or(f64::NEG_INFINITY) + } + + fn select( + &self, + query: &ScoredPostsQuery, + candidates: Vec, + ) -> SelectResult { + if !query.params.get(params::EnableSlateDiversity) { + return TopKScoreSelector.select(query, candidates); + } + + let mut remaining = self.sort(candidates); + let selection_size = params::TOP_K_CANDIDATES_TO_SELECT.min(remaining.len()); + let author_window = query.params.get(params::SlateDiversityAuthorWindowSize) as usize; + let max_posts_per_author = + query.params.get(params::SlateDiversityMaxPostsPerAuthor) as usize; + let enable_semantic_diversity = query.params.get(params::EnableSlateSemanticDiversity); + let mut selected = Vec::with_capacity(selection_size); + let mut author_counts: FxHashMap = FxHashMap::default(); + + while selected.len() < selection_size { + let previous = selected.last(); + let enforce_author_cap = selected.len() < author_window; + let strict = remaining.iter().position(|candidate| { + (!enforce_author_cap + || is_within_author_cap(candidate, &author_counts, max_posts_per_author)) + && (!enable_semantic_diversity || !shares_semantic_cluster(previous, candidate)) + }); + let author_only = remaining.iter().position(|candidate| { + !enforce_author_cap + || is_within_author_cap(candidate, &author_counts, max_posts_per_author) + }); + let index = strict.or(author_only).unwrap_or(0); + let candidate = remaining.remove(index); + if enforce_author_cap { + *author_counts + .entry(candidate.get_original_author_id()) + .or_default() += 1; + } + selected.push(candidate); + } + + SelectResult { + selected, + non_selected: remaining, + } + } +} + +fn is_within_author_cap( + candidate: &PostCandidate, + author_counts: &FxHashMap, + max_posts_per_author: usize, +) -> bool { + author_counts + .get(&candidate.get_original_author_id()) + .copied() + .unwrap_or_default() + < max_posts_per_author +} + +fn shares_semantic_cluster(previous: Option<&PostCandidate>, candidate: &PostCandidate) -> bool { + let Some(previous_ids) = previous.and_then(|post| post.semantic_ids.as_ref()) else { + return false; + }; + let Some(candidate_ids) = candidate.semantic_ids.as_ref() else { + return false; + }; + + previous_ids + .iter() + .any(|semantic_id| candidate_ids.contains(semantic_id)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn candidate(tweet_id: u64, author_id: u64, score: f64, semantic_ids: &[i32]) -> PostCandidate { + PostCandidate { + tweet_id, + author_id, + score: Some(score), + semantic_ids: Some(semantic_ids.to_vec()), + ..Default::default() + } + } + + fn select(candidates: Vec) -> SelectResult { + SlateDiversitySelector.select(&ScoredPostsQuery::default(), candidates) + } + + #[test] + fn caps_each_author_at_two_in_first_twenty_when_pool_allows() { + let mut candidates = vec![ + candidate(1, 1, 100.0, &[1]), + candidate(2, 1, 99.0, &[2]), + candidate(3, 1, 98.0, &[3]), + ]; + candidates.extend( + (2..=11).map(|author| { + candidate(author + 10, author, 90.0 - author as f64, &[author as i32]) + }), + ); + candidates.extend( + (12..=20).map(|author| { + candidate(author + 10, author, 50.0 - author as f64, &[author as i32]) + }), + ); + + let result = select(candidates); + let first_twenty = &result.selected[..20]; + let author_one_count = first_twenty + .iter() + .filter(|candidate| candidate.author_id == 1) + .count(); + + assert_eq!(author_one_count, 2); + assert!(!first_twenty.iter().any(|candidate| candidate.tweet_id == 3)); + } + + #[test] + fn avoids_adjacent_shared_semantic_clusters() { + let result = select(vec![ + candidate(1, 1, 10.0, &[7]), + candidate(2, 2, 9.0, &[7]), + candidate(3, 3, 8.0, &[8]), + ]); + + let ids: Vec = result + .selected + .iter() + .map(|candidate| candidate.tweet_id) + .collect(); + assert_eq!(ids, vec![1, 3, 2]); + } + + #[test] + fn relaxes_semantic_constraint_before_author_cap() { + let result = select(vec![ + candidate(1, 1, 10.0, &[1]), + candidate(2, 1, 9.0, &[2]), + candidate(3, 2, 8.0, &[2]), + candidate(4, 1, 7.0, &[3]), + ]); + + let ids: Vec = result + .selected + .iter() + .map(|candidate| candidate.tweet_id) + .collect(); + assert_eq!(ids, vec![1, 2, 3, 4]); + } + + #[test] + fn falls_back_to_score_order_when_author_cap_is_impossible() { + let result = select(vec![ + candidate(1, 1, 10.0, &[1]), + candidate(2, 1, 9.0, &[2]), + candidate(3, 1, 8.0, &[3]), + ]); + + let ids: Vec = result + .selected + .iter() + .map(|candidate| candidate.tweet_id) + .collect(); + assert_eq!(ids, vec![1, 2, 3]); + } + + #[test] + fn retains_top_k_size_and_reports_non_selected_candidates() { + let candidates = (0..60) + .map(|index| candidate(index, index, 100.0 - index as f64, &[index as i32])) + .collect(); + + let result = select(candidates); + + assert_eq!(result.selected.len(), params::TOP_K_CANDIDATES_TO_SELECT); + assert_eq!(result.non_selected.len(), 10); + assert_eq!(result.selected[0].score, Some(100.0)); + } +} From 03bedfed86dd07bc773ec84e018ff2e995b56d13 Mon Sep 17 00:00:00 2001 From: Rufet Arzumanov Date: Fri, 14 Aug 2026 05:22:08 +0200 Subject: [PATCH 2/3] Harden and locally test slate diversity --- home-mixer/params/param.rs | 8 +- home-mixer/selectors/mod.rs | 1 + home-mixer/selectors/slate_diversity.rs | 369 +++++++++++++++++++ home-mixer/selectors/top_k_score_selector.rs | 211 ++--------- local-tests/slate-diversity/Cargo.lock | 7 + local-tests/slate-diversity/Cargo.toml | 8 + local-tests/slate-diversity/README.md | 16 + local-tests/slate-diversity/src/lib.rs | 2 + 8 files changed, 445 insertions(+), 177 deletions(-) create mode 100644 home-mixer/selectors/slate_diversity.rs create mode 100644 local-tests/slate-diversity/Cargo.lock create mode 100644 local-tests/slate-diversity/Cargo.toml create mode 100644 local-tests/slate-diversity/README.md create mode 100644 local-tests/slate-diversity/src/lib.rs diff --git a/home-mixer/params/param.rs b/home-mixer/params/param.rs index 14f8024c..33da6cff 100644 --- a/home-mixer/params/param.rs +++ b/home-mixer/params/param.rs @@ -280,7 +280,7 @@ param!( EnableSlateDiversity, bool, "rust_home_mixer_enable_slate_diversity", - true + false ); param!( SlateDiversityAuthorWindowSize, @@ -300,6 +300,12 @@ param!( "rust_home_mixer_enable_slate_semantic_diversity", true ); +param!( + SlateDiversityMaxLookahead, + u32, + "rust_home_mixer_slate_diversity_max_lookahead", + 10 +); // These weights reflect a combination of how much an action is // valued in ranking and typical propensities of these actions diff --git a/home-mixer/selectors/mod.rs b/home-mixer/selectors/mod.rs index dec1b918..ac84265d 100644 --- a/home-mixer/selectors/mod.rs +++ b/home-mixer/selectors/mod.rs @@ -1,6 +1,7 @@ mod blender_selector; mod following_blender_selector; mod passthrough_selector; +mod slate_diversity; mod top_k_score_selector; pub use blender_selector::BlenderSelector; diff --git a/home-mixer/selectors/slate_diversity.rs b/home-mixer/selectors/slate_diversity.rs new file mode 100644 index 00000000..86729ab9 --- /dev/null +++ b/home-mixer/selectors/slate_diversity.rs @@ -0,0 +1,369 @@ +//! Dependency-free slate-diversity selection. +//! +//! Input must already be ranked from highest to lowest score. Keeping this +//! module independent of Home Mixer types lets the standalone local harness +//! compile and test the exact algorithm used by production. + +use std::collections::HashMap; +use std::hash::Hash; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DiversityConfig { + pub selection_size: usize, + pub author_window_size: usize, + pub max_posts_per_author: usize, + pub enable_semantic_diversity: bool, + pub max_lookahead: usize, +} + +pub struct SelectionResult { + pub selected: Vec, + pub non_selected: Vec, +} + +pub trait SlateItem { + type AuthorId: Copy + Eq + Hash; + + fn original_author_id(&self) -> Self::AuthorId; + fn serving_author_id(&self) -> Self::AuthorId; + fn semantic_ids(&self) -> Option<&[i32]>; +} + +pub fn select_from_ranked( + mut remaining: Vec, + config: DiversityConfig, +) -> SelectionResult { + let selection_size = config.selection_size.min(remaining.len()); + let mut selected = Vec::with_capacity(selection_size); + let mut original_author_counts: HashMap = HashMap::new(); + let mut serving_author_counts: HashMap = HashMap::new(); + + while selected.len() < selection_size { + let previous = selected.last(); + let enforce_author_cap = selected.len() < config.author_window_size; + let window = &remaining[..remaining.len().min(config.max_lookahead)]; + let strict = window.iter().position(|candidate| { + (!enforce_author_cap + || is_within_author_caps( + candidate, + &original_author_counts, + &serving_author_counts, + config.max_posts_per_author, + )) + && (!config.enable_semantic_diversity + || !shares_semantic_cluster(previous, candidate)) + }); + let index = strict + .or_else(|| { + window.iter().position(|candidate| { + !enforce_author_cap + || is_within_author_caps( + candidate, + &original_author_counts, + &serving_author_counts, + config.max_posts_per_author, + ) + }) + }) + .unwrap_or(0); + let candidate = remaining.remove(index); + if enforce_author_cap { + *original_author_counts + .entry(candidate.original_author_id()) + .or_default() += 1; + *serving_author_counts + .entry(candidate.serving_author_id()) + .or_default() += 1; + } + selected.push(candidate); + } + + SelectionResult { + selected, + non_selected: remaining, + } +} + +fn is_within_author_caps( + candidate: &T, + original_author_counts: &HashMap, + serving_author_counts: &HashMap, + max_posts_per_author: usize, +) -> bool { + let within_original_cap = original_author_counts + .get(&candidate.original_author_id()) + .copied() + .unwrap_or_default() + < max_posts_per_author; + let within_serving_cap = serving_author_counts + .get(&candidate.serving_author_id()) + .copied() + .unwrap_or_default() + < max_posts_per_author; + within_original_cap && within_serving_cap +} + +fn shares_semantic_cluster(previous: Option<&T>, candidate: &T) -> bool { + let Some(previous_ids) = previous.and_then(SlateItem::semantic_ids) else { + return false; + }; + let Some(candidate_ids) = candidate.semantic_ids() else { + return false; + }; + + !previous_ids.is_empty() && previous_ids == candidate_ids +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::{HashMap, HashSet}; + + #[derive(Clone, Debug)] + struct Candidate { + id: u64, + original_author_id: u64, + serving_author_id: u64, + semantic_ids: Option>, + } + + impl SlateItem for Candidate { + type AuthorId = u64; + + fn original_author_id(&self) -> Self::AuthorId { + self.original_author_id + } + + fn serving_author_id(&self) -> Self::AuthorId { + self.serving_author_id + } + + fn semantic_ids(&self) -> Option<&[i32]> { + self.semantic_ids.as_deref() + } + } + + fn candidate(id: u64, author_id: u64, semantic_ids: &[i32]) -> Candidate { + Candidate { + id, + original_author_id: author_id, + serving_author_id: author_id, + semantic_ids: Some(semantic_ids.to_vec()), + } + } + + fn config(selection_size: usize) -> DiversityConfig { + DiversityConfig { + selection_size, + author_window_size: 20, + max_posts_per_author: 2, + enable_semantic_diversity: true, + max_lookahead: 10, + } + } + + fn selected_ids(result: &SelectionResult) -> Vec { + result + .selected + .iter() + .map(|candidate| candidate.id) + .collect() + } + + #[test] + fn caps_original_author_in_first_twenty_when_pool_allows() { + let mut candidates = vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 1, &[3]), + ]; + candidates.extend((2..=20).map(|author| candidate(author + 10, author, &[author as i32]))); + + let result = select_from_ranked(candidates, config(22)); + let first_twenty = &result.selected[..20]; + assert_eq!( + first_twenty + .iter() + .filter(|candidate| candidate.original_author_id == 1) + .count(), + 2 + ); + assert!(!first_twenty.iter().any(|candidate| candidate.id == 3)); + } + + #[test] + fn caps_serving_author_independently() { + let mut candidates = vec![ + candidate(1, 1, &[1]), + candidate(2, 2, &[2]), + candidate(3, 3, &[3]), + candidate(4, 4, &[4]), + ]; + for candidate in &mut candidates[..3] { + candidate.serving_author_id = 100; + } + + let result = select_from_ranked(candidates, config(4)); + assert_eq!(selected_ids(&result), vec![1, 2, 4, 3]); + } + + #[test] + fn separates_identical_non_empty_semantic_vectors() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[7]), + candidate(2, 2, &[7]), + candidate(3, 3, &[8]), + ], + config(3), + ); + assert_eq!(selected_ids(&result), vec![1, 3, 2]); + } + + #[test] + fn shared_semantic_component_is_not_the_same_cluster() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[7, 100]), + candidate(2, 2, &[7, 200]), + candidate(3, 3, &[8, 300]), + ], + config(3), + ); + assert_eq!(selected_ids(&result), vec![1, 2, 3]); + } + + #[test] + fn relaxes_semantics_before_author_caps() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 2, &[2]), + candidate(4, 1, &[3]), + ], + config(4), + ); + assert_eq!(selected_ids(&result), vec![1, 2, 3, 4]); + } + + #[test] + fn lookahead_bounds_rank_displacement() { + let mut candidates = vec![candidate(0, 1, &[1])]; + candidates.extend((1..=10).map(|id| candidate(id, id + 1, &[1]))); + candidates.push(candidate(11, 12, &[2])); + + let result = select_from_ranked(candidates, config(12)); + assert_eq!(&selected_ids(&result)[..2], &[0, 1]); + } + + #[test] + fn degenerate_configs_preserve_candidates() { + for config in [ + DiversityConfig { + selection_size: 3, + author_window_size: 0, + max_posts_per_author: 0, + enable_semantic_diversity: true, + max_lookahead: 0, + }, + DiversityConfig { + selection_size: 3, + author_window_size: usize::MAX, + max_posts_per_author: 0, + enable_semantic_diversity: false, + max_lookahead: usize::MAX, + }, + ] { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 1, &[3]), + ], + config, + ); + assert_eq!(selected_ids(&result), vec![1, 2, 3]); + assert!(result.non_selected.is_empty()); + } + } + + #[test] + fn reports_expected_rank_displacement() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 1, &[3]), + candidate(4, 2, &[4]), + ], + config(4), + ); + let ids = selected_ids(&result); + assert_eq!(ids, vec![1, 2, 4, 3]); + + let original_rank: HashMap = [1, 2, 3, 4] + .into_iter() + .enumerate() + .map(|(i, id)| (id, i)) + .collect(); + let displacement: Vec = ids + .iter() + .enumerate() + .map(|(new_rank, id)| new_rank as isize - original_rank[id] as isize) + .collect(); + assert_eq!(displacement, vec![0, 0, -1, 1]); + } + + #[test] + fn randomized_inputs_never_panic_or_lose_candidates() { + let mut state = 0x4d595df4d0f33173_u64; + let mut next = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + state + }; + + for case in 0..500 { + let candidate_count = (next() % 101) as usize; + let selection_size = (next() % 121) as usize; + let candidates: Vec = (0..candidate_count) + .map(|id| Candidate { + id: id as u64, + original_author_id: next() % 17, + serving_author_id: next() % 17, + semantic_ids: (next() % 4 != 0) + .then(|| vec![(next() % 6) as i32, (next() % 13) as i32]), + }) + .collect(); + let expected_ids: HashSet = + candidates.iter().map(|candidate| candidate.id).collect(); + let config = DiversityConfig { + selection_size, + author_window_size: (next() % 30) as usize, + max_posts_per_author: (next() % 5) as usize, + enable_semantic_diversity: next() % 2 == 0, + max_lookahead: (next() % 20) as usize, + }; + + let result = select_from_ranked(candidates, config); + let actual_ids: Vec = result + .selected + .iter() + .chain(&result.non_selected) + .map(|candidate| candidate.id) + .collect(); + let actual_set: HashSet = actual_ids.iter().copied().collect(); + + assert_eq!( + result.selected.len(), + selection_size.min(candidate_count), + "case {case}" + ); + assert_eq!(actual_ids.len(), candidate_count, "case {case}"); + assert_eq!(actual_set.len(), candidate_count, "case {case}"); + assert_eq!(actual_set, expected_ids, "case {case}"); + } + } +} diff --git a/home-mixer/selectors/top_k_score_selector.rs b/home-mixer/selectors/top_k_score_selector.rs index 83066ca2..e7810174 100644 --- a/home-mixer/selectors/top_k_score_selector.rs +++ b/home-mixer/selectors/top_k_score_selector.rs @@ -1,8 +1,7 @@ -use crate::models::candidate::CandidateHelpers; -use crate::models::candidate::PostCandidate; +use super::slate_diversity::{self, DiversityConfig, SlateItem}; +use crate::models::candidate::{CandidateHelpers, PostCandidate}; use crate::models::query::ScoredPostsQuery; use crate::params; -use rustc_hash::FxHashMap; use xai_candidate_pipeline::selector::{SelectResult, Selector}; pub struct TopKScoreSelector; @@ -11,20 +10,33 @@ impl Selector for TopKScoreSelector { fn score(&self, candidate: &PostCandidate) -> f64 { candidate.score.unwrap_or(f64::NEG_INFINITY) } + fn size(&self) -> Option { Some(params::TOP_K_CANDIDATES_TO_SELECT) } } /// Selects the highest-scoring candidates while improving slate diversity. -/// The author cap applies near the top; semantic adjacency applies throughout. -/// -/// Constraints are applied greedily in score order. When the remaining pool -/// cannot satisfy every constraint, semantic adjacency is relaxed first and -/// the author cap second. This keeps the response full while making any -/// constraint violation an explicit best-effort fallback. +/// Author caps apply near the top; semantic adjacency applies throughout. +/// Lookahead bounds how far a constraint may displace a ranked candidate. pub struct SlateDiversitySelector; +impl SlateItem for PostCandidate { + type AuthorId = u64; + + fn original_author_id(&self) -> Self::AuthorId { + self.get_original_author_id() + } + + fn serving_author_id(&self) -> Self::AuthorId { + self.author_id + } + + fn semantic_ids(&self) -> Option<&[i32]> { + self.semantic_ids.as_deref() + } +} + impl Selector for SlateDiversitySelector { fn score(&self, candidate: &PostCandidate) -> f64 { candidate.score.unwrap_or(f64::NEG_INFINITY) @@ -39,175 +51,22 @@ impl Selector for SlateDiversitySelector { return TopKScoreSelector.select(query, candidates); } - let mut remaining = self.sort(candidates); - let selection_size = params::TOP_K_CANDIDATES_TO_SELECT.min(remaining.len()); - let author_window = query.params.get(params::SlateDiversityAuthorWindowSize) as usize; - let max_posts_per_author = - query.params.get(params::SlateDiversityMaxPostsPerAuthor) as usize; - let enable_semantic_diversity = query.params.get(params::EnableSlateSemanticDiversity); - let mut selected = Vec::with_capacity(selection_size); - let mut author_counts: FxHashMap = FxHashMap::default(); - - while selected.len() < selection_size { - let previous = selected.last(); - let enforce_author_cap = selected.len() < author_window; - let strict = remaining.iter().position(|candidate| { - (!enforce_author_cap - || is_within_author_cap(candidate, &author_counts, max_posts_per_author)) - && (!enable_semantic_diversity || !shares_semantic_cluster(previous, candidate)) - }); - let author_only = remaining.iter().position(|candidate| { - !enforce_author_cap - || is_within_author_cap(candidate, &author_counts, max_posts_per_author) - }); - let index = strict.or(author_only).unwrap_or(0); - let candidate = remaining.remove(index); - if enforce_author_cap { - *author_counts - .entry(candidate.get_original_author_id()) - .or_default() += 1; - } - selected.push(candidate); - } + let result = slate_diversity::select_from_ranked( + self.sort(candidates), + DiversityConfig { + selection_size: params::TOP_K_CANDIDATES_TO_SELECT, + author_window_size: query.params.get(params::SlateDiversityAuthorWindowSize) + as usize, + max_posts_per_author: query.params.get(params::SlateDiversityMaxPostsPerAuthor) + as usize, + enable_semantic_diversity: query.params.get(params::EnableSlateSemanticDiversity), + max_lookahead: query.params.get(params::SlateDiversityMaxLookahead) as usize, + }, + ); SelectResult { - selected, - non_selected: remaining, - } - } -} - -fn is_within_author_cap( - candidate: &PostCandidate, - author_counts: &FxHashMap, - max_posts_per_author: usize, -) -> bool { - author_counts - .get(&candidate.get_original_author_id()) - .copied() - .unwrap_or_default() - < max_posts_per_author -} - -fn shares_semantic_cluster(previous: Option<&PostCandidate>, candidate: &PostCandidate) -> bool { - let Some(previous_ids) = previous.and_then(|post| post.semantic_ids.as_ref()) else { - return false; - }; - let Some(candidate_ids) = candidate.semantic_ids.as_ref() else { - return false; - }; - - previous_ids - .iter() - .any(|semantic_id| candidate_ids.contains(semantic_id)) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn candidate(tweet_id: u64, author_id: u64, score: f64, semantic_ids: &[i32]) -> PostCandidate { - PostCandidate { - tweet_id, - author_id, - score: Some(score), - semantic_ids: Some(semantic_ids.to_vec()), - ..Default::default() + selected: result.selected, + non_selected: result.non_selected, } } - - fn select(candidates: Vec) -> SelectResult { - SlateDiversitySelector.select(&ScoredPostsQuery::default(), candidates) - } - - #[test] - fn caps_each_author_at_two_in_first_twenty_when_pool_allows() { - let mut candidates = vec![ - candidate(1, 1, 100.0, &[1]), - candidate(2, 1, 99.0, &[2]), - candidate(3, 1, 98.0, &[3]), - ]; - candidates.extend( - (2..=11).map(|author| { - candidate(author + 10, author, 90.0 - author as f64, &[author as i32]) - }), - ); - candidates.extend( - (12..=20).map(|author| { - candidate(author + 10, author, 50.0 - author as f64, &[author as i32]) - }), - ); - - let result = select(candidates); - let first_twenty = &result.selected[..20]; - let author_one_count = first_twenty - .iter() - .filter(|candidate| candidate.author_id == 1) - .count(); - - assert_eq!(author_one_count, 2); - assert!(!first_twenty.iter().any(|candidate| candidate.tweet_id == 3)); - } - - #[test] - fn avoids_adjacent_shared_semantic_clusters() { - let result = select(vec![ - candidate(1, 1, 10.0, &[7]), - candidate(2, 2, 9.0, &[7]), - candidate(3, 3, 8.0, &[8]), - ]); - - let ids: Vec = result - .selected - .iter() - .map(|candidate| candidate.tweet_id) - .collect(); - assert_eq!(ids, vec![1, 3, 2]); - } - - #[test] - fn relaxes_semantic_constraint_before_author_cap() { - let result = select(vec![ - candidate(1, 1, 10.0, &[1]), - candidate(2, 1, 9.0, &[2]), - candidate(3, 2, 8.0, &[2]), - candidate(4, 1, 7.0, &[3]), - ]); - - let ids: Vec = result - .selected - .iter() - .map(|candidate| candidate.tweet_id) - .collect(); - assert_eq!(ids, vec![1, 2, 3, 4]); - } - - #[test] - fn falls_back_to_score_order_when_author_cap_is_impossible() { - let result = select(vec![ - candidate(1, 1, 10.0, &[1]), - candidate(2, 1, 9.0, &[2]), - candidate(3, 1, 8.0, &[3]), - ]); - - let ids: Vec = result - .selected - .iter() - .map(|candidate| candidate.tweet_id) - .collect(); - assert_eq!(ids, vec![1, 2, 3]); - } - - #[test] - fn retains_top_k_size_and_reports_non_selected_candidates() { - let candidates = (0..60) - .map(|index| candidate(index, index, 100.0 - index as f64, &[index as i32])) - .collect(); - - let result = select(candidates); - - assert_eq!(result.selected.len(), params::TOP_K_CANDIDATES_TO_SELECT); - assert_eq!(result.non_selected.len(), 10); - assert_eq!(result.selected[0].score, Some(100.0)); - } } diff --git a/local-tests/slate-diversity/Cargo.lock b/local-tests/slate-diversity/Cargo.lock new file mode 100644 index 00000000..08ac0f7e --- /dev/null +++ b/local-tests/slate-diversity/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "slate-diversity-local-tests" +version = "0.1.0" diff --git a/local-tests/slate-diversity/Cargo.toml b/local-tests/slate-diversity/Cargo.toml new file mode 100644 index 00000000..f18a517d --- /dev/null +++ b/local-tests/slate-diversity/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "slate-diversity-local-tests" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +path = "src/lib.rs" diff --git a/local-tests/slate-diversity/README.md b/local-tests/slate-diversity/README.md new file mode 100644 index 00000000..ff2ec2e6 --- /dev/null +++ b/local-tests/slate-diversity/README.md @@ -0,0 +1,16 @@ +# Slate diversity local tests + +This crate compiles the dependency-free selection engine used by Home Mixer's +`SlateDiversitySelector`. It does not compile the Home Mixer adapter or its +internal X dependencies. + +Run from the repository root: + +```sh +CARGO_TARGET_DIR=/tmp/xai-slate-diversity-target \ + cargo test --manifest-path local-tests/slate-diversity/Cargo.toml +``` + +The external target directory is required because this checkout's directory +name contains `:`, which macOS cannot represent as one entry in +`DYLD_FALLBACK_LIBRARY_PATH`. diff --git a/local-tests/slate-diversity/src/lib.rs b/local-tests/slate-diversity/src/lib.rs new file mode 100644 index 00000000..21b7e863 --- /dev/null +++ b/local-tests/slate-diversity/src/lib.rs @@ -0,0 +1,2 @@ +#[path = "../../../home-mixer/selectors/slate_diversity.rs"] +pub mod slate_diversity; From fb244cf815f3fcd24fdd5efed11597dc2082d656 Mon Sep 17 00:00:00 2001 From: Rufet Arzumanov Date: Fri, 14 Aug 2026 05:45:38 +0200 Subject: [PATCH 3/3] Document author diversity interaction --- home-mixer/selectors/top_k_score_selector.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/home-mixer/selectors/top_k_score_selector.rs b/home-mixer/selectors/top_k_score_selector.rs index e7810174..709ebc72 100644 --- a/home-mixer/selectors/top_k_score_selector.rs +++ b/home-mixer/selectors/top_k_score_selector.rs @@ -19,6 +19,13 @@ impl Selector for TopKScoreSelector { /// Selects the highest-scoring candidates while improving slate diversity. /// Author caps apply near the top; semantic adjacency applies throughout. /// Lookahead bounds how far a constraint may displace a ranked candidate. +/// +/// `RankingScorer` may already have applied a soft serving-author decay and +/// populated `SlateContext.k`. This selector intentionally recomputes counts: +/// VMRanker can change the order after that context is recorded, `k` describes +/// the earlier pool order rather than this selector's chosen prefix, and it +/// does not track the original author of a retweeted post. The hard cap is a +/// final guardrail layered on top of that independently configurable decay. pub struct SlateDiversitySelector; impl SlateItem for PostCandidate {