diff --git a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs index dd38792c..ff0d9e97 100644 --- a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs +++ b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs @@ -78,7 +78,7 @@ use crate::query_hydrators::user_installed_apps_query_hydrator::UserInstalledApp use crate::scorers::phoenix_scorer::PhoenixScorer; use crate::scorers::ranking_scorer::RankingScorer; use crate::scorers::vm_ranker::VMRanker; -use crate::selectors::TopKScoreSelector; +use crate::selectors::SlateDiversitySelector; use crate::side_effects::author_served_metrics_side_effect::AuthorServedMetricsSideEffect; use crate::side_effects::debug_side_effect::DebugSideEffect; use crate::side_effects::mutual_follow_stats_side_effect::MutualFollowStatsSideEffect; @@ -106,8 +106,8 @@ use xai_candidate_pipeline::component_library::clients::gender_prediction_client GenderPredictionGrpcClient, MockGenderPredictionGrpcClient, ProdGenderPredictionGrpcClient, }; use xai_candidate_pipeline::component_library::clients::kafka_publisher_client::{ - KafkaCluster, KafkaPublisherClient, MockKafkaPublisherClient, ProdKafkaPublisherClient, - PHOENIX_SCORES_TOPIC, RERANKING_TOPIC, + KafkaCluster, KafkaPublisherClient, MockKafkaPublisherClient, PHOENIX_SCORES_TOPIC, + ProdKafkaPublisherClient, RERANKING_TOPIC, }; use xai_candidate_pipeline::component_library::clients::media_info_cache_client::{ MediaInfoCacheClient, MockMediaInfoCacheClient, ProdMediaInfoCacheClient, @@ -176,7 +176,7 @@ pub struct PhoenixCandidatePipeline { hydrators: Vec>>, filters: Vec>>, scorers: Vec>>, - selector: TopKScoreSelector, + selector: SlateDiversitySelector, post_selection_hydrators: Vec>>, post_selection_filters: Vec>>, side_effects: Arc>>>, @@ -395,7 +395,7 @@ impl PhoenixCandidatePipeline { let scorers: Vec>> = vec![phoenix_scorer, ranking_scorer, vm_ranker]; - let selector = TopKScoreSelector; + let selector = SlateDiversitySelector; let post_selection_hydrators: Vec>> = vec![ Box::new( diff --git a/home-mixer/params/param.rs b/home-mixer/params/param.rs index dc3a12e4..33da6cff 100644 --- a/home-mixer/params/param.rs +++ b/home-mixer/params/param.rs @@ -276,6 +276,37 @@ param!( 0 ); +param!( + EnableSlateDiversity, + bool, + "rust_home_mixer_enable_slate_diversity", + false +); +param!( + SlateDiversityAuthorWindowSize, + u32, + "rust_home_mixer_slate_diversity_author_window_size", + 20 +); +param!( + SlateDiversityMaxPostsPerAuthor, + u32, + "rust_home_mixer_slate_diversity_max_posts_per_author", + 2 +); +param!( + EnableSlateSemanticDiversity, + bool, + "rust_home_mixer_enable_slate_semantic_diversity", + true +); +param!( + SlateDiversityMaxLookahead, + u32, + "rust_home_mixer_slate_diversity_max_lookahead", + 10 +); + // These weights reflect a combination of how much an action is // valued in ranking and typical propensities of these actions // across the X network (e.g. negative feedback is overall rare). diff --git a/home-mixer/selectors/mod.rs b/home-mixer/selectors/mod.rs index f3ebd995..ac84265d 100644 --- a/home-mixer/selectors/mod.rs +++ b/home-mixer/selectors/mod.rs @@ -1,9 +1,10 @@ mod blender_selector; mod following_blender_selector; mod passthrough_selector; +mod slate_diversity; mod top_k_score_selector; pub use blender_selector::BlenderSelector; pub use following_blender_selector::FollowingBlenderSelector; pub use passthrough_selector::PassthroughSelector; -pub use top_k_score_selector::TopKScoreSelector; +pub use top_k_score_selector::{SlateDiversitySelector, TopKScoreSelector}; diff --git a/home-mixer/selectors/slate_diversity.rs b/home-mixer/selectors/slate_diversity.rs new file mode 100644 index 00000000..86729ab9 --- /dev/null +++ b/home-mixer/selectors/slate_diversity.rs @@ -0,0 +1,369 @@ +//! Dependency-free slate-diversity selection. +//! +//! Input must already be ranked from highest to lowest score. Keeping this +//! module independent of Home Mixer types lets the standalone local harness +//! compile and test the exact algorithm used by production. + +use std::collections::HashMap; +use std::hash::Hash; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DiversityConfig { + pub selection_size: usize, + pub author_window_size: usize, + pub max_posts_per_author: usize, + pub enable_semantic_diversity: bool, + pub max_lookahead: usize, +} + +pub struct SelectionResult { + pub selected: Vec, + pub non_selected: Vec, +} + +pub trait SlateItem { + type AuthorId: Copy + Eq + Hash; + + fn original_author_id(&self) -> Self::AuthorId; + fn serving_author_id(&self) -> Self::AuthorId; + fn semantic_ids(&self) -> Option<&[i32]>; +} + +pub fn select_from_ranked( + mut remaining: Vec, + config: DiversityConfig, +) -> SelectionResult { + let selection_size = config.selection_size.min(remaining.len()); + let mut selected = Vec::with_capacity(selection_size); + let mut original_author_counts: HashMap = HashMap::new(); + let mut serving_author_counts: HashMap = HashMap::new(); + + while selected.len() < selection_size { + let previous = selected.last(); + let enforce_author_cap = selected.len() < config.author_window_size; + let window = &remaining[..remaining.len().min(config.max_lookahead)]; + let strict = window.iter().position(|candidate| { + (!enforce_author_cap + || is_within_author_caps( + candidate, + &original_author_counts, + &serving_author_counts, + config.max_posts_per_author, + )) + && (!config.enable_semantic_diversity + || !shares_semantic_cluster(previous, candidate)) + }); + let index = strict + .or_else(|| { + window.iter().position(|candidate| { + !enforce_author_cap + || is_within_author_caps( + candidate, + &original_author_counts, + &serving_author_counts, + config.max_posts_per_author, + ) + }) + }) + .unwrap_or(0); + let candidate = remaining.remove(index); + if enforce_author_cap { + *original_author_counts + .entry(candidate.original_author_id()) + .or_default() += 1; + *serving_author_counts + .entry(candidate.serving_author_id()) + .or_default() += 1; + } + selected.push(candidate); + } + + SelectionResult { + selected, + non_selected: remaining, + } +} + +fn is_within_author_caps( + candidate: &T, + original_author_counts: &HashMap, + serving_author_counts: &HashMap, + max_posts_per_author: usize, +) -> bool { + let within_original_cap = original_author_counts + .get(&candidate.original_author_id()) + .copied() + .unwrap_or_default() + < max_posts_per_author; + let within_serving_cap = serving_author_counts + .get(&candidate.serving_author_id()) + .copied() + .unwrap_or_default() + < max_posts_per_author; + within_original_cap && within_serving_cap +} + +fn shares_semantic_cluster(previous: Option<&T>, candidate: &T) -> bool { + let Some(previous_ids) = previous.and_then(SlateItem::semantic_ids) else { + return false; + }; + let Some(candidate_ids) = candidate.semantic_ids() else { + return false; + }; + + !previous_ids.is_empty() && previous_ids == candidate_ids +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::{HashMap, HashSet}; + + #[derive(Clone, Debug)] + struct Candidate { + id: u64, + original_author_id: u64, + serving_author_id: u64, + semantic_ids: Option>, + } + + impl SlateItem for Candidate { + type AuthorId = u64; + + fn original_author_id(&self) -> Self::AuthorId { + self.original_author_id + } + + fn serving_author_id(&self) -> Self::AuthorId { + self.serving_author_id + } + + fn semantic_ids(&self) -> Option<&[i32]> { + self.semantic_ids.as_deref() + } + } + + fn candidate(id: u64, author_id: u64, semantic_ids: &[i32]) -> Candidate { + Candidate { + id, + original_author_id: author_id, + serving_author_id: author_id, + semantic_ids: Some(semantic_ids.to_vec()), + } + } + + fn config(selection_size: usize) -> DiversityConfig { + DiversityConfig { + selection_size, + author_window_size: 20, + max_posts_per_author: 2, + enable_semantic_diversity: true, + max_lookahead: 10, + } + } + + fn selected_ids(result: &SelectionResult) -> Vec { + result + .selected + .iter() + .map(|candidate| candidate.id) + .collect() + } + + #[test] + fn caps_original_author_in_first_twenty_when_pool_allows() { + let mut candidates = vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 1, &[3]), + ]; + candidates.extend((2..=20).map(|author| candidate(author + 10, author, &[author as i32]))); + + let result = select_from_ranked(candidates, config(22)); + let first_twenty = &result.selected[..20]; + assert_eq!( + first_twenty + .iter() + .filter(|candidate| candidate.original_author_id == 1) + .count(), + 2 + ); + assert!(!first_twenty.iter().any(|candidate| candidate.id == 3)); + } + + #[test] + fn caps_serving_author_independently() { + let mut candidates = vec![ + candidate(1, 1, &[1]), + candidate(2, 2, &[2]), + candidate(3, 3, &[3]), + candidate(4, 4, &[4]), + ]; + for candidate in &mut candidates[..3] { + candidate.serving_author_id = 100; + } + + let result = select_from_ranked(candidates, config(4)); + assert_eq!(selected_ids(&result), vec![1, 2, 4, 3]); + } + + #[test] + fn separates_identical_non_empty_semantic_vectors() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[7]), + candidate(2, 2, &[7]), + candidate(3, 3, &[8]), + ], + config(3), + ); + assert_eq!(selected_ids(&result), vec![1, 3, 2]); + } + + #[test] + fn shared_semantic_component_is_not_the_same_cluster() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[7, 100]), + candidate(2, 2, &[7, 200]), + candidate(3, 3, &[8, 300]), + ], + config(3), + ); + assert_eq!(selected_ids(&result), vec![1, 2, 3]); + } + + #[test] + fn relaxes_semantics_before_author_caps() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 2, &[2]), + candidate(4, 1, &[3]), + ], + config(4), + ); + assert_eq!(selected_ids(&result), vec![1, 2, 3, 4]); + } + + #[test] + fn lookahead_bounds_rank_displacement() { + let mut candidates = vec![candidate(0, 1, &[1])]; + candidates.extend((1..=10).map(|id| candidate(id, id + 1, &[1]))); + candidates.push(candidate(11, 12, &[2])); + + let result = select_from_ranked(candidates, config(12)); + assert_eq!(&selected_ids(&result)[..2], &[0, 1]); + } + + #[test] + fn degenerate_configs_preserve_candidates() { + for config in [ + DiversityConfig { + selection_size: 3, + author_window_size: 0, + max_posts_per_author: 0, + enable_semantic_diversity: true, + max_lookahead: 0, + }, + DiversityConfig { + selection_size: 3, + author_window_size: usize::MAX, + max_posts_per_author: 0, + enable_semantic_diversity: false, + max_lookahead: usize::MAX, + }, + ] { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 1, &[3]), + ], + config, + ); + assert_eq!(selected_ids(&result), vec![1, 2, 3]); + assert!(result.non_selected.is_empty()); + } + } + + #[test] + fn reports_expected_rank_displacement() { + let result = select_from_ranked( + vec![ + candidate(1, 1, &[1]), + candidate(2, 1, &[2]), + candidate(3, 1, &[3]), + candidate(4, 2, &[4]), + ], + config(4), + ); + let ids = selected_ids(&result); + assert_eq!(ids, vec![1, 2, 4, 3]); + + let original_rank: HashMap = [1, 2, 3, 4] + .into_iter() + .enumerate() + .map(|(i, id)| (id, i)) + .collect(); + let displacement: Vec = ids + .iter() + .enumerate() + .map(|(new_rank, id)| new_rank as isize - original_rank[id] as isize) + .collect(); + assert_eq!(displacement, vec![0, 0, -1, 1]); + } + + #[test] + fn randomized_inputs_never_panic_or_lose_candidates() { + let mut state = 0x4d595df4d0f33173_u64; + let mut next = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + state + }; + + for case in 0..500 { + let candidate_count = (next() % 101) as usize; + let selection_size = (next() % 121) as usize; + let candidates: Vec = (0..candidate_count) + .map(|id| Candidate { + id: id as u64, + original_author_id: next() % 17, + serving_author_id: next() % 17, + semantic_ids: (next() % 4 != 0) + .then(|| vec![(next() % 6) as i32, (next() % 13) as i32]), + }) + .collect(); + let expected_ids: HashSet = + candidates.iter().map(|candidate| candidate.id).collect(); + let config = DiversityConfig { + selection_size, + author_window_size: (next() % 30) as usize, + max_posts_per_author: (next() % 5) as usize, + enable_semantic_diversity: next() % 2 == 0, + max_lookahead: (next() % 20) as usize, + }; + + let result = select_from_ranked(candidates, config); + let actual_ids: Vec = result + .selected + .iter() + .chain(&result.non_selected) + .map(|candidate| candidate.id) + .collect(); + let actual_set: HashSet = actual_ids.iter().copied().collect(); + + assert_eq!( + result.selected.len(), + selection_size.min(candidate_count), + "case {case}" + ); + assert_eq!(actual_ids.len(), candidate_count, "case {case}"); + assert_eq!(actual_set.len(), candidate_count, "case {case}"); + assert_eq!(actual_set, expected_ids, "case {case}"); + } + } +} diff --git a/home-mixer/selectors/top_k_score_selector.rs b/home-mixer/selectors/top_k_score_selector.rs index 6f0ba37b..709ebc72 100644 --- a/home-mixer/selectors/top_k_score_selector.rs +++ b/home-mixer/selectors/top_k_score_selector.rs @@ -1,7 +1,8 @@ -use crate::models::candidate::PostCandidate; +use super::slate_diversity::{self, DiversityConfig, SlateItem}; +use crate::models::candidate::{CandidateHelpers, PostCandidate}; use crate::models::query::ScoredPostsQuery; use crate::params; -use xai_candidate_pipeline::selector::Selector; +use xai_candidate_pipeline::selector::{SelectResult, Selector}; pub struct TopKScoreSelector; @@ -9,7 +10,70 @@ impl Selector for TopKScoreSelector { fn score(&self, candidate: &PostCandidate) -> f64 { candidate.score.unwrap_or(f64::NEG_INFINITY) } + fn size(&self) -> Option { Some(params::TOP_K_CANDIDATES_TO_SELECT) } } + +/// Selects the highest-scoring candidates while improving slate diversity. +/// Author caps apply near the top; semantic adjacency applies throughout. +/// Lookahead bounds how far a constraint may displace a ranked candidate. +/// +/// `RankingScorer` may already have applied a soft serving-author decay and +/// populated `SlateContext.k`. This selector intentionally recomputes counts: +/// VMRanker can change the order after that context is recorded, `k` describes +/// the earlier pool order rather than this selector's chosen prefix, and it +/// does not track the original author of a retweeted post. The hard cap is a +/// final guardrail layered on top of that independently configurable decay. +pub struct SlateDiversitySelector; + +impl SlateItem for PostCandidate { + type AuthorId = u64; + + fn original_author_id(&self) -> Self::AuthorId { + self.get_original_author_id() + } + + fn serving_author_id(&self) -> Self::AuthorId { + self.author_id + } + + fn semantic_ids(&self) -> Option<&[i32]> { + self.semantic_ids.as_deref() + } +} + +impl Selector for SlateDiversitySelector { + fn score(&self, candidate: &PostCandidate) -> f64 { + candidate.score.unwrap_or(f64::NEG_INFINITY) + } + + fn select( + &self, + query: &ScoredPostsQuery, + candidates: Vec, + ) -> SelectResult { + if !query.params.get(params::EnableSlateDiversity) { + return TopKScoreSelector.select(query, candidates); + } + + let result = slate_diversity::select_from_ranked( + self.sort(candidates), + DiversityConfig { + selection_size: params::TOP_K_CANDIDATES_TO_SELECT, + author_window_size: query.params.get(params::SlateDiversityAuthorWindowSize) + as usize, + max_posts_per_author: query.params.get(params::SlateDiversityMaxPostsPerAuthor) + as usize, + enable_semantic_diversity: query.params.get(params::EnableSlateSemanticDiversity), + max_lookahead: query.params.get(params::SlateDiversityMaxLookahead) as usize, + }, + ); + + SelectResult { + selected: result.selected, + non_selected: result.non_selected, + } + } +} diff --git a/local-tests/slate-diversity/Cargo.lock b/local-tests/slate-diversity/Cargo.lock new file mode 100644 index 00000000..08ac0f7e --- /dev/null +++ b/local-tests/slate-diversity/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "slate-diversity-local-tests" +version = "0.1.0" diff --git a/local-tests/slate-diversity/Cargo.toml b/local-tests/slate-diversity/Cargo.toml new file mode 100644 index 00000000..f18a517d --- /dev/null +++ b/local-tests/slate-diversity/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "slate-diversity-local-tests" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +path = "src/lib.rs" diff --git a/local-tests/slate-diversity/README.md b/local-tests/slate-diversity/README.md new file mode 100644 index 00000000..ff2ec2e6 --- /dev/null +++ b/local-tests/slate-diversity/README.md @@ -0,0 +1,16 @@ +# Slate diversity local tests + +This crate compiles the dependency-free selection engine used by Home Mixer's +`SlateDiversitySelector`. It does not compile the Home Mixer adapter or its +internal X dependencies. + +Run from the repository root: + +```sh +CARGO_TARGET_DIR=/tmp/xai-slate-diversity-target \ + cargo test --manifest-path local-tests/slate-diversity/Cargo.toml +``` + +The external target directory is required because this checkout's directory +name contains `:`, which macOS cannot represent as one entry in +`DYLD_FALLBACK_LIBRARY_PATH`. diff --git a/local-tests/slate-diversity/src/lib.rs b/local-tests/slate-diversity/src/lib.rs new file mode 100644 index 00000000..21b7e863 --- /dev/null +++ b/local-tests/slate-diversity/src/lib.rs @@ -0,0 +1,2 @@ +#[path = "../../../home-mixer/selectors/slate_diversity.rs"] +pub mod slate_diversity;