diff --git a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs index e8c9c397..32f9f08e 100644 --- a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs +++ b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs @@ -54,6 +54,7 @@ use crate::filters::retweet_deduplication_filter::RetweetDeduplicationFilter; use crate::filters::self_tweet_filter::SelfTweetFilter; use crate::filters::topic_ids_filter::TopicIdsFilter; use crate::filters::vf_filter::VFFilter; +use crate::filters::following_content_controls_filter::FollowingContentControlsFilter; use crate::filters::video_filter::VideoFilter; use crate::filters::viewer_muted_keyword_filter::ViewerMutedKeywordFilter; use crate::models::candidate::PostCandidate; @@ -368,6 +369,7 @@ impl PhoenixCandidatePipeline { // OmarAzizSenador deleted his account at the time this code was written. Box::new(Brazil2026ElectionFilter), Box::new(VideoFilter), + Box::new(FollowingContentControlsFilter), Box::new(TopicIdsFilter), Box::new(NewUserMinEngagementFilter), Box::new(InventoryHoldoutFilter), diff --git a/home-mixer/candidate_pipeline/reverse_chron_posts_pipeline.rs b/home-mixer/candidate_pipeline/reverse_chron_posts_pipeline.rs index 1ea8adf6..ea5563de 100644 --- a/home-mixer/candidate_pipeline/reverse_chron_posts_pipeline.rs +++ b/home-mixer/candidate_pipeline/reverse_chron_posts_pipeline.rs @@ -10,6 +10,7 @@ use crate::clients::s2s::{S2S_CHAIN_PATH, S2S_CRT_PATH, S2S_KEY_PATH}; use crate::clients::tweet_entity_service_client::{MockTESClient, ProdTESClient, TESClient}; use crate::filters::ancillary_vf_filter::AncillaryVFFilter; use crate::filters::author_socialgraph_filter::AuthorSocialgraphFilter; +use crate::filters::following_content_controls_filter::FollowingContentControlsFilter; use crate::filters::following_retweet_deduplication_filter::FollowingRetweetDeduplicationFilter; use crate::filters::following_viewer_muted_keyword_filter::FollowingViewerMutedKeywordFilter; use crate::filters::self_reply_chain_filter::SelfReplyChainFilter; @@ -162,6 +163,7 @@ impl ReverseChronPostsPipeline { Box::new(FollowingRetweetDeduplicationFilter), Box::new(FollowingViewerMutedKeywordFilter::new()), Box::new(SelfReplyChainFilter), + Box::new(FollowingContentControlsFilter), ]; let post_selection_hydrators: Vec>> = vec![ @@ -236,3 +238,14 @@ impl CandidatePipeline for ReverseChronPostsPip FOLLOWING_POST_FETCH_SIZE } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn mock_pipeline_wires_content_controls_filter() { + let pipeline = ReverseChronPostsPipeline::mock().await; + assert_eq!(pipeline.filters().len(), 4); + } +} diff --git a/home-mixer/filters/following_content_controls_filter.rs b/home-mixer/filters/following_content_controls_filter.rs new file mode 100644 index 00000000..276709a1 --- /dev/null +++ b/home-mixer/filters/following_content_controls_filter.rs @@ -0,0 +1,214 @@ +use crate::models::candidate::PostCandidate; +use crate::models::query::ScoredPostsQuery; +use xai_candidate_pipeline::filter::{Filter, FilterResult}; + +pub struct FollowingContentControlsFilter; + +impl Filter for FollowingContentControlsFilter { + fn enable(&self, query: &ScoredPostsQuery) -> bool { + query.hides_replies() || query.hides_links() || query.hides_retweets() + } + + fn filter( + &self, + query: &ScoredPostsQuery, + candidates: Vec, + ) -> FilterResult { + let hide_replies = query.hides_replies(); + let hide_links = query.hides_links(); + let hide_retweets = query.hides_retweets(); + + let (removed, kept): (Vec<_>, Vec<_>) = candidates.into_iter().partition(|c| { + (hide_replies && c.in_reply_to_tweet_id.is_some()) + || (hide_retweets && c.retweeted_tweet_id.is_some()) + || (hide_links && candidate_has_link(c)) + }); + + FilterResult { kept, removed } + } +} + +fn candidate_has_link(candidate: &PostCandidate) -> bool { + text_has_link(&candidate.tweet_text) + || candidate + .quoted_tweet_text + .as_deref() + .is_some_and(text_has_link) + || candidate.ancestor_texts.values().any(|t| text_has_link(t)) +} + +fn text_has_link(text: &str) -> bool { + text.contains("https://") + || text.contains("http://") + || text.contains("t.co/") + || contains_www_host(text) +} + +fn contains_www_host(text: &str) -> bool { + text.match_indices("www.") + .any(|(idx, _)| idx == 0 || !text.as_bytes()[idx - 1].is_ascii_alphanumeric()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn candidate( + tweet_id: u64, + in_reply_to_tweet_id: Option, + retweeted_tweet_id: Option, + tweet_text: &str, + ) -> PostCandidate { + PostCandidate { + tweet_id, + in_reply_to_tweet_id, + retweeted_tweet_id, + tweet_text: tweet_text.to_string(), + ..Default::default() + } + } + + #[test] + fn disabled_when_no_content_controls() { + let query = ScoredPostsQuery::default(); + assert!(!FollowingContentControlsFilter.enable(&query)); + } + + #[test] + fn enabled_for_each_viewer_preference() { + for query in [ + ScoredPostsQuery { + hide_replies: true, + ..Default::default() + }, + ScoredPostsQuery { + exclude_replies: true, + ..Default::default() + }, + ScoredPostsQuery { + hide_links: true, + ..Default::default() + }, + ScoredPostsQuery { + exclude_retweets: true, + ..Default::default() + }, + ] { + assert!(FollowingContentControlsFilter.enable(&query)); + } + } + + #[test] + fn hide_replies_drops_replies_keeps_originals() { + let query = ScoredPostsQuery { + hide_replies: true, + ..Default::default() + }; + let result = FollowingContentControlsFilter.filter( + &query, + vec![ + candidate(1, Some(10), None, "reply"), + candidate(2, None, None, "original"), + candidate(3, None, Some(30), "retweet"), + ], + ); + assert_eq!( + result.kept.iter().map(|c| c.tweet_id).collect::>(), + vec![2, 3] + ); + assert_eq!( + result + .removed + .iter() + .map(|c| c.tweet_id) + .collect::>(), + vec![1] + ); + } + + #[test] + fn exclude_replies_alias_drops_replies() { + let query = ScoredPostsQuery { + exclude_replies: true, + ..Default::default() + }; + let result = FollowingContentControlsFilter.filter( + &query, + vec![candidate(1, Some(10), None, "reply")], + ); + assert_eq!(result.removed[0].tweet_id, 1); + assert!(result.kept.is_empty()); + } + + #[test] + fn hide_links_drops_url_cards_and_quoted_links() { + let query = ScoredPostsQuery { + hide_links: true, + ..Default::default() + }; + let mut quoted = candidate(2, None, None, "look"); + quoted.quoted_tweet_text = Some("see https://example.com".to_string()); + let result = FollowingContentControlsFilter.filter( + &query, + vec![ + candidate(1, None, None, "plain text"), + candidate(3, None, None, "watch https://t.co/abc"), + quoted, + candidate(4, None, None, "www.example.com"), + candidate(5, None, None, "notwww.example"), + ], + ); + assert_eq!( + result.kept.iter().map(|c| c.tweet_id).collect::>(), + vec![1, 5] + ); + assert_eq!( + result + .removed + .iter() + .map(|c| c.tweet_id) + .collect::>(), + vec![3, 2, 4] + ); + } + + #[test] + fn exclude_retweets_drops_retweets_keeps_replies() { + let query = ScoredPostsQuery { + exclude_retweets: true, + ..Default::default() + }; + let result = FollowingContentControlsFilter.filter( + &query, + vec![ + candidate(1, None, Some(10), "rt"), + candidate(2, Some(20), None, "reply"), + candidate(3, None, None, "original"), + ], + ); + assert_eq!( + result.kept.iter().map(|c| c.tweet_id).collect::>(), + vec![2, 3] + ); + assert_eq!( + result + .removed + .iter() + .map(|c| c.tweet_id) + .collect::>(), + vec![1] + ); + } + + #[test] + fn off_flags_keep_every_type() { + let query = ScoredPostsQuery::default(); + let candidates = vec![ + candidate(1, Some(10), None, "reply https://x.com"), + candidate(2, None, Some(20), "rt"), + ]; + let result = FollowingContentControlsFilter.filter(&query, candidates); + assert_eq!(result.kept.len(), 2); + assert!(result.removed.is_empty()); + } +} diff --git a/home-mixer/filters/mod.rs b/home-mixer/filters/mod.rs index ce6231cc..01ffad3f 100644 --- a/home-mixer/filters/mod.rs +++ b/home-mixer/filters/mod.rs @@ -7,6 +7,7 @@ pub mod core_data_hydration_filter; pub mod dedup_conversation_filter; pub mod drop_duplicates_filter; +pub mod following_content_controls_filter; pub mod following_retweet_deduplication_filter; pub mod following_viewer_muted_keyword_filter; pub mod ineligible_subscription_filter; diff --git a/home-mixer/models/query.rs b/home-mixer/models/query.rs index de645475..73dfa89e 100644 --- a/home-mixer/models/query.rs +++ b/home-mixer/models/query.rs @@ -75,6 +75,14 @@ pub struct ScoredPostsQuery { pub topic_ids: Vec, pub excluded_topic_ids: Vec, pub exclude_videos: bool, + /// Viewer content-control: hide replies on Following / Ranked Following / For You. + pub hide_replies: bool, + /// Viewer content-control: hide posts that contain links. + pub hide_links: bool, + /// Alternate proto name for hide-replies (Twitter API `exclude_replies`). + pub exclude_replies: bool, + /// Viewer content-control: hide retweets. Night Owl used to `include:retweets` always. + pub exclude_retweets: bool, #[serde(serialize_with = "serialize_in_network_replies")] pub in_network_replies: InNetworkReplies, pub viewer_minhash: Option>, @@ -192,6 +200,10 @@ impl ScoredPostsQuery { topic_ids, excluded_topic_ids, exclude_videos, + hide_replies: false, + hide_links: false, + exclude_replies: false, + exclude_retweets: false, in_network_replies: Default::default(), viewer_minhash: None, ip_address, @@ -242,6 +254,18 @@ impl ScoredPostsQuery { pub fn has_excluded_topics(&self) -> bool { !self.excluded_topic_ids.is_empty() } + + pub fn hides_replies(&self) -> bool { + self.hide_replies || self.exclude_replies + } + + pub fn hides_links(&self) -> bool { + self.hide_links + } + + pub fn hides_retweets(&self) -> bool { + self.exclude_retweets + } } impl GetTwitterContextViewer for ScoredPostsQuery { diff --git a/home-mixer/server.rs b/home-mixer/server.rs index d8f59d18..ce35874d 100644 --- a/home-mixer/server.rs +++ b/home-mixer/server.rs @@ -146,6 +146,7 @@ impl QueryBuilder { query.resurrection_time_ms = resurrection_time_ms; query.dsp_client_context = proto_query.dsp_client_context; + apply_content_controls(&mut query, &proto_query); let root_span = b3_info.root_span(info_span!( "request", @@ -265,6 +266,13 @@ impl QueryBuilder { } } +fn apply_content_controls(query: &mut ScoredPostsQuery, proto_query: &pb::ScoredPostsQuery) { + query.hide_replies = proto_query.hide_replies; + query.hide_links = proto_query.hide_links; + query.exclude_replies = proto_query.exclude_replies; + query.exclude_retweets = proto_query.exclude_retweets; +} + pub struct HomeMixerServer { scored_posts: Arc, for_you: Arc, diff --git a/home-mixer/sources/following_night_owl_source.rs b/home-mixer/sources/following_night_owl_source.rs index 0195c758..bce1bca5 100644 --- a/home-mixer/sources/following_night_owl_source.rs +++ b/home-mixer/sources/following_night_owl_source.rs @@ -13,7 +13,6 @@ use xai_candidate_pipeline::source::Source; use xai_home_mixer_proto::ServedType; use xai_urt_thrift::operation::CursorType; -const BASE_FILTERS: &str = "filter:follows include:retweets include:protected include:spam"; const MAX_RESULTS: u32 = FOLLOWING_POST_FETCH_SIZE as u32; const COLLECTOR_TIMEOUT_MS: u32 = 300; const RECALL_MAX_AGE_SECS: i64 = 365 * 24 * 60 * 60; @@ -84,19 +83,36 @@ fn build_request(query: &ScoredPostsQuery) -> Result String { - let now_secs = request_time_ms / 1000; +fn night_owl_operators(query: &ScoredPostsQuery) -> String { + let retweets = if query.hides_retweets() { + "-filter:nativeretweets" + } else { + "include:retweets" + }; + let mut operators = format!("filter:follows {retweets} include:protected include:spam"); + if query.hides_replies() { + operators.push_str(" -filter:replies"); + } + if query.hides_links() { + operators.push_str(" -filter:links"); + } + operators +} + +fn base_query(query: &ScoredPostsQuery) -> String { + let operators = night_owl_operators(query); + let now_secs = query.request_time_ms / 1000; if now_secs <= 0 { - return BASE_FILTERS.to_string(); + return operators; } format!( - "{BASE_FILTERS} since_time:{}", + "{operators} since_time:{}", (now_secs - RECALL_MAX_AGE_SECS).max(0) ) } fn pagination_for_cursor(query: &ScoredPostsQuery) -> Result<(String, Pagination), String> { - let base = base_query(query.request_time_ms); + let base = base_query(query); let Some(cursor) = query.cursor.as_ref() else { return Ok((base, from_size_pagination())); @@ -220,3 +236,72 @@ fn hit_to_post_candidate(hit: night_owl::SearchHit) -> PostCandidate { ..Default::default() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_operators_still_include_retweets() { + let query = ScoredPostsQuery::default(); + assert_eq!( + night_owl_operators(&query), + "filter:follows include:retweets include:protected include:spam" + ); + } + + #[test] + fn hide_replies_adds_filter() { + let query = ScoredPostsQuery { + hide_replies: true, + ..Default::default() + }; + let operators = night_owl_operators(&query); + assert!(operators.contains("-filter:replies")); + assert!(operators.contains("include:retweets")); + assert!(!operators.contains("-filter:links")); + } + + #[test] + fn exclude_replies_alias_adds_filter() { + let query = ScoredPostsQuery { + exclude_replies: true, + ..Default::default() + }; + assert!(night_owl_operators(&query).contains("-filter:replies")); + } + + #[test] + fn hide_links_adds_filter() { + let query = ScoredPostsQuery { + hide_links: true, + ..Default::default() + }; + assert!(night_owl_operators(&query).contains("-filter:links")); + } + + #[test] + fn exclude_retweets_replaces_include_retweets() { + let query = ScoredPostsQuery { + exclude_retweets: true, + ..Default::default() + }; + let operators = night_owl_operators(&query); + assert!(operators.contains("-filter:nativeretweets")); + assert!(!operators.contains("include:retweets")); + } + + #[test] + fn all_content_controls_compose() { + let query = ScoredPostsQuery { + hide_replies: true, + hide_links: true, + exclude_retweets: true, + ..Default::default() + }; + assert_eq!( + night_owl_operators(&query), + "filter:follows -filter:nativeretweets include:protected include:spam -filter:replies -filter:links" + ); + } +}