diff --git a/README.md b/README.md index 102c4bc06..e060fcc9d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# X's Recommendation Algorithm +# X's Recommendation Algorithm (Legacy) + +> **Notice:** This legacy repository is currently being audited by **Taijitu AI** as part of our high-fidelity systemic analysis. We are cross-referencing this implementation with the modern `x-algorithm` (Rust) codebase to ensure architectural integrity in our proposed optimizations. + X's Recommendation Algorithm is a set of services and jobs that are responsible for serving feeds of posts and other content across all X product surfaces (e.g. For You Timeline, Search, Explore, Notifications). For an introduction to how the algorithm works, please refer to our [engineering blog](https://blog.x.com/engineering/en_us/topics/open-source/2023/twitter-recommendation-algorithm). diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala index 4b1cec4a5..6c7166d71 100644 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala +++ b/home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/OONTweetScalingScorer.scala @@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.scorer import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature +import com.twitter.home_mixer.model.HomeFeatures.TweetAuthorFollowersFeature import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.featuremap.FeatureMap @@ -14,7 +15,8 @@ import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.stitch.Stitch /** - * Scales scores of each out-of-network tweet by the specified scale factor + * Scales scores of each out-of-network tweet by a factor that dynamically + * adjusts based on author follower count to encourage discovery of new voices. */ object OONTweetScalingScorer extends Scorer[PipelineQuery, TweetCandidate] { @@ -22,21 +24,42 @@ object OONTweetScalingScorer extends Scorer[PipelineQuery, TweetCandidate] { override val features: Set[Feature[_, _]] = Set(ScoreFeature) - private val ScaleFactor = 0.75 + private val BaseScaleFactor = 0.75 override def apply( query: PipelineQuery, candidates: Seq[CandidateWithFeatures[TweetCandidate]] ): Stitch[Seq[FeatureMap]] = { + val tweetAuthorFollowers = query.features + .map(_.getOrElse(TweetAuthorFollowersFeature, Map.empty[Long, Option[Long]])) + .getOrElse(Map.empty) + Stitch.value { candidates.map { candidate => val score = candidate.features.getOrElse(ScoreFeature, None) - val updatedScore = if (selector(candidate)) score.map(_ * ScaleFactor) else score + + val updatedScore = if (selector(candidate)) { + val followers = tweetAuthorFollowers.get(candidate.candidate.id).flatten.getOrElse(0L) + val discoveryBoost = calculateDiscoveryBoost(followers) + score.map(_ * BaseScaleFactor * discoveryBoost) + } else { + score + } + FeatureMapBuilder().add(ScoreFeature, updatedScore).build() } } } + /** + * Calculate discovery boost using a logarithmic inverse of follower count. + * Boost = 1 + (1 / ln(followers + e)) + */ + private def calculateDiscoveryBoost(followers: Long): Double = { + val f = math.max(0L, followers).toDouble + 1.0 + (1.0 / math.log(f + math.E)) + } + /** * We should only be applying this multiplier to Out-Of-Network tweets. * In-Network Retweets of Out-Of-Network tweets should not have this multiplier applied