├── .editorconfig ├── .gitignore ├── .prettierrc ├── license ├── package.json ├── readme.md ├── src ├── algorithmic-feed.ts ├── index.ts └── types.ts ├── tsconfig.json └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | node_modules 5 | 6 | # builds 7 | build 8 | dist 9 | 10 | # misc 11 | .DS_Store 12 | .env 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | .cache 18 | tsconfig.tsbuildinfo 19 | 20 | npm-debug.log* 21 | yarn-debug.log* 22 | yarn-error.log* 23 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "jsxSingleQuote": true, 4 | "semi": false, 5 | "useTabs": false, 6 | "tabWidth": 2, 7 | "bracketSpacing": true, 8 | "bracketSameLine": false, 9 | "arrowParens": "always", 10 | "trailingComma": "none" 11 | } 12 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Travis Fischer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "twitter-feed-algorithm", 3 | "private": true, 4 | "version": "0.0.1", 5 | "description": "TypeScript code exploring what an open source version of Twitter's algorithmic feed might look like.", 6 | "author": "Travis Fischer ", 7 | "repository": "transitive-bullshit/nextjs-notion-starter-kit", 8 | "license": "MIT", 9 | "engines": { 10 | "node": ">=14" 11 | }, 12 | "main": "index.js", 13 | "dependencies": { 14 | "twitter-api-sdk": "^1.0.3" 15 | }, 16 | "devDependencies": { 17 | "typescript": "^4.6.3" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Twitter Feed Algorithm 2 | 3 | > TypeScript code exploring what an open source version of Twitter's algorithmic feed might look like. 4 | 5 | ## Intro 6 | 7 | This repo contains code samples for a series of articles I'm working on with the aim of exploring what “open sourcing Twitter's algorithm” might look like from a more practical perspective. 8 | 9 | ## Goals, Non-Goals, and Caveats 10 | 11 | - We are only focused on Twitter. 12 | - We will not consider the more general space of algorithmic recommendation engines or other social networks. 13 | - We are only focused on the Twitter's core algorithmic feed. 14 | - Other algorithmically-generated recommendations such as “Who to follow”, “Topics you may be interested in”, “Search”, “Trending”, etc are all out of scope for now. 15 | - Any work towards open sourcing aspects of the Twitter’s core feed algorithm are likely to naturally extend to open sourcing these other recommendation algorithms in the future. 16 | - We are only focused on high-level business logic and pseudocode. 17 | - Our goal is to drive a more transparent discussion around the Twitter feed’s core business logic and data models without worrying too much about the underlying engineering complexity. 18 | - The engineering challenges inherent in running a global, low-latency, highly reliable production system like Twitter are immensely important in practice but will be considered mostly out of scope for the purposes of this discussion. 19 | - It’s important to keep in mind, however, that there is an incredible amount of engineering complexity hidden behind a platform like Twitter. We won’t be diving into databases, services, machine learning techniques, networking, infrastructure, devops, etc. 20 | - We will defer to the core data models from [v2 of Twitter’s public API](https://developer.twitter.com/en/docs/twitter-api) wherever possible, including the [Tweet](https://developer.twitter.com/en/docs/twitter-api/data-dictionary/object-model/tweet) and [User](https://developer.twitter.com/en/docs/twitter-api/data-dictionary/object-model/user) objects. 21 | - Twitter likely uses many different abstractions for these resources under the hood, but for the purposes of an open source algorithm, we should remain consistent with the resource models published by the latest version of [their public API](https://developer.twitter.com/en/products/twitter-api). 22 | - This project uses TypeScript because it's what I'm most comfortable with, but it could easily be ported to other languages. 23 | - Any real open source solution would need to come from Twitter itself. 24 | - E.g., you can consider this project for education purposes only. 25 | 26 | ## License 27 | 28 | MIT © [Travis Fischer](https://transitivebullsh.it) 29 | 30 | Support my open source work by following me on twitter twitter 31 | -------------------------------------------------------------------------------- /src/algorithmic-feed.ts: -------------------------------------------------------------------------------- 1 | import { Timeline, User } from './types' 2 | 3 | export abstract class TwitterAlgorithmicFeed { 4 | /** 5 | * Pseudo-code for understanding how Twitter's algorithmic feed works. 6 | */ 7 | async getAlgorithmicTimelineForUser(user: User): Promise { 8 | const rawTimeline = await this.getRawTimelineForUser(user) 9 | const relevantTweets = await this.getPotentiallyRelevantTweetsForUser(user) 10 | 11 | const mergedTimeline = await this.mergeTimelinesForUserBasedOnRelevancy( 12 | user, 13 | rawTimeline, 14 | relevantTweets 15 | ) 16 | 17 | return this.injectAdsForUserIntoTimeline(user, mergedTimeline) 18 | } 19 | 20 | /** 21 | * Returns a reverse-chronological stream of tweets from users directly 22 | * followed by a given user. 23 | */ 24 | abstract getRawTimelineForUser(user: User): Promise 25 | 26 | /** 27 | * Returns a stream of tweets ordered by relevancy for a given user at a 28 | * given time. 29 | * 30 | * This will only consider tweets from users the given user is not already 31 | * following. 32 | */ 33 | abstract getPotentiallyRelevantTweetsForUser(user: User): Promise 34 | 35 | /** 36 | * Returns a stream of tweets ordered by relevancy to a given user, taking 37 | * into account both their raw timeline of latest tweets and a subset of 38 | * the network graph timeline containing potentially relevant tweets. 39 | */ 40 | abstract mergeTimelinesForUserBasedOnRelevancy( 41 | user: User, 42 | rawTimeline: Timeline, 43 | relevantTweets: Timeline 44 | ): Promise 45 | 46 | /** 47 | * Returns a stream of tweets which injects ads into a given timeline for a 48 | * given user. 49 | */ 50 | abstract injectAdsForUserIntoTimeline( 51 | user: User, 52 | timeline: Timeline 53 | ): Promise 54 | } 55 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './types' 2 | export * from './algorithmic-feed' 3 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import { types as twitter } from 'twitter-api-sdk' 2 | 3 | export type Tweet = twitter.components['schemas']['Tweet'] 4 | export type User = twitter.components['schemas']['User'] 5 | export type Timeline = 6 | twitter.components['schemas']['GenericTweetsTimelineResponse'] 7 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016", 4 | "module": "esnext", 5 | "moduleResolution": "node", 6 | "resolveJsonModule": true, 7 | "isolatedModules": true, 8 | "baseUrl": "." 9 | }, 10 | "exclude": ["node_modules"] 11 | } 12 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | node-fetch@^2.6.1: 6 | version "2.6.7" 7 | resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" 8 | integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== 9 | dependencies: 10 | whatwg-url "^5.0.0" 11 | 12 | tr46@~0.0.3: 13 | version "0.0.3" 14 | resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" 15 | integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= 16 | 17 | twitter-api-sdk@^1.0.3: 18 | version "1.0.3" 19 | resolved "https://registry.yarnpkg.com/twitter-api-sdk/-/twitter-api-sdk-1.0.3.tgz#f298cbcb15e891027e41cab1fc07710198c2ae51" 20 | integrity sha512-82tGMkg8ova+EpQ8sc5iEBlu5YZxKWtGz3I2pP99+NzTr28K6BQvUj3kdg0M+lU6+HiPpoX6v5N/4ikGQorQlg== 21 | dependencies: 22 | node-fetch "^2.6.1" 23 | 24 | typescript@^4.6.3: 25 | version "4.6.3" 26 | resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.6.3.tgz#eefeafa6afdd31d725584c67a0eaba80f6fc6c6c" 27 | integrity sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw== 28 | 29 | webidl-conversions@^3.0.0: 30 | version "3.0.1" 31 | resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" 32 | integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= 33 | 34 | whatwg-url@^5.0.0: 35 | version "5.0.0" 36 | resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" 37 | integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= 38 | dependencies: 39 | tr46 "~0.0.3" 40 | webidl-conversions "^3.0.0" 41 | --------------------------------------------------------------------------------