├── .DS_Store ├── .env.example ├── .eslintrc.js ├── .github └── workflows │ ├── node.js.yml │ └── publish.yml ├── .prettierignore ├── .prettierrc ├── LICENSE ├── README.md ├── SampleAgent.js ├── jest.config.js ├── package-lock.json ├── package.json ├── rollup.config.mjs ├── src ├── _module.ts ├── api-data.ts ├── api.ts ├── auth-user.ts ├── auth.test.ts ├── auth.ts ├── command.ts ├── errors.ts ├── messages.test.ts ├── messages.ts ├── platform │ ├── index.ts │ ├── node │ │ ├── index.ts │ │ └── randomize-ciphers.ts │ └── platform-interface.ts ├── profile.test.ts ├── profile.ts ├── relationships.test.ts ├── relationships.ts ├── requests.ts ├── scraper.test.ts ├── scraper.ts ├── search.test.ts ├── search.ts ├── spaces.ts ├── spaces │ ├── core │ │ ├── ChatClient.ts │ │ ├── JanusAudio.ts │ │ ├── JanusClient.ts │ │ └── Space.ts │ ├── logger.ts │ ├── plugins │ │ ├── HlsRecordPlugin.ts │ │ ├── IdleMonitorPlugin.ts │ │ ├── MonitorAudioPlugin.ts │ │ ├── RecordToDiskPlugin.ts │ │ └── SttTtsPlugin.ts │ ├── test.ts │ ├── types.ts │ └── utils.ts ├── test-utils.ts ├── timeline-async.ts ├── timeline-following.ts ├── timeline-home.ts ├── timeline-list.ts ├── timeline-relationship.ts ├── timeline-search.ts ├── timeline-tweet-util.ts ├── timeline-v1.ts ├── timeline-v2.ts ├── trends.test.ts ├── trends.ts ├── tweets.test.ts ├── tweets.ts ├── type-util.ts └── types │ └── spaces.ts ├── test-assets ├── test-image.jpeg └── test-video.mp4 ├── test-setup.js ├── tsconfig.json └── typedoc.json /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingbootoshi/goat-x/ffcfed49c35b1130540c45eef95086a9fb6dff8a/.DS_Store -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # for v1 api support 2 | TWITTER_USERNAME=myaccount 3 | TWITTER_PASSWORD=MyPassword!!! 4 | TWITTER_EMAIL=myemail@gmail.com 5 | 6 | # for v2 api support 7 | TWITTER_API_KEY=key 8 | TWITTER_API_SECRET_KEY=secret 9 | TWITTER_ACCESS_TOKEN=token 10 | TWITTER_ACCESS_TOKEN_SECRET=tokensecret 11 | 12 | # optional 13 | PROXY_URL= # HTTP(s) proxy for requests 14 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | parserOptions: { 4 | tsconfigRootDir: __dirname, 5 | sourceType: 'module', 6 | }, 7 | plugins: ['@typescript-eslint/eslint-plugin'], 8 | extends: [ 9 | 'plugin:@typescript-eslint/recommended', 10 | 'plugin:prettier/recommended', 11 | ], 12 | root: true, 13 | env: { 14 | node: true, 15 | jest: true, 16 | }, 17 | ignorePatterns: ['**/*.js'], 18 | rules: { 19 | '@typescript-eslint/interface-name-prefix': 'off', 20 | '@typescript-eslint/explicit-function-return-type': 'off', 21 | '@typescript-eslint/explicit-module-boundary-types': 'off', 22 | '@typescript-eslint/no-explicit-any': 'off', 23 | }, 24 | }; 25 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: [main] 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Install Node.js 20 16 | uses: actions/setup-node@v3 17 | with: 18 | node-version: 20.x 19 | - name: Install 20 | run: npm install 21 | # - name: Test 22 | # env: 23 | # TWITTER_USERNAME: ${{ secrets.TWITTER_USERNAME }} 24 | # TWITTER_PASSWORD: ${{ secrets.TWITTER_PASSWORD }} 25 | # TWITTER_EMAIL: ${{ secrets.TWITTER_EMAIL }} 26 | # run: npm run test 27 | - name: Build 28 | run: npm run build 29 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | release: 5 | types: 6 | - created 7 | 8 | jobs: 9 | publish: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - name: Install Node.js 14 | uses: actions/setup-node@v3 15 | with: 16 | node-version: 20.x 17 | - name: Install 18 | run: npm install 19 | - name: Build 20 | run: npm run build 21 | - name: Publish 22 | env: 23 | NPM_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 24 | run: npm publish --access public 25 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | *.json -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all", 4 | "semi": true 5 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Ruby Research 4 | Copyright (c) 2022 karashiiro 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # agent-twitter-client 2 | 3 | This is a modified version of [@the-convocation/twitter-scraper](https://github.com/the-convocation/twitter-scraper) with added functionality for sending tweets and retweets. This package does not require the Twitter API to use and will run in both the browser and server. 4 | 5 | ## Installation 6 | 7 | ```sh 8 | npm install agent-twitter-client 9 | ``` 10 | 11 | ## Setup 12 | 13 | Configure environment variables for authentication. 14 | 15 | ``` 16 | TWITTER_USERNAME= # Account username 17 | TWITTER_PASSWORD= # Account password 18 | TWITTER_EMAIL= # Account email 19 | PROXY_URL= # HTTP(s) proxy for requests (necessary for browsers) 20 | 21 | # Twitter API v2 credentials for tweet and poll functionality 22 | TWITTER_API_KEY= # Twitter API Key 23 | TWITTER_API_SECRET_KEY= # Twitter API Secret Key 24 | TWITTER_ACCESS_TOKEN= # Access Token for Twitter API v2 25 | TWITTER_ACCESS_TOKEN_SECRET= # Access Token Secret for Twitter API v2 26 | ``` 27 | 28 | ### Getting Twitter Cookies 29 | 30 | It is important to use Twitter cookies to avoid sending a new login request to Twitter every time you want to perform an action. 31 | 32 | In your application, you will likely want to check for existing cookies. If cookies are not available, log in with user authentication credentials and cache the cookies for future use. 33 | 34 | ```ts 35 | const scraper = await getScraper({ authMethod: 'password' }); 36 | 37 | scraper.getCookies().then((cookies) => { 38 | console.log(cookies); 39 | // Remove 'Cookies' and save the cookies as a JSON array 40 | }); 41 | ``` 42 | 43 | ## Getting Started 44 | 45 | ```ts 46 | const scraper = new Scraper(); 47 | await scraper.login('username', 'password'); 48 | 49 | // If using v2 functionality (currently required to support polls) 50 | await scraper.login( 51 | 'username', 52 | 'password', 53 | 'email', 54 | 'appKey', 55 | 'appSecret', 56 | 'accessToken', 57 | 'accessSecret', 58 | ); 59 | 60 | const tweets = await scraper.getTweets('elonmusk', 10); 61 | const tweetsAndReplies = scraper.getTweetsAndReplies('elonmusk'); 62 | const latestTweet = await scraper.getLatestTweet('elonmusk'); 63 | const tweet = await scraper.getTweet('1234567890123456789'); 64 | await scraper.sendTweet('Hello world!'); 65 | 66 | // Create a poll 67 | await scraper.sendTweetV2( 68 | `What's got you most hyped? Let us know! 🤖💸`, 69 | undefined, 70 | { 71 | poll: { 72 | options: [ 73 | { label: 'AI Innovations 🤖' }, 74 | { label: 'Crypto Craze 💸' }, 75 | { label: 'Both! 🌌' }, 76 | { label: 'Neither for Me 😅' }, 77 | ], 78 | durationMinutes: 120, // Duration of the poll in minutes 79 | }, 80 | }, 81 | ); 82 | ``` 83 | 84 | ### Fetching Specific Tweet Data (V2) 85 | 86 | ```ts 87 | // Fetch a single tweet with poll details 88 | const tweet = await scraper.getTweetV2('1856441982811529619', { 89 | expansions: ['attachments.poll_ids'], 90 | pollFields: ['options', 'end_datetime'], 91 | }); 92 | console.log('tweet', tweet); 93 | 94 | // Fetch multiple tweets with poll and media details 95 | const tweets = await scraper.getTweetsV2( 96 | ['1856441982811529619', '1856429655215260130'], 97 | { 98 | expansions: ['attachments.poll_ids', 'attachments.media_keys'], 99 | pollFields: ['options', 'end_datetime'], 100 | mediaFields: ['url', 'preview_image_url'], 101 | }, 102 | ); 103 | console.log('tweets', tweets); 104 | ``` 105 | 106 | ## API 107 | 108 | ### Authentication 109 | 110 | ```ts 111 | // Log in 112 | await scraper.login('username', 'password'); 113 | 114 | // Log out 115 | await scraper.logout(); 116 | 117 | // Check if logged in 118 | const isLoggedIn = await scraper.isLoggedIn(); 119 | 120 | // Get current session cookies 121 | const cookies = await scraper.getCookies(); 122 | 123 | // Set current session cookies 124 | await scraper.setCookies(cookies); 125 | 126 | // Clear current cookies 127 | await scraper.clearCookies(); 128 | ``` 129 | 130 | ### Profile 131 | 132 | ```ts 133 | // Get a user's profile 134 | const profile = await scraper.getProfile('TwitterDev'); 135 | 136 | // Get a user ID from their screen name 137 | const userId = await scraper.getUserIdByScreenName('TwitterDev'); 138 | 139 | // Get logged-in user's profile 140 | const me = await scraper.me(); 141 | ``` 142 | 143 | ### Search 144 | 145 | ```ts 146 | import { SearchMode } from 'agent-twitter-client'; 147 | 148 | // Search for recent tweets 149 | const tweets = scraper.searchTweets('#nodejs', 20, SearchMode.Latest); 150 | 151 | // Search for profiles 152 | const profiles = scraper.searchProfiles('John', 10); 153 | 154 | // Fetch a page of tweet results 155 | const results = await scraper.fetchSearchTweets('#nodejs', 20, SearchMode.Top); 156 | 157 | // Fetch a page of profile results 158 | const profileResults = await scraper.fetchSearchProfiles('John', 10); 159 | ``` 160 | 161 | ### Relationships 162 | 163 | ```ts 164 | // Get a user's followers 165 | const followers = scraper.getFollowers('12345', 100); 166 | 167 | // Get who a user is following 168 | const following = scraper.getFollowing('12345', 100); 169 | 170 | // Fetch a page of a user's followers 171 | const followerResults = await scraper.fetchProfileFollowers('12345', 100); 172 | 173 | // Fetch a page of who a user is following 174 | const followingResults = await scraper.fetchProfileFollowing('12345', 100); 175 | 176 | // Follow a user 177 | const followUserResults = await scraper.followUser('elonmusk'); 178 | ``` 179 | 180 | ### Trends 181 | 182 | ```ts 183 | // Get current trends 184 | const trends = await scraper.getTrends(); 185 | 186 | // Fetch tweets from a list 187 | const listTweets = await scraper.fetchListTweets('1234567890', 50); 188 | ``` 189 | 190 | ### Tweets 191 | 192 | ```ts 193 | // Get a user's tweets 194 | const tweets = scraper.getTweets('TwitterDev'); 195 | 196 | // Get a user's liked tweets 197 | const likedTweets = scraper.getLikedTweets('TwitterDev'); 198 | 199 | // Get a user's tweets and replies 200 | const tweetsAndReplies = scraper.getTweetsAndReplies('TwitterDev'); 201 | 202 | // Get tweets matching specific criteria 203 | const timeline = scraper.getTweets('TwitterDev', 100); 204 | const retweets = await scraper.getTweetsWhere( 205 | timeline, 206 | (tweet) => tweet.isRetweet, 207 | ); 208 | 209 | // Get a user's latest tweet 210 | const latestTweet = await scraper.getLatestTweet('TwitterDev'); 211 | 212 | // Get a specific tweet by ID 213 | const tweet = await scraper.getTweet('1234567890123456789'); 214 | 215 | // Send a tweet 216 | const sendTweetResults = await scraper.sendTweet('Hello world!'); 217 | 218 | // Send a quote tweet - Media files are optional 219 | const sendQuoteTweetResults = await scraper.sendQuoteTweet('Hello world!', '1234567890123456789', ['mediaFile1', 'mediaFile2']); 220 | 221 | // Retweet a tweet 222 | const retweetResults = await scraper.retweet('1234567890123456789'); 223 | 224 | // Like a tweet 225 | const likeTweetResults = await scraper.likeTweet('1234567890123456789'); 226 | ``` 227 | 228 | ## Sending Tweets with Media 229 | 230 | ### Media Handling 231 | The scraper requires media files to be processed into a specific format before sending: 232 | - Media must be converted to Buffer format 233 | - Each media file needs its MIME type specified 234 | - This helps the scraper distinguish between image and video processing models 235 | 236 | ### Basic Tweet with Media 237 | ```ts 238 | // Example: Sending a tweet with media attachments 239 | const mediaData = [ 240 | { 241 | data: fs.readFileSync('path/to/image.jpg'), 242 | mediaType: 'image/jpeg' 243 | }, 244 | { 245 | data: fs.readFileSync('path/to/video.mp4'), 246 | mediaType: 'video/mp4' 247 | } 248 | ]; 249 | 250 | await scraper.sendTweet('Hello world!', undefined, mediaData); 251 | ``` 252 | 253 | ### Supported Media Types 254 | ```ts 255 | // Image formats and their MIME types 256 | const imageTypes = { 257 | '.jpg': 'image/jpeg', 258 | '.jpeg': 'image/jpeg', 259 | '.png': 'image/png', 260 | '.gif': 'image/gif' 261 | }; 262 | 263 | // Video format 264 | const videoTypes = { 265 | '.mp4': 'video/mp4' 266 | }; 267 | ``` 268 | 269 | 270 | ### Media Upload Limitations 271 | - Maximum 4 images per tweet 272 | - Only 1 video per tweet 273 | - Maximum video file size: 512MB 274 | - Supported image formats: JPG, PNG, GIF 275 | - Supported video format: MP4 276 | -------------------------------------------------------------------------------- /SampleAgent.js: -------------------------------------------------------------------------------- 1 | import { Scraper } from 'agent-twitter-client'; 2 | import dotenv from 'dotenv'; 3 | dotenv.config(); 4 | 5 | async function main() { 6 | // const scraper = new Scraper(); 7 | // // v1 login 8 | // await scraper.login( 9 | // process.env.TWITTER_USERNAME, 10 | // process.env.TWITTER_PASSWORD, 11 | // ); 12 | // // v2 login 13 | // await scraper.login( 14 | // process.env.TWITTER_USERNAME, 15 | // process.env.TWITTER_PASSWORD, 16 | // undefined, 17 | // undefined, 18 | // process.env.TWITTER_API_KEY, 19 | // process.env.TWITTER_API_SECRET_KEY, 20 | // process.env.TWITTER_ACCESS_TOKEN, 21 | // process.env.TWITTER_ACCESS_TOKEN_SECRET, 22 | // ); 23 | // console.log('Logged in successfully!'); 24 | // // Example: Posting a new tweet with a poll 25 | // await scraper.sendTweetV2( 26 | // `When do you think we'll achieve AGI (Artificial General Intelligence)? 🤖 Cast your prediction!`, 27 | // undefined, 28 | // { 29 | // poll: { 30 | // options: [ 31 | // { label: '2025 🗓️' }, 32 | // { label: '2026 📅' }, 33 | // { label: '2027 🛠️' }, 34 | // { label: '2030+ 🚀' }, 35 | // ], 36 | // durationMinutes: 1440, 37 | // }, 38 | // }, 39 | // ); 40 | // console.log(await scraper.getTweet('1856441982811529619')); 41 | // const tweet = await scraper.getTweetV2('1856441982811529619'); 42 | // console.log({ tweet }); 43 | // console.log('tweet', tweet); 44 | // const tweets = await scraper.getTweetsV2([ 45 | // '1856441982811529619', 46 | // '1856429655215260130', 47 | // ]); 48 | // console.log('tweets', tweets); 49 | } 50 | 51 | main(); 52 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | module.exports = { 3 | preset: 'ts-jest', 4 | testEnvironment: 'node', 5 | setupFiles: ['dotenv/config', './test-setup.js'], 6 | }; 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agent-twitter-client", 3 | "description": "A twitter client for agents", 4 | "keywords": [], 5 | "version": "0.0.18", 6 | "main": "dist/default/cjs/index.js", 7 | "types": "./dist/types/index.d.ts", 8 | "exports": { 9 | "types": "./dist/types/index.d.ts", 10 | "node": { 11 | "import": "./dist/node/esm/index.mjs", 12 | "require": "./dist/node/cjs/index.cjs" 13 | }, 14 | "default": { 15 | "import": "./dist/default/esm/index.mjs", 16 | "require": "./dist/default/cjs/index.js" 17 | } 18 | }, 19 | "author": "elizaOS", 20 | "license": "MIT", 21 | "scripts": { 22 | "build": "rimraf dist && rollup -c", 23 | "docs:generate": "typedoc --options typedoc.json", 24 | "docs:deploy": "npm run docs:generate && gh-pages -d docs", 25 | "format": "prettier --write src/**/*.ts", 26 | "test": "jest" 27 | }, 28 | "dependencies": { 29 | "@roamhq/wrtc": "^0.8.0", 30 | "@sinclair/typebox": "^0.32.20", 31 | "headers-polyfill": "^3.1.2", 32 | "json-stable-stringify": "^1.0.2", 33 | "otpauth": "^9.2.2", 34 | "set-cookie-parser": "^2.6.0", 35 | "tough-cookie": "^4.1.2", 36 | "twitter-api-v2": "^1.18.2", 37 | "undici": "^7.1.1", 38 | "undici-types": "^7.2.0", 39 | "ws": "^8.18.0" 40 | }, 41 | "devDependencies": { 42 | "@commitlint/cli": "^17.6.3", 43 | "@commitlint/config-conventional": "^17.6.3", 44 | "@tsconfig/node16": "^16.1.3", 45 | "@types/jest": "^29.5.1", 46 | "@types/json-stable-stringify": "^1.0.34", 47 | "@types/node": "^22.10.2", 48 | "@types/set-cookie-parser": "^2.4.2", 49 | "@types/tough-cookie": "^4.0.2", 50 | "@types/ws": "^8.5.13", 51 | "@typescript-eslint/eslint-plugin": "^5.59.7", 52 | "@typescript-eslint/parser": "^5.59.7", 53 | "dotenv": "^16.4.5", 54 | "esbuild": "^0.21.5", 55 | "eslint": "^8.41.0", 56 | "eslint-config-prettier": "^8.8.0", 57 | "eslint-plugin-prettier": "^4.2.1", 58 | "gh-pages": "^5.0.0", 59 | "jest": "^29.7.0", 60 | "lint-staged": "^13.2.2", 61 | "prettier": "^2.8.8", 62 | "rimraf": "^5.0.7", 63 | "rollup": "^4.18.0", 64 | "rollup-plugin-dts": "^6.1.1", 65 | "rollup-plugin-esbuild": "^6.1.1", 66 | "ts-jest": "^29.1.0", 67 | "typedoc": "^0.24.7", 68 | "typescript": "^5.0.4" 69 | }, 70 | "lint-staged": { 71 | "*.{js,ts}": [ 72 | "eslint --cache --fix", 73 | "prettier --write" 74 | ] 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /rollup.config.mjs: -------------------------------------------------------------------------------- 1 | import dts from 'rollup-plugin-dts'; 2 | import esbuild from 'rollup-plugin-esbuild'; 3 | 4 | export default [ 5 | { 6 | input: 'src/_module.ts', 7 | plugins: [ 8 | esbuild({ 9 | define: { 10 | PLATFORM_NODE: 'false', 11 | PLATFORM_NODE_JEST: 'false', 12 | }, 13 | }), 14 | ], 15 | output: [ 16 | { 17 | file: 'dist/default/cjs/index.js', 18 | format: 'cjs', 19 | sourcemap: true, 20 | }, 21 | { 22 | file: 'dist/default/esm/index.mjs', 23 | format: 'es', 24 | sourcemap: true, 25 | }, 26 | ], 27 | }, 28 | { 29 | input: 'src/_module.ts', 30 | plugins: [ 31 | esbuild({ 32 | define: { 33 | PLATFORM_NODE: 'true', 34 | PLATFORM_NODE_JEST: 'false', 35 | }, 36 | }), 37 | ], 38 | output: [ 39 | { 40 | file: 'dist/node/cjs/index.cjs', 41 | format: 'cjs', 42 | sourcemap: true, 43 | inlineDynamicImports: true, 44 | }, 45 | { 46 | file: 'dist/node/esm/index.mjs', 47 | format: 'es', 48 | sourcemap: true, 49 | inlineDynamicImports: true, 50 | }, 51 | ], 52 | }, 53 | { 54 | input: 'src/_module.ts', 55 | plugins: [dts()], 56 | output: { 57 | file: 'dist/types/index.d.ts', 58 | format: 'es', 59 | }, 60 | }, 61 | ]; 62 | -------------------------------------------------------------------------------- /src/_module.ts: -------------------------------------------------------------------------------- 1 | export type { Profile } from './profile'; 2 | export { Scraper } from './scraper'; 3 | export { SearchMode } from './search'; 4 | export type { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1'; 5 | export type { Tweet } from './tweets'; 6 | 7 | export { Space } from './spaces/core/Space' 8 | export { SttTtsPlugin } from './spaces/plugins/SttTtsPlugin' 9 | export { RecordToDiskPlugin } from './spaces/plugins/RecordToDiskPlugin' 10 | export { MonitorAudioPlugin } from './spaces/plugins/MonitorAudioPlugin' 11 | export { IdleMonitorPlugin } from './spaces/plugins/IdleMonitorPlugin' 12 | export { HlsRecordPlugin } from './spaces/plugins/HlsRecordPlugin' 13 | 14 | export * from './types/spaces' 15 | -------------------------------------------------------------------------------- /src/api.ts: -------------------------------------------------------------------------------- 1 | import { TwitterAuth } from './auth'; 2 | import { ApiError } from './errors'; 3 | import { Platform, PlatformExtensions } from './platform'; 4 | import { updateCookieJar } from './requests'; 5 | import { Headers } from 'headers-polyfill'; 6 | 7 | // For some reason using Parameters reduces the request transform function to 8 | // `(url: string) => string` in tests. 9 | type FetchParameters = [input: RequestInfo | URL, init?: RequestInit]; 10 | 11 | export interface FetchTransformOptions { 12 | /** 13 | * Transforms the request options before a request is made. This executes after all of the default 14 | * parameters have been configured, and is stateless. It is safe to return new request options 15 | * objects. 16 | * @param args The request options. 17 | * @returns The transformed request options. 18 | */ 19 | request: ( 20 | ...args: FetchParameters 21 | ) => FetchParameters | Promise; 22 | 23 | /** 24 | * Transforms the response after a request completes. This executes immediately after the request 25 | * completes, and is stateless. It is safe to return a new response object. 26 | * @param response The response object. 27 | * @returns The transformed response object. 28 | */ 29 | response: (response: Response) => Response | Promise; 30 | } 31 | 32 | export const bearerToken = 33 | 'AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF'; 34 | 35 | /** 36 | * An API result container. 37 | */ 38 | export type RequestApiResult = 39 | | { success: true; value: T } 40 | | { success: false; err: Error }; 41 | 42 | /** 43 | * Used internally to send HTTP requests to the Twitter API. 44 | * @internal 45 | * @param url - The URL to send the request to. 46 | * @param auth - The instance of {@link TwitterAuth} that will be used to authorize this request. 47 | * @param method - The HTTP method used when sending this request. 48 | */ 49 | export async function requestApi( 50 | url: string, 51 | auth: TwitterAuth, 52 | method: 'GET' | 'POST' = 'GET', 53 | platform: PlatformExtensions = new Platform(), 54 | ): Promise> { 55 | const headers = new Headers(); 56 | await auth.installTo(headers, url); 57 | await platform.randomizeCiphers(); 58 | 59 | let res: Response; 60 | do { 61 | try { 62 | res = await auth.fetch(url, { 63 | method, 64 | headers, 65 | credentials: 'include', 66 | }); 67 | } catch (err) { 68 | if (!(err instanceof Error)) { 69 | throw err; 70 | } 71 | 72 | return { 73 | success: false, 74 | err: new Error('Failed to perform request.'), 75 | }; 76 | } 77 | 78 | await updateCookieJar(auth.cookieJar(), res.headers); 79 | 80 | if (res.status === 429) { 81 | /* 82 | Known headers at this point: 83 | - x-rate-limit-limit: Maximum number of requests per time period? 84 | - x-rate-limit-reset: UNIX timestamp when the current rate limit will be reset. 85 | - x-rate-limit-remaining: Number of requests remaining in current time period? 86 | */ 87 | const xRateLimitRemaining = res.headers.get('x-rate-limit-remaining'); 88 | const xRateLimitReset = res.headers.get('x-rate-limit-reset'); 89 | if (xRateLimitRemaining == '0' && xRateLimitReset) { 90 | const currentTime = new Date().valueOf() / 1000; 91 | const timeDeltaMs = 1000 * (parseInt(xRateLimitReset) - currentTime); 92 | 93 | // I have seen this block for 800s (~13 *minutes*) 94 | await new Promise((resolve) => setTimeout(resolve, timeDeltaMs)); 95 | } 96 | } 97 | } while (res.status === 429); 98 | 99 | if (!res.ok) { 100 | return { 101 | success: false, 102 | err: await ApiError.fromResponse(res), 103 | }; 104 | } 105 | 106 | const value: T = await res.json(); 107 | if (res.headers.get('x-rate-limit-incoming') == '0') { 108 | auth.deleteToken(); 109 | return { success: true, value }; 110 | } else { 111 | return { success: true, value }; 112 | } 113 | } 114 | 115 | /** @internal */ 116 | export function addApiFeatures(o: object) { 117 | return { 118 | ...o, 119 | rweb_lists_timeline_redesign_enabled: true, 120 | responsive_web_graphql_exclude_directive_enabled: true, 121 | verified_phone_label_enabled: false, 122 | creator_subscriptions_tweet_preview_api_enabled: true, 123 | responsive_web_graphql_timeline_navigation_enabled: true, 124 | responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, 125 | tweetypie_unmention_optimization_enabled: true, 126 | responsive_web_edit_tweet_api_enabled: true, 127 | graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, 128 | view_counts_everywhere_api_enabled: true, 129 | longform_notetweets_consumption_enabled: true, 130 | tweet_awards_web_tipping_enabled: false, 131 | freedom_of_speech_not_reach_fetch_enabled: true, 132 | standardized_nudges_misinfo: true, 133 | longform_notetweets_rich_text_read_enabled: true, 134 | responsive_web_enhance_cards_enabled: false, 135 | subscriptions_verification_info_enabled: true, 136 | subscriptions_verification_info_reason_enabled: true, 137 | subscriptions_verification_info_verified_since_enabled: true, 138 | super_follow_badge_privacy_enabled: false, 139 | super_follow_exclusive_tweet_notifications_enabled: false, 140 | super_follow_tweet_api_enabled: false, 141 | super_follow_user_api_enabled: false, 142 | android_graphql_skip_api_media_color_palette: false, 143 | creator_subscriptions_subscription_count_enabled: false, 144 | blue_business_profile_image_shape_enabled: false, 145 | unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: 146 | false, 147 | }; 148 | } 149 | 150 | export function addApiParams( 151 | params: URLSearchParams, 152 | includeTweetReplies: boolean, 153 | ): URLSearchParams { 154 | params.set('include_profile_interstitial_type', '1'); 155 | params.set('include_blocking', '1'); 156 | params.set('include_blocked_by', '1'); 157 | params.set('include_followed_by', '1'); 158 | params.set('include_want_retweets', '1'); 159 | params.set('include_mute_edge', '1'); 160 | params.set('include_can_dm', '1'); 161 | params.set('include_can_media_tag', '1'); 162 | params.set('include_ext_has_nft_avatar', '1'); 163 | params.set('include_ext_is_blue_verified', '1'); 164 | params.set('include_ext_verified_type', '1'); 165 | params.set('skip_status', '1'); 166 | params.set('cards_platform', 'Web-12'); 167 | params.set('include_cards', '1'); 168 | params.set('include_ext_alt_text', 'true'); 169 | params.set('include_ext_limited_action_results', 'false'); 170 | params.set('include_quote_count', 'true'); 171 | params.set('include_reply_count', '1'); 172 | params.set('tweet_mode', 'extended'); 173 | params.set('include_ext_collab_control', 'true'); 174 | params.set('include_ext_views', 'true'); 175 | params.set('include_entities', 'true'); 176 | params.set('include_user_entities', 'true'); 177 | params.set('include_ext_media_color', 'true'); 178 | params.set('include_ext_media_availability', 'true'); 179 | params.set('include_ext_sensitive_media_warning', 'true'); 180 | params.set('include_ext_trusted_friends_metadata', 'true'); 181 | params.set('send_error_codes', 'true'); 182 | params.set('simple_quoted_tweet', 'true'); 183 | params.set('include_tweet_replies', `${includeTweetReplies}`); 184 | params.set( 185 | 'ext', 186 | 'mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe', 187 | ); 188 | return params; 189 | } 190 | -------------------------------------------------------------------------------- /src/auth-user.ts: -------------------------------------------------------------------------------- 1 | import { TwitterAuthOptions, TwitterGuestAuth } from './auth'; 2 | import { requestApi } from './api'; 3 | import { CookieJar } from 'tough-cookie'; 4 | import { updateCookieJar } from './requests'; 5 | import { Headers } from 'headers-polyfill'; 6 | import { TwitterApiErrorRaw } from './errors'; 7 | import { Type, type Static } from '@sinclair/typebox'; 8 | import { Check } from '@sinclair/typebox/value'; 9 | import * as OTPAuth from 'otpauth'; 10 | import { LegacyUserRaw, parseProfile, type Profile } from './profile'; 11 | 12 | interface TwitterUserAuthFlowInitRequest { 13 | flow_name: string; 14 | input_flow_data: Record; 15 | } 16 | 17 | interface TwitterUserAuthFlowSubtaskRequest { 18 | flow_token: string; 19 | subtask_inputs: ({ 20 | subtask_id: string; 21 | } & Record)[]; 22 | } 23 | 24 | type TwitterUserAuthFlowRequest = 25 | | TwitterUserAuthFlowInitRequest 26 | | TwitterUserAuthFlowSubtaskRequest; 27 | 28 | interface TwitterUserAuthFlowResponse { 29 | errors?: TwitterApiErrorRaw[]; 30 | flow_token?: string; 31 | status?: string; 32 | subtasks?: TwitterUserAuthSubtask[]; 33 | } 34 | 35 | interface TwitterUserAuthVerifyCredentials { 36 | errors?: TwitterApiErrorRaw[]; 37 | } 38 | 39 | const TwitterUserAuthSubtask = Type.Object({ 40 | subtask_id: Type.String(), 41 | enter_text: Type.Optional(Type.Object({})), 42 | }); 43 | type TwitterUserAuthSubtask = Static; 44 | 45 | type FlowTokenResultSuccess = { 46 | status: 'success'; 47 | flowToken: string; 48 | subtask?: TwitterUserAuthSubtask; 49 | }; 50 | 51 | type FlowTokenResult = FlowTokenResultSuccess | { status: 'error'; err: Error }; 52 | 53 | /** 54 | * A user authentication token manager. 55 | */ 56 | export class TwitterUserAuth extends TwitterGuestAuth { 57 | private userProfile: Profile | undefined; 58 | 59 | constructor(bearerToken: string, options?: Partial) { 60 | super(bearerToken, options); 61 | } 62 | 63 | async isLoggedIn(): Promise { 64 | const res = await requestApi( 65 | 'https://api.twitter.com/1.1/account/verify_credentials.json', 66 | this, 67 | ); 68 | if (!res.success) { 69 | return false; 70 | } 71 | 72 | const { value: verify } = res; 73 | this.userProfile = parseProfile( 74 | verify as LegacyUserRaw, 75 | (verify as unknown as { verified: boolean }).verified, 76 | ); 77 | return verify && !verify.errors?.length; 78 | } 79 | 80 | async me(): Promise { 81 | if (this.userProfile) { 82 | return this.userProfile; 83 | } 84 | await this.isLoggedIn(); 85 | return this.userProfile; 86 | } 87 | 88 | async login( 89 | username: string, 90 | password: string, 91 | email?: string, 92 | twoFactorSecret?: string, 93 | appKey?: string, 94 | appSecret?: string, 95 | accessToken?: string, 96 | accessSecret?: string, 97 | ): Promise { 98 | await this.updateGuestToken(); 99 | 100 | let next = await this.initLogin(); 101 | while ('subtask' in next && next.subtask) { 102 | if (next.subtask.subtask_id === 'LoginJsInstrumentationSubtask') { 103 | next = await this.handleJsInstrumentationSubtask(next); 104 | } else if (next.subtask.subtask_id === 'LoginEnterUserIdentifierSSO') { 105 | next = await this.handleEnterUserIdentifierSSO(next, username); 106 | } else if ( 107 | next.subtask.subtask_id === 'LoginEnterAlternateIdentifierSubtask' 108 | ) { 109 | next = await this.handleEnterAlternateIdentifierSubtask( 110 | next, 111 | email as string, 112 | ); 113 | } else if (next.subtask.subtask_id === 'LoginEnterPassword') { 114 | next = await this.handleEnterPassword(next, password); 115 | } else if (next.subtask.subtask_id === 'AccountDuplicationCheck') { 116 | next = await this.handleAccountDuplicationCheck(next); 117 | } else if (next.subtask.subtask_id === 'LoginTwoFactorAuthChallenge') { 118 | if (twoFactorSecret) { 119 | next = await this.handleTwoFactorAuthChallenge(next, twoFactorSecret); 120 | } else { 121 | throw new Error( 122 | 'Requested two factor authentication code but no secret provided', 123 | ); 124 | } 125 | } else if (next.subtask.subtask_id === 'LoginAcid') { 126 | next = await this.handleAcid(next, email); 127 | } else if (next.subtask.subtask_id === 'LoginSuccessSubtask') { 128 | next = await this.handleSuccessSubtask(next); 129 | } else { 130 | throw new Error(`Unknown subtask ${next.subtask.subtask_id}`); 131 | } 132 | } 133 | if (appKey && appSecret && accessToken && accessSecret) { 134 | this.loginWithV2(appKey, appSecret, accessToken, accessSecret); 135 | } 136 | if ('err' in next) { 137 | throw next.err; 138 | } 139 | } 140 | 141 | async logout(): Promise { 142 | if (!this.isLoggedIn()) { 143 | return; 144 | } 145 | 146 | await requestApi( 147 | 'https://api.twitter.com/1.1/account/logout.json', 148 | this, 149 | 'POST', 150 | ); 151 | this.deleteToken(); 152 | this.jar = new CookieJar(); 153 | } 154 | 155 | async installCsrfToken(headers: Headers): Promise { 156 | const cookies = await this.getCookies(); 157 | const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); 158 | if (xCsrfToken) { 159 | headers.set('x-csrf-token', xCsrfToken.value); 160 | } 161 | } 162 | 163 | async installTo(headers: Headers): Promise { 164 | headers.set('authorization', `Bearer ${this.bearerToken}`); 165 | headers.set('cookie', await this.getCookieString()); 166 | await this.installCsrfToken(headers); 167 | } 168 | 169 | private async initLogin() { 170 | // Reset certain session-related cookies because Twitter complains sometimes if we don't 171 | this.removeCookie('twitter_ads_id='); 172 | this.removeCookie('ads_prefs='); 173 | this.removeCookie('_twitter_sess='); 174 | this.removeCookie('zipbox_forms_auth_token='); 175 | this.removeCookie('lang='); 176 | this.removeCookie('bouncer_reset_cookie='); 177 | this.removeCookie('twid='); 178 | this.removeCookie('twitter_ads_idb='); 179 | this.removeCookie('email_uid='); 180 | this.removeCookie('external_referer='); 181 | this.removeCookie('ct0='); 182 | this.removeCookie('aa_u='); 183 | 184 | return await this.executeFlowTask({ 185 | flow_name: 'login', 186 | input_flow_data: { 187 | flow_context: { 188 | debug_overrides: {}, 189 | start_location: { 190 | location: 'splash_screen', 191 | }, 192 | }, 193 | }, 194 | }); 195 | } 196 | 197 | private async handleJsInstrumentationSubtask(prev: FlowTokenResultSuccess) { 198 | return await this.executeFlowTask({ 199 | flow_token: prev.flowToken, 200 | subtask_inputs: [ 201 | { 202 | subtask_id: 'LoginJsInstrumentationSubtask', 203 | js_instrumentation: { 204 | response: '{}', 205 | link: 'next_link', 206 | }, 207 | }, 208 | ], 209 | }); 210 | } 211 | 212 | private async handleEnterAlternateIdentifierSubtask( 213 | prev: FlowTokenResultSuccess, 214 | email: string, 215 | ) { 216 | return await this.executeFlowTask({ 217 | flow_token: prev.flowToken, 218 | subtask_inputs: [ 219 | { 220 | subtask_id: 'LoginEnterAlternateIdentifierSubtask', 221 | enter_text: { 222 | text: email, 223 | link: 'next_link', 224 | }, 225 | }, 226 | ], 227 | }); 228 | } 229 | 230 | private async handleEnterUserIdentifierSSO( 231 | prev: FlowTokenResultSuccess, 232 | username: string, 233 | ) { 234 | return await this.executeFlowTask({ 235 | flow_token: prev.flowToken, 236 | subtask_inputs: [ 237 | { 238 | subtask_id: 'LoginEnterUserIdentifierSSO', 239 | settings_list: { 240 | setting_responses: [ 241 | { 242 | key: 'user_identifier', 243 | response_data: { 244 | text_data: { result: username }, 245 | }, 246 | }, 247 | ], 248 | link: 'next_link', 249 | }, 250 | }, 251 | ], 252 | }); 253 | } 254 | 255 | private async handleEnterPassword( 256 | prev: FlowTokenResultSuccess, 257 | password: string, 258 | ) { 259 | return await this.executeFlowTask({ 260 | flow_token: prev.flowToken, 261 | subtask_inputs: [ 262 | { 263 | subtask_id: 'LoginEnterPassword', 264 | enter_password: { 265 | password, 266 | link: 'next_link', 267 | }, 268 | }, 269 | ], 270 | }); 271 | } 272 | 273 | private async handleAccountDuplicationCheck(prev: FlowTokenResultSuccess) { 274 | return await this.executeFlowTask({ 275 | flow_token: prev.flowToken, 276 | subtask_inputs: [ 277 | { 278 | subtask_id: 'AccountDuplicationCheck', 279 | check_logged_in_account: { 280 | link: 'AccountDuplicationCheck_false', 281 | }, 282 | }, 283 | ], 284 | }); 285 | } 286 | 287 | private async handleTwoFactorAuthChallenge( 288 | prev: FlowTokenResultSuccess, 289 | secret: string, 290 | ) { 291 | const totp = new OTPAuth.TOTP({ secret }); 292 | let error; 293 | for (let attempts = 1; attempts < 4; attempts += 1) { 294 | try { 295 | return await this.executeFlowTask({ 296 | flow_token: prev.flowToken, 297 | subtask_inputs: [ 298 | { 299 | subtask_id: 'LoginTwoFactorAuthChallenge', 300 | enter_text: { 301 | link: 'next_link', 302 | text: totp.generate(), 303 | }, 304 | }, 305 | ], 306 | }); 307 | } catch (err) { 308 | error = err; 309 | await new Promise((resolve) => setTimeout(resolve, 2000 * attempts)); 310 | } 311 | } 312 | throw error; 313 | } 314 | 315 | private async handleAcid( 316 | prev: FlowTokenResultSuccess, 317 | email: string | undefined, 318 | ) { 319 | return await this.executeFlowTask({ 320 | flow_token: prev.flowToken, 321 | subtask_inputs: [ 322 | { 323 | subtask_id: 'LoginAcid', 324 | enter_text: { 325 | text: email, 326 | link: 'next_link', 327 | }, 328 | }, 329 | ], 330 | }); 331 | } 332 | 333 | private async handleSuccessSubtask(prev: FlowTokenResultSuccess) { 334 | return await this.executeFlowTask({ 335 | flow_token: prev.flowToken, 336 | subtask_inputs: [], 337 | }); 338 | } 339 | 340 | private async executeFlowTask( 341 | data: TwitterUserAuthFlowRequest, 342 | ): Promise { 343 | const onboardingTaskUrl = 344 | 'https://api.twitter.com/1.1/onboarding/task.json'; 345 | 346 | const token = this.guestToken; 347 | if (token == null) { 348 | throw new Error('Authentication token is null or undefined.'); 349 | } 350 | 351 | const headers = new Headers({ 352 | authorization: `Bearer ${this.bearerToken}`, 353 | cookie: await this.getCookieString(), 354 | 'content-type': 'application/json', 355 | 'User-Agent': 356 | 'Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36', 357 | 'x-guest-token': token, 358 | 'x-twitter-auth-type': 'OAuth2Client', 359 | 'x-twitter-active-user': 'yes', 360 | 'x-twitter-client-language': 'en', 361 | }); 362 | await this.installCsrfToken(headers); 363 | 364 | const res = await this.fetch(onboardingTaskUrl, { 365 | credentials: 'include', 366 | method: 'POST', 367 | headers: headers, 368 | body: JSON.stringify(data), 369 | }); 370 | 371 | await updateCookieJar(this.jar, res.headers); 372 | 373 | if (!res.ok) { 374 | return { status: 'error', err: new Error(await res.text()) }; 375 | } 376 | 377 | const flow: TwitterUserAuthFlowResponse = await res.json(); 378 | if (flow?.flow_token == null) { 379 | return { status: 'error', err: new Error('flow_token not found.') }; 380 | } 381 | 382 | if (flow.errors?.length) { 383 | return { 384 | status: 'error', 385 | err: new Error( 386 | `Authentication error (${flow.errors[0].code}): ${flow.errors[0].message}`, 387 | ), 388 | }; 389 | } 390 | 391 | if (typeof flow.flow_token !== 'string') { 392 | return { 393 | status: 'error', 394 | err: new Error('flow_token was not a string.'), 395 | }; 396 | } 397 | 398 | const subtask = flow.subtasks?.length ? flow.subtasks[0] : undefined; 399 | Check(TwitterUserAuthSubtask, subtask); 400 | 401 | if (subtask && subtask.subtask_id === 'DenyLoginSubtask') { 402 | return { 403 | status: 'error', 404 | err: new Error('Authentication error: DenyLoginSubtask'), 405 | }; 406 | } 407 | 408 | return { 409 | status: 'success', 410 | subtask, 411 | flowToken: flow.flow_token, 412 | }; 413 | } 414 | } 415 | -------------------------------------------------------------------------------- /src/auth.test.ts: -------------------------------------------------------------------------------- 1 | import { getScraper } from './test-utils'; 2 | 3 | const testLogin = process.env['TWITTER_PASSWORD'] ? test : test.skip; 4 | 5 | testLogin( 6 | 'scraper can log in', 7 | async () => { 8 | const scraper = await getScraper({ authMethod: 'password' }); 9 | await expect(scraper.isLoggedIn()).resolves.toBeTruthy(); 10 | }, 11 | 15000, 12 | ); 13 | 14 | test('scraper can log in with cookies', async () => { 15 | const scraper = await getScraper(); 16 | await expect(scraper.isLoggedIn()).resolves.toBeTruthy(); 17 | }); 18 | 19 | test('scraper can restore its login state from cookies', async () => { 20 | const scraper = await getScraper(); 21 | await expect(scraper.isLoggedIn()).resolves.toBeTruthy(); 22 | const scraper2 = await getScraper({ authMethod: 'anonymous' }); 23 | await expect(scraper2.isLoggedIn()).resolves.toBeFalsy(); 24 | 25 | const cookies = await scraper.getCookies(); 26 | await scraper2.setCookies(cookies); 27 | 28 | await expect(scraper2.isLoggedIn()).resolves.toBeTruthy(); 29 | }); 30 | 31 | testLogin( 32 | 'scraper can log out', 33 | async () => { 34 | const scraper = await getScraper({ authMethod: 'password' }); 35 | await expect(scraper.isLoggedIn()).resolves.toBeTruthy(); 36 | 37 | await scraper.logout(); 38 | 39 | await expect(scraper.isLoggedIn()).resolves.toBeFalsy(); 40 | }, 41 | 15000, 42 | ); 43 | -------------------------------------------------------------------------------- /src/auth.ts: -------------------------------------------------------------------------------- 1 | import { Cookie, CookieJar, MemoryCookieStore } from 'tough-cookie'; 2 | import { updateCookieJar } from './requests'; 3 | import { Headers } from 'headers-polyfill'; 4 | import { FetchTransformOptions } from './api'; 5 | import { TwitterApi } from 'twitter-api-v2'; 6 | import { Profile } from './profile'; 7 | 8 | export interface TwitterAuthOptions { 9 | fetch: typeof fetch; 10 | transform: Partial; 11 | } 12 | 13 | export interface TwitterAuth { 14 | fetch: typeof fetch; 15 | 16 | /** 17 | * Returns the current cookie jar. 18 | */ 19 | cookieJar(): CookieJar; 20 | 21 | /** 22 | * Logs into a Twitter account using the v2 API 23 | */ 24 | loginWithV2( 25 | appKey: string, 26 | appSecret: string, 27 | accessToken: string, 28 | accessSecret: string, 29 | ): void; 30 | 31 | /** 32 | * Get v2 API client if it exists 33 | */ 34 | getV2Client(): TwitterApi | null; 35 | 36 | /** 37 | * Returns if a user is logged-in to Twitter through this instance. 38 | * @returns `true` if a user is logged-in; otherwise `false`. 39 | */ 40 | isLoggedIn(): Promise; 41 | 42 | /** 43 | * Fetches the current user's profile. 44 | */ 45 | me(): Promise; 46 | 47 | /** 48 | * Logs into a Twitter account. 49 | * @param username The username to log in with. 50 | * @param password The password to log in with. 51 | * @param email The email to log in with, if you have email confirmation enabled. 52 | * @param twoFactorSecret The secret to generate two factor authentication tokens with, if you have two factor authentication enabled. 53 | */ 54 | login( 55 | username: string, 56 | password: string, 57 | email?: string, 58 | twoFactorSecret?: string, 59 | ): Promise; 60 | 61 | /** 62 | * Logs out of the current session. 63 | */ 64 | logout(): Promise; 65 | 66 | /** 67 | * Deletes the current guest token token. 68 | */ 69 | deleteToken(): void; 70 | 71 | /** 72 | * Returns if the authentication state has a token. 73 | * @returns `true` if the authentication state has a token; `false` otherwise. 74 | */ 75 | hasToken(): boolean; 76 | 77 | /** 78 | * Returns the time that authentication was performed. 79 | * @returns The time at which the authentication token was created, or `null` if it hasn't been created yet. 80 | */ 81 | authenticatedAt(): Date | null; 82 | 83 | /** 84 | * Installs the authentication information into a headers-like object. If needed, the 85 | * authentication token will be updated from the API automatically. 86 | * @param headers A Headers instance representing a request's headers. 87 | */ 88 | installTo(headers: Headers, url: string): Promise; 89 | } 90 | 91 | /** 92 | * Wraps the provided fetch function with transforms. 93 | * @param fetchFn The fetch function. 94 | * @param transform The transform options. 95 | * @returns The input fetch function, wrapped with the provided transforms. 96 | */ 97 | function withTransform( 98 | fetchFn: typeof fetch, 99 | transform?: Partial, 100 | ): typeof fetch { 101 | return async (input, init) => { 102 | const fetchArgs = (await transform?.request?.(input, init)) ?? [ 103 | input, 104 | init, 105 | ]; 106 | const res = await fetchFn(...fetchArgs); 107 | return (await transform?.response?.(res)) ?? res; 108 | }; 109 | } 110 | 111 | /** 112 | * A guest authentication token manager. Automatically handles token refreshes. 113 | */ 114 | export class TwitterGuestAuth implements TwitterAuth { 115 | protected bearerToken: string; 116 | protected jar: CookieJar; 117 | protected guestToken?: string; 118 | protected guestCreatedAt?: Date; 119 | protected v2Client: TwitterApi | null; 120 | 121 | fetch: typeof fetch; 122 | 123 | constructor( 124 | bearerToken: string, 125 | protected readonly options?: Partial, 126 | ) { 127 | this.fetch = withTransform(options?.fetch ?? fetch, options?.transform); 128 | this.bearerToken = bearerToken; 129 | this.jar = new CookieJar(); 130 | this.v2Client = null; 131 | } 132 | 133 | cookieJar(): CookieJar { 134 | return this.jar; 135 | } 136 | 137 | getV2Client(): TwitterApi | null { 138 | return this.v2Client ?? null; 139 | } 140 | 141 | loginWithV2( 142 | appKey: string, 143 | appSecret: string, 144 | accessToken: string, 145 | accessSecret: string, 146 | ): void { 147 | const v2Client = new TwitterApi({ 148 | appKey, 149 | appSecret, 150 | accessToken, 151 | accessSecret, 152 | }); 153 | this.v2Client = v2Client; 154 | } 155 | 156 | isLoggedIn(): Promise { 157 | return Promise.resolve(false); 158 | } 159 | 160 | async me(): Promise { 161 | return undefined; 162 | } 163 | 164 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 165 | login(_username: string, _password: string, _email?: string): Promise { 166 | return this.updateGuestToken(); 167 | } 168 | 169 | logout(): Promise { 170 | this.deleteToken(); 171 | this.jar = new CookieJar(); 172 | return Promise.resolve(); 173 | } 174 | 175 | deleteToken() { 176 | delete this.guestToken; 177 | delete this.guestCreatedAt; 178 | } 179 | 180 | hasToken(): boolean { 181 | return this.guestToken != null; 182 | } 183 | 184 | authenticatedAt(): Date | null { 185 | if (this.guestCreatedAt == null) { 186 | return null; 187 | } 188 | 189 | return new Date(this.guestCreatedAt); 190 | } 191 | 192 | async installTo(headers: Headers): Promise { 193 | if (this.shouldUpdate()) { 194 | await this.updateGuestToken(); 195 | } 196 | 197 | const token = this.guestToken; 198 | if (token == null) { 199 | throw new Error('Authentication token is null or undefined.'); 200 | } 201 | 202 | headers.set('authorization', `Bearer ${this.bearerToken}`); 203 | headers.set('x-guest-token', token); 204 | 205 | const cookies = await this.getCookies(); 206 | const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); 207 | if (xCsrfToken) { 208 | headers.set('x-csrf-token', xCsrfToken.value); 209 | } 210 | 211 | headers.set('cookie', await this.getCookieString()); 212 | } 213 | 214 | protected getCookies(): Promise { 215 | return this.jar.getCookies(this.getCookieJarUrl()); 216 | } 217 | 218 | protected getCookieString(): Promise { 219 | return this.jar.getCookieString(this.getCookieJarUrl()); 220 | } 221 | 222 | protected async removeCookie(key: string): Promise { 223 | //@ts-expect-error don't care 224 | const store: MemoryCookieStore = this.jar.store; 225 | const cookies = await this.jar.getCookies(this.getCookieJarUrl()); 226 | for (const cookie of cookies) { 227 | if (!cookie.domain || !cookie.path) continue; 228 | store.removeCookie(cookie.domain, cookie.path, key); 229 | 230 | if (typeof document !== 'undefined') { 231 | document.cookie = `${cookie.key}=; Max-Age=0; path=${cookie.path}; domain=${cookie.domain}`; 232 | } 233 | } 234 | } 235 | 236 | private getCookieJarUrl(): string { 237 | return typeof document !== 'undefined' 238 | ? document.location.toString() 239 | : 'https://twitter.com'; 240 | } 241 | 242 | /** 243 | * Updates the authentication state with a new guest token from the Twitter API. 244 | */ 245 | protected async updateGuestToken() { 246 | const guestActivateUrl = 'https://api.twitter.com/1.1/guest/activate.json'; 247 | 248 | const headers = new Headers({ 249 | Authorization: `Bearer ${this.bearerToken}`, 250 | Cookie: await this.getCookieString(), 251 | }); 252 | 253 | const res = await this.fetch(guestActivateUrl, { 254 | method: 'POST', 255 | headers: headers, 256 | referrerPolicy: 'no-referrer', 257 | }); 258 | 259 | await updateCookieJar(this.jar, res.headers); 260 | 261 | if (!res.ok) { 262 | throw new Error(await res.text()); 263 | } 264 | 265 | const o = await res.json(); 266 | if (o == null || o['guest_token'] == null) { 267 | throw new Error('guest_token not found.'); 268 | } 269 | 270 | const newGuestToken = o['guest_token']; 271 | if (typeof newGuestToken !== 'string') { 272 | throw new Error('guest_token was not a string.'); 273 | } 274 | 275 | this.guestToken = newGuestToken; 276 | this.guestCreatedAt = new Date(); 277 | } 278 | 279 | /** 280 | * Returns if the authentication token needs to be updated or not. 281 | * @returns `true` if the token needs to be updated; `false` otherwise. 282 | */ 283 | private shouldUpdate(): boolean { 284 | return ( 285 | !this.hasToken() || 286 | (this.guestCreatedAt != null && 287 | this.guestCreatedAt < 288 | new Date(new Date().valueOf() - 3 * 60 * 60 * 1000)) 289 | ); 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /src/errors.ts: -------------------------------------------------------------------------------- 1 | export class ApiError extends Error { 2 | private constructor( 3 | readonly response: Response, 4 | readonly data: any, 5 | message: string, 6 | ) { 7 | super(message); 8 | } 9 | 10 | static async fromResponse(response: Response) { 11 | // Try our best to parse the result, but don't bother if we can't 12 | let data: string | object | undefined = undefined; 13 | try { 14 | data = await response.json(); 15 | } catch { 16 | try { 17 | data = await response.text(); 18 | } catch {} 19 | } 20 | 21 | return new ApiError(response, data, `Response status: ${response.status}`); 22 | } 23 | } 24 | 25 | interface Position { 26 | line: number; 27 | column: number; 28 | } 29 | 30 | interface TraceInfo { 31 | trace_id: string; 32 | } 33 | 34 | interface TwitterApiErrorExtensions { 35 | code?: number; 36 | kind?: string; 37 | name?: string; 38 | source?: string; 39 | tracing?: TraceInfo; 40 | } 41 | 42 | export interface TwitterApiErrorRaw extends TwitterApiErrorExtensions { 43 | message?: string; 44 | locations?: Position[]; 45 | path?: string[]; 46 | extensions?: TwitterApiErrorExtensions; 47 | } 48 | -------------------------------------------------------------------------------- /src/messages.test.ts: -------------------------------------------------------------------------------- 1 | import { getScraper } from './test-utils'; 2 | import { jest } from '@jest/globals'; 3 | 4 | let shouldSkipV2Tests = false; 5 | let testUserId: string; 6 | let testConversationId: string; 7 | 8 | beforeAll(async () => { 9 | const { 10 | TWITTER_API_KEY, 11 | TWITTER_API_SECRET_KEY, 12 | TWITTER_ACCESS_TOKEN, 13 | TWITTER_ACCESS_TOKEN_SECRET, 14 | TWITTER_USERNAME, 15 | } = process.env; 16 | 17 | if ( 18 | !TWITTER_API_KEY || 19 | !TWITTER_API_SECRET_KEY || 20 | !TWITTER_ACCESS_TOKEN || 21 | !TWITTER_ACCESS_TOKEN_SECRET || 22 | !TWITTER_USERNAME 23 | ) { 24 | console.warn( 25 | 'Skipping tests: Twitter API v2 keys are not available in environment variables.', 26 | ); 27 | shouldSkipV2Tests = true; 28 | return; 29 | } 30 | 31 | try { 32 | // Get the user ID from username 33 | const scraper = await getScraper(); 34 | const profile = await scraper.getProfile(TWITTER_USERNAME); 35 | 36 | if (!profile.userId) { 37 | throw new Error('User ID not found'); 38 | } 39 | 40 | testUserId = profile.userId; 41 | 42 | // Get first conversation ID for testing 43 | const conversations = await scraper.getDirectMessageConversations( 44 | testUserId, 45 | ); 46 | 47 | if ( 48 | !conversations.conversations.length && 49 | !conversations.conversations[0].conversationId 50 | ) { 51 | throw new Error('No conversations found'); 52 | } 53 | 54 | // testConversationId = conversations.conversations[0].conversationId; 55 | testConversationId = '1025530896651362304-1247854858931040258'; 56 | } catch (error) { 57 | console.error('Failed to initialize test data:', error); 58 | shouldSkipV2Tests = true; 59 | } 60 | }); 61 | 62 | describe('Direct Message Tests', () => { 63 | beforeEach(() => { 64 | if (shouldSkipV2Tests || !testUserId || !testConversationId) { 65 | console.warn('Skipping test: Required test data not available'); 66 | return; 67 | } 68 | }); 69 | 70 | test('should get DM conversations', async () => { 71 | if (shouldSkipV2Tests) return; 72 | 73 | const scraper = await getScraper(); 74 | const conversations = await scraper.getDirectMessageConversations( 75 | testUserId, 76 | ); 77 | 78 | expect(conversations).toBeDefined(); 79 | expect(conversations.conversations).toBeInstanceOf(Array); 80 | expect(conversations.users).toBeInstanceOf(Array); 81 | }, 30000); 82 | 83 | test('should handle DM send failure gracefully', async () => { 84 | if (shouldSkipV2Tests) return; 85 | 86 | const scraper = await getScraper(); 87 | const invalidConversationId = 'invalid-id'; 88 | 89 | await expect( 90 | scraper.sendDirectMessage(invalidConversationId, 'test message'), 91 | ).rejects.toThrow(); 92 | }, 30000); 93 | 94 | test('should verify DM conversation structure', async () => { 95 | if (shouldSkipV2Tests) return; 96 | 97 | const scraper = await getScraper(); 98 | const conversations = await scraper.getDirectMessageConversations( 99 | testUserId, 100 | ); 101 | 102 | if (conversations.conversations.length > 0) { 103 | const conversation = conversations.conversations[0]; 104 | 105 | // Test conversation structure 106 | expect(conversation).toHaveProperty('conversationId'); 107 | expect(conversation).toHaveProperty('messages'); 108 | expect(conversation).toHaveProperty('participants'); 109 | 110 | // Test participants structure 111 | expect(conversation.participants[0]).toHaveProperty('id'); 112 | expect(conversation.participants[0]).toHaveProperty('screenName'); 113 | 114 | // Test message structure if messages exist 115 | if (conversation.messages.length > 0) { 116 | const message = conversation.messages[0]; 117 | expect(message).toHaveProperty('id'); 118 | expect(message).toHaveProperty('text'); 119 | expect(message).toHaveProperty('senderId'); 120 | expect(message).toHaveProperty('recipientId'); 121 | expect(message).toHaveProperty('createdAt'); 122 | } 123 | } 124 | }, 30000); 125 | }); 126 | -------------------------------------------------------------------------------- /src/messages.ts: -------------------------------------------------------------------------------- 1 | import { TwitterAuth } from './auth'; 2 | import { updateCookieJar } from './requests'; 3 | 4 | export interface DirectMessage { 5 | id: string; 6 | text: string; 7 | senderId: string; 8 | recipientId: string; 9 | createdAt: string; 10 | mediaUrls?: string[]; 11 | senderScreenName?: string; 12 | recipientScreenName?: string; 13 | } 14 | 15 | export interface DirectMessageConversation { 16 | conversationId: string; 17 | messages: DirectMessage[]; 18 | participants: { 19 | id: string; 20 | screenName: string; 21 | }[]; 22 | } 23 | 24 | export interface DirectMessageEvent { 25 | id: string; 26 | type: string; 27 | message_create: { 28 | sender_id: string; 29 | target: { 30 | recipient_id: string; 31 | }; 32 | message_data: { 33 | text: string; 34 | created_at: string; 35 | entities?: { 36 | urls?: Array<{ 37 | url: string; 38 | expanded_url: string; 39 | display_url: string; 40 | }>; 41 | media?: Array<{ 42 | url: string; 43 | type: string; 44 | }>; 45 | }; 46 | }; 47 | }; 48 | } 49 | 50 | export interface DirectMessagesResponse { 51 | conversations: DirectMessageConversation[]; 52 | users: TwitterUser[]; 53 | cursor?: string; 54 | lastSeenEventId?: string; 55 | trustedLastSeenEventId?: string; 56 | untrustedLastSeenEventId?: string; 57 | inboxTimelines?: { 58 | trusted?: { 59 | status: string; 60 | minEntryId?: string; 61 | }; 62 | untrusted?: { 63 | status: string; 64 | minEntryId?: string; 65 | }; 66 | }; 67 | userId: string; 68 | } 69 | 70 | export interface TwitterUser { 71 | id: string; 72 | screenName: string; 73 | name: string; 74 | profileImageUrl: string; 75 | description?: string; 76 | verified?: boolean; 77 | protected?: boolean; 78 | followersCount?: number; 79 | friendsCount?: number; 80 | } 81 | 82 | export interface SendDirectMessageResponse { 83 | entries: { 84 | message: { 85 | id: string; 86 | time: string; 87 | affects_sort: boolean; 88 | conversation_id: string; 89 | message_data: { 90 | id: string; 91 | time: string; 92 | recipient_id: string; 93 | sender_id: string; 94 | text: string; 95 | }; 96 | }; 97 | }[]; 98 | users: Record; 99 | } 100 | 101 | function parseDirectMessageConversations( 102 | data: any, 103 | userId: string, 104 | ): DirectMessagesResponse { 105 | try { 106 | const inboxState = data?.inbox_initial_state; 107 | const conversations = inboxState?.conversations || {}; 108 | const entries = inboxState?.entries || []; 109 | const users = inboxState?.users || {}; 110 | 111 | // Parse users first 112 | const parsedUsers: TwitterUser[] = Object.values(users).map( 113 | (user: any) => ({ 114 | id: user.id_str, 115 | screenName: user.screen_name, 116 | name: user.name, 117 | profileImageUrl: user.profile_image_url_https, 118 | description: user.description, 119 | verified: user.verified, 120 | protected: user.protected, 121 | followersCount: user.followers_count, 122 | friendsCount: user.friends_count, 123 | }), 124 | ); 125 | 126 | // Group messages by conversation_id 127 | const messagesByConversation: Record = {}; 128 | entries.forEach((entry: any) => { 129 | if (entry.message) { 130 | const convId = entry.message.conversation_id; 131 | if (!messagesByConversation[convId]) { 132 | messagesByConversation[convId] = []; 133 | } 134 | messagesByConversation[convId].push(entry.message); 135 | } 136 | }); 137 | 138 | // Convert to DirectMessageConversation array 139 | const parsedConversations = Object.entries(conversations).map( 140 | ([convId, conv]: [string, any]) => { 141 | const messages = messagesByConversation[convId] || []; 142 | 143 | // Sort messages by time in ascending order 144 | messages.sort((a, b) => Number(a.time) - Number(b.time)); 145 | 146 | return { 147 | conversationId: convId, 148 | messages: parseDirectMessages(messages, users), 149 | participants: conv.participants.map((p: any) => ({ 150 | id: p.user_id, 151 | screenName: users[p.user_id]?.screen_name || p.user_id, 152 | })), 153 | }; 154 | }, 155 | ); 156 | 157 | return { 158 | conversations: parsedConversations, 159 | users: parsedUsers, 160 | cursor: inboxState?.cursor, 161 | lastSeenEventId: inboxState?.last_seen_event_id, 162 | trustedLastSeenEventId: inboxState?.trusted_last_seen_event_id, 163 | untrustedLastSeenEventId: inboxState?.untrusted_last_seen_event_id, 164 | inboxTimelines: { 165 | trusted: inboxState?.inbox_timelines?.trusted && { 166 | status: inboxState.inbox_timelines.trusted.status, 167 | minEntryId: inboxState.inbox_timelines.trusted.min_entry_id, 168 | }, 169 | untrusted: inboxState?.inbox_timelines?.untrusted && { 170 | status: inboxState.inbox_timelines.untrusted.status, 171 | minEntryId: inboxState.inbox_timelines.untrusted.min_entry_id, 172 | }, 173 | }, 174 | userId, 175 | }; 176 | } catch (error) { 177 | console.error('Error parsing DM conversations:', error); 178 | return { 179 | conversations: [], 180 | users: [], 181 | userId, 182 | }; 183 | } 184 | } 185 | 186 | function parseDirectMessages(messages: any[], users: any): DirectMessage[] { 187 | try { 188 | return messages.map((msg: any) => ({ 189 | id: msg.message_data.id, 190 | text: msg.message_data.text, 191 | senderId: msg.message_data.sender_id, 192 | recipientId: msg.message_data.recipient_id, 193 | createdAt: msg.message_data.time, 194 | mediaUrls: extractMediaUrls(msg.message_data), 195 | senderScreenName: users[msg.message_data.sender_id]?.screen_name, 196 | recipientScreenName: users[msg.message_data.recipient_id]?.screen_name, 197 | })); 198 | } catch (error) { 199 | console.error('Error parsing DMs:', error); 200 | return []; 201 | } 202 | } 203 | 204 | function extractMediaUrls(messageData: any): string[] | undefined { 205 | const urls: string[] = []; 206 | 207 | // Extract URLs from entities if they exist 208 | if (messageData.entities?.urls) { 209 | messageData.entities.urls.forEach((url: any) => { 210 | urls.push(url.expanded_url); 211 | }); 212 | } 213 | 214 | // Extract media URLs if they exist 215 | if (messageData.entities?.media) { 216 | messageData.entities.media.forEach((media: any) => { 217 | urls.push(media.media_url_https || media.media_url); 218 | }); 219 | } 220 | 221 | return urls.length > 0 ? urls : undefined; 222 | } 223 | 224 | export async function getDirectMessageConversations( 225 | userId: string, 226 | auth: TwitterAuth, 227 | cursor?: string, 228 | ): Promise { 229 | if (!auth.isLoggedIn()) { 230 | throw new Error('Authentication required to fetch direct messages'); 231 | } 232 | 233 | const url = 234 | 'https://twitter.com/i/api/graphql/7s3kOODhC5vgXlO0OlqYdA/DMInboxTimeline'; 235 | const messageListUrl = 'https://x.com/i/api/1.1/dm/inbox_initial_state.json'; 236 | 237 | const params = new URLSearchParams(); 238 | 239 | if (cursor) { 240 | params.append('cursor', cursor); 241 | } 242 | 243 | const finalUrl = `${messageListUrl}${ 244 | params.toString() ? '?' + params.toString() : '' 245 | }`; 246 | const cookies = await auth.cookieJar().getCookies(url); 247 | const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); 248 | 249 | const headers = new Headers({ 250 | authorization: `Bearer ${(auth as any).bearerToken}`, 251 | cookie: await auth.cookieJar().getCookieString(url), 252 | 'content-type': 'application/json', 253 | 'User-Agent': 254 | 'Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36', 255 | 'x-guest-token': (auth as any).guestToken, 256 | 'x-twitter-auth-type': 'OAuth2Client', 257 | 'x-twitter-active-user': 'yes', 258 | 'x-csrf-token': xCsrfToken?.value as string, 259 | }); 260 | 261 | const response = await fetch(finalUrl, { 262 | method: 'GET', 263 | headers, 264 | }); 265 | 266 | await updateCookieJar(auth.cookieJar(), response.headers); 267 | 268 | if (!response.ok) { 269 | throw new Error(await response.text()); 270 | } 271 | 272 | // parse the response 273 | const data = await response.json(); 274 | return parseDirectMessageConversations(data, userId); 275 | } 276 | 277 | export async function sendDirectMessage( 278 | auth: TwitterAuth, 279 | conversation_id: string, 280 | text: string, 281 | ): Promise { 282 | if (!auth.isLoggedIn()) { 283 | throw new Error('Authentication required to send direct messages'); 284 | } 285 | 286 | const url = 287 | 'https://twitter.com/i/api/graphql/7s3kOODhC5vgXlO0OlqYdA/DMInboxTimeline'; 288 | const messageDmUrl = 'https://x.com/i/api/1.1/dm/new2.json'; 289 | 290 | const cookies = await auth.cookieJar().getCookies(url); 291 | const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); 292 | 293 | const headers = new Headers({ 294 | authorization: `Bearer ${(auth as any).bearerToken}`, 295 | cookie: await auth.cookieJar().getCookieString(url), 296 | 'content-type': 'application/json', 297 | 'User-Agent': 298 | 'Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36', 299 | 'x-guest-token': (auth as any).guestToken, 300 | 'x-twitter-auth-type': 'OAuth2Client', 301 | 'x-twitter-active-user': 'yes', 302 | 'x-csrf-token': xCsrfToken?.value as string, 303 | }); 304 | 305 | const payload = { 306 | conversation_id: `${conversation_id}`, 307 | recipient_ids: false, 308 | text: text, 309 | cards_platform: 'Web-12', 310 | include_cards: 1, 311 | include_quote_count: true, 312 | dm_users: false, 313 | }; 314 | 315 | const response = await fetch(messageDmUrl, { 316 | method: 'POST', 317 | headers, 318 | body: JSON.stringify(payload), 319 | }); 320 | 321 | await updateCookieJar(auth.cookieJar(), response.headers); 322 | 323 | if (!response.ok) { 324 | throw new Error(await response.text()); 325 | } 326 | 327 | return await response.json(); 328 | } 329 | -------------------------------------------------------------------------------- /src/platform/index.ts: -------------------------------------------------------------------------------- 1 | import { PlatformExtensions, genericPlatform } from './platform-interface'; 2 | 3 | export * from './platform-interface'; 4 | 5 | declare const PLATFORM_NODE: boolean; 6 | declare const PLATFORM_NODE_JEST: boolean; 7 | 8 | export class Platform implements PlatformExtensions { 9 | async randomizeCiphers() { 10 | const platform = await Platform.importPlatform(); 11 | await platform?.randomizeCiphers(); 12 | } 13 | 14 | private static async importPlatform(): Promise { 15 | if (PLATFORM_NODE) { 16 | const { platform } = await import('./node/index.js'); 17 | return platform as PlatformExtensions; 18 | } else if (PLATFORM_NODE_JEST) { 19 | // Jest gets unhappy when using an await import here, so we just use require instead. 20 | // eslint-disable-next-line @typescript-eslint/no-var-requires 21 | const { platform } = require('./node'); 22 | return platform as PlatformExtensions; 23 | } 24 | 25 | return genericPlatform; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/platform/node/index.ts: -------------------------------------------------------------------------------- 1 | import { PlatformExtensions } from '../platform-interface'; 2 | import { randomizeCiphers } from './randomize-ciphers'; 3 | 4 | class NodePlatform implements PlatformExtensions { 5 | randomizeCiphers(): Promise { 6 | randomizeCiphers(); 7 | return Promise.resolve(); 8 | } 9 | } 10 | 11 | export const platform = new NodePlatform(); 12 | -------------------------------------------------------------------------------- /src/platform/node/randomize-ciphers.ts: -------------------------------------------------------------------------------- 1 | import tls from 'node:tls'; 2 | import { randomBytes } from 'node:crypto'; 3 | 4 | const ORIGINAL_CIPHERS = tls.DEFAULT_CIPHERS; 5 | 6 | // How many ciphers from the top of the list to shuffle. 7 | // The remaining ciphers are left in the original order. 8 | const TOP_N_SHUFFLE = 8; 9 | 10 | // Modified variation of https://stackoverflow.com/a/12646864 11 | const shuffleArray = (array: unknown[]) => { 12 | for (let i = array.length - 1; i > 0; i--) { 13 | const j = randomBytes(4).readUint32LE() % array.length; 14 | [array[i], array[j]] = [array[j], array[i]]; 15 | } 16 | 17 | return array; 18 | }; 19 | 20 | // https://github.com/imputnet/cobalt/pull/574 21 | export const randomizeCiphers = () => { 22 | do { 23 | const cipherList = ORIGINAL_CIPHERS.split(':'); 24 | const shuffled = shuffleArray(cipherList.slice(0, TOP_N_SHUFFLE)); 25 | const retained = cipherList.slice(TOP_N_SHUFFLE); 26 | 27 | tls.DEFAULT_CIPHERS = [...shuffled, ...retained].join(':'); 28 | } while (tls.DEFAULT_CIPHERS === ORIGINAL_CIPHERS); 29 | }; 30 | -------------------------------------------------------------------------------- /src/platform/platform-interface.ts: -------------------------------------------------------------------------------- 1 | export interface PlatformExtensions { 2 | /** 3 | * Randomizes the runtime's TLS ciphers to bypass TLS client fingerprinting, which 4 | * hopefully avoids random 404s on some requests. 5 | * 6 | * **References:** 7 | * - https://github.com/imputnet/cobalt/pull/574 8 | */ 9 | randomizeCiphers(): Promise; 10 | } 11 | 12 | export const genericPlatform = new (class implements PlatformExtensions { 13 | randomizeCiphers(): Promise { 14 | return Promise.resolve(); 15 | } 16 | })(); 17 | -------------------------------------------------------------------------------- /src/profile.test.ts: -------------------------------------------------------------------------------- 1 | import { Profile } from './profile'; 2 | import { getScraper } from './test-utils'; 3 | 4 | test('scraper can get screen name by user id', async () => { 5 | const scraper = await getScraper(); 6 | const screenName = await scraper.getScreenNameByUserId('1586562503865008129'); 7 | expect(screenName).toEqual('ligma__sigma'); 8 | }); 9 | 10 | test('scraper can get profile', async () => { 11 | const expected: Profile = { 12 | avatar: 13 | 'https://pbs.twimg.com/profile_images/436075027193004032/XlDa2oaz.jpeg', 14 | banner: 'https://pbs.twimg.com/profile_banners/106037940/1541084318', 15 | biography: 'nothing', 16 | isPrivate: false, 17 | isVerified: false, 18 | joined: new Date(Date.UTC(2010, 0, 18, 8, 49, 30, 0)), 19 | location: 'Ukraine', 20 | name: 'Nomadic', 21 | pinnedTweetIds: [], 22 | url: 'https://twitter.com/nomadic_ua', 23 | userId: '106037940', 24 | username: 'nomadic_ua', 25 | website: 'https://nomadic.name', 26 | }; 27 | 28 | const scraper = await getScraper(); 29 | 30 | const actual = await scraper.getProfile('nomadic_ua'); 31 | expect(actual.avatar).toEqual(expected.avatar); 32 | expect(actual.banner).toEqual(expected.banner); 33 | expect(actual.biography).toEqual(expected.biography); 34 | expect(actual.isPrivate).toEqual(expected.isPrivate); 35 | expect(actual.isVerified).toEqual(expected.isVerified); 36 | expect(actual.joined).toEqual(expected.joined); 37 | expect(actual.location).toEqual(expected.location); 38 | expect(actual.name).toEqual(expected.name); 39 | expect(actual.pinnedTweetIds).toEqual(expected.pinnedTweetIds); 40 | expect(actual.url).toEqual(expected.url); 41 | expect(actual.userId).toEqual(expected.userId); 42 | expect(actual.username).toEqual(expected.username); 43 | expect(actual.website).toEqual(expected.website); 44 | }); 45 | 46 | test('scraper can get partial private profile', async () => { 47 | const expected: Profile = { 48 | avatar: 49 | 'https://pbs.twimg.com/profile_images/1612213936082030594/_HEsjv7Q.jpg', 50 | banner: 51 | 'https://pbs.twimg.com/profile_banners/1221221876849995777/1673110776', 52 | biography: `t h e h e r m i t`, 53 | isPrivate: true, 54 | isVerified: false, 55 | joined: new Date(Date.UTC(2020, 0, 26, 0, 3, 5, 0)), 56 | location: 'sometimes', 57 | name: 'private account', 58 | pinnedTweetIds: [], 59 | url: 'https://twitter.com/tomdumont', 60 | userId: '1221221876849995777', 61 | username: 'tomdumont', 62 | website: undefined, 63 | }; 64 | 65 | const scraper = await getScraper(); 66 | 67 | const actual = await scraper.getProfile('tomdumont'); 68 | expect(actual.avatar).toEqual(expected.avatar); 69 | expect(actual.banner).toEqual(expected.banner); 70 | expect(actual.biography).toEqual(expected.biography); 71 | expect(actual.isPrivate).toEqual(expected.isPrivate); 72 | expect(actual.isVerified).toEqual(expected.isVerified); 73 | expect(actual.joined).toEqual(expected.joined); 74 | expect(actual.location).toEqual(expected.location); 75 | expect(actual.name).toEqual(expected.name); 76 | expect(actual.pinnedTweetIds).toEqual(expected.pinnedTweetIds); 77 | expect(actual.url).toEqual(expected.url); 78 | expect(actual.userId).toEqual(expected.userId); 79 | expect(actual.username).toEqual(expected.username); 80 | expect(actual.website).toEqual(expected.website); 81 | }); 82 | 83 | test('scraper cannot get suspended profile', async () => { 84 | const scraper = await getScraper(); 85 | // taken from https://en.wikipedia.org/wiki/Twitter_suspensions#List_of_notable_suspensions 86 | expect(scraper.getProfile('RobertC20041800')).rejects.toThrow(); 87 | }); 88 | 89 | test('scraper cannot get not found profile', async () => { 90 | const scraper = await getScraper(); 91 | expect(scraper.getProfile('sample3123131')).rejects.toThrow(); 92 | }); 93 | 94 | test('scraper can get profile by screen name', async () => { 95 | const scraper = await getScraper(); 96 | await scraper.getProfile('Twitter'); 97 | }); 98 | -------------------------------------------------------------------------------- /src/profile.ts: -------------------------------------------------------------------------------- 1 | import stringify from 'json-stable-stringify'; 2 | import { requestApi, RequestApiResult } from './api'; 3 | import { TwitterAuth } from './auth'; 4 | import { TwitterApiErrorRaw } from './errors'; 5 | 6 | export interface LegacyUserRaw { 7 | created_at?: string; 8 | description?: string; 9 | entities?: { 10 | url?: { 11 | urls?: { 12 | expanded_url?: string; 13 | }[]; 14 | }; 15 | }; 16 | favourites_count?: number; 17 | followers_count?: number; 18 | friends_count?: number; 19 | media_count?: number; 20 | statuses_count?: number; 21 | id_str?: string; 22 | listed_count?: number; 23 | name?: string; 24 | location: string; 25 | geo_enabled?: boolean; 26 | pinned_tweet_ids_str?: string[]; 27 | profile_background_color?: string; 28 | profile_banner_url?: string; 29 | profile_image_url_https?: string; 30 | protected?: boolean; 31 | screen_name?: string; 32 | verified?: boolean; 33 | has_custom_timelines?: boolean; 34 | has_extended_profile?: boolean; 35 | url?: string; 36 | can_dm?: boolean; 37 | } 38 | 39 | /** 40 | * A parsed profile object. 41 | */ 42 | export interface Profile { 43 | avatar?: string; 44 | banner?: string; 45 | biography?: string; 46 | birthday?: string; 47 | followersCount?: number; 48 | followingCount?: number; 49 | friendsCount?: number; 50 | mediaCount?: number; 51 | statusesCount?: number; 52 | isPrivate?: boolean; 53 | isVerified?: boolean; 54 | isBlueVerified?: boolean; 55 | joined?: Date; 56 | likesCount?: number; 57 | listedCount?: number; 58 | location: string; 59 | name?: string; 60 | pinnedTweetIds?: string[]; 61 | tweetsCount?: number; 62 | url?: string; 63 | userId?: string; 64 | username?: string; 65 | website?: string; 66 | canDm?: boolean; 67 | } 68 | 69 | export interface UserRaw { 70 | data: { 71 | user: { 72 | result: { 73 | rest_id?: string; 74 | is_blue_verified?: boolean; 75 | legacy: LegacyUserRaw; 76 | }; 77 | }; 78 | }; 79 | errors?: TwitterApiErrorRaw[]; 80 | } 81 | 82 | function getAvatarOriginalSizeUrl(avatarUrl: string | undefined) { 83 | return avatarUrl ? avatarUrl.replace('_normal', '') : undefined; 84 | } 85 | 86 | export function parseProfile( 87 | user: LegacyUserRaw, 88 | isBlueVerified?: boolean, 89 | ): Profile { 90 | const profile: Profile = { 91 | avatar: getAvatarOriginalSizeUrl(user.profile_image_url_https), 92 | banner: user.profile_banner_url, 93 | biography: user.description, 94 | followersCount: user.followers_count, 95 | followingCount: user.friends_count, 96 | friendsCount: user.friends_count, 97 | mediaCount: user.media_count, 98 | isPrivate: user.protected ?? false, 99 | isVerified: user.verified, 100 | likesCount: user.favourites_count, 101 | listedCount: user.listed_count, 102 | location: user.location, 103 | name: user.name, 104 | pinnedTweetIds: user.pinned_tweet_ids_str, 105 | tweetsCount: user.statuses_count, 106 | url: `https://twitter.com/${user.screen_name}`, 107 | userId: user.id_str, 108 | username: user.screen_name, 109 | isBlueVerified: isBlueVerified ?? false, 110 | canDm: user.can_dm, 111 | }; 112 | 113 | if (user.created_at != null) { 114 | profile.joined = new Date(Date.parse(user.created_at)); 115 | } 116 | 117 | const urls = user.entities?.url?.urls; 118 | if (urls?.length != null && urls?.length > 0) { 119 | profile.website = urls[0].expanded_url; 120 | } 121 | 122 | return profile; 123 | } 124 | 125 | export async function getProfile( 126 | username: string, 127 | auth: TwitterAuth, 128 | ): Promise> { 129 | const params = new URLSearchParams(); 130 | params.set( 131 | 'variables', 132 | stringify({ 133 | screen_name: username, 134 | withSafetyModeUserFields: true, 135 | }) ?? '', 136 | ); 137 | 138 | params.set( 139 | 'features', 140 | stringify({ 141 | hidden_profile_likes_enabled: false, 142 | hidden_profile_subscriptions_enabled: false, // Auth-restricted 143 | responsive_web_graphql_exclude_directive_enabled: true, 144 | verified_phone_label_enabled: false, 145 | subscriptions_verification_info_is_identity_verified_enabled: false, 146 | subscriptions_verification_info_verified_since_enabled: true, 147 | highlights_tweets_tab_ui_enabled: true, 148 | creator_subscriptions_tweet_preview_api_enabled: true, 149 | responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, 150 | responsive_web_graphql_timeline_navigation_enabled: true, 151 | }) ?? '', 152 | ); 153 | 154 | params.set('fieldToggles', stringify({ withAuxiliaryUserLabels: false }) ?? ''); 155 | 156 | const res = await requestApi( 157 | `https://twitter.com/i/api/graphql/G3KGOASz96M-Qu0nwmGXNg/UserByScreenName?${params.toString()}`, 158 | auth, 159 | ); 160 | if (!res.success) { 161 | return res; 162 | } 163 | 164 | const { value } = res; 165 | const { errors } = value; 166 | if (errors != null && errors.length > 0) { 167 | return { 168 | success: false, 169 | err: new Error(errors[0].message), 170 | }; 171 | } 172 | 173 | if (!value.data || !value.data.user || !value.data.user.result) { 174 | return { 175 | success: false, 176 | err: new Error('User not found.'), 177 | }; 178 | } 179 | const { result: user } = value.data.user; 180 | const { legacy } = user; 181 | 182 | if (user.rest_id == null || user.rest_id.length === 0) { 183 | return { 184 | success: false, 185 | err: new Error('rest_id not found.'), 186 | }; 187 | } 188 | 189 | legacy.id_str = user.rest_id; 190 | 191 | if (legacy.screen_name == null || legacy.screen_name.length === 0) { 192 | return { 193 | success: false, 194 | err: new Error(`Either ${username} does not exist or is private.`), 195 | }; 196 | } 197 | 198 | return { 199 | success: true, 200 | value: parseProfile(user.legacy, user.is_blue_verified), 201 | }; 202 | } 203 | 204 | const idCache = new Map(); 205 | 206 | export async function getScreenNameByUserId( 207 | userId: string, 208 | auth: TwitterAuth, 209 | ): Promise> { 210 | const params = new URLSearchParams(); 211 | params.set( 212 | 'variables', 213 | stringify({ 214 | userId: userId, 215 | withSafetyModeUserFields: true, 216 | }) ?? '', 217 | ); 218 | 219 | params.set( 220 | 'features', 221 | stringify({ 222 | hidden_profile_subscriptions_enabled: true, 223 | rweb_tipjar_consumption_enabled: true, 224 | responsive_web_graphql_exclude_directive_enabled: true, 225 | verified_phone_label_enabled: false, 226 | highlights_tweets_tab_ui_enabled: true, 227 | responsive_web_twitter_article_notes_tab_enabled: true, 228 | subscriptions_feature_can_gift_premium: false, 229 | creator_subscriptions_tweet_preview_api_enabled: true, 230 | responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, 231 | responsive_web_graphql_timeline_navigation_enabled: true, 232 | }) ?? '', 233 | ); 234 | 235 | const res = await requestApi( 236 | `https://twitter.com/i/api/graphql/xf3jd90KKBCUxdlI_tNHZw/UserByRestId?${params.toString()}`, 237 | auth, 238 | ); 239 | 240 | if (!res.success) { 241 | return res; 242 | } 243 | 244 | const { value } = res; 245 | const { errors } = value; 246 | if (errors != null && errors.length > 0) { 247 | return { 248 | success: false, 249 | err: new Error(errors[0].message), 250 | }; 251 | } 252 | 253 | if (!value.data || !value.data.user || !value.data.user.result) { 254 | return { 255 | success: false, 256 | err: new Error('User not found.'), 257 | }; 258 | } 259 | 260 | const { result: user } = value.data.user; 261 | const { legacy } = user; 262 | 263 | if (legacy.screen_name == null || legacy.screen_name.length === 0) { 264 | return { 265 | success: false, 266 | err: new Error( 267 | `Either user with ID ${userId} does not exist or is private.`, 268 | ), 269 | }; 270 | } 271 | 272 | return { 273 | success: true, 274 | value: legacy.screen_name, 275 | }; 276 | } 277 | 278 | export async function getUserIdByScreenName( 279 | screenName: string, 280 | auth: TwitterAuth, 281 | ): Promise> { 282 | const cached = idCache.get(screenName); 283 | if (cached != null) { 284 | return { success: true, value: cached }; 285 | } 286 | 287 | const profileRes = await getProfile(screenName, auth); 288 | if (!profileRes.success) { 289 | return profileRes; 290 | } 291 | 292 | const profile = profileRes.value; 293 | if (profile.userId != null) { 294 | idCache.set(screenName, profile.userId); 295 | 296 | return { 297 | success: true, 298 | value: profile.userId, 299 | }; 300 | } 301 | 302 | return { 303 | success: false, 304 | err: new Error('User ID is undefined.'), 305 | }; 306 | } 307 | -------------------------------------------------------------------------------- /src/relationships.test.ts: -------------------------------------------------------------------------------- 1 | import { getScraper } from './test-utils'; 2 | 3 | test('scraper can get profile followers', async () => { 4 | const scraper = await getScraper(); 5 | 6 | const seenProfiles = new Map(); 7 | const maxProfiles = 50; 8 | let nProfiles = 0; 9 | 10 | const profiles = await scraper.getFollowers( 11 | '1425600122885394432', 12 | maxProfiles, 13 | ); 14 | 15 | for await (const profile of profiles) { 16 | nProfiles++; 17 | 18 | const id = profile.userId; 19 | expect(id).toBeTruthy(); 20 | 21 | if (id != null) { 22 | expect(seenProfiles.has(id)).toBeFalsy(); 23 | seenProfiles.set(id, true); 24 | } 25 | 26 | expect(profile.username).toBeTruthy(); 27 | } 28 | 29 | expect(nProfiles).toEqual(maxProfiles); 30 | }); 31 | 32 | test('scraper can get profile following', async () => { 33 | const scraper = await getScraper(); 34 | 35 | const seenProfiles = new Map(); 36 | const maxProfiles = 50; 37 | let nProfiles = 0; 38 | 39 | const profiles = await scraper.getFollowing( 40 | '1425600122885394432', 41 | maxProfiles, 42 | ); 43 | 44 | for await (const profile of profiles) { 45 | nProfiles++; 46 | 47 | const id = profile.userId; 48 | expect(id).toBeTruthy(); 49 | 50 | if (id != null) { 51 | expect(seenProfiles.has(id)).toBeFalsy(); 52 | seenProfiles.set(id, true); 53 | } 54 | 55 | expect(profile.username).toBeTruthy(); 56 | } 57 | 58 | expect(nProfiles).toEqual(maxProfiles); 59 | }); 60 | -------------------------------------------------------------------------------- /src/relationships.ts: -------------------------------------------------------------------------------- 1 | import { addApiFeatures, requestApi, bearerToken } from './api'; 2 | import { Headers } from 'headers-polyfill'; 3 | import { TwitterAuth } from './auth'; 4 | import { Profile, getUserIdByScreenName } from './profile'; 5 | import { QueryProfilesResponse } from './timeline-v1'; 6 | import { getUserTimeline } from './timeline-async'; 7 | import { 8 | RelationshipTimeline, 9 | parseRelationshipTimeline, 10 | } from './timeline-relationship'; 11 | import stringify from 'json-stable-stringify'; 12 | 13 | export function getFollowing( 14 | userId: string, 15 | maxProfiles: number, 16 | auth: TwitterAuth, 17 | ): AsyncGenerator { 18 | return getUserTimeline(userId, maxProfiles, (q, mt, c) => { 19 | return fetchProfileFollowing(q, mt, auth, c); 20 | }); 21 | } 22 | 23 | export function getFollowers( 24 | userId: string, 25 | maxProfiles: number, 26 | auth: TwitterAuth, 27 | ): AsyncGenerator { 28 | return getUserTimeline(userId, maxProfiles, (q, mt, c) => { 29 | return fetchProfileFollowers(q, mt, auth, c); 30 | }); 31 | } 32 | 33 | export async function fetchProfileFollowing( 34 | userId: string, 35 | maxProfiles: number, 36 | auth: TwitterAuth, 37 | cursor?: string, 38 | ): Promise { 39 | const timeline = await getFollowingTimeline( 40 | userId, 41 | maxProfiles, 42 | auth, 43 | cursor, 44 | ); 45 | 46 | return parseRelationshipTimeline(timeline); 47 | } 48 | 49 | export async function fetchProfileFollowers( 50 | userId: string, 51 | maxProfiles: number, 52 | auth: TwitterAuth, 53 | cursor?: string, 54 | ): Promise { 55 | const timeline = await getFollowersTimeline( 56 | userId, 57 | maxProfiles, 58 | auth, 59 | cursor, 60 | ); 61 | 62 | return parseRelationshipTimeline(timeline); 63 | } 64 | 65 | async function getFollowingTimeline( 66 | userId: string, 67 | maxItems: number, 68 | auth: TwitterAuth, 69 | cursor?: string, 70 | ): Promise { 71 | if (!auth.isLoggedIn()) { 72 | throw new Error('Scraper is not logged-in for profile following.'); 73 | } 74 | 75 | if (maxItems > 50) { 76 | maxItems = 50; 77 | } 78 | 79 | const variables: Record = { 80 | userId, 81 | count: maxItems, 82 | includePromotedContent: false, 83 | }; 84 | 85 | const features = addApiFeatures({ 86 | responsive_web_twitter_article_tweet_consumption_enabled: false, 87 | tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: 88 | true, 89 | longform_notetweets_inline_media_enabled: true, 90 | responsive_web_media_download_video_enabled: false, 91 | }); 92 | 93 | if (cursor != null && cursor != '') { 94 | variables['cursor'] = cursor; 95 | } 96 | 97 | const params = new URLSearchParams(); 98 | params.set('features', stringify(features) ?? ''); 99 | params.set('variables', stringify(variables) ?? ''); 100 | 101 | const res = await requestApi( 102 | `https://twitter.com/i/api/graphql/iSicc7LrzWGBgDPL0tM_TQ/Following?${params.toString()}`, 103 | auth, 104 | ); 105 | 106 | if (!res.success) { 107 | throw res.err; 108 | } 109 | 110 | return res.value; 111 | } 112 | 113 | async function getFollowersTimeline( 114 | userId: string, 115 | maxItems: number, 116 | auth: TwitterAuth, 117 | cursor?: string, 118 | ): Promise { 119 | if (!auth.isLoggedIn()) { 120 | throw new Error('Scraper is not logged-in for profile followers.'); 121 | } 122 | 123 | if (maxItems > 50) { 124 | maxItems = 50; 125 | } 126 | 127 | const variables: Record = { 128 | userId, 129 | count: maxItems, 130 | includePromotedContent: false, 131 | }; 132 | 133 | const features = addApiFeatures({ 134 | responsive_web_twitter_article_tweet_consumption_enabled: false, 135 | tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: 136 | true, 137 | longform_notetweets_inline_media_enabled: true, 138 | responsive_web_media_download_video_enabled: false, 139 | }); 140 | 141 | if (cursor != null && cursor != '') { 142 | variables['cursor'] = cursor; 143 | } 144 | 145 | const params = new URLSearchParams(); 146 | params.set('features', stringify(features) ?? ''); 147 | params.set('variables', stringify(variables) ?? ''); 148 | 149 | const res = await requestApi( 150 | `https://twitter.com/i/api/graphql/rRXFSG5vR6drKr5M37YOTw/Followers?${params.toString()}`, 151 | auth, 152 | ); 153 | 154 | if (!res.success) { 155 | throw res.err; 156 | } 157 | 158 | return res.value; 159 | } 160 | 161 | export async function followUser( 162 | username: string, 163 | auth: TwitterAuth, 164 | ): Promise { 165 | 166 | // Check if the user is logged in 167 | if (!(await auth.isLoggedIn())) { 168 | throw new Error('Must be logged in to follow users'); 169 | } 170 | // Get user ID from username 171 | const userIdResult = await getUserIdByScreenName(username, auth); 172 | 173 | if (!userIdResult.success) { 174 | throw new Error(`Failed to get user ID: ${userIdResult.err.message}`); 175 | } 176 | 177 | const userId = userIdResult.value; 178 | 179 | // Prepare the request body 180 | const requestBody = { 181 | include_profile_interstitial_type: '1', 182 | skip_status: 'true', 183 | user_id: userId, 184 | }; 185 | 186 | // Prepare the headers 187 | const headers = new Headers({ 188 | 'Content-Type': 'application/x-www-form-urlencoded', 189 | Referer: `https://twitter.com/${username}`, 190 | 'X-Twitter-Active-User': 'yes', 191 | 'X-Twitter-Auth-Type': 'OAuth2Session', 192 | 'X-Twitter-Client-Language': 'en', 193 | Authorization: `Bearer ${bearerToken}`, 194 | }); 195 | 196 | // Install auth headers 197 | await auth.installTo(headers, 'https://api.twitter.com/1.1/friendships/create.json'); 198 | 199 | // Make the follow request using auth.fetch 200 | const res = await auth.fetch( 201 | 'https://api.twitter.com/1.1/friendships/create.json', 202 | { 203 | method: 'POST', 204 | headers, 205 | body: new URLSearchParams(requestBody).toString(), 206 | credentials: 'include', 207 | }, 208 | ); 209 | 210 | if (!res.ok) { 211 | throw new Error(`Failed to follow user: ${res.statusText}`); 212 | } 213 | 214 | const data = await res.json(); 215 | 216 | return new Response(JSON.stringify(data), { 217 | status: 200, 218 | headers: { 219 | 'Content-Type': 'application/json', 220 | }, 221 | }); 222 | } -------------------------------------------------------------------------------- /src/requests.ts: -------------------------------------------------------------------------------- 1 | import { Cookie, CookieJar } from 'tough-cookie'; 2 | import setCookie from 'set-cookie-parser'; 3 | import type { Headers as HeadersPolyfill } from 'headers-polyfill'; 4 | 5 | /** 6 | * Updates a cookie jar with the Set-Cookie headers from the provided Headers instance. 7 | * @param cookieJar The cookie jar to update. 8 | * @param headers The response headers to populate the cookie jar with. 9 | */ 10 | export async function updateCookieJar( 11 | cookieJar: CookieJar, 12 | headers: Headers | HeadersPolyfill, 13 | ) { 14 | const setCookieHeader = headers.get('set-cookie'); 15 | if (setCookieHeader) { 16 | const cookies = setCookie.splitCookiesString(setCookieHeader); 17 | for (const cookie of cookies.map((c) => Cookie.parse(c))) { 18 | if (!cookie) continue; 19 | await cookieJar.setCookie( 20 | cookie, 21 | `${cookie.secure ? 'https' : 'http'}://${cookie.domain}${cookie.path}`, 22 | ); 23 | } 24 | } else if (typeof document !== 'undefined') { 25 | for (const cookie of document.cookie.split(';')) { 26 | const hardCookie = Cookie.parse(cookie); 27 | if (hardCookie) { 28 | await cookieJar.setCookie(hardCookie, document.location.toString()); 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/scraper.test.ts: -------------------------------------------------------------------------------- 1 | import { Scraper } from './scraper'; 2 | import { getScraper } from './test-utils'; 3 | 4 | test('scraper can fetch home timeline', async () => { 5 | const scraper = await getScraper(); 6 | 7 | const count = 20; 8 | const seenTweetIds: string[] = []; 9 | 10 | const homeTimeline = await scraper.fetchHomeTimeline(count, seenTweetIds); 11 | console.log(homeTimeline); 12 | expect(homeTimeline).toBeDefined(); 13 | expect(homeTimeline?.length).toBeGreaterThan(0); 14 | expect(homeTimeline[0]?.rest_id).toBeDefined(); 15 | }, 30000); 16 | 17 | test('scraper can fetch following timeline', async () => { 18 | const scraper = await getScraper(); 19 | 20 | const count = 20; 21 | const seenTweetIds: string[] = []; 22 | 23 | const homeTimeline = await scraper.fetchFollowingTimeline(count, seenTweetIds); 24 | console.log(homeTimeline); 25 | expect(homeTimeline).toBeDefined(); 26 | expect(homeTimeline?.length).toBeGreaterThan(0); 27 | expect(homeTimeline[0]?.rest_id).toBeDefined(); 28 | }, 30000); 29 | 30 | test('scraper uses response transform when provided', async () => { 31 | const scraper = new Scraper({ 32 | transform: { 33 | response: (response) => 34 | new Proxy(response, { 35 | get(target, p, receiver) { 36 | if (p === 'status') { 37 | return 400; 38 | } 39 | 40 | if (p === 'ok') { 41 | return false; 42 | } 43 | 44 | return Reflect.get(target, p, receiver); 45 | }, 46 | }), 47 | }, 48 | }); 49 | 50 | await expect(scraper.getLatestTweet('twitter')).rejects.toThrow(); 51 | }); 52 | -------------------------------------------------------------------------------- /src/search.test.ts: -------------------------------------------------------------------------------- 1 | import { getScraper } from './test-utils'; 2 | import { SearchMode } from './search'; 3 | import { QueryTweetsResponse } from './timeline-v1'; 4 | 5 | test('scraper can process search cursor', async () => { 6 | const scraper = await getScraper(); 7 | 8 | let cursor: string | undefined = undefined; 9 | const maxTweets = 30; 10 | let nTweets = 0; 11 | while (nTweets < maxTweets) { 12 | const res: QueryTweetsResponse = await scraper.fetchSearchTweets( 13 | 'twitter', 14 | maxTweets, 15 | SearchMode.Top, 16 | cursor, 17 | ); 18 | 19 | expect(res.next).toBeTruthy(); 20 | 21 | nTweets += res.tweets.length; 22 | cursor = res.next; 23 | } 24 | }, 30000); 25 | 26 | test('scraper can search profiles', async () => { 27 | const scraper = await getScraper(); 28 | 29 | const seenProfiles = new Map(); 30 | const maxProfiles = 150; 31 | let nProfiles = 0; 32 | 33 | const profiles = scraper.searchProfiles('Twitter', maxProfiles); 34 | for await (const profile of profiles) { 35 | nProfiles++; 36 | 37 | const profileId = profile.userId; 38 | expect(profileId).toBeTruthy(); 39 | 40 | if (profileId != null) { 41 | expect(seenProfiles.has(profileId)).toBeFalsy(); 42 | seenProfiles.set(profileId, true); 43 | } 44 | } 45 | 46 | expect(nProfiles).toEqual(maxProfiles); 47 | }, 30000); 48 | 49 | test('scraper can search tweets', async () => { 50 | const scraper = await getScraper(); 51 | 52 | const seenTweets = new Map(); 53 | const maxTweets = 150; 54 | let nTweets = 0; 55 | 56 | const profiles = scraper.searchTweets( 57 | 'twitter', 58 | maxTweets, 59 | SearchMode.Latest, 60 | ); 61 | 62 | for await (const tweet of profiles) { 63 | nTweets++; 64 | 65 | const id = tweet.id; 66 | expect(id).toBeTruthy(); 67 | 68 | if (id != null) { 69 | expect(seenTweets.has(id)).toBeFalsy(); 70 | seenTweets.set(id, true); 71 | } 72 | 73 | expect(tweet.permanentUrl).toBeTruthy(); 74 | expect(tweet.isRetweet).toBeFalsy(); 75 | expect(tweet.text).toBeTruthy(); 76 | } 77 | 78 | expect(nTweets).toEqual(maxTweets); 79 | }, 30000); 80 | -------------------------------------------------------------------------------- /src/search.ts: -------------------------------------------------------------------------------- 1 | import { addApiFeatures, requestApi } from './api'; 2 | import { TwitterAuth } from './auth'; 3 | import { Profile } from './profile'; 4 | import { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1'; 5 | import { getTweetTimeline, getUserTimeline } from './timeline-async'; 6 | import { Tweet } from './tweets'; 7 | import { 8 | SearchTimeline, 9 | parseSearchTimelineTweets, 10 | parseSearchTimelineUsers, 11 | } from './timeline-search'; 12 | import stringify from 'json-stable-stringify'; 13 | 14 | /** 15 | * The categories that can be used in Twitter searches. 16 | */ 17 | export enum SearchMode { 18 | Top, 19 | Latest, 20 | Photos, 21 | Videos, 22 | Users, 23 | } 24 | 25 | export function searchTweets( 26 | query: string, 27 | maxTweets: number, 28 | searchMode: SearchMode, 29 | auth: TwitterAuth, 30 | ): AsyncGenerator { 31 | return getTweetTimeline(query, maxTweets, (q, mt, c) => { 32 | return fetchSearchTweets(q, mt, searchMode, auth, c); 33 | }); 34 | } 35 | 36 | export function searchProfiles( 37 | query: string, 38 | maxProfiles: number, 39 | auth: TwitterAuth, 40 | ): AsyncGenerator { 41 | return getUserTimeline(query, maxProfiles, (q, mt, c) => { 42 | return fetchSearchProfiles(q, mt, auth, c); 43 | }); 44 | } 45 | 46 | export async function fetchSearchTweets( 47 | query: string, 48 | maxTweets: number, 49 | searchMode: SearchMode, 50 | auth: TwitterAuth, 51 | cursor?: string, 52 | ): Promise { 53 | const timeline = await getSearchTimeline( 54 | query, 55 | maxTweets, 56 | searchMode, 57 | auth, 58 | cursor, 59 | ); 60 | 61 | return parseSearchTimelineTweets(timeline); 62 | } 63 | 64 | export async function fetchSearchProfiles( 65 | query: string, 66 | maxProfiles: number, 67 | auth: TwitterAuth, 68 | cursor?: string, 69 | ): Promise { 70 | const timeline = await getSearchTimeline( 71 | query, 72 | maxProfiles, 73 | SearchMode.Users, 74 | auth, 75 | cursor, 76 | ); 77 | 78 | return parseSearchTimelineUsers(timeline); 79 | } 80 | 81 | async function getSearchTimeline( 82 | query: string, 83 | maxItems: number, 84 | searchMode: SearchMode, 85 | auth: TwitterAuth, 86 | cursor?: string, 87 | ): Promise { 88 | if (!auth.isLoggedIn()) { 89 | throw new Error('Scraper is not logged-in for search.'); 90 | } 91 | 92 | if (maxItems > 50) { 93 | maxItems = 50; 94 | } 95 | 96 | const variables: Record = { 97 | rawQuery: query, 98 | count: maxItems, 99 | querySource: 'typed_query', 100 | product: 'Top', 101 | }; 102 | 103 | const features = addApiFeatures({ 104 | longform_notetweets_inline_media_enabled: true, 105 | responsive_web_enhance_cards_enabled: false, 106 | responsive_web_media_download_video_enabled: false, 107 | responsive_web_twitter_article_tweet_consumption_enabled: false, 108 | tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: 109 | true, 110 | interactive_text_enabled: false, 111 | responsive_web_text_conversations_enabled: false, 112 | vibe_api_enabled: false, 113 | }); 114 | 115 | const fieldToggles: Record = { 116 | withArticleRichContentState: false, 117 | }; 118 | 119 | if (cursor != null && cursor != '') { 120 | variables['cursor'] = cursor; 121 | } 122 | 123 | switch (searchMode) { 124 | case SearchMode.Latest: 125 | variables.product = 'Latest'; 126 | break; 127 | case SearchMode.Photos: 128 | variables.product = 'Photos'; 129 | break; 130 | case SearchMode.Videos: 131 | variables.product = 'Videos'; 132 | break; 133 | case SearchMode.Users: 134 | variables.product = 'People'; 135 | break; 136 | default: 137 | break; 138 | } 139 | 140 | const params = new URLSearchParams(); 141 | params.set('features', stringify(features) ?? ''); 142 | params.set('fieldToggles', stringify(fieldToggles) ?? ''); 143 | params.set('variables', stringify(variables) ?? ''); 144 | 145 | const res = await requestApi( 146 | `https://api.twitter.com/graphql/gkjsKepM6gl_HmFWoWKfgg/SearchTimeline?${params.toString()}`, 147 | auth, 148 | ); 149 | 150 | if (!res.success) { 151 | throw res.err; 152 | } 153 | 154 | return res.value; 155 | } 156 | -------------------------------------------------------------------------------- /src/spaces/core/ChatClient.ts: -------------------------------------------------------------------------------- 1 | // src/core/ChatClient.ts 2 | 3 | import WebSocket from 'ws'; 4 | import { EventEmitter } from 'events'; 5 | import type { SpeakerRequest, OccupancyUpdate } from '../types'; 6 | import { Logger } from '../logger'; 7 | 8 | interface ChatClientConfig { 9 | spaceId: string; 10 | accessToken: string; 11 | endpoint: string; 12 | logger: Logger; 13 | } 14 | 15 | export class ChatClient extends EventEmitter { 16 | private ws?: WebSocket; 17 | private connected = false; 18 | private logger: Logger; 19 | private readonly spaceId: string; 20 | private readonly accessToken: string; 21 | private endpoint: string; 22 | 23 | constructor(config: ChatClientConfig) { 24 | super(); 25 | this.spaceId = config.spaceId; 26 | this.accessToken = config.accessToken; 27 | this.endpoint = config.endpoint; 28 | this.logger = config.logger; 29 | } 30 | 31 | async connect() { 32 | const wsUrl = `${this.endpoint}/chatapi/v1/chatnow`.replace( 33 | 'https://', 34 | 'wss://', 35 | ); 36 | this.logger.info('[ChatClient] Connecting =>', wsUrl); 37 | 38 | this.ws = new WebSocket(wsUrl, { 39 | headers: { 40 | Origin: 'https://x.com', 41 | 'User-Agent': 'Mozilla/5.0', 42 | }, 43 | }); 44 | 45 | await this.setupHandlers(); 46 | } 47 | 48 | private setupHandlers(): Promise { 49 | if (!this.ws) { 50 | throw new Error('No WebSocket instance'); 51 | } 52 | 53 | return new Promise((resolve, reject) => { 54 | this.ws!.on('open', () => { 55 | this.logger.info('[ChatClient] Connected'); 56 | this.connected = true; 57 | this.sendAuthAndJoin(); 58 | resolve(); 59 | }); 60 | 61 | this.ws!.on('message', (data: { toString: () => string }) => { 62 | this.handleMessage(data.toString()); 63 | }); 64 | 65 | this.ws!.on('close', () => { 66 | this.logger.info('[ChatClient] Closed'); 67 | this.connected = false; 68 | this.emit('disconnected'); 69 | }); 70 | 71 | this.ws!.on('error', (err) => { 72 | this.logger.error('[ChatClient] Error =>', err); 73 | reject(err); 74 | }); 75 | }); 76 | } 77 | 78 | private sendAuthAndJoin() { 79 | if (!this.ws) return; 80 | 81 | this.ws.send( 82 | JSON.stringify({ 83 | payload: JSON.stringify({ access_token: this.accessToken }), 84 | kind: 3, 85 | }), 86 | ); 87 | 88 | this.ws.send( 89 | JSON.stringify({ 90 | payload: JSON.stringify({ 91 | body: JSON.stringify({ room: this.spaceId }), 92 | kind: 1, 93 | }), 94 | kind: 2, 95 | }), 96 | ); 97 | } 98 | 99 | reactWithEmoji(emoji: string) { 100 | if (!this.ws) return; 101 | const payload = JSON.stringify({ 102 | body: JSON.stringify({ body: emoji, type: 2, v: 2 }), 103 | kind: 1, 104 | /* 105 | // The 'sender' field is not required, it's not even verified by the server 106 | // Instead of passing attributes down here it's easier to ignore it 107 | sender: { 108 | user_id: null, 109 | twitter_id: null, 110 | username: null, 111 | display_name: null, 112 | }, 113 | */ 114 | payload: JSON.stringify({ 115 | room: this.spaceId, 116 | body: JSON.stringify({ body: emoji, type: 2, v: 2 }), 117 | }), 118 | type: 2, 119 | }); 120 | this.ws.send(payload); 121 | } 122 | 123 | private handleMessage(raw: string) { 124 | let msg: any; 125 | try { 126 | msg = JSON.parse(raw); 127 | } catch { 128 | return; 129 | } 130 | if (!msg.payload) return; 131 | 132 | const payload = safeJson(msg.payload); 133 | if (!payload?.body) return; 134 | 135 | const body = safeJson(payload.body); 136 | 137 | if (body.guestBroadcastingEvent === 1) { 138 | const req: SpeakerRequest = { 139 | userId: body.guestRemoteID, 140 | username: body.guestUsername, 141 | displayName: payload.sender?.display_name || body.guestUsername, 142 | sessionUUID: body.sessionUUID, 143 | }; 144 | this.emit('speakerRequest', req); 145 | } 146 | 147 | if (typeof body.occupancy === 'number') { 148 | const update: OccupancyUpdate = { 149 | occupancy: body.occupancy, 150 | totalParticipants: body.total_participants || 0, 151 | }; 152 | this.emit('occupancyUpdate', update); 153 | } 154 | 155 | if (body.guestBroadcastingEvent === 16) { 156 | this.emit('muteStateChanged', { 157 | userId: body.guestRemoteID, 158 | muted: true, 159 | }); 160 | } 161 | if (body.guestBroadcastingEvent === 17) { 162 | this.emit('muteStateChanged', { 163 | userId: body.guestRemoteID, 164 | muted: false, 165 | }); 166 | } 167 | // Example of guest reaction 168 | if (body?.type === 2) { 169 | this.logger.info('[ChatClient] Emitting guest reaction event =>', body); 170 | this.emit('guestReaction', { 171 | displayName: body.displayName, 172 | emoji: body.body, 173 | }); 174 | } 175 | } 176 | 177 | async disconnect() { 178 | if (this.ws) { 179 | this.logger.info('[ChatClient] Disconnecting...'); 180 | this.ws.close(); 181 | this.ws = undefined; 182 | this.connected = false; 183 | } 184 | } 185 | } 186 | 187 | function safeJson(text: string): any { 188 | try { 189 | return JSON.parse(text); 190 | } catch { 191 | return null; 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/spaces/core/JanusAudio.ts: -------------------------------------------------------------------------------- 1 | // src/core/JanusAudio.ts 2 | 3 | import { EventEmitter } from 'events'; 4 | import wrtc from '@roamhq/wrtc'; 5 | const { nonstandard } = wrtc; 6 | const { RTCAudioSource, RTCAudioSink } = nonstandard; 7 | import { Logger } from '../logger'; 8 | 9 | interface AudioSourceOptions { 10 | logger?: Logger; 11 | } 12 | 13 | interface AudioSinkOptions { 14 | logger?: Logger; 15 | } 16 | 17 | export class JanusAudioSource extends EventEmitter { 18 | private source: any; 19 | private readonly track: MediaStreamTrack; 20 | private logger?: Logger; 21 | 22 | constructor(options?: AudioSourceOptions) { 23 | super(); 24 | this.logger = options?.logger; 25 | this.source = new RTCAudioSource(); 26 | this.track = this.source.createTrack(); 27 | } 28 | 29 | getTrack() { 30 | return this.track; 31 | } 32 | 33 | pushPcmData(samples: Int16Array, sampleRate: number, channels = 1) { 34 | if (this.logger?.isDebugEnabled()) { 35 | this.logger?.debug( 36 | `[JanusAudioSource] pushPcmData => sampleRate=${sampleRate}, channels=${channels}`, 37 | ); 38 | } 39 | this.source.onData({ 40 | samples, 41 | sampleRate, 42 | bitsPerSample: 16, 43 | channelCount: channels, 44 | numberOfFrames: samples.length / channels, 45 | }); 46 | } 47 | } 48 | 49 | export class JanusAudioSink extends EventEmitter { 50 | private sink: any; 51 | private active = true; 52 | private logger?: Logger; 53 | 54 | constructor(track: MediaStreamTrack, options?: AudioSinkOptions) { 55 | super(); 56 | this.logger = options?.logger; 57 | if (track.kind !== 'audio') { 58 | throw new Error('JanusAudioSink must be an audio track'); 59 | } 60 | this.sink = new RTCAudioSink(track); 61 | 62 | this.sink.ondata = (frame: { 63 | samples: Int16Array; 64 | sampleRate: number; 65 | bitsPerSample: number; 66 | channelCount: number; 67 | }) => { 68 | if (!this.active) return; 69 | if (this.logger?.isDebugEnabled()) { 70 | this.logger?.debug( 71 | `[JanusAudioSink] ondata => sampleRate=${frame.sampleRate}, bitsPerSample=${frame.bitsPerSample}, channelCount=${frame.channelCount}`, 72 | ); 73 | } 74 | this.emit('audioData', frame); 75 | }; 76 | } 77 | 78 | stop() { 79 | this.active = false; 80 | if (this.logger?.isDebugEnabled()) { 81 | this.logger?.debug('[JanusAudioSink] stop'); 82 | } 83 | this.sink?.stop(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/spaces/core/Space.ts: -------------------------------------------------------------------------------- 1 | // src/core/Space.ts 2 | 3 | import { EventEmitter } from 'events'; 4 | import { ChatClient } from './ChatClient'; 5 | import { JanusClient } from './JanusClient'; 6 | import { 7 | getTurnServers, 8 | createBroadcast, 9 | publishBroadcast, 10 | authorizeToken, 11 | getRegion, 12 | } from '../utils'; 13 | import type { 14 | BroadcastCreated, 15 | SpeakerRequest, 16 | OccupancyUpdate, 17 | GuestReaction, 18 | Plugin, 19 | AudioDataWithUser, 20 | PluginRegistration, 21 | SpeakerInfo, 22 | } from '../types'; 23 | import { Scraper } from '../../scraper'; 24 | import { Logger } from '../logger'; 25 | 26 | export interface SpaceConfig { 27 | mode: 'BROADCAST' | 'LISTEN' | 'INTERACTIVE'; 28 | title?: string; 29 | description?: string; 30 | languages?: string[]; 31 | debug?: boolean; 32 | } 33 | 34 | /** 35 | * This class orchestrates: 36 | * 1) Creation of the broadcast 37 | * 2) Instantiation of Janus + Chat 38 | * 3) Approve speakers, push audio, etc. 39 | */ 40 | export class Space extends EventEmitter { 41 | private readonly debug: boolean; 42 | private readonly logger: Logger; 43 | 44 | private janusClient?: JanusClient; 45 | private chatClient?: ChatClient; 46 | private authToken?: string; 47 | private broadcastInfo?: BroadcastCreated; 48 | private isInitialized = false; 49 | private plugins = new Set(); 50 | private speakers = new Map(); 51 | 52 | constructor( 53 | private readonly scraper: Scraper, 54 | options?: { debug?: boolean }, 55 | ) { 56 | super(); 57 | this.debug = options?.debug ?? false; 58 | this.logger = new Logger(this.debug); 59 | } 60 | 61 | public use(plugin: Plugin, config?: Record) { 62 | const registration: PluginRegistration = { plugin, config }; 63 | this.plugins.add(registration); 64 | 65 | this.logger.debug('[Space] Plugin added =>', plugin.constructor.name); 66 | 67 | plugin.onAttach?.(this); 68 | 69 | if (this.isInitialized && plugin.init) { 70 | plugin.init({ 71 | space: this, 72 | pluginConfig: config, 73 | }); 74 | } 75 | 76 | return this; 77 | } 78 | 79 | /** 80 | * Main entry point 81 | */ 82 | async initialize(config: SpaceConfig) { 83 | this.logger.debug('[Space] Initializing...'); 84 | 85 | const cookie = await this.scraper.getPeriscopeCookie(); 86 | const region = await getRegion(); 87 | this.logger.debug('[Space] Got region =>', region); 88 | this.logger.debug('[Space] Creating broadcast...'); 89 | const broadcast = await createBroadcast({ 90 | description: config.description, 91 | languages: config.languages, 92 | cookie, 93 | region, 94 | }); 95 | this.broadcastInfo = broadcast; 96 | 97 | this.logger.debug('[Space] Authorizing token...'); 98 | this.authToken = await authorizeToken(cookie); 99 | 100 | this.logger.debug('[Space] Getting turn servers...'); 101 | const turnServers = await getTurnServers(cookie); 102 | 103 | this.janusClient = new JanusClient({ 104 | webrtcUrl: broadcast.webrtc_gw_url, 105 | roomId: broadcast.room_id, 106 | credential: broadcast.credential, 107 | userId: broadcast.broadcast.user_id, 108 | streamName: broadcast.stream_name, 109 | turnServers, 110 | logger: this.logger, 111 | }); 112 | await this.janusClient.initialize(); 113 | 114 | this.janusClient.on('audioDataFromSpeaker', (data: AudioDataWithUser) => { 115 | this.logger.debug('[Space] Received PCM from speaker =>', data.userId); 116 | this.handleAudioData(data); 117 | // You can store or forward to a plugin, run STT, etc. 118 | }); 119 | 120 | this.janusClient.on('subscribedSpeaker', ({ userId, feedId }) => { 121 | const speaker = this.speakers.get(userId); 122 | if (!speaker) { 123 | this.logger.debug( 124 | '[Space] subscribedSpeaker => no speaker found', 125 | userId, 126 | ); 127 | return; 128 | } 129 | speaker.janusParticipantId = feedId; 130 | this.logger.debug( 131 | `[Space] updated speaker info => userId=${userId}, feedId=${feedId}`, 132 | ); 133 | }); 134 | 135 | // 7) Publish the broadcast 136 | this.logger.debug('[Space] Publishing broadcast...'); 137 | await publishBroadcast({ 138 | title: config.title || '', 139 | broadcast, 140 | cookie, 141 | janusSessionId: this.janusClient.getSessionId(), 142 | janusHandleId: this.janusClient.getHandleId(), 143 | janusPublisherId: this.janusClient.getPublisherId(), 144 | }); 145 | 146 | // 8) If interactive, open chat 147 | if (config.mode === 'INTERACTIVE') { 148 | this.logger.debug('[Space] Connecting chat...'); 149 | this.chatClient = new ChatClient({ 150 | spaceId: broadcast.room_id, 151 | accessToken: broadcast.access_token, 152 | endpoint: broadcast.endpoint, 153 | logger: this.logger, 154 | }); 155 | await this.chatClient.connect(); 156 | this.setupChatEvents(); 157 | } 158 | 159 | this.logger.info('[Space] Initialized =>', broadcast.share_url); 160 | 161 | this.isInitialized = true; 162 | 163 | for (const { plugin, config: pluginConfig } of this.plugins) { 164 | if (plugin.init) { 165 | plugin.init({ 166 | space: this, 167 | pluginConfig, 168 | }); 169 | } 170 | } 171 | 172 | this.logger.debug('[Space] All plugins initialized'); 173 | return broadcast; 174 | } 175 | 176 | reactWithEmoji(emoji: string) { 177 | if (!this.chatClient) return; 178 | this.chatClient.reactWithEmoji(emoji); 179 | } 180 | 181 | private setupChatEvents() { 182 | if (!this.chatClient) return; 183 | 184 | this.chatClient.on('speakerRequest', (req: SpeakerRequest) => { 185 | this.logger.info('[Space] Speaker request =>', req); 186 | this.emit('speakerRequest', req); 187 | }); 188 | 189 | this.chatClient.on('occupancyUpdate', (update: OccupancyUpdate) => { 190 | this.logger.debug('[Space] occupancyUpdate =>', update); 191 | this.emit('occupancyUpdate', update); 192 | }); 193 | 194 | this.chatClient.on('muteStateChanged', (evt) => { 195 | this.logger.debug('[Space] muteStateChanged =>', evt); 196 | this.emit('muteStateChanged', evt); 197 | }); 198 | 199 | this.chatClient.on('guestReaction', (reaction: GuestReaction) => { 200 | this.logger.info('[Space] Guest reaction =>', reaction); 201 | this.emit('guestReaction', reaction); 202 | }); 203 | } 204 | 205 | /** 206 | * Approves a speaker on Periscope side, then subscribes on Janus side 207 | */ 208 | async approveSpeaker(userId: string, sessionUUID: string) { 209 | if (!this.isInitialized || !this.broadcastInfo) { 210 | throw new Error('[Space] Not initialized or no broadcastInfo'); 211 | } 212 | 213 | if (!this.authToken) { 214 | throw new Error('[Space] No auth token available'); 215 | } 216 | 217 | this.speakers.set(userId, { userId, sessionUUID }); 218 | 219 | // 1) Call the "request/approve" endpoint 220 | await this.callApproveEndpoint( 221 | this.broadcastInfo, 222 | this.authToken, 223 | userId, 224 | sessionUUID, 225 | ); 226 | 227 | // 2) Subscribe in Janus => receive speaker's audio 228 | await this.janusClient?.subscribeSpeaker(userId); 229 | } 230 | 231 | private async callApproveEndpoint( 232 | broadcast: BroadcastCreated, 233 | authorizationToken: string, 234 | userId: string, 235 | sessionUUID: string, 236 | ): Promise { 237 | const endpoint = 'https://guest.pscp.tv/api/v1/audiospace/request/approve'; 238 | const headers = { 239 | 'Content-Type': 'application/json', 240 | Referer: 'https://x.com/', 241 | Authorization: authorizationToken, 242 | }; 243 | const body = { 244 | ntpForBroadcasterFrame: '2208988800024000300', 245 | ntpForLiveFrame: '2208988800024000300', 246 | chat_token: broadcast.access_token, 247 | session_uuid: sessionUUID, 248 | }; 249 | 250 | this.logger.debug('[Space] Approving speaker =>', endpoint, body); 251 | 252 | const resp = await fetch(endpoint, { 253 | method: 'POST', 254 | headers, 255 | body: JSON.stringify(body), 256 | }); 257 | 258 | if (!resp.ok) { 259 | const error = await resp.text(); 260 | throw new Error( 261 | `[Space] Failed to approve speaker => ${resp.status}: ${error}`, 262 | ); 263 | } 264 | 265 | this.logger.info('[Space] Speaker approved =>', userId); 266 | } 267 | 268 | /** 269 | * Removes a speaker (userId) on the Twitter side (audiospace/stream/eject) 270 | * then unsubscribes in Janus if needed. 271 | */ 272 | public async removeSpeaker(userId: string) { 273 | if (!this.isInitialized || !this.broadcastInfo) { 274 | throw new Error('[Space] Not initialized or no broadcastInfo'); 275 | } 276 | if (!this.authToken) { 277 | throw new Error('[Space] No auth token available'); 278 | } 279 | if (!this.janusClient) { 280 | throw new Error('[Space] No Janus client initialized'); 281 | } 282 | 283 | const speaker = this.speakers.get(userId); 284 | if (!speaker) { 285 | throw new Error( 286 | `[Space] removeSpeaker => no speaker found for userId=${userId}`, 287 | ); 288 | } 289 | 290 | const { sessionUUID, janusParticipantId } = speaker; 291 | this.logger.debug( 292 | '[Space] removeSpeaker =>', 293 | sessionUUID, 294 | janusParticipantId, 295 | speaker, 296 | ); 297 | 298 | if (!sessionUUID || janusParticipantId === undefined) { 299 | throw new Error( 300 | `[Space] removeSpeaker => missing sessionUUID or feedId for userId=${userId}`, 301 | ); 302 | } 303 | 304 | const janusHandleId = this.janusClient.getHandleId(); 305 | const janusSessionId = this.janusClient.getSessionId(); 306 | 307 | if (!janusHandleId || !janusSessionId) { 308 | throw new Error( 309 | `[Space] removeSpeaker => missing Janus handle/session for userId=${userId}`, 310 | ); 311 | } 312 | 313 | // 1) Call the eject endpoint 314 | await this.callRemoveEndpoint( 315 | this.broadcastInfo, 316 | this.authToken, 317 | sessionUUID, 318 | janusParticipantId, 319 | this.broadcastInfo.room_id, 320 | janusHandleId, 321 | janusSessionId, 322 | ); 323 | 324 | // 2) Remove from local speakers map 325 | this.speakers.delete(userId); 326 | 327 | this.logger.info(`[Space] removeSpeaker => removed userId=${userId}`); 328 | } 329 | 330 | /** 331 | * Calls the audiospace/stream/eject endpoint to remove a speaker on Twitter 332 | */ 333 | private async callRemoveEndpoint( 334 | broadcast: BroadcastCreated, 335 | authorizationToken: string, 336 | sessionUUID: string, 337 | janusParticipantId: number, 338 | janusRoomId: string, 339 | webrtcHandleId: number, 340 | webrtcSessionId: number, 341 | ): Promise { 342 | const endpoint = 'https://guest.pscp.tv/api/v1/audiospace/stream/eject'; 343 | const headers = { 344 | 'Content-Type': 'application/json', 345 | Referer: 'https://x.com/', 346 | Authorization: authorizationToken, 347 | }; 348 | const body = { 349 | ntpForBroadcasterFrame: '2208988800024000300', 350 | ntpForLiveFrame: '2208988800024000300', 351 | session_uuid: sessionUUID, 352 | chat_token: broadcast.access_token, 353 | janus_room_id: janusRoomId, 354 | janus_participant_id: janusParticipantId, 355 | webrtc_handle_id: webrtcHandleId, 356 | webrtc_session_id: webrtcSessionId, 357 | }; 358 | 359 | this.logger.debug('[Space] Removing speaker =>', endpoint, body); 360 | 361 | const resp = await fetch(endpoint, { 362 | method: 'POST', 363 | headers, 364 | body: JSON.stringify(body), 365 | }); 366 | 367 | if (!resp.ok) { 368 | const error = await resp.text(); 369 | throw new Error( 370 | `[Space] Failed to remove speaker => ${resp.status}: ${error}`, 371 | ); 372 | } 373 | 374 | this.logger.debug('[Space] Speaker removed => sessionUUID=', sessionUUID); 375 | } 376 | 377 | pushAudio(samples: Int16Array, sampleRate: number) { 378 | this.janusClient?.pushLocalAudio(samples, sampleRate); 379 | } 380 | 381 | /** 382 | * This method is called by JanusClient on 'audioDataFromSpeaker' 383 | * or we do it from the 'initialize(...)' once Janus is set up. 384 | */ 385 | private handleAudioData(data: AudioDataWithUser) { 386 | // Forward to plugins 387 | for (const { plugin } of this.plugins) { 388 | plugin.onAudioData?.(data); 389 | } 390 | } 391 | 392 | /** 393 | * Gracefully end the Space (stop broadcast, destroy Janus room, etc.) 394 | */ 395 | public async finalizeSpace(): Promise { 396 | this.logger.info('[Space] finalizeSpace => stopping broadcast gracefully'); 397 | 398 | const tasks: Array> = []; 399 | 400 | if (this.janusClient) { 401 | tasks.push( 402 | this.janusClient.destroyRoom().catch((err) => { 403 | this.logger.error('[Space] destroyRoom error =>', err); 404 | }), 405 | ); 406 | } 407 | 408 | if (this.broadcastInfo) { 409 | tasks.push( 410 | this.endAudiospace({ 411 | broadcastId: this.broadcastInfo.room_id, 412 | chatToken: this.broadcastInfo.access_token, 413 | }).catch((err) => { 414 | this.logger.error('[Space] endAudiospace error =>', err); 415 | }), 416 | ); 417 | } 418 | 419 | if (this.janusClient) { 420 | tasks.push( 421 | this.janusClient.leaveRoom().catch((err) => { 422 | this.logger.error('[Space] leaveRoom error =>', err); 423 | }), 424 | ); 425 | } 426 | 427 | await Promise.all(tasks); 428 | 429 | this.logger.info('[Space] finalizeSpace => done.'); 430 | } 431 | 432 | /** 433 | * Calls the endAudiospace endpoint from Twitter 434 | */ 435 | private async endAudiospace(params: { 436 | broadcastId: string; 437 | chatToken: string; 438 | }): Promise { 439 | const url = 'https://guest.pscp.tv/api/v1/audiospace/admin/endAudiospace'; 440 | const headers = { 441 | 'Content-Type': 'application/json', 442 | Referer: 'https://x.com/', 443 | Authorization: this.authToken || '', 444 | }; 445 | const body = { 446 | broadcast_id: params.broadcastId, 447 | chat_token: params.chatToken, 448 | }; 449 | 450 | this.logger.debug('[Space] endAudiospace =>', body); 451 | 452 | const resp = await fetch(url, { 453 | method: 'POST', 454 | headers, 455 | body: JSON.stringify(body), 456 | }); 457 | 458 | if (!resp.ok) { 459 | const errText = await resp.text(); 460 | throw new Error(`[Space] endAudiospace => ${resp.status} ${errText}`); 461 | } 462 | 463 | const json = await resp.json(); 464 | this.logger.debug('[Space] endAudiospace => success =>', json); 465 | } 466 | 467 | public getSpeakers(): SpeakerInfo[] { 468 | return Array.from(this.speakers.values()); 469 | } 470 | 471 | public async stop() { 472 | this.logger.info('[Space] Stopping...'); 473 | 474 | await this.finalizeSpace().catch((err) => { 475 | this.logger.error('[Space] finalizeBroadcast error =>', err); 476 | }); 477 | 478 | if (this.chatClient) { 479 | await this.chatClient.disconnect(); 480 | this.chatClient = undefined; 481 | } 482 | if (this.janusClient) { 483 | await this.janusClient.stop(); 484 | this.janusClient = undefined; 485 | } 486 | for (const { plugin } of this.plugins) { 487 | plugin.cleanup?.(); 488 | } 489 | this.plugins.clear(); 490 | 491 | this.isInitialized = false; 492 | } 493 | } 494 | -------------------------------------------------------------------------------- /src/spaces/logger.ts: -------------------------------------------------------------------------------- 1 | // src/logger.ts 2 | 3 | export class Logger { 4 | private readonly debugEnabled: boolean; 5 | 6 | constructor(debugEnabled: boolean) { 7 | this.debugEnabled = debugEnabled; 8 | } 9 | 10 | info(msg: string, ...args: any[]) { 11 | console.log(msg, ...args); 12 | } 13 | 14 | debug(msg: string, ...args: any[]) { 15 | if (this.debugEnabled) { 16 | console.log(msg, ...args); 17 | } 18 | } 19 | 20 | warn(msg: string, ...args: any[]) { 21 | console.warn('[WARN]', msg, ...args); 22 | } 23 | 24 | error(msg: string, ...args: any[]) { 25 | console.error(msg, ...args); 26 | } 27 | 28 | isDebugEnabled(): boolean { 29 | return this.debugEnabled; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/spaces/plugins/HlsRecordPlugin.ts: -------------------------------------------------------------------------------- 1 | // src/plugins/HlsRecordPlugin.ts 2 | 3 | import { spawn, ChildProcessWithoutNullStreams } from 'child_process'; 4 | import { Plugin, OccupancyUpdate } from '../types'; 5 | import { Space } from '../core/Space'; 6 | import { Logger } from '../logger'; 7 | 8 | /** 9 | * Plugin that records the final Twitter Spaces HLS mix to a local file. 10 | * It waits for at least one listener to join (occupancy > 0), 11 | * then repeatedly attempts to get the HLS URL from Twitter 12 | * until it is available (HTTP 200), and finally spawns ffmpeg. 13 | */ 14 | export class HlsRecordPlugin implements Plugin { 15 | private logger?: Logger; 16 | private recordingProcess?: ChildProcessWithoutNullStreams; 17 | private isRecording = false; 18 | private outputPath?: string; 19 | private mediaKey?: string; 20 | private space?: Space; 21 | 22 | constructor(outputPath?: string) { 23 | this.outputPath = outputPath; 24 | } 25 | 26 | /** 27 | * Called once the Space has fully initialized (broadcastInfo is ready). 28 | * We store references and subscribe to "occupancyUpdate". 29 | */ 30 | async init(params: { space: Space; pluginConfig?: Record }) { 31 | const spaceLogger = (params.space as any).logger as Logger | undefined; 32 | if (spaceLogger) { 33 | this.logger = spaceLogger; 34 | } 35 | 36 | if (params.pluginConfig?.outputPath) { 37 | this.outputPath = params.pluginConfig.outputPath; 38 | } 39 | 40 | this.space = params.space; 41 | 42 | const broadcastInfo = (params.space as any).broadcastInfo; 43 | if (!broadcastInfo || !broadcastInfo.broadcast?.media_key) { 44 | this.logger?.warn( 45 | '[HlsRecordPlugin] No media_key found in broadcastInfo', 46 | ); 47 | return; 48 | } 49 | this.mediaKey = broadcastInfo.broadcast.media_key; 50 | 51 | const roomId = broadcastInfo.room_id || 'unknown_room'; 52 | if (!this.outputPath) { 53 | this.outputPath = `/tmp/record_${roomId}.ts`; 54 | } 55 | 56 | // Subscribe to occupancyUpdate 57 | this.space.on('occupancyUpdate', (update: OccupancyUpdate) => { 58 | this.handleOccupancyUpdate(update).catch((err) => { 59 | this.logger?.error( 60 | '[HlsRecordPlugin] handleOccupancyUpdate error =>', 61 | err, 62 | ); 63 | }); 64 | }); 65 | } 66 | 67 | /** 68 | * Called each time occupancyUpdate is emitted. 69 | * If occupancy > 0 and we're not recording yet, we attempt to fetch the HLS URL. 70 | * If the URL is valid (HTTP 200), we launch ffmpeg. 71 | */ 72 | private async handleOccupancyUpdate(update: OccupancyUpdate) { 73 | if (!this.space || !this.mediaKey) return; 74 | if (this.isRecording) return; 75 | 76 | // We only care if occupancy > 0 (at least one listener). 77 | if (update.occupancy <= 0) { 78 | this.logger?.debug('[HlsRecordPlugin] occupancy=0 => ignoring'); 79 | return; 80 | } 81 | 82 | const scraper = (this.space as any).scraper; 83 | if (!scraper) { 84 | this.logger?.warn('[HlsRecordPlugin] No scraper found on space'); 85 | return; 86 | } 87 | 88 | this.logger?.debug( 89 | `[HlsRecordPlugin] occupancy=${update.occupancy} => trying to fetch HLS URL...`, 90 | ); 91 | 92 | try { 93 | const status = await scraper.getAudioSpaceStreamStatus(this.mediaKey); 94 | if (!status?.source?.location) { 95 | this.logger?.debug( 96 | '[HlsRecordPlugin] occupancy>0 but no HLS URL => wait next update', 97 | ); 98 | return; 99 | } 100 | 101 | const hlsUrl = status.source.location; 102 | const isReady = await this.waitForHlsReady(hlsUrl, 1); 103 | if (!isReady) { 104 | this.logger?.debug( 105 | '[HlsRecordPlugin] HLS URL 404 => waiting next occupancy update...', 106 | ); 107 | return; 108 | } 109 | 110 | await this.startRecording(hlsUrl); 111 | } catch (err) { 112 | this.logger?.error('[HlsRecordPlugin] handleOccupancyUpdate =>', err); 113 | } 114 | } 115 | 116 | /** 117 | * Spawns ffmpeg to record the HLS stream at the given URL. 118 | */ 119 | private async startRecording(hlsUrl: string): Promise { 120 | if (this.isRecording) { 121 | this.logger?.debug('[HlsRecordPlugin] Already recording'); 122 | return; 123 | } 124 | this.isRecording = true; 125 | 126 | if (!this.outputPath) { 127 | this.logger?.warn( 128 | '[HlsRecordPlugin] No output path set, using /tmp/space_record.ts', 129 | ); 130 | this.outputPath = '/tmp/space_record.ts'; 131 | } 132 | 133 | this.logger?.info('[HlsRecordPlugin] Starting HLS recording =>', hlsUrl); 134 | 135 | this.recordingProcess = spawn('ffmpeg', [ 136 | '-y', 137 | '-i', 138 | hlsUrl, 139 | '-c', 140 | 'copy', 141 | this.outputPath, 142 | ]); 143 | 144 | this.recordingProcess.stderr.on('data', (chunk) => { 145 | const msg = chunk.toString(); 146 | if (msg.toLowerCase().includes('error')) { 147 | this.logger?.error('[HlsRecordPlugin][ffmpeg error] =>', msg.trim()); 148 | } else { 149 | this.logger?.debug('[HlsRecordPlugin][ffmpeg]', msg.trim()); 150 | } 151 | }); 152 | 153 | this.recordingProcess.on('close', (code) => { 154 | this.isRecording = false; 155 | this.logger?.info( 156 | '[HlsRecordPlugin] Recording process closed => code=', 157 | code, 158 | ); 159 | }); 160 | 161 | this.recordingProcess.on('error', (err) => { 162 | this.logger?.error('[HlsRecordPlugin] Recording process failed =>', err); 163 | }); 164 | } 165 | 166 | /** 167 | * HEAD request to see if the HLS URL is returning 200 OK. 168 | * maxRetries=1 means we'll just try once here, and rely on occupancyUpdate re-calls for further tries. 169 | */ 170 | private async waitForHlsReady( 171 | hlsUrl: string, 172 | maxRetries: number, 173 | ): Promise { 174 | let attempt = 0; 175 | while (attempt < maxRetries) { 176 | try { 177 | const resp = await fetch(hlsUrl, { method: 'HEAD' }); 178 | if (resp.ok) { 179 | this.logger?.debug( 180 | `[HlsRecordPlugin] HLS is ready (attempt #${attempt + 1})`, 181 | ); 182 | return true; 183 | } else { 184 | this.logger?.debug( 185 | `[HlsRecordPlugin] HLS status=${resp.status}, retrying...`, 186 | ); 187 | } 188 | } catch (error) { 189 | this.logger?.debug( 190 | '[HlsRecordPlugin] HLS fetch error:', 191 | (error as Error).message, 192 | ); 193 | } 194 | 195 | attempt++; 196 | await new Promise((r) => setTimeout(r, 2000)); 197 | } 198 | return false; 199 | } 200 | 201 | /** 202 | * Called when the plugin is cleaned up (e.g. space.stop()). 203 | */ 204 | cleanup(): void { 205 | if (this.isRecording && this.recordingProcess) { 206 | this.logger?.info('[HlsRecordPlugin] Stopping HLS recording...'); 207 | this.recordingProcess.kill(); 208 | this.recordingProcess = undefined; 209 | this.isRecording = false; 210 | } 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/spaces/plugins/IdleMonitorPlugin.ts: -------------------------------------------------------------------------------- 1 | // src/plugins/IdleMonitorPlugin.ts 2 | 3 | import { Plugin, AudioDataWithUser } from '../types'; 4 | import { Space } from '../core/Space'; 5 | 6 | /** 7 | * Plugin that tracks the last speaker audio timestamp 8 | * and the last local audio timestamp to detect overall silence. 9 | */ 10 | export class IdleMonitorPlugin implements Plugin { 11 | private space?: Space; 12 | private lastSpeakerAudioMs = Date.now(); 13 | private lastLocalAudioMs = Date.now(); 14 | private checkInterval?: NodeJS.Timeout; 15 | 16 | /** 17 | * @param idleTimeoutMs How many ms of silence before triggering idle (default 60s) 18 | * @param checkEveryMs Interval for checking silence (default 10s) 19 | */ 20 | constructor( 21 | private idleTimeoutMs: number = 60_000, 22 | private checkEveryMs: number = 10_000, 23 | ) {} 24 | 25 | onAttach(space: Space) { 26 | this.space = space; 27 | console.log('[IdleMonitorPlugin] onAttach => plugin attached'); 28 | } 29 | 30 | init(params: { space: Space; pluginConfig?: Record }): void { 31 | this.space = params.space; 32 | console.log('[IdleMonitorPlugin] init => setting up idle checks'); 33 | 34 | // Update lastSpeakerAudioMs on incoming speaker audio 35 | this.space.on('audioDataFromSpeaker', (data: AudioDataWithUser) => { 36 | this.lastSpeakerAudioMs = Date.now(); 37 | }); 38 | 39 | // Patch space.pushAudio to update lastLocalAudioMs 40 | const originalPushAudio = this.space.pushAudio.bind(this.space); 41 | this.space.pushAudio = (samples, sampleRate) => { 42 | this.lastLocalAudioMs = Date.now(); 43 | originalPushAudio(samples, sampleRate); 44 | }; 45 | 46 | // Periodically check for silence 47 | this.checkInterval = setInterval(() => this.checkIdle(), this.checkEveryMs); 48 | } 49 | 50 | private checkIdle() { 51 | const now = Date.now(); 52 | const lastAudio = Math.max(this.lastSpeakerAudioMs, this.lastLocalAudioMs); 53 | const idleMs = now - lastAudio; 54 | 55 | if (idleMs >= this.idleTimeoutMs) { 56 | console.log( 57 | '[IdleMonitorPlugin] idleTimeout => no audio for', 58 | idleMs, 59 | 'ms', 60 | ); 61 | this.space?.emit('idleTimeout', { idleMs }); 62 | } 63 | } 64 | 65 | /** 66 | * Returns how many ms have passed since any audio was detected. 67 | */ 68 | public getIdleTimeMs(): number { 69 | const now = Date.now(); 70 | const lastAudio = Math.max(this.lastSpeakerAudioMs, this.lastLocalAudioMs); 71 | return now - lastAudio; 72 | } 73 | 74 | cleanup(): void { 75 | console.log('[IdleMonitorPlugin] cleanup => stopping idle checks'); 76 | if (this.checkInterval) { 77 | clearInterval(this.checkInterval); 78 | this.checkInterval = undefined; 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/spaces/plugins/MonitorAudioPlugin.ts: -------------------------------------------------------------------------------- 1 | // src/plugins/MonitorAudioPlugin.ts 2 | 3 | import { spawn, ChildProcessWithoutNullStreams } from 'child_process'; 4 | import { Plugin, AudioDataWithUser } from '../types'; 5 | 6 | export class MonitorAudioPlugin implements Plugin { 7 | private ffplay?: ChildProcessWithoutNullStreams; 8 | 9 | constructor(private readonly sampleRate = 48000) { 10 | // spawn ffplay reading raw PCM s16le from stdin 11 | // "-nodisp" hides any video window, "-loglevel quiet" reduces console spam 12 | this.ffplay = spawn('ffplay', [ 13 | '-f', 14 | 's16le', 15 | '-ar', 16 | this.sampleRate.toString(), // e.g. "16000" 17 | '-ac', 18 | '1', // mono 19 | '-nodisp', 20 | '-loglevel', 21 | 'quiet', 22 | '-i', 23 | 'pipe:0', 24 | ]); 25 | 26 | this.ffplay.on('error', (err) => { 27 | console.error('[MonitorAudioPlugin] ffplay error =>', err); 28 | }); 29 | this.ffplay.on('close', (code) => { 30 | console.log('[MonitorAudioPlugin] ffplay closed => code=', code); 31 | this.ffplay = undefined; 32 | }); 33 | 34 | console.log('[MonitorAudioPlugin] Started ffplay for real-time monitoring'); 35 | } 36 | 37 | onAudioData(data: AudioDataWithUser): void { 38 | // TODO: REMOVE DEBUG 39 | // console.log( 40 | // '[MonitorAudioPlugin] onAudioData => user=', 41 | // data.userId, 42 | // 'samples=', 43 | // data.samples.length, 44 | // 'sampleRate=', 45 | // data.sampleRate, 46 | // ); 47 | 48 | // Check sampleRate if needed 49 | if (!this.ffplay?.stdin.writable) return; 50 | 51 | // Suppose data.sampleRate = this.sampleRate 52 | // Convert Int16Array => Buffer 53 | const buf = Buffer.from(data.samples.buffer); 54 | 55 | // Write raw 16-bit PCM samples to ffplay stdin 56 | this.ffplay.stdin.write(buf); 57 | } 58 | 59 | cleanup(): void { 60 | console.log('[MonitorAudioPlugin] Cleanup => stopping ffplay'); 61 | if (this.ffplay) { 62 | this.ffplay.stdin.end(); // close the pipe 63 | this.ffplay.kill(); // kill ffplay process 64 | this.ffplay = undefined; 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/spaces/plugins/RecordToDiskPlugin.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs'; 2 | import { AudioDataWithUser, Plugin } from '../types'; 3 | 4 | export class RecordToDiskPlugin implements Plugin { 5 | private outStream = fs.createWriteStream('/tmp/speaker_audio.raw'); 6 | 7 | onAudioData(data: AudioDataWithUser): void { 8 | // Convert Int16Array -> Buffer 9 | const buf = Buffer.from(data.samples.buffer); 10 | this.outStream.write(buf); 11 | } 12 | 13 | cleanup(): void { 14 | this.outStream.end(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/spaces/test.ts: -------------------------------------------------------------------------------- 1 | // src/test.ts 2 | 3 | import 'dotenv/config'; 4 | import { Space, SpaceConfig } from './core/Space'; 5 | import { Scraper } from '../scraper'; 6 | import { RecordToDiskPlugin } from './plugins/RecordToDiskPlugin'; 7 | import { SttTtsPlugin } from './plugins/SttTtsPlugin'; 8 | import { IdleMonitorPlugin } from './plugins/IdleMonitorPlugin'; 9 | import { HlsRecordPlugin } from './plugins/HlsRecordPlugin'; 10 | 11 | /** 12 | * Main test entry point 13 | */ 14 | async function main() { 15 | console.log('[Test] Starting...'); 16 | 17 | // 1) Twitter login with your scraper 18 | const scraper = new Scraper(); 19 | await scraper.login( 20 | process.env.TWITTER_USERNAME!, 21 | process.env.TWITTER_PASSWORD!, 22 | ); 23 | 24 | // 2) Create the Space instance 25 | // Set debug=true if you want more logs 26 | const space = new Space(scraper, { debug: false }); 27 | 28 | // -------------------------------------------------------------------------------- 29 | // EXAMPLE 1: Record raw speaker audio via RecordToDiskPlugin (local plugin approach) 30 | // -------------------------------------------------------------------------------- 31 | const recordPlugin = new RecordToDiskPlugin(); 32 | space.use(recordPlugin); 33 | 34 | // -------------------------------------------------------------------------------- 35 | // EXAMPLE 2: HLSRecordPlugin => record final Space mix as .ts file via HLS 36 | // (Requires the "scraper" to fetch the HLS URL, and ffmpeg installed.) 37 | // -------------------------------------------------------------------------------- 38 | const hlsPlugin = new HlsRecordPlugin(); 39 | // If you want, you can override the default output path in pluginConfig, for example: 40 | // space.use(hlsPlugin, { outputPath: '/tmp/my_custom_space.ts' }); 41 | space.use(hlsPlugin); 42 | 43 | // Create our TTS/STT plugin instance 44 | const sttTtsPlugin = new SttTtsPlugin(); 45 | space.use(sttTtsPlugin, { 46 | openAiApiKey: process.env.OPENAI_API_KEY, 47 | elevenLabsApiKey: process.env.ELEVENLABS_API_KEY, 48 | voiceId: 'D38z5RcWu1voky8WS1ja', // example 49 | // You can also initialize systemPrompt, chatContext, etc. here if you wish 50 | // systemPrompt: "You are a calm and friendly AI assistant." 51 | }); 52 | 53 | // Create an IdleMonitorPlugin to stop after 60s of silence 54 | const idlePlugin = new IdleMonitorPlugin(60_000, 10_000); 55 | space.use(idlePlugin); 56 | 57 | // If idle occurs, say goodbye and end the Space 58 | space.on('idleTimeout', async (info) => { 59 | console.log(`[Test] idleTimeout => no audio for ${info.idleMs}ms.`); 60 | await sttTtsPlugin.speakText('Ending Space due to inactivity. Goodbye!'); 61 | await new Promise((r) => setTimeout(r, 10_000)); 62 | await space.stop(); 63 | console.log('[Test] Space stopped due to silence.'); 64 | process.exit(0); 65 | }); 66 | 67 | // 3) Initialize the Space 68 | const config: SpaceConfig = { 69 | mode: 'INTERACTIVE', 70 | title: 'AI Chat - Dynamic GPT Config', 71 | description: 'Space that demonstrates dynamic GPT personalities.', 72 | languages: ['en'], 73 | }; 74 | 75 | const broadcastInfo = await space.initialize(config); 76 | const spaceUrl = broadcastInfo.share_url.replace('broadcasts', 'spaces'); 77 | console.log('[Test] Space created =>', spaceUrl); 78 | 79 | // (Optional) Tweet out the Space link 80 | await scraper.sendTweet(`${config.title} ${spaceUrl}`); 81 | console.log('[Test] Tweet sent'); 82 | 83 | // --------------------------------------- 84 | // Example of dynamic GPT usage: 85 | // You can change the system prompt at runtime 86 | setTimeout(() => { 87 | console.log('[Test] Changing system prompt to a new persona...'); 88 | sttTtsPlugin.setSystemPrompt( 89 | 'You are a very sarcastic AI who uses short answers.', 90 | ); 91 | }, 45_000); 92 | 93 | // Another example: after some time, switch to GPT-4 94 | setTimeout(() => { 95 | console.log('[Test] Switching GPT model to "gpt-4" (if available)...'); 96 | sttTtsPlugin.setGptModel('gpt-4'); 97 | }, 60_000); 98 | 99 | // Also, demonstrate how to manually call askChatGPT and speak the result 100 | setTimeout(async () => { 101 | console.log('[Test] Asking GPT for an introduction...'); 102 | try { 103 | const response = await sttTtsPlugin['askChatGPT']('Introduce yourself'); 104 | console.log('[Test] ChatGPT introduction =>', response); 105 | 106 | // Then speak it 107 | await sttTtsPlugin.speakText(response); 108 | } catch (err) { 109 | console.error('[Test] askChatGPT error =>', err); 110 | } 111 | }, 75_000); 112 | 113 | // Example: periodically speak a greeting every 60s 114 | setInterval(() => { 115 | sttTtsPlugin 116 | .speakText('Hello everyone, this is an automated greeting.') 117 | .catch((err) => console.error('[Test] speakText() =>', err)); 118 | }, 20_000); 119 | 120 | // 4) Some event listeners 121 | space.on('speakerRequest', async (req) => { 122 | console.log('[Test] Speaker request =>', req); 123 | await space.approveSpeaker(req.userId, req.sessionUUID); 124 | 125 | // Remove the speaker after 10 seconds (testing only) 126 | setTimeout(() => { 127 | console.log( 128 | `[Test] Removing speaker => userId=${req.userId} (after 60s)`, 129 | ); 130 | space.removeSpeaker(req.userId).catch((err) => { 131 | console.error('[Test] removeSpeaker error =>', err); 132 | }); 133 | }, 60_000); 134 | }); 135 | 136 | // When a user reacts, send back an emoji to test the flow 137 | space.on('guestReaction', (evt) => { 138 | // Pick a random emoji from the list 139 | const emojis = ['💯', '✨', '🙏', '🎮']; 140 | const emoji = emojis[Math.floor(Math.random() * emojis.length)]; 141 | space.reactWithEmoji(emoji); 142 | }); 143 | 144 | space.on('error', (err) => { 145 | console.error('[Test] Space Error =>', err); 146 | }); 147 | 148 | // ================================================== 149 | // BEEP GENERATION (500 ms) @16kHz => 8000 samples 150 | // ================================================== 151 | const beepDurationMs = 500; 152 | const sampleRate = 16000; 153 | const totalSamples = (sampleRate * beepDurationMs) / 1000; // 8000 154 | const beepFull = new Int16Array(totalSamples); 155 | 156 | // Sine wave: 440Hz, amplitude ~12000 157 | const freq = 440; 158 | const amplitude = 12000; 159 | for (let i = 0; i < beepFull.length; i++) { 160 | const t = i / sampleRate; 161 | beepFull[i] = amplitude * Math.sin(2 * Math.PI * freq * t); 162 | } 163 | 164 | const FRAME_SIZE = 160; 165 | /** 166 | * Send a beep by slicing beepFull into frames of 160 samples 167 | */ 168 | async function sendBeep() { 169 | console.log('[Test] Starting beep...'); 170 | for (let offset = 0; offset < beepFull.length; offset += FRAME_SIZE) { 171 | const portion = beepFull.subarray(offset, offset + FRAME_SIZE); 172 | const frame = new Int16Array(FRAME_SIZE); 173 | frame.set(portion); 174 | space.pushAudio(frame, sampleRate); 175 | await new Promise((r) => setTimeout(r, 10)); 176 | } 177 | console.log('[Test] Finished beep'); 178 | } 179 | 180 | // Example: Send beep every 5s (currently commented out) 181 | // setInterval(() => { 182 | // sendBeep().catch((err) => console.error('[Test] beep error =>', err)); 183 | // }, 5000); 184 | 185 | console.log('[Test] Space is running... press Ctrl+C to exit.'); 186 | 187 | // Graceful shutdown 188 | process.on('SIGINT', async () => { 189 | console.log('\n[Test] Caught interrupt signal, stopping...'); 190 | await space.stop(); 191 | console.log('[Test] Space stopped. Bye!'); 192 | process.exit(0); 193 | }); 194 | } 195 | 196 | main().catch((err) => { 197 | console.error('[Test] Unhandled main error =>', err); 198 | process.exit(1); 199 | }); 200 | -------------------------------------------------------------------------------- /src/spaces/types.ts: -------------------------------------------------------------------------------- 1 | // src/types.ts 2 | 3 | import { Space } from './core/Space'; 4 | 5 | export interface AudioData { 6 | bitsPerSample: number; // e.g., 16 7 | sampleRate: number; // e.g., 48000 8 | channelCount: number; // e.g., 1 for mono, 2 for stereo 9 | numberOfFrames: number; // how many samples per channel 10 | samples: Int16Array; // the raw PCM data 11 | } 12 | 13 | export interface AudioDataWithUser extends AudioData { 14 | userId: string; // The ID of the speaker or user 15 | } 16 | 17 | export interface SpeakerRequest { 18 | userId: string; 19 | username: string; 20 | displayName: string; 21 | sessionUUID: string; 22 | } 23 | 24 | export interface OccupancyUpdate { 25 | occupancy: number; 26 | totalParticipants: number; 27 | } 28 | 29 | export interface GuestReaction { 30 | displayName: string; 31 | emoji: string; 32 | } 33 | 34 | export interface BroadcastCreated { 35 | room_id: string; 36 | credential: string; 37 | stream_name: string; 38 | webrtc_gw_url: string; 39 | broadcast: { 40 | user_id: string; 41 | twitter_id: string; 42 | media_key: string; 43 | }; 44 | access_token: string; 45 | endpoint: string; 46 | share_url: string; 47 | stream_url: string; 48 | } 49 | 50 | export interface TurnServersInfo { 51 | ttl: string; 52 | username: string; 53 | password: string; 54 | uris: string[]; 55 | } 56 | 57 | export interface Plugin { 58 | /** 59 | * onAttach is called immediately when .use(plugin) is invoked, 60 | * passing the Space instance (if needed for immediate usage). 61 | */ 62 | onAttach?(space: Space): void; 63 | 64 | /** 65 | * init is called once the Space has *fully* initialized (Janus, broadcast, etc.) 66 | * so the plugin can get references to Janus or final config, etc. 67 | */ 68 | init?(params: { space: Space; pluginConfig?: Record }): void; 69 | 70 | onAudioData?(data: AudioDataWithUser): void; 71 | cleanup?(): void; 72 | } 73 | 74 | export interface PluginRegistration { 75 | plugin: Plugin; 76 | config?: Record; 77 | } 78 | 79 | export interface SpeakerInfo { 80 | userId: string; 81 | sessionUUID: string; 82 | janusParticipantId?: number; 83 | } 84 | -------------------------------------------------------------------------------- /src/spaces/utils.ts: -------------------------------------------------------------------------------- 1 | // src/utils.ts 2 | 3 | import { Headers } from 'headers-polyfill'; 4 | import type { BroadcastCreated, TurnServersInfo } from './types'; 5 | 6 | export async function authorizeToken(cookie: string): Promise { 7 | const headers = new Headers({ 8 | 'X-Periscope-User-Agent': 'Twitter/m5', 9 | 'Content-Type': 'application/json', 10 | 'X-Idempotence': Date.now().toString(), 11 | Referer: 'https://x.com/', 12 | 'X-Attempt': '1', 13 | }); 14 | 15 | const resp = await fetch('https://proxsee.pscp.tv/api/v2/authorizeToken', { 16 | method: 'POST', 17 | headers, 18 | body: JSON.stringify({ 19 | service: 'guest', 20 | cookie: cookie, 21 | }), 22 | }); 23 | 24 | if (!resp.ok) { 25 | throw new Error(`Failed to authorize token => ${resp.status}`); 26 | } 27 | 28 | const data = (await resp.json()) as { authorization_token: string }; 29 | if (!data.authorization_token) { 30 | throw new Error('authorizeToken: Missing authorization_token in response'); 31 | } 32 | 33 | return data.authorization_token; 34 | } 35 | 36 | export async function publishBroadcast(params: { 37 | title: string; 38 | broadcast: BroadcastCreated; 39 | cookie: string; 40 | janusSessionId?: number; 41 | janusHandleId?: number; 42 | janusPublisherId?: number; 43 | }) { 44 | const headers = new Headers({ 45 | 'X-Periscope-User-Agent': 'Twitter/m5', 46 | 'Content-Type': 'application/json', 47 | Referer: 'https://x.com/', 48 | 'X-Idempotence': Date.now().toString(), 49 | 'X-Attempt': '1', 50 | }); 51 | 52 | await fetch('https://proxsee.pscp.tv/api/v2/publishBroadcast', { 53 | method: 'POST', 54 | headers, 55 | body: JSON.stringify({ 56 | accept_guests: true, 57 | broadcast_id: params.broadcast.room_id, 58 | webrtc_handle_id: params.janusHandleId, 59 | webrtc_session_id: params.janusSessionId, 60 | janus_publisher_id: params.janusPublisherId, 61 | janus_room_id: params.broadcast.room_id, 62 | cookie: params.cookie, 63 | status: params.title, 64 | conversation_controls: 0, 65 | }), 66 | }); 67 | } 68 | 69 | export async function getTurnServers(cookie: string): Promise { 70 | const headers = new Headers({ 71 | 'X-Periscope-User-Agent': 'Twitter/m5', 72 | 'Content-Type': 'application/json', 73 | Referer: 'https://x.com/', 74 | 'X-Idempotence': Date.now().toString(), 75 | 'X-Attempt': '1', 76 | }); 77 | 78 | const resp = await fetch('https://proxsee.pscp.tv/api/v2/turnServers', { 79 | method: 'POST', 80 | headers, 81 | body: JSON.stringify({ cookie }), 82 | }); 83 | if (!resp.ok) throw new Error('Failed to get turn servers => ' + resp.status); 84 | return resp.json(); 85 | } 86 | 87 | /** 88 | * Get region from signer.pscp.tv 89 | */ 90 | export async function getRegion(): Promise { 91 | const resp = await fetch('https://signer.pscp.tv/region', { 92 | method: 'POST', 93 | headers: { 94 | 'Content-Type': 'application/json', 95 | Referer: 'https://x.com', 96 | }, 97 | body: JSON.stringify({}), 98 | }); 99 | if (!resp.ok) { 100 | throw new Error(`Failed to get region => ${resp.status}`); 101 | } 102 | const data = (await resp.json()) as { region: string }; 103 | return data.region; 104 | } 105 | 106 | /** 107 | * Create broadcast on Periscope 108 | */ 109 | export async function createBroadcast(params: { 110 | description?: string; 111 | languages?: string[]; 112 | cookie: string; 113 | region: string; 114 | }): Promise { 115 | const headers = new Headers({ 116 | 'X-Periscope-User-Agent': 'Twitter/m5', 117 | 'Content-Type': 'application/json', 118 | 'X-Idempotence': Date.now().toString(), 119 | Referer: 'https://x.com/', 120 | 'X-Attempt': '1', 121 | }); 122 | 123 | const resp = await fetch('https://proxsee.pscp.tv/api/v2/createBroadcast', { 124 | method: 'POST', 125 | headers, 126 | body: JSON.stringify({ 127 | app_component: 'audio-room', 128 | content_type: 'visual_audio', 129 | cookie: params.cookie, 130 | conversation_controls: 0, 131 | description: params.description || '', 132 | height: 1080, 133 | is_360: false, 134 | is_space_available_for_replay: false, 135 | is_webrtc: true, 136 | languages: params.languages ?? [], 137 | region: params.region, 138 | width: 1920, 139 | }), 140 | }); 141 | 142 | if (!resp.ok) { 143 | const text = await resp.text(); 144 | throw new Error(`Failed to create broadcast => ${resp.status} ${text}`); 145 | } 146 | 147 | const data = await resp.json(); 148 | return data as BroadcastCreated; 149 | } 150 | -------------------------------------------------------------------------------- /src/test-utils.ts: -------------------------------------------------------------------------------- 1 | import { ProxyAgent,setGlobalDispatcher } from 'undici'; 2 | import { Scraper } from './scraper'; 3 | import fs from 'fs'; 4 | 5 | export interface ScraperTestOptions { 6 | /** 7 | * Authentication method preference for the scraper. 8 | * - 'api': Use Twitter API keys and tokens. 9 | * - 'cookies': Resume session using cookies. 10 | * - 'password': Use username/password for login. 11 | * - 'anonymous': No authentication. 12 | */ 13 | authMethod: 'api' | 'cookies' | 'password' | 'anonymous'; 14 | } 15 | 16 | export async function getScraper( 17 | options: Partial = { authMethod: 'cookies' }, 18 | ) { 19 | const username = process.env['TWITTER_USERNAME']; 20 | const password = process.env['TWITTER_PASSWORD']; 21 | const email = process.env['TWITTER_EMAIL']; 22 | const twoFactorSecret = process.env['TWITTER_2FA_SECRET']; 23 | 24 | const apiKey = process.env['TWITTER_API_KEY']; 25 | const apiSecretKey = process.env['TWITTER_API_SECRET_KEY']; 26 | const accessToken = process.env['TWITTER_ACCESS_TOKEN']; 27 | const accessTokenSecret = process.env['TWITTER_ACCESS_TOKEN_SECRET']; 28 | 29 | let cookiesArray: any = null; 30 | 31 | // try to read cookies by reading cookies.json with fs and parsing 32 | // check if cookies.json exists 33 | if (!fs.existsSync('./cookies.json')) { 34 | console.error( 35 | 'cookies.json not found, using password auth - this is NOT recommended!', 36 | ); 37 | } else { 38 | try { 39 | const cookiesText = fs.readFileSync('./cookies.json', 'utf8'); 40 | cookiesArray = JSON.parse(cookiesText); 41 | } catch (e) { 42 | console.error('Error parsing cookies.json', e); 43 | } 44 | } 45 | 46 | const cookieStrings = cookiesArray?.map( 47 | (cookie: any) => 48 | `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${ 49 | cookie.path 50 | }; ${cookie.secure ? 'Secure' : ''}; ${ 51 | cookie.httpOnly ? 'HttpOnly' : '' 52 | }; SameSite=${cookie.sameSite || 'Lax'}`, 53 | ); 54 | 55 | const proxyUrl = process.env['PROXY_URL']; 56 | let agent: any; 57 | 58 | if ( 59 | options.authMethod === 'cookies' && 60 | (!cookieStrings || cookieStrings.length === 0) 61 | ) { 62 | console.warn( 63 | 'TWITTER_COOKIES variable is not defined, reverting to password auth (not recommended)', 64 | ); 65 | options.authMethod = 'password'; 66 | } 67 | 68 | if (options.authMethod === 'password' && !(username && password)) { 69 | throw new Error( 70 | 'TWITTER_USERNAME and TWITTER_PASSWORD variables must be defined.', 71 | ); 72 | } 73 | 74 | if (proxyUrl) { 75 | // Parse the proxy URL 76 | const url = new URL(proxyUrl); 77 | const username = url.username; 78 | const password = url.password; 79 | 80 | // Strip auth from URL if present 81 | url.username = ''; 82 | url.password = ''; 83 | 84 | const agentOptions: any = { 85 | uri: url.toString(), 86 | requestTls: { 87 | rejectUnauthorized: false, 88 | }, 89 | }; 90 | 91 | // Add Basic auth if credentials exist 92 | if (username && password) { 93 | agentOptions.token = `Basic ${Buffer.from( 94 | `${username}:${password}`, 95 | ).toString('base64')}`; 96 | } 97 | 98 | agent = new ProxyAgent(agentOptions); 99 | 100 | setGlobalDispatcher(agent) 101 | } 102 | 103 | const scraper = new Scraper({ 104 | transform: { 105 | request: (input, init) => { 106 | if (agent) { 107 | return [input, { ...init, dispatcher: agent }]; 108 | } 109 | return [input, init]; 110 | }, 111 | }, 112 | }); 113 | 114 | if ( 115 | options.authMethod === 'api' && 116 | username && 117 | password && 118 | apiKey && 119 | apiSecretKey && 120 | accessToken && 121 | accessTokenSecret 122 | ) { 123 | await scraper.login( 124 | username, 125 | password, 126 | email, 127 | twoFactorSecret, 128 | apiKey, 129 | apiSecretKey, 130 | accessToken, 131 | accessTokenSecret, 132 | ); 133 | } else if (options.authMethod === 'cookies' && cookieStrings?.length) { 134 | await scraper.setCookies(cookieStrings); 135 | } else if (options.authMethod === 'password' && username && password) { 136 | await scraper.login(username, password, email, twoFactorSecret); 137 | } else { 138 | console.warn( 139 | 'No valid authentication method available. Ensure at least one of the following is configured: API credentials, cookies, or username/password.', 140 | ); 141 | } 142 | 143 | return scraper; 144 | } 145 | -------------------------------------------------------------------------------- /src/timeline-async.ts: -------------------------------------------------------------------------------- 1 | import { Profile } from './profile'; 2 | import { Tweet } from './tweets'; 3 | 4 | export interface FetchProfilesResponse { 5 | profiles: Profile[]; 6 | next?: string; 7 | } 8 | 9 | export type FetchProfiles = ( 10 | query: string, 11 | maxProfiles: number, 12 | cursor: string | undefined, 13 | ) => Promise; 14 | 15 | export interface FetchTweetsResponse { 16 | tweets: Tweet[]; 17 | next?: string; 18 | } 19 | 20 | export type FetchTweets = ( 21 | query: string, 22 | maxTweets: number, 23 | cursor: string | undefined, 24 | ) => Promise; 25 | 26 | export async function* getUserTimeline( 27 | query: string, 28 | maxProfiles: number, 29 | fetchFunc: FetchProfiles, 30 | ): AsyncGenerator { 31 | let nProfiles = 0; 32 | let cursor: string | undefined = undefined; 33 | let consecutiveEmptyBatches = 0; 34 | while (nProfiles < maxProfiles) { 35 | const batch: FetchProfilesResponse = await fetchFunc( 36 | query, 37 | maxProfiles, 38 | cursor, 39 | ); 40 | 41 | const { profiles, next } = batch; 42 | cursor = next; 43 | 44 | if (profiles.length === 0) { 45 | consecutiveEmptyBatches++; 46 | if (consecutiveEmptyBatches > 5) break; 47 | } else consecutiveEmptyBatches = 0; 48 | 49 | for (const profile of profiles) { 50 | if (nProfiles < maxProfiles) yield profile; 51 | else break; 52 | nProfiles++; 53 | } 54 | 55 | if (!next) break; 56 | } 57 | } 58 | 59 | export async function* getTweetTimeline( 60 | query: string, 61 | maxTweets: number, 62 | fetchFunc: FetchTweets, 63 | ): AsyncGenerator { 64 | let nTweets = 0; 65 | let cursor: string | undefined = undefined; 66 | while (nTweets < maxTweets) { 67 | const batch: FetchTweetsResponse = await fetchFunc( 68 | query, 69 | maxTweets, 70 | cursor, 71 | ); 72 | 73 | const { tweets, next } = batch; 74 | 75 | if (tweets.length === 0) { 76 | break; 77 | } 78 | 79 | for (const tweet of tweets) { 80 | if (nTweets < maxTweets) { 81 | cursor = next; 82 | yield tweet; 83 | } else { 84 | break; 85 | } 86 | 87 | nTweets++; 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/timeline-following.ts: -------------------------------------------------------------------------------- 1 | import { requestApi } from './api'; 2 | import { TwitterAuth } from './auth'; 3 | import { ApiError } from './errors'; 4 | import { TimelineInstruction } from './timeline-v2'; 5 | 6 | export interface HomeLatestTimelineResponse { 7 | data?: { 8 | home: { 9 | home_timeline_urt: { 10 | instructions: TimelineInstruction[]; 11 | }; 12 | }; 13 | }; 14 | } 15 | 16 | export async function fetchFollowingTimeline( 17 | count: number, 18 | seenTweetIds: string[], 19 | auth: TwitterAuth, 20 | ): Promise { 21 | const variables = { 22 | count, 23 | includePromotedContent: true, 24 | latestControlAvailable: true, 25 | requestContext: 'launch', 26 | seenTweetIds, 27 | }; 28 | 29 | const features = { 30 | profile_label_improvements_pcf_label_in_post_enabled: true, 31 | rweb_tipjar_consumption_enabled: true, 32 | responsive_web_graphql_exclude_directive_enabled: true, 33 | verified_phone_label_enabled: false, 34 | creator_subscriptions_tweet_preview_api_enabled: true, 35 | responsive_web_graphql_timeline_navigation_enabled: true, 36 | responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, 37 | communities_web_enable_tweet_community_results_fetch: true, 38 | c9s_tweet_anatomy_moderator_badge_enabled: true, 39 | articles_preview_enabled: true, 40 | responsive_web_edit_tweet_api_enabled: true, 41 | graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, 42 | view_counts_everywhere_api_enabled: true, 43 | longform_notetweets_consumption_enabled: true, 44 | responsive_web_twitter_article_tweet_consumption_enabled: true, 45 | tweet_awards_web_tipping_enabled: false, 46 | creator_subscriptions_quote_tweet_preview_enabled: false, 47 | freedom_of_speech_not_reach_fetch_enabled: true, 48 | standardized_nudges_misinfo: true, 49 | tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: 50 | true, 51 | rweb_video_timestamps_enabled: true, 52 | longform_notetweets_rich_text_read_enabled: true, 53 | longform_notetweets_inline_media_enabled: true, 54 | responsive_web_enhance_cards_enabled: false, 55 | }; 56 | 57 | const res = await requestApi( 58 | `https://x.com/i/api/graphql/K0X1xbCZUjttdK8RazKAlw/HomeLatestTimeline?variables=${encodeURIComponent( 59 | JSON.stringify(variables), 60 | )}&features=${encodeURIComponent(JSON.stringify(features))}`, 61 | auth, 62 | 'GET', 63 | ); 64 | 65 | if (!res.success) { 66 | if (res.err instanceof ApiError) { 67 | console.error('Error details:', res.err.data); 68 | } 69 | throw res.err; 70 | } 71 | 72 | const home = res.value?.data?.home.home_timeline_urt?.instructions; 73 | 74 | if (!home) { 75 | return []; 76 | } 77 | 78 | const entries: any[] = []; 79 | 80 | for (const instruction of home) { 81 | if (instruction.type === 'TimelineAddEntries') { 82 | for (const entry of instruction.entries ?? []) { 83 | entries.push(entry); 84 | } 85 | } 86 | } 87 | // get the itemContnent from each entry 88 | const tweets = entries 89 | .map((entry) => entry.content.itemContent?.tweet_results?.result) 90 | .filter((tweet) => tweet !== undefined); 91 | 92 | return tweets; 93 | } 94 | -------------------------------------------------------------------------------- /src/timeline-home.ts: -------------------------------------------------------------------------------- 1 | import { requestApi } from './api'; 2 | import { TwitterAuth } from './auth'; 3 | import { ApiError } from './errors'; 4 | import { TimelineInstruction } from './timeline-v2'; 5 | 6 | export interface HomeTimelineResponse { 7 | data?: { 8 | home: { 9 | home_timeline_urt: { 10 | instructions: TimelineInstruction[]; 11 | }; 12 | }; 13 | }; 14 | } 15 | 16 | export async function fetchHomeTimeline( 17 | count: number, 18 | seenTweetIds: string[], 19 | auth: TwitterAuth, 20 | ): Promise { 21 | const variables = { 22 | count, 23 | includePromotedContent: true, 24 | latestControlAvailable: true, 25 | requestContext: 'launch', 26 | withCommunity: true, 27 | seenTweetIds, 28 | }; 29 | 30 | const features = { 31 | rweb_tipjar_consumption_enabled: true, 32 | responsive_web_graphql_exclude_directive_enabled: true, 33 | verified_phone_label_enabled: false, 34 | creator_subscriptions_tweet_preview_api_enabled: true, 35 | responsive_web_graphql_timeline_navigation_enabled: true, 36 | responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, 37 | communities_web_enable_tweet_community_results_fetch: true, 38 | c9s_tweet_anatomy_moderator_badge_enabled: true, 39 | articles_preview_enabled: true, 40 | responsive_web_edit_tweet_api_enabled: true, 41 | graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, 42 | view_counts_everywhere_api_enabled: true, 43 | longform_notetweets_consumption_enabled: true, 44 | responsive_web_twitter_article_tweet_consumption_enabled: true, 45 | tweet_awards_web_tipping_enabled: false, 46 | creator_subscriptions_quote_tweet_preview_enabled: false, 47 | freedom_of_speech_not_reach_fetch_enabled: true, 48 | standardized_nudges_misinfo: true, 49 | tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: 50 | true, 51 | rweb_video_timestamps_enabled: true, 52 | longform_notetweets_rich_text_read_enabled: true, 53 | longform_notetweets_inline_media_enabled: true, 54 | responsive_web_enhance_cards_enabled: false, 55 | }; 56 | 57 | const res = await requestApi( 58 | `https://x.com/i/api/graphql/HJFjzBgCs16TqxewQOeLNg/HomeTimeline?variables=${encodeURIComponent( 59 | JSON.stringify(variables), 60 | )}&features=${encodeURIComponent(JSON.stringify(features))}`, 61 | auth, 62 | 'GET', 63 | ); 64 | 65 | if (!res.success) { 66 | if (res.err instanceof ApiError) { 67 | console.error('Error details:', res.err.data); 68 | } 69 | throw res.err; 70 | } 71 | 72 | const home = res.value?.data?.home.home_timeline_urt?.instructions; 73 | 74 | if (!home) { 75 | return []; 76 | } 77 | 78 | const entries: any[] = []; 79 | 80 | for (const instruction of home) { 81 | if (instruction.type === 'TimelineAddEntries') { 82 | for (const entry of instruction.entries ?? []) { 83 | entries.push(entry); 84 | } 85 | } 86 | } 87 | // get the itemContnent from each entry 88 | const tweets = entries 89 | .map((entry) => entry.content.itemContent?.tweet_results?.result) 90 | .filter((tweet) => tweet !== undefined); 91 | 92 | return tweets; 93 | } 94 | -------------------------------------------------------------------------------- /src/timeline-list.ts: -------------------------------------------------------------------------------- 1 | import { QueryTweetsResponse } from './timeline-v1'; 2 | import { parseAndPush, TimelineEntryRaw } from './timeline-v2'; 3 | import { Tweet } from './tweets'; 4 | 5 | export interface ListTimeline { 6 | data?: { 7 | list?: { 8 | tweets_timeline?: { 9 | timeline?: { 10 | instructions?: { 11 | entries?: TimelineEntryRaw[]; 12 | entry?: TimelineEntryRaw; 13 | type?: string; 14 | }[]; 15 | }; 16 | }; 17 | }; 18 | }; 19 | } 20 | 21 | export function parseListTimelineTweets( 22 | timeline: ListTimeline, 23 | ): QueryTweetsResponse { 24 | let bottomCursor: string | undefined; 25 | let topCursor: string | undefined; 26 | const tweets: Tweet[] = []; 27 | const instructions = 28 | timeline.data?.list?.tweets_timeline?.timeline?.instructions ?? []; 29 | for (const instruction of instructions) { 30 | const entries = instruction.entries ?? []; 31 | 32 | for (const entry of entries) { 33 | const entryContent = entry.content; 34 | if (!entryContent) continue; 35 | 36 | if (entryContent.cursorType === 'Bottom') { 37 | bottomCursor = entryContent.value; 38 | continue; 39 | } else if (entryContent.cursorType === 'Top') { 40 | topCursor = entryContent.value; 41 | continue; 42 | } 43 | 44 | const idStr = entry.entryId; 45 | if ( 46 | !idStr.startsWith('tweet') && 47 | !idStr.startsWith('list-conversation') 48 | ) { 49 | continue; 50 | } 51 | 52 | if (entryContent.itemContent) { 53 | parseAndPush(tweets, entryContent.itemContent, idStr); 54 | } else if (entryContent.items) { 55 | for (const contentItem of entryContent.items) { 56 | if ( 57 | contentItem.item && 58 | contentItem.item.itemContent && 59 | contentItem.entryId 60 | ) { 61 | parseAndPush( 62 | tweets, 63 | contentItem.item.itemContent, 64 | contentItem.entryId.split('tweet-')[1], 65 | ); 66 | } 67 | } 68 | } 69 | } 70 | } 71 | 72 | return { tweets, next: bottomCursor, previous: topCursor }; 73 | } 74 | -------------------------------------------------------------------------------- /src/timeline-relationship.ts: -------------------------------------------------------------------------------- 1 | import { Profile, parseProfile } from './profile'; 2 | import { QueryProfilesResponse } from './timeline-v1'; 3 | import { TimelineUserResultRaw } from './timeline-v2'; 4 | 5 | export interface RelationshipEntryItemContentRaw { 6 | itemType?: string; 7 | userDisplayType?: string; 8 | user_results?: { 9 | result?: TimelineUserResultRaw; 10 | }; 11 | } 12 | 13 | export interface RelationshipEntryRaw { 14 | entryId: string; 15 | sortIndex: string; 16 | content?: { 17 | cursorType?: string; 18 | entryType?: string; 19 | __typename?: string; 20 | value?: string; 21 | itemContent?: RelationshipEntryItemContentRaw; 22 | }; 23 | } 24 | 25 | export interface RelationshipTimeline { 26 | data?: { 27 | user?: { 28 | result?: { 29 | timeline?: { 30 | timeline?: { 31 | instructions?: { 32 | entries?: RelationshipEntryRaw[]; 33 | entry?: RelationshipEntryRaw; 34 | type?: string; 35 | }[]; 36 | }; 37 | }; 38 | }; 39 | }; 40 | }; 41 | } 42 | 43 | export function parseRelationshipTimeline( 44 | timeline: RelationshipTimeline, 45 | ): QueryProfilesResponse { 46 | let bottomCursor: string | undefined; 47 | let topCursor: string | undefined; 48 | const profiles: Profile[] = []; 49 | const instructions = 50 | timeline.data?.user?.result?.timeline?.timeline?.instructions ?? []; 51 | 52 | for (const instruction of instructions) { 53 | if ( 54 | instruction.type === 'TimelineAddEntries' || 55 | instruction.type === 'TimelineReplaceEntry' 56 | ) { 57 | if (instruction.entry?.content?.cursorType === 'Bottom') { 58 | bottomCursor = instruction.entry.content.value; 59 | continue; 60 | } 61 | 62 | if (instruction.entry?.content?.cursorType === 'Top') { 63 | topCursor = instruction.entry.content.value; 64 | continue; 65 | } 66 | 67 | const entries = instruction.entries ?? []; 68 | for (const entry of entries) { 69 | const itemContent = entry.content?.itemContent; 70 | if (itemContent?.userDisplayType === 'User') { 71 | const userResultRaw = itemContent.user_results?.result; 72 | 73 | if (userResultRaw?.legacy) { 74 | const profile = parseProfile( 75 | userResultRaw.legacy, 76 | userResultRaw.is_blue_verified, 77 | ); 78 | 79 | if (!profile.userId) { 80 | profile.userId = userResultRaw.rest_id; 81 | } 82 | 83 | profiles.push(profile); 84 | } 85 | } else if (entry.content?.cursorType === 'Bottom') { 86 | bottomCursor = entry.content.value; 87 | } else if (entry.content?.cursorType === 'Top') { 88 | topCursor = entry.content.value; 89 | } 90 | } 91 | } 92 | } 93 | 94 | return { profiles, next: bottomCursor, previous: topCursor }; 95 | } 96 | -------------------------------------------------------------------------------- /src/timeline-search.ts: -------------------------------------------------------------------------------- 1 | import { Profile, parseProfile } from './profile'; 2 | import { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1'; 3 | import { SearchEntryRaw, parseLegacyTweet } from './timeline-v2'; 4 | import { Tweet } from './tweets'; 5 | 6 | export interface SearchTimeline { 7 | data?: { 8 | search_by_raw_query?: { 9 | search_timeline?: { 10 | timeline?: { 11 | instructions?: { 12 | entries?: SearchEntryRaw[]; 13 | entry?: SearchEntryRaw; 14 | type?: string; 15 | }[]; 16 | }; 17 | }; 18 | }; 19 | }; 20 | } 21 | 22 | export function parseSearchTimelineTweets( 23 | timeline: SearchTimeline, 24 | ): QueryTweetsResponse { 25 | let bottomCursor: string | undefined; 26 | let topCursor: string | undefined; 27 | const tweets: Tweet[] = []; 28 | const instructions = 29 | timeline.data?.search_by_raw_query?.search_timeline?.timeline 30 | ?.instructions ?? []; 31 | for (const instruction of instructions) { 32 | if ( 33 | instruction.type === 'TimelineAddEntries' || 34 | instruction.type === 'TimelineReplaceEntry' 35 | ) { 36 | if (instruction.entry?.content?.cursorType === 'Bottom') { 37 | bottomCursor = instruction.entry.content.value; 38 | continue; 39 | } else if (instruction.entry?.content?.cursorType === 'Top') { 40 | topCursor = instruction.entry.content.value; 41 | continue; 42 | } 43 | 44 | const entries = instruction.entries ?? []; 45 | for (const entry of entries) { 46 | const itemContent = entry.content?.itemContent; 47 | if (itemContent?.tweetDisplayType === 'Tweet') { 48 | const tweetResultRaw = itemContent.tweet_results?.result; 49 | const tweetResult = parseLegacyTweet( 50 | tweetResultRaw?.core?.user_results?.result?.legacy, 51 | tweetResultRaw?.legacy, 52 | ); 53 | 54 | if (tweetResult.success) { 55 | if (!tweetResult.tweet.views && tweetResultRaw?.views?.count) { 56 | const views = parseInt(tweetResultRaw.views.count); 57 | if (!isNaN(views)) { 58 | tweetResult.tweet.views = views; 59 | } 60 | } 61 | 62 | tweets.push(tweetResult.tweet); 63 | } 64 | } else if (entry.content?.cursorType === 'Bottom') { 65 | bottomCursor = entry.content.value; 66 | } else if (entry.content?.cursorType === 'Top') { 67 | topCursor = entry.content.value; 68 | } 69 | } 70 | } 71 | } 72 | 73 | return { tweets, next: bottomCursor, previous: topCursor }; 74 | } 75 | 76 | export function parseSearchTimelineUsers( 77 | timeline: SearchTimeline, 78 | ): QueryProfilesResponse { 79 | let bottomCursor: string | undefined; 80 | let topCursor: string | undefined; 81 | const profiles: Profile[] = []; 82 | const instructions = 83 | timeline.data?.search_by_raw_query?.search_timeline?.timeline 84 | ?.instructions ?? []; 85 | 86 | for (const instruction of instructions) { 87 | if ( 88 | instruction.type === 'TimelineAddEntries' || 89 | instruction.type === 'TimelineReplaceEntry' 90 | ) { 91 | if (instruction.entry?.content?.cursorType === 'Bottom') { 92 | bottomCursor = instruction.entry.content.value; 93 | continue; 94 | } else if (instruction.entry?.content?.cursorType === 'Top') { 95 | topCursor = instruction.entry.content.value; 96 | continue; 97 | } 98 | 99 | const entries = instruction.entries ?? []; 100 | for (const entry of entries) { 101 | const itemContent = entry.content?.itemContent; 102 | if (itemContent?.userDisplayType === 'User') { 103 | const userResultRaw = itemContent.user_results?.result; 104 | 105 | if (userResultRaw?.legacy) { 106 | const profile = parseProfile( 107 | userResultRaw.legacy, 108 | userResultRaw.is_blue_verified, 109 | ); 110 | 111 | if (!profile.userId) { 112 | profile.userId = userResultRaw.rest_id; 113 | } 114 | 115 | profiles.push(profile); 116 | } 117 | } else if (entry.content?.cursorType === 'Bottom') { 118 | bottomCursor = entry.content.value; 119 | } else if (entry.content?.cursorType === 'Top') { 120 | topCursor = entry.content.value; 121 | } 122 | } 123 | } 124 | } 125 | 126 | return { profiles, next: bottomCursor, previous: topCursor }; 127 | } 128 | -------------------------------------------------------------------------------- /src/timeline-tweet-util.ts: -------------------------------------------------------------------------------- 1 | import { LegacyTweetRaw, TimelineMediaExtendedRaw } from './timeline-v1'; 2 | import { Photo, Video } from './tweets'; 3 | import { isFieldDefined, NonNullableField } from './type-util'; 4 | 5 | const reHashtag = /\B(\#\S+\b)/g; 6 | const reCashtag = /\B(\$\S+\b)/g; 7 | const reTwitterUrl = /https:(\/\/t\.co\/([A-Za-z0-9]|[A-Za-z]){10})/g; 8 | const reUsername = /\B(\@\S{1,15}\b)/g; 9 | 10 | export function parseMediaGroups(media: TimelineMediaExtendedRaw[]): { 11 | sensitiveContent?: boolean; 12 | photos: Photo[]; 13 | videos: Video[]; 14 | } { 15 | const photos: Photo[] = []; 16 | const videos: Video[] = []; 17 | let sensitiveContent: boolean | undefined = undefined; 18 | 19 | for (const m of media 20 | .filter(isFieldDefined('id_str')) 21 | .filter(isFieldDefined('media_url_https'))) { 22 | if (m.type === 'photo') { 23 | photos.push({ 24 | id: m.id_str, 25 | url: m.media_url_https, 26 | alt_text: m.ext_alt_text, 27 | }); 28 | } else if (m.type === 'video') { 29 | videos.push(parseVideo(m)); 30 | } 31 | 32 | const sensitive = m.ext_sensitive_media_warning; 33 | if (sensitive != null) { 34 | sensitiveContent = 35 | sensitive.adult_content || 36 | sensitive.graphic_violence || 37 | sensitive.other; 38 | } 39 | } 40 | 41 | return { sensitiveContent, photos, videos }; 42 | } 43 | 44 | function parseVideo( 45 | m: NonNullableField, 46 | ): Video { 47 | const video: Video = { 48 | id: m.id_str, 49 | preview: m.media_url_https, 50 | }; 51 | 52 | let maxBitrate = 0; 53 | const variants = m.video_info?.variants ?? []; 54 | for (const variant of variants) { 55 | const bitrate = variant.bitrate; 56 | if (bitrate != null && bitrate > maxBitrate && variant.url != null) { 57 | let variantUrl = variant.url; 58 | const stringStart = 0; 59 | const tagSuffixIdx = variantUrl.indexOf('?tag=10'); 60 | if (tagSuffixIdx !== -1) { 61 | variantUrl = variantUrl.substring(stringStart, tagSuffixIdx + 1); 62 | } 63 | 64 | video.url = variantUrl; 65 | maxBitrate = bitrate; 66 | } 67 | } 68 | 69 | return video; 70 | } 71 | 72 | export function reconstructTweetHtml( 73 | tweet: LegacyTweetRaw, 74 | photos: Photo[], 75 | videos: Video[], 76 | ): string { 77 | const media: string[] = []; 78 | 79 | // HTML parsing with regex :) 80 | let html = tweet.full_text ?? ''; 81 | 82 | html = html.replace(reHashtag, linkHashtagHtml); 83 | html = html.replace(reCashtag, linkCashtagHtml); 84 | html = html.replace(reUsername, linkUsernameHtml); 85 | html = html.replace(reTwitterUrl, unwrapTcoUrlHtml(tweet, media)); 86 | 87 | for (const { url } of photos) { 88 | if (media.indexOf(url) !== -1) { 89 | continue; 90 | } 91 | 92 | html += `
`; 93 | } 94 | 95 | for (const { preview: url } of videos) { 96 | if (media.indexOf(url) !== -1) { 97 | continue; 98 | } 99 | 100 | html += `
`; 101 | } 102 | 103 | html = html.replace(/\n/g, '
'); 104 | 105 | return html; 106 | } 107 | 108 | function linkHashtagHtml(hashtag: string) { 109 | return `${hashtag}`; 113 | } 114 | 115 | function linkCashtagHtml(cashtag: string) { 116 | return `${cashtag}`; 120 | } 121 | 122 | function linkUsernameHtml(username: string) { 123 | return `${username}`; 127 | } 128 | 129 | function unwrapTcoUrlHtml(tweet: LegacyTweetRaw, foundedMedia: string[]) { 130 | return function (tco: string) { 131 | for (const entity of tweet.entities?.urls ?? []) { 132 | if (tco === entity.url && entity.expanded_url != null) { 133 | return `${tco}`; 134 | } 135 | } 136 | 137 | for (const entity of tweet.extended_entities?.media ?? []) { 138 | if (tco === entity.url && entity.media_url_https != null) { 139 | foundedMedia.push(entity.media_url_https); 140 | return `
`; 141 | } 142 | } 143 | 144 | return tco; 145 | }; 146 | } 147 | -------------------------------------------------------------------------------- /src/timeline-v1.ts: -------------------------------------------------------------------------------- 1 | import { LegacyUserRaw, parseProfile, Profile } from './profile'; 2 | import { parseMediaGroups, reconstructTweetHtml } from './timeline-tweet-util'; 3 | import { PlaceRaw, Tweet } from './tweets'; 4 | import { isFieldDefined } from './type-util'; 5 | 6 | export interface Hashtag { 7 | text?: string; 8 | } 9 | 10 | export interface TimelineUserMentionBasicRaw { 11 | id_str?: string; 12 | name?: string; 13 | screen_name?: string; 14 | } 15 | 16 | export interface TimelineMediaBasicRaw { 17 | media_url_https?: string; 18 | type?: string; 19 | url?: string; 20 | } 21 | 22 | export interface TimelineUrlBasicRaw { 23 | expanded_url?: string; 24 | url?: string; 25 | } 26 | 27 | export interface ExtSensitiveMediaWarningRaw { 28 | adult_content?: boolean; 29 | graphic_violence?: boolean; 30 | other?: boolean; 31 | } 32 | 33 | export interface VideoVariant { 34 | bitrate?: number; 35 | url?: string; 36 | } 37 | 38 | export interface VideoInfo { 39 | variants?: VideoVariant[]; 40 | } 41 | 42 | export interface TimelineMediaExtendedRaw { 43 | id_str?: string; 44 | media_url_https?: string; 45 | ext_sensitive_media_warning?: ExtSensitiveMediaWarningRaw; 46 | type?: string; 47 | url?: string; 48 | video_info?: VideoInfo; 49 | ext_alt_text: string | undefined; 50 | } 51 | 52 | export interface SearchResultRaw { 53 | rest_id?: string; 54 | __typename?: string; 55 | core?: { 56 | user_results?: { 57 | result?: { 58 | is_blue_verified?: boolean; 59 | legacy?: LegacyUserRaw; 60 | }; 61 | }; 62 | }; 63 | views?: { 64 | count?: string; 65 | }; 66 | note_tweet?: { 67 | note_tweet_results?: { 68 | result?: { 69 | text?: string; 70 | }; 71 | }; 72 | }; 73 | quoted_status_result?: { 74 | result?: SearchResultRaw; 75 | }; 76 | legacy?: LegacyTweetRaw; 77 | } 78 | 79 | export interface TimelineArticleResultRaw { 80 | id?: string; 81 | rest_id?: string; 82 | title?: string; 83 | preview_text?: string; 84 | cover_media?: { 85 | media_id?: string; 86 | media_info?: { 87 | original_img_url?: string; 88 | original_img_height?: number; 89 | original_img_width?: number; 90 | }; 91 | }; 92 | content_state?: { 93 | blocks?: { 94 | key?: string; 95 | data?: string; 96 | text?: string; 97 | entityRanges?: { 98 | key?: number; 99 | length?: number; 100 | offset?: number; 101 | }[]; 102 | }[]; 103 | }; 104 | entityMap?: { 105 | key?: string; 106 | value?: { 107 | type?: string; // LINK, MEDIA, TWEET 108 | mutability?: string; 109 | data?: { 110 | entityKey?: string; 111 | url?: string; 112 | tweetId?: string; 113 | mediaItems?: { 114 | localMediaId?: string; 115 | mediaCategory?: string; 116 | mediaId?: string; 117 | }[]; 118 | }; 119 | }; 120 | }[]; 121 | } 122 | 123 | export interface TimelineResultRaw { 124 | rest_id?: string; 125 | __typename?: string; 126 | core?: { 127 | user_results?: { 128 | result?: { 129 | is_blue_verified?: boolean; 130 | legacy?: LegacyUserRaw; 131 | }; 132 | }; 133 | }; 134 | views?: { 135 | count?: string; 136 | }; 137 | note_tweet?: { 138 | note_tweet_results?: { 139 | result?: { 140 | text?: string; 141 | }; 142 | }; 143 | }; 144 | article?: { 145 | article_results?: { 146 | result?: TimelineArticleResultRaw; 147 | }; 148 | }; 149 | quoted_status_result?: { 150 | result?: TimelineResultRaw; 151 | }; 152 | legacy?: LegacyTweetRaw; 153 | tweet?: TimelineResultRaw; 154 | } 155 | 156 | export interface LegacyTweetRaw { 157 | bookmark_count?: number; 158 | conversation_id_str?: string; 159 | created_at?: string; 160 | favorite_count?: number; 161 | full_text?: string; 162 | entities?: { 163 | hashtags?: Hashtag[]; 164 | media?: TimelineMediaBasicRaw[]; 165 | urls?: TimelineUrlBasicRaw[]; 166 | user_mentions?: TimelineUserMentionBasicRaw[]; 167 | }; 168 | extended_entities?: { 169 | media?: TimelineMediaExtendedRaw[]; 170 | }; 171 | id_str?: string; 172 | in_reply_to_status_id_str?: string; 173 | place?: PlaceRaw; 174 | reply_count?: number; 175 | retweet_count?: number; 176 | retweeted_status_id_str?: string; 177 | retweeted_status_result?: { 178 | result?: TimelineResultRaw; 179 | }; 180 | quoted_status_id_str?: string; 181 | time?: string; 182 | user_id_str?: string; 183 | ext_views?: { 184 | state?: string; 185 | count?: string; 186 | }; 187 | } 188 | 189 | export interface TimelineGlobalObjectsRaw { 190 | tweets?: { [key: string]: LegacyTweetRaw | undefined }; 191 | users?: { [key: string]: LegacyUserRaw | undefined }; 192 | } 193 | 194 | export interface TimelineDataRawCursor { 195 | value?: string; 196 | cursorType?: string; 197 | } 198 | 199 | export interface TimelineDataRawEntity { 200 | id?: string; 201 | } 202 | 203 | export interface TimelineDataRawModuleItem { 204 | clientEventInfo?: { 205 | details?: { 206 | guideDetails?: { 207 | transparentGuideDetails?: { 208 | trendMetadata?: { 209 | trendName?: string; 210 | }; 211 | }; 212 | }; 213 | }; 214 | }; 215 | } 216 | 217 | export interface TimelineDataRawAddEntry { 218 | content?: { 219 | item?: { 220 | content?: { 221 | tweet?: TimelineDataRawEntity; 222 | user?: TimelineDataRawEntity; 223 | }; 224 | }; 225 | operation?: { 226 | cursor?: TimelineDataRawCursor; 227 | }; 228 | timelineModule?: { 229 | items?: { 230 | item?: TimelineDataRawModuleItem; 231 | }[]; 232 | }; 233 | }; 234 | } 235 | 236 | export interface TimelineDataRawPinEntry { 237 | content?: { 238 | item?: { 239 | content?: { 240 | tweet?: TimelineDataRawEntity; 241 | }; 242 | }; 243 | }; 244 | } 245 | 246 | export interface TimelineDataRawReplaceEntry { 247 | content?: { 248 | operation?: { 249 | cursor?: TimelineDataRawCursor; 250 | }; 251 | }; 252 | } 253 | 254 | export interface TimelineDataRawInstruction { 255 | addEntries?: { 256 | entries?: TimelineDataRawAddEntry[]; 257 | }; 258 | pinEntry?: { 259 | entry?: TimelineDataRawPinEntry; 260 | }; 261 | replaceEntry?: { 262 | entry?: TimelineDataRawReplaceEntry; 263 | }; 264 | } 265 | 266 | export interface TimelineDataRaw { 267 | instructions?: TimelineDataRawInstruction[]; 268 | } 269 | 270 | export interface TimelineV1 { 271 | globalObjects?: TimelineGlobalObjectsRaw; 272 | timeline?: TimelineDataRaw; 273 | } 274 | 275 | export type ParseTweetResult = 276 | | { success: true; tweet: Tweet } 277 | | { success: false; err: Error }; 278 | 279 | function parseTimelineTweet( 280 | timeline: TimelineV1, 281 | id: string, 282 | ): ParseTweetResult { 283 | const tweets = timeline.globalObjects?.tweets ?? {}; 284 | const tweet = tweets[id]; 285 | if (tweet?.user_id_str == null) { 286 | return { 287 | success: false, 288 | err: new Error(`Tweet "${id}" was not found in the timeline object.`), 289 | }; 290 | } 291 | 292 | const users = timeline.globalObjects?.users ?? {}; 293 | const user = users[tweet.user_id_str]; 294 | if (user?.screen_name == null) { 295 | return { 296 | success: false, 297 | err: new Error(`User "${tweet.user_id_str}" has no username data.`), 298 | }; 299 | } 300 | 301 | const hashtags = tweet.entities?.hashtags ?? []; 302 | const mentions = tweet.entities?.user_mentions ?? []; 303 | const media = tweet.extended_entities?.media ?? []; 304 | const pinnedTweets = new Set( 305 | user.pinned_tweet_ids_str ?? [], 306 | ); 307 | const urls = tweet.entities?.urls ?? []; 308 | const { photos, videos, sensitiveContent } = parseMediaGroups(media); 309 | 310 | const tw: Tweet = { 311 | conversationId: tweet.conversation_id_str, 312 | id, 313 | hashtags: hashtags 314 | .filter(isFieldDefined('text')) 315 | .map((hashtag) => hashtag.text), 316 | likes: tweet.favorite_count, 317 | mentions: mentions.filter(isFieldDefined('id_str')).map((mention) => ({ 318 | id: mention.id_str, 319 | username: mention.screen_name, 320 | name: mention.name, 321 | })), 322 | name: user.name, 323 | permanentUrl: `https://twitter.com/${user.screen_name}/status/${id}`, 324 | photos, 325 | replies: tweet.reply_count, 326 | retweets: tweet.retweet_count, 327 | text: tweet.full_text, 328 | thread: [], 329 | urls: urls 330 | .filter(isFieldDefined('expanded_url')) 331 | .map((url) => url.expanded_url), 332 | userId: tweet.user_id_str, 333 | username: user.screen_name, 334 | videos, 335 | }; 336 | 337 | if (tweet.created_at) { 338 | tw.timeParsed = new Date(Date.parse(tweet.created_at)); 339 | tw.timestamp = Math.floor(tw.timeParsed.valueOf() / 1000); 340 | } 341 | 342 | if (tweet.place?.id) { 343 | tw.place = tweet.place; 344 | } 345 | 346 | if (tweet.quoted_status_id_str) { 347 | tw.isQuoted = true; 348 | tw.quotedStatusId = tweet.quoted_status_id_str; 349 | 350 | const quotedStatusResult = parseTimelineTweet( 351 | timeline, 352 | tweet.quoted_status_id_str, 353 | ); 354 | if (quotedStatusResult.success) { 355 | tw.quotedStatus = quotedStatusResult.tweet; 356 | } 357 | } 358 | 359 | if (tweet.in_reply_to_status_id_str) { 360 | tw.isReply = true; 361 | tw.inReplyToStatusId = tweet.in_reply_to_status_id_str; 362 | 363 | const replyStatusResult = parseTimelineTweet( 364 | timeline, 365 | tweet.in_reply_to_status_id_str, 366 | ); 367 | if (replyStatusResult.success) { 368 | tw.inReplyToStatus = replyStatusResult.tweet; 369 | } 370 | } 371 | 372 | if (tweet.retweeted_status_id_str != null) { 373 | tw.isRetweet = true; 374 | tw.retweetedStatusId = tweet.retweeted_status_id_str; 375 | 376 | const retweetedStatusResult = parseTimelineTweet( 377 | timeline, 378 | tweet.retweeted_status_id_str, 379 | ); 380 | if (retweetedStatusResult.success) { 381 | tw.retweetedStatus = retweetedStatusResult.tweet; 382 | } 383 | } 384 | 385 | const views = parseInt(tweet.ext_views?.count ?? ''); 386 | if (!isNaN(views)) { 387 | tw.views = views; 388 | } 389 | 390 | if (pinnedTweets.has(tweet.id_str)) { 391 | // TODO: Update tests so this can be assigned at the tweet declaration 392 | tw.isPin = true; 393 | } 394 | 395 | if (sensitiveContent) { 396 | // TODO: Update tests so this can be assigned at the tweet declaration 397 | tw.sensitiveContent = true; 398 | } 399 | 400 | tw.html = reconstructTweetHtml(tweet, tw.photos, tw.videos); 401 | 402 | return { success: true, tweet: tw }; 403 | } 404 | 405 | /** 406 | * A paginated tweets API response. The `next` field can be used to fetch the next page of results, 407 | * and the `previous` can be used to fetch the previous results (or results created after the 408 | * inital request) 409 | */ 410 | export interface QueryTweetsResponse { 411 | tweets: Tweet[]; 412 | next?: string; 413 | previous?: string; 414 | } 415 | 416 | export function parseTimelineTweetsV1( 417 | timeline: TimelineV1, 418 | ): QueryTweetsResponse { 419 | let bottomCursor: string | undefined; 420 | let topCursor: string | undefined; 421 | let pinnedTweet: Tweet | undefined; 422 | let orderedTweets: Tweet[] = []; 423 | for (const instruction of timeline.timeline?.instructions ?? []) { 424 | const { pinEntry, addEntries, replaceEntry } = instruction; 425 | 426 | // Handle pin instruction 427 | const pinnedTweetId = pinEntry?.entry?.content?.item?.content?.tweet?.id; 428 | if (pinnedTweetId != null) { 429 | const tweetResult = parseTimelineTweet(timeline, pinnedTweetId); 430 | if (tweetResult.success) { 431 | pinnedTweet = tweetResult.tweet; 432 | } 433 | } 434 | 435 | // Handle add instructions 436 | for (const { content } of addEntries?.entries ?? []) { 437 | const tweetId = content?.item?.content?.tweet?.id; 438 | if (tweetId != null) { 439 | const tweetResult = parseTimelineTweet(timeline, tweetId); 440 | if (tweetResult.success) { 441 | orderedTweets.push(tweetResult.tweet); 442 | } 443 | } 444 | 445 | const operation = content?.operation; 446 | if (operation?.cursor?.cursorType === 'Bottom') { 447 | bottomCursor = operation?.cursor?.value; 448 | } else if (operation?.cursor?.cursorType === 'Top') { 449 | topCursor = operation?.cursor?.value; 450 | } 451 | } 452 | 453 | // Handle replace instruction 454 | const operation = replaceEntry?.entry?.content?.operation; 455 | if (operation?.cursor?.cursorType === 'Bottom') { 456 | bottomCursor = operation.cursor.value; 457 | } else if (operation?.cursor?.cursorType === 'Top') { 458 | topCursor = operation.cursor.value; 459 | } 460 | } 461 | 462 | if (pinnedTweet != null && orderedTweets.length > 0) { 463 | orderedTweets = [pinnedTweet, ...orderedTweets]; 464 | } 465 | 466 | return { 467 | tweets: orderedTweets, 468 | next: bottomCursor, 469 | previous: topCursor, 470 | }; 471 | } 472 | 473 | /** 474 | * A paginated profiles API response. The `next` field can be used to fetch the next page of results. 475 | */ 476 | export interface QueryProfilesResponse { 477 | profiles: Profile[]; 478 | next?: string; 479 | previous?: string; 480 | } 481 | 482 | export function parseUsers(timeline: TimelineV1): QueryProfilesResponse { 483 | const users = new Map(); 484 | 485 | const userObjects = timeline.globalObjects?.users ?? {}; 486 | for (const id in userObjects) { 487 | const legacy = userObjects[id]; 488 | if (legacy == null) { 489 | continue; 490 | } 491 | 492 | const user = parseProfile(legacy); 493 | users.set(id, user); 494 | } 495 | 496 | let bottomCursor: string | undefined; 497 | let topCursor: string | undefined; 498 | const orderedProfiles: Profile[] = []; 499 | for (const instruction of timeline.timeline?.instructions ?? []) { 500 | for (const entry of instruction.addEntries?.entries ?? []) { 501 | const userId = entry.content?.item?.content?.user?.id; 502 | const profile = users.get(userId); 503 | if (profile != null) { 504 | orderedProfiles.push(profile); 505 | } 506 | 507 | const operation = entry.content?.operation; 508 | if (operation?.cursor?.cursorType === 'Bottom') { 509 | bottomCursor = operation?.cursor?.value; 510 | } else if (operation?.cursor?.cursorType === 'Top') { 511 | topCursor = operation?.cursor?.value; 512 | } 513 | } 514 | 515 | const operation = instruction.replaceEntry?.entry?.content?.operation; 516 | if (operation?.cursor?.cursorType === 'Bottom') { 517 | bottomCursor = operation.cursor.value; 518 | } else if (operation?.cursor?.cursorType === 'Top') { 519 | topCursor = operation.cursor.value; 520 | } 521 | } 522 | 523 | return { 524 | profiles: orderedProfiles, 525 | next: bottomCursor, 526 | previous: topCursor, 527 | }; 528 | } 529 | -------------------------------------------------------------------------------- /src/timeline-v2.ts: -------------------------------------------------------------------------------- 1 | import { LegacyUserRaw } from './profile'; 2 | import { parseMediaGroups, reconstructTweetHtml } from './timeline-tweet-util'; 3 | import { 4 | LegacyTweetRaw, 5 | ParseTweetResult, 6 | QueryTweetsResponse, 7 | SearchResultRaw, 8 | TimelineResultRaw, 9 | } from './timeline-v1'; 10 | import { Tweet } from './tweets'; 11 | import { isFieldDefined } from './type-util'; 12 | 13 | export interface TimelineUserResultRaw { 14 | rest_id?: string; 15 | legacy?: LegacyUserRaw; 16 | is_blue_verified?: boolean; 17 | } 18 | 19 | export interface TimelineEntryItemContentRaw { 20 | itemType?: string; 21 | tweetDisplayType?: string; 22 | tweetResult?: { 23 | result?: TimelineResultRaw; 24 | }; 25 | tweet_results?: { 26 | result?: TimelineResultRaw; 27 | }; 28 | userDisplayType?: string; 29 | user_results?: { 30 | result?: TimelineUserResultRaw; 31 | }; 32 | } 33 | 34 | export interface TimelineEntryRaw { 35 | entryId: string; 36 | content?: { 37 | cursorType?: string; 38 | value?: string; 39 | items?: { 40 | entryId?: string; 41 | item?: { 42 | content?: TimelineEntryItemContentRaw; 43 | itemContent?: SearchEntryItemContentRaw; 44 | }; 45 | }[]; 46 | itemContent?: TimelineEntryItemContentRaw; 47 | }; 48 | } 49 | 50 | export interface SearchEntryItemContentRaw { 51 | tweetDisplayType?: string; 52 | tweet_results?: { 53 | result?: SearchResultRaw; 54 | }; 55 | userDisplayType?: string; 56 | user_results?: { 57 | result?: TimelineUserResultRaw; 58 | }; 59 | } 60 | 61 | export interface SearchEntryRaw { 62 | entryId: string; 63 | sortIndex: string; 64 | content?: { 65 | cursorType?: string; 66 | entryType?: string; 67 | __typename?: string; 68 | value?: string; 69 | items?: { 70 | item?: { 71 | content?: SearchEntryItemContentRaw; 72 | }; 73 | }[]; 74 | itemContent?: SearchEntryItemContentRaw; 75 | }; 76 | } 77 | 78 | export interface TimelineInstruction { 79 | entries?: TimelineEntryRaw[]; 80 | entry?: TimelineEntryRaw; 81 | type?: string; 82 | } 83 | 84 | export interface TimelineV2 { 85 | data?: { 86 | user?: { 87 | result?: { 88 | timeline_v2?: { 89 | timeline?: { 90 | instructions?: TimelineInstruction[]; 91 | }; 92 | }; 93 | }; 94 | }; 95 | }; 96 | } 97 | 98 | export interface ThreadedConversation { 99 | data?: { 100 | threaded_conversation_with_injections_v2?: { 101 | instructions?: TimelineInstruction[]; 102 | }; 103 | }; 104 | } 105 | 106 | export function parseLegacyTweet( 107 | user?: LegacyUserRaw, 108 | tweet?: LegacyTweetRaw, 109 | ): ParseTweetResult { 110 | if (tweet == null) { 111 | return { 112 | success: false, 113 | err: new Error('Tweet was not found in the timeline object.'), 114 | }; 115 | } 116 | 117 | if (user == null) { 118 | return { 119 | success: false, 120 | err: new Error('User was not found in the timeline object.'), 121 | }; 122 | } 123 | 124 | if (!tweet.id_str) { 125 | if (!tweet.conversation_id_str) { 126 | return { 127 | success: false, 128 | err: new Error('Tweet ID was not found in object.'), 129 | }; 130 | } 131 | 132 | tweet.id_str = tweet.conversation_id_str; 133 | } 134 | 135 | const hashtags = tweet.entities?.hashtags ?? []; 136 | const mentions = tweet.entities?.user_mentions ?? []; 137 | const media = tweet.extended_entities?.media ?? []; 138 | const pinnedTweets = new Set( 139 | user.pinned_tweet_ids_str ?? [], 140 | ); 141 | const urls = tweet.entities?.urls ?? []; 142 | const { photos, videos, sensitiveContent } = parseMediaGroups(media); 143 | 144 | const tw: Tweet = { 145 | bookmarkCount: tweet.bookmark_count, 146 | conversationId: tweet.conversation_id_str, 147 | id: tweet.id_str, 148 | hashtags: hashtags 149 | .filter(isFieldDefined('text')) 150 | .map((hashtag) => hashtag.text), 151 | likes: tweet.favorite_count, 152 | mentions: mentions.filter(isFieldDefined('id_str')).map((mention) => ({ 153 | id: mention.id_str, 154 | username: mention.screen_name, 155 | name: mention.name, 156 | })), 157 | name: user.name, 158 | permanentUrl: `https://twitter.com/${user.screen_name}/status/${tweet.id_str}`, 159 | photos, 160 | replies: tweet.reply_count, 161 | retweets: tweet.retweet_count, 162 | text: tweet.full_text, 163 | thread: [], 164 | urls: urls 165 | .filter(isFieldDefined('expanded_url')) 166 | .map((url) => url.expanded_url), 167 | userId: tweet.user_id_str, 168 | username: user.screen_name, 169 | videos, 170 | isQuoted: false, 171 | isReply: false, 172 | isRetweet: false, 173 | isPin: false, 174 | sensitiveContent: false, 175 | }; 176 | 177 | if (tweet.created_at) { 178 | tw.timeParsed = new Date(Date.parse(tweet.created_at)); 179 | tw.timestamp = Math.floor(tw.timeParsed.valueOf() / 1000); 180 | } 181 | 182 | if (tweet.place?.id) { 183 | tw.place = tweet.place; 184 | } 185 | 186 | const quotedStatusIdStr = tweet.quoted_status_id_str; 187 | const inReplyToStatusIdStr = tweet.in_reply_to_status_id_str; 188 | const retweetedStatusIdStr = tweet.retweeted_status_id_str; 189 | const retweetedStatusResult = tweet.retweeted_status_result?.result; 190 | 191 | if (quotedStatusIdStr) { 192 | tw.isQuoted = true; 193 | tw.quotedStatusId = quotedStatusIdStr; 194 | } 195 | 196 | if (inReplyToStatusIdStr) { 197 | tw.isReply = true; 198 | tw.inReplyToStatusId = inReplyToStatusIdStr; 199 | } 200 | 201 | if (retweetedStatusIdStr || retweetedStatusResult) { 202 | tw.isRetweet = true; 203 | tw.retweetedStatusId = retweetedStatusIdStr; 204 | 205 | if (retweetedStatusResult) { 206 | const parsedResult = parseLegacyTweet( 207 | retweetedStatusResult?.core?.user_results?.result?.legacy, 208 | retweetedStatusResult?.legacy, 209 | ); 210 | 211 | if (parsedResult.success) { 212 | tw.retweetedStatus = parsedResult.tweet; 213 | } 214 | } 215 | } 216 | 217 | const views = parseInt(tweet.ext_views?.count ?? ''); 218 | if (!isNaN(views)) { 219 | tw.views = views; 220 | } 221 | 222 | if (pinnedTweets.has(tweet.id_str)) { 223 | // TODO: Update tests so this can be assigned at the tweet declaration 224 | tw.isPin = true; 225 | } 226 | 227 | if (sensitiveContent) { 228 | // TODO: Update tests so this can be assigned at the tweet declaration 229 | tw.sensitiveContent = true; 230 | } 231 | 232 | tw.html = reconstructTweetHtml(tweet, tw.photos, tw.videos); 233 | 234 | return { success: true, tweet: tw }; 235 | } 236 | 237 | function parseResult(result?: TimelineResultRaw): ParseTweetResult { 238 | const noteTweetResultText = 239 | result?.note_tweet?.note_tweet_results?.result?.text; 240 | 241 | if (result?.legacy && noteTweetResultText) { 242 | result.legacy.full_text = noteTweetResultText; 243 | } 244 | 245 | const tweetResult = parseLegacyTweet( 246 | result?.core?.user_results?.result?.legacy, 247 | result?.legacy, 248 | ); 249 | if (!tweetResult.success) { 250 | return tweetResult; 251 | } 252 | 253 | if (!tweetResult.tweet.views && result?.views?.count) { 254 | const views = parseInt(result.views.count); 255 | if (!isNaN(views)) { 256 | tweetResult.tweet.views = views; 257 | } 258 | } 259 | 260 | const quotedResult = result?.quoted_status_result?.result; 261 | if (quotedResult) { 262 | if (quotedResult.legacy && quotedResult.rest_id) { 263 | quotedResult.legacy.id_str = quotedResult.rest_id; 264 | } 265 | 266 | const quotedTweetResult = parseResult(quotedResult); 267 | if (quotedTweetResult.success) { 268 | tweetResult.tweet.quotedStatus = quotedTweetResult.tweet; 269 | } 270 | } 271 | 272 | return tweetResult; 273 | } 274 | 275 | const expectedEntryTypes = ['tweet', 'profile-conversation']; 276 | 277 | export function parseTimelineTweetsV2( 278 | timeline: TimelineV2, 279 | ): QueryTweetsResponse { 280 | let bottomCursor: string | undefined; 281 | let topCursor: string | undefined; 282 | const tweets: Tweet[] = []; 283 | const instructions = 284 | timeline.data?.user?.result?.timeline_v2?.timeline?.instructions ?? []; 285 | for (const instruction of instructions) { 286 | const entries = instruction.entries ?? []; 287 | 288 | for (const entry of entries) { 289 | const entryContent = entry.content; 290 | if (!entryContent) continue; 291 | 292 | // Handle pagination 293 | if (entryContent.cursorType === 'Bottom') { 294 | bottomCursor = entryContent.value; 295 | continue; 296 | } else if (entryContent.cursorType === 'Top') { 297 | topCursor = entryContent.value; 298 | continue; 299 | } 300 | 301 | const idStr = entry.entryId; 302 | if ( 303 | !expectedEntryTypes.some((entryType) => idStr.startsWith(entryType)) 304 | ) { 305 | continue; 306 | } 307 | 308 | if (entryContent.itemContent) { 309 | // Typically TimelineTimelineTweet entries 310 | parseAndPush(tweets, entryContent.itemContent, idStr); 311 | } else if (entryContent.items) { 312 | // Typically TimelineTimelineModule entries 313 | for (const item of entryContent.items) { 314 | if (item.item?.itemContent) { 315 | parseAndPush(tweets, item.item.itemContent, idStr); 316 | } 317 | } 318 | } 319 | } 320 | } 321 | 322 | return { tweets, next: bottomCursor, previous: topCursor }; 323 | } 324 | 325 | export function parseTimelineEntryItemContentRaw( 326 | content: TimelineEntryItemContentRaw, 327 | entryId: string, 328 | isConversation = false, 329 | ) { 330 | let result = content.tweet_results?.result ?? content.tweetResult?.result; 331 | if ( 332 | result?.__typename === 'Tweet' || 333 | (result?.__typename === 'TweetWithVisibilityResults' && result?.tweet) 334 | ) { 335 | if (result?.__typename === 'TweetWithVisibilityResults') 336 | result = result.tweet; 337 | 338 | if (result?.legacy) { 339 | result.legacy.id_str = 340 | result.rest_id ?? 341 | entryId.replace('conversation-', '').replace('tweet-', ''); 342 | } 343 | 344 | const tweetResult = parseResult(result); 345 | if (tweetResult.success) { 346 | if (isConversation) { 347 | if (content?.tweetDisplayType === 'SelfThread') { 348 | tweetResult.tweet.isSelfThread = true; 349 | } 350 | } 351 | 352 | return tweetResult.tweet; 353 | } 354 | } 355 | 356 | return null; 357 | } 358 | 359 | export function parseAndPush( 360 | tweets: Tweet[], 361 | content: TimelineEntryItemContentRaw, 362 | entryId: string, 363 | isConversation = false, 364 | ) { 365 | const tweet = parseTimelineEntryItemContentRaw( 366 | content, 367 | entryId, 368 | isConversation, 369 | ); 370 | 371 | if (tweet) { 372 | tweets.push(tweet); 373 | } 374 | } 375 | 376 | export function parseThreadedConversation( 377 | conversation: ThreadedConversation, 378 | ): Tweet[] { 379 | const tweets: Tweet[] = []; 380 | const instructions = 381 | conversation.data?.threaded_conversation_with_injections_v2?.instructions ?? 382 | []; 383 | 384 | for (const instruction of instructions) { 385 | const entries = instruction.entries ?? []; 386 | for (const entry of entries) { 387 | const entryContent = entry.content?.itemContent; 388 | if (entryContent) { 389 | parseAndPush(tweets, entryContent, entry.entryId, true); 390 | } 391 | 392 | for (const item of entry.content?.items ?? []) { 393 | const itemContent = item.item?.itemContent; 394 | if (itemContent) { 395 | parseAndPush(tweets, itemContent, entry.entryId, true); 396 | } 397 | } 398 | } 399 | } 400 | 401 | for (const tweet of tweets) { 402 | if (tweet.inReplyToStatusId) { 403 | for (const parentTweet of tweets) { 404 | if (parentTweet.id === tweet.inReplyToStatusId) { 405 | tweet.inReplyToStatus = parentTweet; 406 | break; 407 | } 408 | } 409 | } 410 | 411 | if (tweet.isSelfThread && tweet.conversationId === tweet.id) { 412 | for (const childTweet of tweets) { 413 | if (childTweet.isSelfThread && childTweet.id !== tweet.id) { 414 | tweet.thread.push(childTweet); 415 | } 416 | } 417 | 418 | if (tweet.thread.length === 0) { 419 | tweet.isSelfThread = false; 420 | } 421 | } 422 | } 423 | 424 | return tweets; 425 | } 426 | 427 | export interface TimelineArticle { 428 | id: string; 429 | articleId: string; 430 | title: string; 431 | previewText: string; 432 | coverMediaUrl?: string; 433 | text: string; 434 | } 435 | 436 | export function parseArticle( 437 | conversation: ThreadedConversation, 438 | ): TimelineArticle[] { 439 | const articles: TimelineArticle[] = []; 440 | for (const instruction of conversation.data 441 | ?.threaded_conversation_with_injections_v2?.instructions ?? []) { 442 | for (const entry of instruction.entries ?? []) { 443 | const id = entry.content?.itemContent?.tweet_results?.result?.rest_id; 444 | const article = 445 | entry.content?.itemContent?.tweet_results?.result?.article 446 | ?.article_results?.result; 447 | if (!id || !article) continue; 448 | const text = 449 | article.content_state?.blocks 450 | ?.map((block) => block.text) 451 | .join('\n\n') ?? ''; 452 | articles.push({ 453 | id, 454 | articleId: article.rest_id || '', 455 | coverMediaUrl: article.cover_media?.media_info?.original_img_url, 456 | previewText: article.preview_text || '', 457 | text, 458 | title: article.title || '', 459 | }); 460 | } 461 | } 462 | return articles; 463 | } 464 | -------------------------------------------------------------------------------- /src/trends.test.ts: -------------------------------------------------------------------------------- 1 | import { getScraper } from './test-utils'; 2 | 3 | test('scraper can get trends', async () => { 4 | const scraper = await getScraper(); 5 | const trends = await scraper.getTrends(); 6 | expect(trends).toHaveLength(20); 7 | trends.forEach((trend) => expect(trend).not.toBeFalsy()); 8 | }, 15000); 9 | -------------------------------------------------------------------------------- /src/trends.ts: -------------------------------------------------------------------------------- 1 | import { addApiParams, requestApi } from './api'; 2 | import { TwitterAuth } from './auth'; 3 | import { TimelineV1 } from './timeline-v1'; 4 | 5 | export async function getTrends(auth: TwitterAuth): Promise { 6 | const params = new URLSearchParams(); 7 | addApiParams(params, false); 8 | 9 | params.set('count', '20'); 10 | params.set('candidate_source', 'trends'); 11 | params.set('include_page_configuration', 'false'); 12 | params.set('entity_tokens', 'false'); 13 | 14 | const res = await requestApi( 15 | `https://api.twitter.com/2/guide.json?${params.toString()}`, 16 | auth, 17 | ); 18 | if (!res.success) { 19 | throw res.err; 20 | } 21 | 22 | const instructions = res.value.timeline?.instructions ?? []; 23 | if (instructions.length < 2) { 24 | throw new Error('No trend entries found.'); 25 | } 26 | 27 | // Some of this is silly, but for now we're assuming we know nothing about the 28 | // data, and that anything can be missing. Go has non-nilable strings and empty 29 | // slices are nil, so it largely doesn't need to worry about this. 30 | const entries = instructions[1].addEntries?.entries ?? []; 31 | if (entries.length < 2) { 32 | throw new Error('No trend entries found.'); 33 | } 34 | 35 | const items = entries[1].content?.timelineModule?.items ?? []; 36 | const trends: string[] = []; 37 | for (const item of items) { 38 | const trend = 39 | item.item?.clientEventInfo?.details?.guideDetails?.transparentGuideDetails 40 | ?.trendMetadata?.trendName; 41 | if (trend != null) { 42 | trends.push(trend); 43 | } 44 | } 45 | 46 | return trends; 47 | } 48 | -------------------------------------------------------------------------------- /src/type-util.ts: -------------------------------------------------------------------------------- 1 | export type NonNullableField = { 2 | [P in K]-?: T[P]; 3 | } & T; 4 | 5 | export function isFieldDefined(key: K) { 6 | return function (value: T): value is NonNullableField { 7 | return isDefined(value[key]); 8 | }; 9 | } 10 | 11 | export function isDefined(value: T | null | undefined): value is T { 12 | return value != null; 13 | } 14 | -------------------------------------------------------------------------------- /src/types/spaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Represents a Community that can host Spaces. 3 | */ 4 | export interface Community { 5 | id: string; 6 | name: string; 7 | rest_id: string; 8 | } 9 | 10 | /** 11 | * Represents the response structure for the CommunitySelectQuery. 12 | */ 13 | export interface CommunitySelectQueryResponse { 14 | data: { 15 | space_hostable_communities: Community[]; 16 | }; 17 | errors?: any[]; 18 | } 19 | 20 | /** 21 | * Represents a Subtopic within a Category. 22 | */ 23 | export interface Subtopic { 24 | icon_url: string; 25 | name: string; 26 | topic_id: string; 27 | } 28 | 29 | /** 30 | * Represents a Category containing multiple Subtopics. 31 | */ 32 | export interface Category { 33 | icon: string; 34 | name: string; 35 | semantic_core_entity_id: string; 36 | subtopics: Subtopic[]; 37 | } 38 | 39 | /** 40 | * Represents the data structure for BrowseSpaceTopics. 41 | */ 42 | export interface BrowseSpaceTopics { 43 | categories: Category[]; 44 | } 45 | 46 | /** 47 | * Represents the response structure for the BrowseSpaceTopics query. 48 | */ 49 | export interface BrowseSpaceTopicsResponse { 50 | data: { 51 | browse_space_topics: BrowseSpaceTopics; 52 | }; 53 | errors?: any[]; 54 | } 55 | 56 | /** 57 | * Represents the result details of a Creator. 58 | */ 59 | export interface CreatorResult { 60 | __typename: string; 61 | id: string; 62 | rest_id: string; 63 | affiliates_highlighted_label: Record; 64 | has_graduated_access: boolean; 65 | is_blue_verified: boolean; 66 | profile_image_shape: string; 67 | legacy: { 68 | following: boolean; 69 | can_dm: boolean; 70 | can_media_tag: boolean; 71 | created_at: string; 72 | default_profile: boolean; 73 | default_profile_image: boolean; 74 | description: string; 75 | entities: { 76 | description: { 77 | urls: any[]; 78 | }; 79 | }; 80 | fast_followers_count: number; 81 | favourites_count: number; 82 | followers_count: number; 83 | friends_count: number; 84 | has_custom_timelines: boolean; 85 | is_translator: boolean; 86 | listed_count: number; 87 | location: string; 88 | media_count: number; 89 | name: string; 90 | needs_phone_verification: boolean; 91 | normal_followers_count: number; 92 | pinned_tweet_ids_str: string[]; 93 | possibly_sensitive: boolean; 94 | profile_image_url_https: string; 95 | profile_interstitial_type: string; 96 | screen_name: string; 97 | statuses_count: number; 98 | translator_type: string; 99 | verified: boolean; 100 | want_retweets: boolean; 101 | withheld_in_countries: string[]; 102 | }; 103 | tipjar_settings: Record; 104 | } 105 | 106 | /** 107 | * Represents user results within an Admin. 108 | */ 109 | export interface UserResults { 110 | rest_id: string; 111 | result: { 112 | __typename: string; 113 | identity_profile_labels_highlighted_label: Record; 114 | is_blue_verified: boolean; 115 | legacy: Record; 116 | }; 117 | } 118 | 119 | /** 120 | * Represents an Admin participant in an Audio Space. 121 | */ 122 | export interface Admin { 123 | periscope_user_id: string; 124 | start: number; 125 | twitter_screen_name: string; 126 | display_name: string; 127 | avatar_url: string; 128 | is_verified: boolean; 129 | is_muted_by_admin: boolean; 130 | is_muted_by_guest: boolean; 131 | user_results: UserResults; 132 | } 133 | 134 | /** 135 | * Represents Participants in an Audio Space. 136 | */ 137 | export interface Participants { 138 | total: number; 139 | admins: Admin[]; 140 | speakers: any[]; 141 | listeners: any[]; 142 | } 143 | 144 | /** 145 | * Represents Metadata of an Audio Space. 146 | */ 147 | export interface Metadata { 148 | rest_id: string; 149 | state: string; 150 | media_key: string; 151 | created_at: number; 152 | started_at: number; 153 | ended_at: string; 154 | updated_at: number; 155 | content_type: string; 156 | creator_results: { 157 | result: CreatorResult; 158 | }; 159 | conversation_controls: number; 160 | disallow_join: boolean; 161 | is_employee_only: boolean; 162 | is_locked: boolean; 163 | is_muted: boolean; 164 | is_space_available_for_clipping: boolean; 165 | is_space_available_for_replay: boolean; 166 | narrow_cast_space_type: number; 167 | no_incognito: boolean; 168 | total_replay_watched: number; 169 | total_live_listeners: number; 170 | tweet_results: Record; 171 | max_guest_sessions: number; 172 | max_admin_capacity: number; 173 | } 174 | 175 | /** 176 | * Represents Sharings within an Audio Space. 177 | */ 178 | export interface Sharings { 179 | items: any[]; 180 | slice_info: Record; 181 | } 182 | 183 | /** 184 | * Represents an Audio Space. 185 | */ 186 | export interface AudioSpace { 187 | metadata: Metadata; 188 | is_subscribed: boolean; 189 | participants: Participants; 190 | sharings: Sharings; 191 | } 192 | 193 | /** 194 | * Represents the response structure for the AudioSpaceById query. 195 | */ 196 | export interface AudioSpaceByIdResponse { 197 | data: { 198 | audioSpace: AudioSpace; 199 | }; 200 | errors?: any[]; 201 | } 202 | 203 | /** 204 | * Represents the variables required for the AudioSpaceById query. 205 | */ 206 | export interface AudioSpaceByIdVariables { 207 | id: string; 208 | isMetatagsQuery: boolean; 209 | withReplays: boolean; 210 | withListeners: boolean; 211 | } 212 | 213 | export interface LiveVideoSource { 214 | location: string; 215 | noRedirectPlaybackUrl: string; 216 | status: string; 217 | streamType: string; 218 | } 219 | 220 | export interface LiveVideoStreamStatus { 221 | source: LiveVideoSource; 222 | sessionId: string; 223 | chatToken: string; 224 | lifecycleToken: string; 225 | shareUrl: string; 226 | chatPermissionType: string; 227 | } 228 | 229 | export interface AuthenticatePeriscopeResponse { 230 | data: { 231 | authenticate_periscope: string; 232 | }; 233 | errors?: any[]; 234 | } 235 | 236 | export interface LoginTwitterTokenResponse { 237 | cookie: string; 238 | user: { 239 | class_name: string; 240 | id: string; 241 | created_at: string; 242 | is_beta_user: boolean; 243 | is_employee: boolean; 244 | is_twitter_verified: boolean; 245 | verified_type: number; 246 | is_bluebird_user: boolean; 247 | twitter_screen_name: string; 248 | username: string; 249 | display_name: string; 250 | description: string; 251 | profile_image_urls: { 252 | url: string; 253 | ssl_url: string; 254 | width: number; 255 | height: number; 256 | }[]; 257 | twitter_id: string; 258 | initials: string; 259 | n_followers: number; 260 | n_following: number; 261 | }; 262 | type: string; 263 | } 264 | -------------------------------------------------------------------------------- /test-assets/test-image.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingbootoshi/goat-x/ffcfed49c35b1130540c45eef95086a9fb6dff8a/test-assets/test-image.jpeg -------------------------------------------------------------------------------- /test-assets/test-video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingbootoshi/goat-x/ffcfed49c35b1130540c45eef95086a9fb6dff8a/test-assets/test-video.mp4 -------------------------------------------------------------------------------- /test-setup.js: -------------------------------------------------------------------------------- 1 | globalThis.PLATFORM_NODE = false; 2 | globalThis.PLATFORM_NODE_JEST = true; 3 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node16/tsconfig.json", 3 | "exclude": ["node_modules", "dist", "**/*.test.ts", "src/test-utils.ts"], 4 | "compilerOptions": { 5 | // TODO: Remove "dom" from this when support for Node 16 is dropped 6 | "lib": ["es2021", "dom"], 7 | "outDir": "./dist", 8 | "rootDir": "./src", 9 | "sourceMap": true, 10 | "declaration": true, 11 | "declarationMap": true, 12 | "declarationDir": "./dist", 13 | "strict": true, 14 | "noImplicitAny": true, 15 | "esModuleInterop": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "skipLibCheck": true, 18 | "stripInternal": true, 19 | "types": ["jest"] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "entryPointStrategy": "expand", 3 | "entryPoints": ["src"], 4 | "exclude": ["**/*.test.ts", "**/_*.ts"], 5 | "hideGenerator": true 6 | } --------------------------------------------------------------------------------