├── .commitlintrc ├── .gitignore ├── .husky ├── commit-msg └── pre-commit ├── .nvmrc ├── .prettierrc ├── CHANGELOG.md ├── LICENSE ├── README.md ├── bun.lock ├── cliff.toml ├── docs ├── 01-user-interface.png ├── 02-export-media.png ├── 03-menu-commands.png └── README.zh-Hans.md ├── eslint.config.js ├── package.json ├── src ├── components │ ├── common.tsx │ ├── error-boundary.tsx │ ├── modals │ │ ├── export-data.tsx │ │ └── export-media.tsx │ ├── module-ui.tsx │ └── table │ │ ├── base.tsx │ │ ├── columns-tweet.tsx │ │ ├── columns-user.tsx │ │ ├── pagination.tsx │ │ └── table-view.tsx ├── core │ ├── app.tsx │ ├── database │ │ ├── hooks.ts │ │ ├── index.ts │ │ └── manager.ts │ ├── extensions │ │ ├── extension.ts │ │ ├── index.ts │ │ └── manager.ts │ ├── options │ │ ├── index.ts │ │ └── manager.ts │ └── settings.tsx ├── i18n │ ├── detector.ts │ ├── i18next.d.ts │ ├── index.tsx │ ├── init.ts │ └── locales │ │ ├── en │ │ ├── common.json │ │ └── exporter.json │ │ └── zh-Hans │ │ ├── common.json │ │ └── exporter.json ├── index.css ├── main.tsx ├── modules │ ├── bookmarks │ │ ├── api.ts │ │ └── index.tsx │ ├── direct-messages │ │ ├── api.ts │ │ ├── columns.tsx │ │ ├── index.tsx │ │ ├── types.ts │ │ └── ui.tsx │ ├── followers │ │ ├── api.ts │ │ └── index.tsx │ ├── following │ │ ├── api.ts │ │ └── index.tsx │ ├── home-timeline │ │ ├── api.ts │ │ └── index.tsx │ ├── likes │ │ ├── api.ts │ │ └── index.tsx │ ├── list-members │ │ ├── api.ts │ │ └── index.tsx │ ├── list-subscribers │ │ ├── api.ts │ │ └── index.tsx │ ├── list-timeline │ │ ├── api.ts │ │ └── index.tsx │ ├── runtime-logs │ │ ├── index.ts │ │ └── ui.tsx │ ├── search-timeline │ │ ├── api.ts │ │ └── index.tsx │ ├── tweet-detail │ │ ├── api.ts │ │ └── index.tsx │ ├── user-detail │ │ ├── api.ts │ │ └── index.tsx │ ├── user-media │ │ ├── api.ts │ │ └── index.tsx │ └── user-tweets │ │ ├── api.ts │ │ └── index.tsx ├── types │ ├── index.ts │ ├── list.ts │ ├── tweet.ts │ └── user.ts ├── utils │ ├── api.ts │ ├── common.ts │ ├── download.ts │ ├── exporter.ts │ ├── logger.ts │ ├── media.ts │ ├── observable.ts │ ├── react-table.tsx │ └── zip-stream.ts └── vite-env.d.ts ├── tailwind.config.js ├── tsconfig.json └── vite.config.ts /.commitlintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "@commitlint/config-conventional" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | npm run commitlint ${1} 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | npm run lint 2 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | lts/iron 2 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "tabWidth": 2, 4 | "printWidth": 100 5 | } 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 prin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | twitter-web-exporter 4 | 5 |

6 | 7 |

8 | 9 | UserScript 10 | 11 | 12 | Latest Release 13 | 14 | 15 | License 16 | 17 | 18 | TypeScript 19 | 20 |

21 | 22 |

23 | English | 24 | 简体中文 25 |

26 | 27 | ## Features 28 | 29 | - 🚚 Export tweets, replies and likes of any user as JSON/CSV/HTML 30 | - 🔖 Export your bookmarks (without the max 800 limit!) 31 | - 💞 Export following, followers list of any user 32 | - 👥 Export list members and subscribers 33 | - 🌪️ Export tweets from home timeline and list timeline 34 | - 🔍 Export search results 35 | - ✉️ Export direct messages 36 | - 📦 Download images and videos from tweets in bulk at original size 37 | - 🚀 No developer account or API key required 38 | - 🛠️ Ship as a UserScript and everything is done in your browser 39 | - 💾 Your data never leaves your computer 40 | - 💚 Completely free and open-source 41 | 42 | ## Installation 43 | 44 | 1. Install the browser extension [Tampermonkey](https://www.tampermonkey.net/) or [Violentmonkey](https://violentmonkey.github.io/) 45 | 2. Click [HERE](https://github.com/prinsss/twitter-web-exporter/releases/latest/download/twitter-web-exporter.user.js) to install the user script 46 | 47 | ## Usage 48 | 49 | Once the script is installed, you can find a floating panel on the left side of the page. Click the 🐈 Cat button to close the panel or open it again. You can also open the control panel by clicking the Tampermonkey/Violentmonkey icon in the browser menu bar and then selecting it from the script menu. 50 | 51 | If you do not see the cat button or the menu options as shown in the image, please check if the script is properly installed and enabled. 52 | 53 | ![03-menu-commands](https://github.com/prinsss/twitter-web-exporter/raw/main/docs/03-menu-commands.png) 54 | 55 | Click the ⚙️ Cog button to open the settings panel. You can change the UI theme and enable/disable features of script here. 56 | 57 | Then open the page that you want to export data from. The script will automatically capture on the following pages: 58 | 59 | - User profile page (tweets, replies, media, likes) 60 | - Bookmark page 61 | - Search results page 62 | - User following/followers page 63 | - List members/subscribers page 64 | 65 | The numbers of captured data will be displayed on the floating panel. Click the ↗️ Arrow button to open the data table view. You can preview the data captured here and select which items should be exported. 66 | 67 | ![01-user-interface](https://github.com/prinsss/twitter-web-exporter/raw/main/docs/01-user-interface.png) 68 | 69 | Click "Export Data" to export captured data to the selected file format. Currently, the script supports exporting to JSON, CSV and HTML. The exported file will be downloaded to your computer. 70 | 71 | By checking the "Include all metadata" option, all available fields from the API will be included in the exported file, giving you the most complete dataset. This could significantly increase the size of the exported data. 72 | 73 | Click "Export Media" to bulk download images and videos from tweets. 74 | 75 | All media files will be downloaded at its original size in a zip archive. You can also copy the URLs of the media files if you want to download them with a external download manager. 76 | 77 | Please set a reasonable value for the "Rate limit" option to avoid downloading too many files at once. The default value is 1000 which means the script will wait for 1 second after downloading each file. 78 | 79 | ![02-export-media.png](https://github.com/prinsss/twitter-web-exporter/raw/main/docs/02-export-media.png) 80 | 81 | ## Limitation 82 | 83 | The script only works on the web app (twitter.com). It does not work on the mobile app. 84 | 85 | Basically, **the script "sees" what you see on the page**. If you can't see the data on the page, the script can't access it either. For example, Twitter displays only the latest 3200 tweets on the profile page and the script can't export tweets older than that. 86 | 87 | Data on the web page is loaded dynamically, which means the script can't access the data until it is loaded. You need to keep scrolling down to load more data. Make sure that all data is loaded before exporting. 88 | 89 | The export process is not automated (without the help of 3rd party tools). It relies on human interaction to trigger the data fetching process of the Twitter web app. The script itself does not send any request to Twitter API. 90 | 91 | The script does not rely on the official Twitter API and thus does not have the same rate limit. However, the Twitter web app does have its own limit. If you hit that rate limit, try again after a few minutes. 92 | 93 | On the contrary, the script can export data that is not available from the official API. For example, the official API has a 800 limit when accessing the bookmarks. The script can export all bookmarks without that limit until it's restricted by the Twitter web app itself. 94 | 95 | There is also a limitation on downloading media files. Currently, the script downloads pictures and videos to the browser memory and then zip them into a single archive. This could cause the browser to crash if the size of the media files is too large. The maximum archive size it can handle depends on the browser and the available memory of your computer. (2GB on Chrome and 800MB on Firefox) 96 | 97 | ## FAQ 98 | 99 | **Q. How do you get the data?**
100 | A. The script itself does not send any request to Twitter API. It installs an network interceptor to capture the response of GraphQL request that initiated by the Twitter web app. The script then parses the response and extracts data from it. 101 | 102 | **Q. The script captures nothing!**
103 | A. See [Content-Security-Policy (CSP) Issues #19](https://github.com/prinsss/twitter-web-exporter/issues/19). 104 | 105 | **Q. The exported data is incomplete.**
106 | A. The script can only export data that is loaded by the Twitter web app. Since the data is lazy-loaded, you need to keep scrolling down to load more data. For long lists, you may need to scroll down to the bottom of the page to make sure that all data is loaded before exporting. 107 | 108 | **Q. Can the exporting process be automated?**
109 | A. No. At least not without the help of 3rd party tools like auto scrolling. 110 | 111 | **Q. Do I need a developer account?**
112 | A. No. The script does not send any request to Twitter API. 113 | 114 | **Q. Is there an API rate limit?**
115 | A. No. Not until you hit the rate limit of the Twitter web app itself. 116 | 117 | **Q. Will my account be suspended?**
118 | A. Not likely. There is no automatic botting involved and the behavior is similar to manually copying the data from the web page. 119 | 120 | **Q: What about privacy?**
121 | A: Everything is processed on your local browser. No data is sent to the cloud. 122 | 123 | **Q: Why do you build this?**
124 | A: For archival usage. Twitter's archive only contains the numeric user ID of your following/followers which is not human-readable. The archive also does not contain your bookmarks. 125 | 126 | **Q: What's the difference between this and other alternatives?**
127 | A: You don't need a developer account for accessing the Twitter API. You don't need to send your private data to someone's server. The script is completely free and open-source. 128 | 129 | **Q: The script does not work!**
130 | A: A platform upgrade will possibly breaks the script's functionality. Please file an [issue](https://github.com/prinsss/twitter-web-exporter/issues) if you encountered any problem. 131 | 132 | ## License 133 | 134 | [MIT](LICENSE) 135 | -------------------------------------------------------------------------------- /cliff.toml: -------------------------------------------------------------------------------- 1 | # git-cliff ~ configuration file 2 | # https://git-cliff.org/docs/configuration 3 | # 4 | # Lines starting with "#" are comments. 5 | # Configuration options are organized into tables and keys. 6 | # See documentation for more information on available options. 7 | 8 | [changelog] 9 | # changelog header 10 | header = """ 11 | # Changelog\n 12 | All notable changes to this project will be documented in this file.\n 13 | """ 14 | # template for the changelog body 15 | # https://keats.github.io/tera/docs/#introduction 16 | body = """ 17 | {% if version %}\ 18 | {% if previous.version %}\ 19 | ## [{{ version | trim_start_matches(pat="v") }}](/compare/{{ previous.version }}..{{ version }}) - {{ timestamp | date(format="%Y-%m-%d") }} 20 | {% else %}\ 21 | ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} 22 | {% endif %}\ 23 | {% else %}\ 24 | ## [unreleased] 25 | {% endif %}\ 26 | 27 | {% macro commit(commit) -%} 28 | - {% if commit.scope %}*({{ commit.scope }})* {% endif %}{% if commit.breaking %}[**breaking**] {% endif %}\ 29 | {{ commit.message | upper_first }} - ([{{ commit.id | truncate(length=7, end="") }}](/commit/{{ commit.id }}))\ 30 | {% endmacro -%} 31 | 32 | {% for group, commits in commits | group_by(attribute="group") %} 33 | ### {{ group | striptags | trim | upper_first }} 34 | {% for commit in commits 35 | | filter(attribute="scope") 36 | | sort(attribute="scope") %} 37 | {{ self::commit(commit=commit) }} 38 | {%- endfor -%} 39 | {% raw %}\n{% endraw %}\ 40 | {%- for commit in commits %} 41 | {%- if not commit.scope -%} 42 | {{ self::commit(commit=commit) }} 43 | {% endif -%} 44 | {% endfor -%} 45 | {% endfor %}\n 46 | """ 47 | # template for the changelog footer 48 | footer = """ 49 | 50 | """ 51 | # remove the leading and trailing whitespace from the templates 52 | trim = true 53 | # postprocessors 54 | postprocessors = [ 55 | { pattern = '', replace = "https://github.com/prinsss/twitter-web-exporter" }, # replace repository URL 56 | ] 57 | 58 | [git] 59 | # parse the commits based on https://www.conventionalcommits.org 60 | conventional_commits = true 61 | # filter out the commits that are not conventional 62 | filter_unconventional = true 63 | # process each line of a commit as an individual commit 64 | split_commits = false 65 | # regex for preprocessing the commit messages 66 | commit_preprocessors = [ 67 | { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))" }, 68 | # Check spelling of the commit with https://github.com/crate-ci/typos 69 | # If the spelling is incorrect, it will be automatically fixed. 70 | # { pattern = '.*', replace_command = 'typos --write-changes -' }, 71 | ] 72 | # regex for parsing and grouping commits 73 | commit_parsers = [ 74 | { message = "^feat", group = "⛰️ Features" }, 75 | { message = "^fix", group = "🐛 Bug Fixes" }, 76 | { message = "^doc", group = "📚 Documentation" }, 77 | { message = "^perf", group = "⚡ Performance" }, 78 | { message = "^refactor", group = "🚜 Refactor" }, 79 | { message = "^style", group = "🎨 Styling" }, 80 | { message = "^test", group = "🧪 Testing" }, 81 | { message = "^chore\\(release\\): prepare for", skip = true }, 82 | { message = "^chore\\(deps\\)", skip = true }, 83 | { message = "^chore\\(pr\\)", skip = true }, 84 | { message = "^chore\\(pull\\)", skip = true }, 85 | { message = "^chore: bump version", skip = true }, 86 | { message = "^chore|ci", group = "⚙️ Miscellaneous Tasks" }, 87 | { body = ".*security", group = "🛡️ Security" }, 88 | { message = "^revert", group = "◀️ Revert" }, 89 | ] 90 | # protect breaking changes from being skipped due to matching a skipping commit_parser 91 | protect_breaking_commits = false 92 | # filter out the commits that are not matched by commit parsers 93 | filter_commits = false 94 | # regex for matching git tags 95 | tag_pattern = "v[0-9].*" 96 | # regex for skipping tags 97 | skip_tags = "beta|alpha" 98 | # regex for ignoring tags 99 | ignore_tags = "rc" 100 | # sort the tags topologically 101 | topo_order = false 102 | # sort the commits inside sections by oldest/newest order 103 | sort_commits = "newest" 104 | -------------------------------------------------------------------------------- /docs/01-user-interface.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prinsss/twitter-web-exporter/0308628206fabc5c299566073789de899b82f9d0/docs/01-user-interface.png -------------------------------------------------------------------------------- /docs/02-export-media.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prinsss/twitter-web-exporter/0308628206fabc5c299566073789de899b82f9d0/docs/02-export-media.png -------------------------------------------------------------------------------- /docs/03-menu-commands.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prinsss/twitter-web-exporter/0308628206fabc5c299566073789de899b82f9d0/docs/03-menu-commands.png -------------------------------------------------------------------------------- /docs/README.zh-Hans.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | twitter-web-exporter 4 | 5 |

6 | 7 |

8 | 9 | UserScript 10 | 11 | 12 | Latest Release 13 | 14 | 15 | License 16 | 17 | 18 | TypeScript 19 | 20 |

21 | 22 |

23 | English 24 | | 简体中文 25 |

26 | 27 | ## 功能 28 | 29 | - 🚚 以 JSON/CSV/HTML 格式导出用户的推文、回复和喜欢 30 | - 🔖 导出你的书签（没有最多 800 条的数量限制！） 31 | - 💞 导出任意用户的关注者、粉丝列表 32 | - 👥 导出列表成员和订阅者 33 | - 🌪️ 导出主页时间线和列表时间线中的推文 34 | - 🔍 导出搜索结果 35 | - ✉️ 导出私信 36 | - 📦 以原始尺寸批量下载推文中的图片和视频 37 | - 🚀 无需开发者账号或 API 密钥 38 | - 🛠️ 以油猴脚本的形式提供，所有操作均在浏览器内完成 39 | - 💾 你的数据永远不会离开你的计算机 40 | - 💚 完全免费开源 41 | 42 | ## 安装 43 | 44 | 1. 安装浏览器扩展 [Tampermonkey](https://www.tampermonkey.net/) 或 [Violentmonkey](https://violentmonkey.github.io/) 45 | 2. 点击 [这里](https://github.com/prinsss/twitter-web-exporter/releases/latest/download/twitter-web-exporter.user.js) 安装用户脚本 46 | 47 | ## 使用方法 48 | 49 | 安装脚本后，你会在页面左侧看到一个浮动面板。点击 🐈 猫咪按钮关闭面板或重新打开。你也可以通过点击浏览器菜单栏上的 Tampermonkey/Violentmonkey 图标，然后在脚本菜单中打开控制面板。 50 | 51 | 如果你看不到猫咪按钮，也看不到如图所示的脚本菜单选项，请检查脚本是否已正确安装并启用。 52 | 53 | ![03-menu-commands](https://github.com/prinsss/twitter-web-exporter/raw/main/docs/03-menu-commands.png) 54 | 55 | 点击 ⚙️ 齿轮按钮打开设置面板。你可以在此处更改 UI 主题并启用/禁用脚本功能。 56 | 57 | 然后打开要从中导出数据的页面。脚本会自动捕获以下页面上的数据： 58 | 59 | - 用户个人资料页面（推文、回复、媒体、喜欢） 60 | - 书签页面 61 | - 搜索结果页面 62 | - 用户关注者/粉丝页面 63 | - 列表成员/订阅者页面 64 | 65 | 在浮动面板上会显示已经捕获的数据条数，点击 ↗️ 箭头按钮可以打开数据表视图。你可以在这里预览捕获到的数据并选择要导出的项。 66 | 67 | ![01-user-interface](https://github.com/prinsss/twitter-web-exporter/raw/main/docs/01-user-interface.png) 68 | 69 | 点击「导出数据」可将捕获到的数据导出为选定的文件格式。目前，脚本支持导出为 JSON、CSV 和 HTML。导出的文件将下载到你的计算机上。 70 | 71 | 如果勾选「包括所有元数据」选项，导出的文件中将会包含 API 中提供的所有字段，为你提供最完整的数据集。这可能会显著增加导出数据的大小。 72 | 73 | 点击「导出媒体」可批量下载推文中的图片和视频。 74 | 75 | 所有媒体文件都将以其原始尺寸打包下载到一个 zip 压缩文件中。如果想要使用外部下载管理器下载它们，也可以点击「复制 URL」复制媒体文件的下载地址。 76 | 77 | 请合理设置「速率限制」选项的值，以避免一次下载太多文件。默认值是 1000，这意味着脚本将在下载每个文件后等待 1 秒。 78 | 79 | ![02-export-media.png](https://github.com/prinsss/twitter-web-exporter/raw/main/docs/02-export-media.png) 80 | 81 | ## 局限性 82 | 83 | 此脚本仅在 Web App (twitter.com) 上运行，在手机 App 上无效。 84 | 85 | 简单来说，**此脚本只能“看到”你在页面上能看到的内容**。如果你在页面上看不到某些数据，脚本也无法访问该数据。例如，Twitter 在个人资料页上仅显示最新的 3200 条推文，那么脚本就无法导出比这更早的推文。 86 | 87 | 网页上的数据是动态加载的，这意味着只有当数据被加载到本地之后，脚本才能访问这些数据。你需要在页面上不断向下滚动以加载更多数据。导出之前，请确保所有数据都已加载完毕。 88 | 89 | 导出过程不是自动化的（除非配合使用第三方辅助工具）。此脚本依赖于用户操作来触发 Twitter Web App 的数据获取过程。脚本本身不会向 Twitter API 发送任何请求。 90 | 91 | 此脚本不依赖于官方 Twitter API，因此没有官方 API 的速率限制。但是，Twitter Web App 自身是有限制的。如果达到了该速率限制，请稍后重试。 92 | 93 | 另一方面，此脚本可以导出官方 API 中不提供的数据。例如，官方 API 在访问书签时有最多获取 800 条的限制。而此脚本可以无限制地导出所有书签，除非 Twitter Web App 本身被限制。 94 | 95 | 另外，媒体文件下载功能也存在一定的限制。目前，脚本会将图片和视频下载到浏览器内存中，然后压缩为单个压缩包。如果媒体文件的大小过大，可能会导致浏览器崩溃。其最大支持的压缩包大小取决于浏览器和计算机可用内存。（Chrome 上为 2GB，Firefox 上为 800MB） 96 | 97 | ## 常见问题 98 | 99 | **问：你是如何获取数据的？**
100 | 答：此脚本本身不会向 Twitter API 发起任何请求。它会安装一个 HTTP 网络拦截器，来捕获 Twitter Web App 发起的 GraphQL 请求的响应，然后解析响应并从中提取数据。 101 | 102 | **问：脚本抓取不到任何数据！**
103 | 答：参见 [Content-Security-Policy (CSP) Issues #19](https://github.com/prinsss/twitter-web-exporter/issues/19)。 104 | 105 | **问：为什么导出的数据不完整？**
106 | 答：脚本只能导出由 Twitter Web App 加载好的数据。由于数据是懒加载的，你需要不断向下滚动以加载更多数据。对于长列表，可能需要滚动到页面底部，确保所有数据加载完毕后再导出。 107 | 108 | **问：导出过程可以自动化吗？**
109 | 答：不可以。除非使用第三方工具辅助配合，比如 Auto Scroll 自动滚动工具。 110 | 111 | **问：我需要申请开发者帐户吗？**
112 | 答：不需要。此脚本不向 Twitter API 发送任何请求。 113 | 114 | **问：是否有 API 速率限制？**
115 | 答：没有。除非你达到了 Twitter Web App 本身的速率限制。 116 | 117 | **问：使用脚本是否会导致封号？**
118 | 答：基本不可能。此脚本中不存在任何自动操作，行为类似于你手动从网页上拷贝数据。 119 | 120 | **问：关于隐私问题？**
121 | 答：所有操作都在你的本地浏览器中完成。不会将数据发送到云端。 122 | 123 | **问：你为什么要做这个？**
124 | 答：用于归档。Twitter 的存档导出仅包含关注者/粉丝的数字用户 ID，根本不是给人看的。而且存档还不包含书签。 125 | 126 | **问：与其他替代方案有何不同？**
127 | 答：无需为 Twitter API 申请开发者帐户。无需将你的私人数据发送到别人的服务器。此脚本完全免费且开源。 128 | 129 | **问：脚本无法运行！**
130 | 答：平台升级可能会导致脚本功能故障。如果遇到任何问题，请提交 [issue](https://github.com/prinsss/twitter-web-exporter/issues) 反馈。 131 | 132 | ## 开源许可 133 | 134 | [MIT](LICENSE) 135 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | import eslint from '@eslint/js'; 3 | import tseslint from 'typescript-eslint'; 4 | import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended'; 5 | 6 | export default tseslint.config( 7 | eslint.configs.recommended, 8 | ...tseslint.configs.recommended, 9 | eslintPluginPrettierRecommended, 10 | { 11 | files: ['**/*.ts', '**/*.tsx'], 12 | }, 13 | { 14 | ignores: ['dist', 'node_modules'], 15 | }, 16 | ); 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "twitter-web-exporter", 3 | "description": "Export tweets, bookmarks, lists and much more from Twitter(X) web app.", 4 | "version": "1.2.2-beta.1", 5 | "author": "prin ", 6 | "license": "MIT", 7 | "homepage": "https://github.com/prinsss/twitter-web-exporter", 8 | "bugs": "https://github.com/prinsss/twitter-web-exporter/issues", 9 | "private": true, 10 | "type": "module", 11 | "scripts": { 12 | "dev": "vite", 13 | "build": "tsc && vite build", 14 | "prepare": "husky", 15 | "lint": "eslint .", 16 | "commitlint": "commitlint --edit", 17 | "changelog": "git-cliff -o CHANGELOG.md", 18 | "preview": "vite preview" 19 | }, 20 | "dependencies": { 21 | "@preact/signals": "2.0.0", 22 | "@preact/signals-core": "1.8.0", 23 | "@tabler/icons-preact": "3.31.0", 24 | "@tanstack/table-core": "8.21.2", 25 | "dayjs": "1.11.13", 26 | "dexie": "4.0.11", 27 | "dexie-export-import": "4.1.4", 28 | "file-saver-es": "2.0.5", 29 | "i18next": "24.2.3", 30 | "preact": "10.26.4" 31 | }, 32 | "devDependencies": { 33 | "@commitlint/cli": "^19.8.0", 34 | "@commitlint/config-conventional": "^19.8.0", 35 | "@eslint/js": "^9.23.0", 36 | "@preact/preset-vite": "^2.10.1", 37 | "@types/file-saver-es": "^2.0.3", 38 | "@types/node": "^22.14.0", 39 | "autoprefixer": "^10.4.21", 40 | "daisyui": "^4", 41 | "eslint": "^9.23.0", 42 | "eslint-config-prettier": "^10.1.1", 43 | "eslint-plugin-prettier": "^5.2.6", 44 | "git-cliff": "^2.8.0", 45 | "husky": "^9.1.7", 46 | "postcss": "^8.5.3", 47 | "postcss-prefix-selector": "^2.1.1", 48 | "postcss-rem-to-pixel-next": "^5.0.3", 49 | "tailwindcss": "^3", 50 | "typescript": "^5.8.2", 51 | "typescript-eslint": "^8.29.0", 52 | "vite": "^6.2.5", 53 | "vite-plugin-i18next-loader": "^3.1.2", 54 | "vite-plugin-monkey": "^5.0.8" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/components/error-boundary.tsx: -------------------------------------------------------------------------------- 1 | import { Component } from 'preact'; 2 | import { IconExclamationCircle } from '@tabler/icons-preact'; 3 | import { Trans } from '@/i18n'; 4 | import logger from '@/utils/logger'; 5 | 6 | export class ErrorBoundary extends Component { 7 | state = { error: null }; 8 | 9 | static getDerivedStateFromError(err: Error) { 10 | return { error: err.message }; 11 | } 12 | 13 | componentDidCatch(err: Error) { 14 | logger.error(err.message, err); 15 | this.setState({ error: err.message }); 16 | } 17 | 18 | render() { 19 | if (this.state.error) { 20 | return ( 21 |
22 | 23 |
24 |

25 | 26 |

27 |

28 | {this.state.error} 29 |

30 |
31 |
32 | ); 33 | } 34 | return this.props.children; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/components/modals/export-data.tsx: -------------------------------------------------------------------------------- 1 | import { Table } from '@tanstack/table-core'; 2 | import { Modal } from '@/components/common'; 3 | import { TranslationKey, useTranslation } from '@/i18n'; 4 | import { useSignalState, cx, useToggle } from '@/utils/common'; 5 | import { DataType, EXPORT_FORMAT, ExportFormatType, exportData } from '@/utils/exporter'; 6 | 7 | type ExportDataModalProps = { 8 | title: string; 9 | table: Table; 10 | show?: boolean; 11 | onClose?: () => void; 12 | }; 13 | 14 | /** 15 | * Modal for exporting data. 16 | */ 17 | export function ExportDataModal({ title, table, show, onClose }: ExportDataModalProps) { 18 | const { t } = useTranslation('exporter'); 19 | 20 | const [selectedFormat, setSelectedFormat] = useSignalState(EXPORT_FORMAT.JSON); 21 | const [loading, setLoading] = useSignalState(false); 22 | 23 | const [includeMetadata, toggleIncludeMetadata] = useToggle(false); 24 | const [currentProgress, setCurrentProgress] = useSignalState(0); 25 | const [totalProgress, setTotalProgress] = useSignalState(0); 26 | 27 | const selectedRows = table.getSelectedRowModel().rows; 28 | 29 | const onExport = async () => { 30 | setLoading(true); 31 | setTotalProgress(selectedRows.length); 32 | 33 | const allRecords: Array = []; 34 | 35 | // Prepare data for exporting by iterating through all selected rows in the table. 36 | for (const row of selectedRows) { 37 | const allCells = row.getAllCells(); 38 | const record: DataType = {}; 39 | 40 | for (const cell of allCells) { 41 | const value = cell.getValue(); 42 | const meta = cell.column.columnDef.meta; 43 | 44 | if (meta?.exportable === false) { 45 | continue; 46 | } 47 | 48 | // Get export value of the cell by calling column definition if available. 49 | let exportValue = meta?.exportValue ? meta.exportValue(row) : value; 50 | 51 | // Avoid exporting undefined values and use null instead. 52 | if (exportValue === undefined) { 53 | exportValue = null; 54 | } 55 | 56 | record[meta?.exportKey || cell.column.id] = exportValue; 57 | } 58 | 59 | if (includeMetadata) { 60 | record.metadata = row.original; 61 | } 62 | 63 | allRecords.push(record); 64 | setCurrentProgress(allRecords.length); 65 | } 66 | 67 | // Prepare header translations for the exported data. 68 | const headerTranslations = table 69 | .getAllColumns() 70 | .reduce>((acc, column) => { 71 | const key = column.columnDef.meta?.exportKey || column.id; 72 | const header = column.columnDef.meta?.exportHeader || column.id; 73 | acc[key] = t(header as TranslationKey); 74 | return acc; 75 | }, {}); 76 | 77 | // Convert data to selected format and download it. 78 | await exportData( 79 | allRecords, 80 | selectedFormat, 81 | `twitter-${title}-${Date.now()}.${selectedFormat.toLowerCase()}`, 82 | headerTranslations, 83 | ); 84 | setLoading(false); 85 | }; 86 | 87 | return ( 88 | 94 | {/* Modal content. */} 95 |
96 |

97 | {t( 98 | 'Export captured data as JSON/HTML/CSV file. This may take a while depending on the amount of data. The exported file does not include media files such as images and videos but only the URLs.', 99 | )} 100 |

101 | {/* Export options. */} 102 |
103 |

{t('Data length:')}

104 | 105 | {selectedRows.length} 106 | 107 |
108 |
109 |

{t('Include all metadata:')}

110 | 116 |
117 |
118 |

{t('Export as:')}

119 | 131 |
132 | {selectedRows.length > 0 ? null : ( 133 |
134 |

{t('No data selected.')}

135 |
136 | )} 137 | {/* Progress bar. */} 138 |
139 | 144 | 145 | {`${currentProgress}/${selectedRows.length}`} 146 | 147 |
148 |
149 | {/* Action buttons. */} 150 |
151 | 152 | 155 | 159 |
160 | 161 | ); 162 | } 163 | -------------------------------------------------------------------------------- /src/components/module-ui.tsx: -------------------------------------------------------------------------------- 1 | import { ExtensionPanel, Modal } from '@/components/common'; 2 | import { TableView } from '@/components/table/table-view'; 3 | import { useCaptureCount } from '@/core/database/hooks'; 4 | import { Extension, ExtensionType } from '@/core/extensions'; 5 | import { TranslationKey, useTranslation } from '@/i18n'; 6 | import { useToggle } from '@/utils/common'; 7 | 8 | export type CommonModuleUIProps = { 9 | extension: Extension; 10 | }; 11 | 12 | /** 13 | * A common UI boilerplate for modules. 14 | */ 15 | export function CommonModuleUI({ extension }: CommonModuleUIProps) { 16 | const { t } = useTranslation(); 17 | const [showModal, toggleShowModal] = useToggle(); 18 | 19 | const title = t(extension.name.replace('Module', '') as TranslationKey); 20 | const count = useCaptureCount(extension.name); 21 | 22 | if (extension.type !== 'tweet' && extension.type !== 'user') { 23 | throw new Error('Incorrect use of CommonModuleUI component.'); 24 | } 25 | 26 | return ( 27 | 0} 31 | onClick={toggleShowModal} 32 | indicatorColor={extension.type === ExtensionType.TWEET ? 'bg-primary' : 'bg-secondary'} 33 | > 34 | 40 | 41 | 42 | 43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /src/components/table/base.tsx: -------------------------------------------------------------------------------- 1 | import { JSX } from 'preact'; 2 | import { useEffect } from 'preact/hooks'; 3 | 4 | import { Modal, SearchArea } from '@/components/common'; 5 | import { useTranslation } from '@/i18n'; 6 | import { useSignalState, useToggle } from '@/utils/common'; 7 | import { flexRender, useReactTable } from '@/utils/react-table'; 8 | import { IconSortAscending, IconSortDescending } from '@tabler/icons-preact'; 9 | import { 10 | ColumnDef, 11 | getCoreRowModel, 12 | getFilteredRowModel, 13 | getPaginationRowModel, 14 | getSortedRowModel, 15 | Row, 16 | RowData, 17 | Table, 18 | } from '@tanstack/table-core'; 19 | 20 | import { Pagination } from './pagination'; 21 | import { ExportDataModal } from '../modals/export-data'; 22 | 23 | // For opening media preview modal in column definitions. 24 | declare module '@tanstack/table-core' { 25 | interface TableMeta { 26 | mediaPreview: string; 27 | setMediaPreview: (url: string) => void; 28 | rawDataPreview: TData | null; 29 | setRawDataPreview: (data: TData | null) => void; 30 | } 31 | 32 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 33 | interface ColumnMeta { 34 | exportable?: boolean; 35 | exportKey?: string; 36 | exportHeader?: string; 37 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 38 | exportValue?: (row: Row) => any; 39 | } 40 | } 41 | 42 | type BaseTableViewProps = { 43 | title: string; 44 | records: T[]; 45 | columns: ColumnDef[]; 46 | clear: () => void; 47 | renderActions?: (table: Table) => JSX.Element; 48 | renderExtra?: (table: Table) => JSX.Element; 49 | }; 50 | 51 | /** 52 | * Basic table view. 53 | */ 54 | export function BaseTableView({ 55 | title, 56 | records, 57 | columns, 58 | clear, 59 | renderActions, 60 | renderExtra, 61 | }: BaseTableViewProps) { 62 | const { t } = useTranslation(); 63 | 64 | // Control modal visibility for previewing media and JSON data. 65 | const [mediaPreview, setMediaPreview] = useSignalState(''); 66 | const [rawDataPreview, setRawDataPreview] = useSignalState(null); 67 | 68 | // Initialize the table instance. 69 | // TODO: implement server-side pagination, sorting, and filtering. 70 | const table = useReactTable({ 71 | data: records ?? [], 72 | columns, 73 | getCoreRowModel: getCoreRowModel(), 74 | getFilteredRowModel: getFilteredRowModel(), 75 | getSortedRowModel: getSortedRowModel(), 76 | getPaginationRowModel: getPaginationRowModel(), 77 | meta: { 78 | mediaPreview, 79 | setMediaPreview: (url) => setMediaPreview(url), 80 | rawDataPreview, 81 | setRawDataPreview: (data) => setRawDataPreview(data), 82 | }, 83 | }); 84 | 85 | // Control modal visibility for exporting data. 86 | const [showExportDataModal, toggleShowExportDataModal] = useToggle(); 87 | 88 | // Select all rows by default. 89 | useEffect(() => { 90 | setTimeout(() => { 91 | if (!table.getIsSomeRowsSelected()) { 92 | table.toggleAllRowsSelected(true); 93 | } 94 | }, 100); 95 | }, [table]); 96 | 97 | return ( 98 | <> 99 | 100 | {/* Data view. */} 101 |
102 | 103 | 104 | {table.getHeaderGroups().map((headerGroup) => ( 105 | 106 | {headerGroup.headers.map((header) => ( 107 | 120 | ))} 121 | 122 | ))} 123 | 124 | 125 | {table.getRowModel().rows.map((row) => ( 126 | 127 | {row.getVisibleCells().map((cell) => ( 128 | 129 | ))} 130 | 131 | ))} 132 | 133 |
112 | {flexRender(header.column.columnDef.header, header.getContext())} 113 | {header.column.getIsSorted() === 'asc' && ( 114 | 115 | )} 116 | {header.column.getIsSorted() === 'desc' && ( 117 | 118 | )} 119 |
{flexRender(cell.column.columnDef.cell, cell.getContext())}
134 | {/* Empty view. */} 135 | {table.getRowModel().rows.length > 0 ? null : ( 136 |
137 |

{t('No data available.')}

138 |
139 | )} 140 |
141 | {/* Page navigation. */} 142 | 143 | {/* Action buttons. */} 144 |
145 | 148 | 149 | {renderActions?.(table)} 150 | 153 |
154 | {/* Extra modal for previewing JSON data. */} 155 | setRawDataPreview(null)} 160 | > 161 |
162 | {typeof rawDataPreview === 'string' ? ( 163 |

{rawDataPreview}

164 | ) : ( 165 | 
{JSON.stringify(rawDataPreview, null, 2)}
166 | )} 167 |
168 | 169 | {/* Extra modal for previewing images and videos. */} 170 | setMediaPreview('')} 175 | > 176 |
177 | {mediaPreview.includes('.mp4') ? ( 178 |
183 | 184 | {/* Extra modal for previewing JSON data. */} 185 | 191 | {/* Extra contents. */} 192 | {renderExtra?.(table)} 193 | 194 | ); 195 | } 196 | -------------------------------------------------------------------------------- /src/components/table/columns-user.tsx: -------------------------------------------------------------------------------- 1 | import { createColumnHelper } from '@tanstack/table-core'; 2 | import { IconLink } from '@tabler/icons-preact'; 3 | import { 4 | formatDateTime, 5 | formatTwitterBirthdate, 6 | parseTwitterDateTime, 7 | strEntitiesToHTML, 8 | } from '@/utils/common'; 9 | import { getProfileImageOriginalUrl, getUserURL } from '@/utils/api'; 10 | import { options } from '@/core/options'; 11 | import { Trans } from '@/i18n'; 12 | import { User } from '@/types'; 13 | 14 | const columnHelper = createColumnHelper(); 15 | 16 | /** 17 | * Table columns definition for users. 18 | */ 19 | export const columns = [ 20 | columnHelper.display({ 21 | id: 'select', 22 | meta: { exportable: false }, 23 | header: ({ table }) => ( 24 | 31 | ), 32 | cell: ({ row }) => ( 33 | 41 | ), 42 | }), 43 | columnHelper.accessor('rest_id', { 44 | meta: { exportKey: 'id', exportHeader: 'ID' }, 45 | header: () => , 46 | cell: (info) =>

{info.getValue()}

, 47 | }), 48 | columnHelper.accessor('legacy.screen_name', { 49 | meta: { exportKey: 'screen_name', exportHeader: 'Screen Name' }, 50 | header: () => , 51 | cell: (info) => ( 52 |

53 | 54 | @{info.getValue()} 55 | 56 |

57 | ), 58 | }), 59 | columnHelper.accessor('legacy.name', { 60 | meta: { exportKey: 'name', exportHeader: 'Profile Name' }, 61 | header: () => , 62 | cell: (info) =>

{info.getValue()}

, 63 | }), 64 | columnHelper.accessor('legacy.description', { 65 | meta: { exportKey: 'description', exportHeader: 'Description' }, 66 | header: () => , 67 | cell: (info) => ( 68 |

77 | ), 78 | }), 79 | columnHelper.accessor('legacy.profile_image_url_https', { 80 | meta: { exportKey: 'profile_image_url', exportHeader: 'Profile Image' }, 81 | header: () => , 82 | cell: (info) => ( 83 |

86 | info.table.options.meta?.setMediaPreview(getProfileImageOriginalUrl(info.getValue())) 87 | } 88 | > 89 | 90 |
91 | ), 92 | }), 93 | columnHelper.accessor('legacy.profile_banner_url', { 94 | meta: { exportKey: 'profile_banner_url', exportHeader: 'Profile Banner' }, 95 | header: () => , 96 | cell: (info) => ( 97 |
info.table.options.meta?.setMediaPreview(info.getValue() ?? '')} 100 | > 101 | {info.getValue() ? ( 102 | 103 | ) : ( 104 | N/A 105 | )} 106 |
107 | ), 108 | }), 109 | columnHelper.accessor('legacy.followers_count', { 110 | meta: { exportKey: 'followers_count', exportHeader: 'Followers' }, 111 | header: () => , 112 | cell: (info) =>

{info.getValue()}

, 113 | }), 114 | columnHelper.accessor('legacy.friends_count', { 115 | meta: { exportKey: 'friends_count', exportHeader: 'FollowingCount' }, 116 | header: () => , 117 | cell: (info) =>

{info.getValue()}

, 118 | }), 119 | columnHelper.accessor('legacy.statuses_count', { 120 | meta: { exportKey: 'statuses_count', exportHeader: 'Statuses' }, 121 | header: () => , 122 | cell: (info) =>

{info.getValue()}

, 123 | }), 124 | columnHelper.accessor('legacy.favourites_count', { 125 | meta: { exportKey: 'favourites_count', exportHeader: 'Favourites' }, 126 | header: () => , 127 | cell: (info) =>

{info.getValue()}

, 128 | }), 129 | columnHelper.accessor('legacy.listed_count', { 130 | meta: { exportKey: 'listed_count', exportHeader: 'Listed' }, 131 | header: () => , 132 | cell: (info) =>

{info.getValue()}

, 133 | }), 134 | columnHelper.accessor('legacy.location', { 135 | meta: { exportKey: 'location', exportHeader: 'Location' }, 136 | header: () => , 137 | cell: (info) =>

{info.getValue() ?? 'N/A'}

, 138 | }), 139 | columnHelper.accessor('legacy.url', { 140 | meta: { exportKey: 'website', exportHeader: 'Website' }, 141 | header: () => , 142 | cell: (info) => ( 143 |

151 | ), 152 | }), 153 | columnHelper.accessor('legacy_extended_profile.birthdate', { 154 | meta: { 155 | exportKey: 'birthdate', 156 | exportHeader: 'Birthdate', 157 | exportValue: (row) => formatTwitterBirthdate(row.original.legacy_extended_profile?.birthdate), 158 | }, 159 | header: () => , 160 | cell: (info) =>

{formatTwitterBirthdate(info.getValue()) ?? 'N/A'}

, 161 | }), 162 | columnHelper.accessor('legacy.verified_type', { 163 | meta: { exportKey: 'verified_type', exportHeader: 'Verified Type' }, 164 | header: () => , 165 | cell: (info) =>

{info.getValue() ?? 'N/A'}

, 166 | }), 167 | columnHelper.accessor('is_blue_verified', { 168 | meta: { exportKey: 'is_blue_verified', exportHeader: 'Blue Verified' }, 169 | header: () => , 170 | cell: (info) =>

{info.getValue() ? 'YES' : 'NO'}

, 171 | }), 172 | columnHelper.accessor('legacy.following', { 173 | meta: { exportKey: 'following', exportHeader: 'Following' }, 174 | header: () => , 175 | cell: (info) =>

{info.getValue() ? 'YES' : 'NO'}

, 176 | }), 177 | columnHelper.accessor('legacy.followed_by', { 178 | meta: { exportKey: 'followed_by', exportHeader: 'Follows You' }, 179 | header: () => , 180 | cell: (info) =>

{info.getValue() ? 'YES' : 'NO'}

, 181 | }), 182 | columnHelper.accessor('legacy.can_dm', { 183 | meta: { exportKey: 'can_dm', exportHeader: 'Can DM' }, 184 | header: () => , 185 | cell: (info) =>

{info.getValue() ? 'YES' : 'NO'}

, 186 | }), 187 | columnHelper.accessor('legacy.protected', { 188 | meta: { exportKey: 'protected', exportHeader: 'Protected' }, 189 | header: () => , 190 | cell: (info) =>

{info.getValue() ? 'YES' : 'NO'}

, 191 | }), 192 | columnHelper.accessor((row) => +parseTwitterDateTime(row.legacy.created_at), { 193 | id: 'created_at', 194 | meta: { 195 | exportKey: 'created_at', 196 | exportHeader: 'Created At', 197 | exportValue: (row) => 198 | formatDateTime( 199 | parseTwitterDateTime(row.original.legacy.created_at), 200 | options.get('dateTimeFormat'), 201 | ), 202 | }, 203 | header: () => , 204 | cell: (info) => ( 205 |

{formatDateTime(info.getValue(), options.get('dateTimeFormat'))}

206 | ), 207 | }), 208 | columnHelper.display({ 209 | id: 'url', 210 | meta: { 211 | exportKey: 'url', 212 | exportHeader: 'URL', 213 | exportValue: (row) => getUserURL(row.original), 214 | }, 215 | header: () => , 216 | cell: (info) => ( 217 | 218 | 219 | 220 | ), 221 | }), 222 | columnHelper.display({ 223 | id: 'actions', 224 | meta: { exportable: false }, 225 | header: () => , 226 | cell: (info) => ( 227 |
228 | 234 |
235 | ), 236 | }), 237 | ]; 238 | -------------------------------------------------------------------------------- /src/components/table/pagination.tsx: -------------------------------------------------------------------------------- 1 | import { Table } from '@tanstack/table-core'; 2 | import { 3 | IconChevronLeft, 4 | IconChevronLeftPipe, 5 | IconChevronRight, 6 | IconChevronRightPipe, 7 | } from '@tabler/icons-preact'; 8 | import { useTranslation } from '@/i18n'; 9 | 10 | type PaginationProps = { 11 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 12 | table: Table; 13 | }; 14 | 15 | export const Pagination = ({ table }: PaginationProps) => { 16 | const { t } = useTranslation(); 17 | const state = table.getState().pagination; 18 | 19 | // With @preact/signals 2.0.1+, this component does not re-render when filtered rows change. 20 | // While the reason is not clear, we will stick at 2.0.0 for now. 21 | 22 | return ( 23 |
24 | {t('Rows per page:')} 25 | 38 | 39 | 40 | {t('A - B of N items', { 41 | from: state.pageSize * state.pageIndex + 1, 42 | to: Math.min( 43 | state.pageSize * (state.pageIndex + 1), 44 | table.getFilteredRowModel().rows.length, 45 | ), 46 | total: table.getFilteredRowModel().rows.length, 47 | })} 48 | 49 | {/* Jump to specific page. */} 50 | { 54 | const value = (e.target as HTMLInputElement).value; 55 | table.setPageIndex(value ? Number(value) - 1 : 0); 56 | }} 57 | className="input input-bordered w-20 input-sm text-center" 58 | /> 59 | {/* Navigation buttons. */} 60 |
61 | 68 | 75 | 82 | 89 |
90 |
91 | ); 92 | }; 93 | -------------------------------------------------------------------------------- /src/components/table/table-view.tsx: -------------------------------------------------------------------------------- 1 | import { ExportMediaModal } from '@/components/modals/export-media'; 2 | import { useCapturedRecords, useClearCaptures } from '@/core/database/hooks'; 3 | import { Extension, ExtensionType } from '@/core/extensions'; 4 | import { useTranslation } from '@/i18n'; 5 | import { Tweet, User } from '@/types'; 6 | import { useToggle } from '@/utils/common'; 7 | import { ColumnDef } from '@tanstack/table-core'; 8 | 9 | import { BaseTableView } from './base'; 10 | import { columns as columnsTweet } from './columns-tweet'; 11 | import { columns as columnsUser } from './columns-user'; 12 | 13 | type TableViewProps = { 14 | title: string; 15 | extension: Extension; 16 | }; 17 | 18 | type InferDataType = T extends ExtensionType.TWEET ? Tweet : User; 19 | 20 | /** 21 | * Common table view. 22 | */ 23 | export function TableView({ title, extension }: TableViewProps) { 24 | const { t } = useTranslation(); 25 | 26 | // Infer data type (Tweet or User) from extension type. 27 | type DataType = InferDataType; 28 | 29 | // Query records from the database. 30 | const { name, type } = extension; 31 | const records = useCapturedRecords(name, type); 32 | const clearCapturedData = useClearCaptures(name); 33 | 34 | // Control modal visibility for exporting media. 35 | const [showExportMediaModal, toggleShowExportMediaModal] = useToggle(); 36 | 37 | const columns = ( 38 | type === ExtensionType.TWEET ? columnsTweet : columnsUser 39 | ) as ColumnDef[]; 40 | 41 | return ( 42 | ( 48 | 51 | )} 52 | renderExtra={(table) => ( 53 | 60 | )} 61 | /> 62 | ); 63 | } 64 | -------------------------------------------------------------------------------- /src/core/app.tsx: -------------------------------------------------------------------------------- 1 | import { Fragment } from 'preact'; 2 | import { useEffect } from 'preact/hooks'; 3 | import { useSignal } from '@preact/signals'; 4 | import { IconBrandTwitterFilled, IconX } from '@tabler/icons-preact'; 5 | import { GM_registerMenuCommand } from '$'; 6 | 7 | import { ErrorBoundary } from '@/components/error-boundary'; 8 | import { CatIcon } from '@/components/common'; 9 | import { useTranslation } from '@/i18n'; 10 | import { cx } from '@/utils/common'; 11 | import logger from '@/utils/logger'; 12 | 13 | import extensionManager, { Extension } from './extensions'; 14 | import { Settings } from './settings'; 15 | import { options } from './options'; 16 | 17 | export function App() { 18 | const { t } = useTranslation(); 19 | 20 | const extensions = useSignal([]); 21 | const currentTheme = useSignal(options.get('theme')); 22 | const showControlPanel = useSignal(options.get('showControlPanel')); 23 | 24 | // Remember the last state of the control panel. 25 | const toggleControlPanel = () => { 26 | showControlPanel.value = !showControlPanel.value; 27 | options.set('showControlPanel', showControlPanel.value); 28 | }; 29 | 30 | // Update UI when extensions or options change. 31 | useEffect(() => { 32 | extensionManager.signal.subscribe(() => { 33 | extensions.value = extensionManager.getExtensions(); 34 | }); 35 | 36 | options.signal.subscribe(() => { 37 | currentTheme.value = options.get('theme'); 38 | }); 39 | 40 | GM_registerMenuCommand(t('Open Control Panel'), toggleControlPanel); 41 | 42 | logger.debug('App useEffect executed'); 43 | }, []); 44 | 45 | return ( 46 | 47 | {/* To show and hide the main UI. */} 48 |
53 |
54 | 55 |
56 |
57 | {/* The main UI block. */} 58 |
65 | {/* Card title. */} 66 |
67 | 68 |

Web Exporter

69 | 70 | 71 | 72 |
76 | 77 |
78 |
79 |

80 | {t('Browse around to capture more data.')} 81 |

82 |
83 | {/* Extensions UI. */} 84 |
85 | {extensions.value.map((ext) => { 86 | const Component = ext.render(); 87 | if (ext.enabled && Component) { 88 | return ( 89 | 90 | 91 | 92 | ); 93 | } 94 | return null; 95 | })} 96 |
97 |
98 | 99 | ); 100 | } 101 | -------------------------------------------------------------------------------- /src/core/database/hooks.ts: -------------------------------------------------------------------------------- 1 | import { ExtensionType } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { Tweet, User } from '@/types'; 4 | import logger from '@/utils/logger'; 5 | import { useLiveQuery } from '@/utils/observable'; 6 | 7 | export function useCaptureCount(extName: string) { 8 | return useLiveQuery(() => db.extGetCaptureCount(extName), [extName], 0); 9 | } 10 | 11 | export function useCapturedRecords(extName: string, type: ExtensionType) { 12 | return useLiveQuery( 13 | () => { 14 | logger.debug('useCapturedRecords liveQuery re-run', extName); 15 | 16 | if (type === ExtensionType.USER) { 17 | return db.extGetCapturedUsers(extName); 18 | } 19 | 20 | if (type === ExtensionType.TWEET) { 21 | return db.extGetCapturedTweets(extName); 22 | } 23 | }, 24 | [extName], 25 | [], 26 | ); 27 | } 28 | 29 | export function useClearCaptures(extName: string) { 30 | return async () => { 31 | logger.debug('Clearing captures for extension:', extName); 32 | return db.extClearCaptures(extName); 33 | }; 34 | } 35 | -------------------------------------------------------------------------------- /src/core/database/index.ts: -------------------------------------------------------------------------------- 1 | import { DatabaseManager } from './manager'; 2 | 3 | export * from './manager'; 4 | 5 | /** 6 | * Global database manager singleton instance. 7 | */ 8 | const databaseManager = new DatabaseManager(); 9 | 10 | export { databaseManager as db }; 11 | -------------------------------------------------------------------------------- /src/core/database/manager.ts: -------------------------------------------------------------------------------- 1 | import Dexie, { KeyPaths } from 'dexie'; 2 | import { exportDB, importInto } from 'dexie-export-import'; 3 | 4 | import packageJson from '@/../package.json'; 5 | import { Capture, Tweet, User } from '@/types'; 6 | import { extractTweetMedia } from '@/utils/api'; 7 | import { parseTwitterDateTime } from '@/utils/common'; 8 | import logger from '@/utils/logger'; 9 | import { ExtensionType } from '../extensions'; 10 | 11 | const DB_NAME = packageJson.name; 12 | const DB_VERSION = 1; 13 | 14 | export class DatabaseManager { 15 | private db: Dexie; 16 | 17 | constructor() { 18 | this.db = new Dexie(DB_NAME); 19 | this.init(); 20 | } 21 | 22 | /* 23 | |-------------------------------------------------------------------------- 24 | | Type-Safe Table Accessors 25 | |-------------------------------------------------------------------------- 26 | */ 27 | 28 | private tweets() { 29 | return this.db.table('tweets'); 30 | } 31 | 32 | private users() { 33 | return this.db.table('users'); 34 | } 35 | 36 | private captures() { 37 | return this.db.table('captures'); 38 | } 39 | 40 | /* 41 | |-------------------------------------------------------------------------- 42 | | Read Methods for Extensions 43 | |-------------------------------------------------------------------------- 44 | */ 45 | 46 | async extGetCaptures(extName: string) { 47 | return this.captures().where('extension').equals(extName).toArray().catch(this.logError); 48 | } 49 | 50 | async extGetCaptureCount(extName: string) { 51 | return this.captures().where('extension').equals(extName).count().catch(this.logError); 52 | } 53 | 54 | async extGetCapturedTweets(extName: string) { 55 | const captures = await this.extGetCaptures(extName); 56 | if (!captures) { 57 | return []; 58 | } 59 | const tweetIds = captures.map((capture) => capture.data_key); 60 | return this.tweets() 61 | .where('rest_id') 62 | .anyOf(tweetIds) 63 | .filter((t) => this.filterEmptyData(t)) 64 | .toArray() 65 | .catch(this.logError); 66 | } 67 | 68 | async extGetCapturedUsers(extName: string) { 69 | const captures = await this.extGetCaptures(extName); 70 | if (!captures) { 71 | return []; 72 | } 73 | const userIds = captures.map((capture) => capture.data_key); 74 | return this.users() 75 | .where('rest_id') 76 | .anyOf(userIds) 77 | .filter((t) => this.filterEmptyData(t)) 78 | .toArray() 79 | .catch(this.logError); 80 | } 81 | 82 | /* 83 | |-------------------------------------------------------------------------- 84 | | Write Methods for Extensions 85 | |-------------------------------------------------------------------------- 86 | */ 87 | 88 | async extAddTweets(extName: string, tweets: Tweet[]) { 89 | await this.upsertTweets(tweets); 90 | await this.upsertCaptures( 91 | tweets.map((tweet) => ({ 92 | id: `${extName}-${tweet.rest_id}`, 93 | extension: extName, 94 | type: ExtensionType.TWEET, 95 | data_key: tweet.rest_id, 96 | created_at: Date.now(), 97 | })), 98 | ); 99 | } 100 | 101 | async extAddUsers(extName: string, users: User[]) { 102 | await this.upsertUsers(users); 103 | await this.upsertCaptures( 104 | users.map((user) => ({ 105 | id: `${extName}-${user.rest_id}`, 106 | extension: extName, 107 | type: ExtensionType.USER, 108 | data_key: user.rest_id, 109 | created_at: Date.now(), 110 | })), 111 | ); 112 | } 113 | 114 | /* 115 | |-------------------------------------------------------------------------- 116 | | Delete Methods for Extensions 117 | |-------------------------------------------------------------------------- 118 | */ 119 | 120 | async extClearCaptures(extName: string) { 121 | const captures = await this.extGetCaptures(extName); 122 | if (!captures) { 123 | return; 124 | } 125 | return this.captures().bulkDelete(captures.map((capture) => capture.id)); 126 | } 127 | 128 | /* 129 | |-------------------------------------------------------------------------- 130 | | Export and Import Methods 131 | |-------------------------------------------------------------------------- 132 | */ 133 | 134 | async export() { 135 | return exportDB(this.db).catch(this.logError); 136 | } 137 | 138 | async import(data: Blob) { 139 | return importInto(this.db, data).catch(this.logError); 140 | } 141 | 142 | async clear() { 143 | await this.deleteAllCaptures(); 144 | await this.deleteAllTweets(); 145 | await this.deleteAllUsers(); 146 | logger.info('Database cleared'); 147 | } 148 | 149 | async count() { 150 | try { 151 | return { 152 | tweets: await this.tweets().count(), 153 | users: await this.users().count(), 154 | captures: await this.captures().count(), 155 | }; 156 | } catch (error) { 157 | this.logError(error); 158 | return null; 159 | } 160 | } 161 | 162 | /* 163 | |-------------------------------------------------------------------------- 164 | | Common Methods 165 | |-------------------------------------------------------------------------- 166 | */ 167 | 168 | async upsertTweets(tweets: Tweet[]) { 169 | return this.db 170 | .transaction('rw', this.tweets(), () => { 171 | const data: Tweet[] = tweets.map((tweet) => ({ 172 | ...tweet, 173 | twe_private_fields: { 174 | created_at: +parseTwitterDateTime(tweet.legacy.created_at), 175 | updated_at: Date.now(), 176 | media_count: extractTweetMedia(tweet).length, 177 | }, 178 | })); 179 | 180 | return this.tweets().bulkPut(data); 181 | }) 182 | .catch(this.logError); 183 | } 184 | 185 | async upsertUsers(users: User[]) { 186 | return this.db 187 | .transaction('rw', this.users(), () => { 188 | const data: User[] = users.map((user) => ({ 189 | ...user, 190 | twe_private_fields: { 191 | created_at: +parseTwitterDateTime(user.legacy.created_at), 192 | updated_at: Date.now(), 193 | }, 194 | })); 195 | 196 | return this.users().bulkPut(data); 197 | }) 198 | .catch(this.logError); 199 | } 200 | 201 | async upsertCaptures(captures: Capture[]) { 202 | return this.db 203 | .transaction('rw', this.captures(), () => { 204 | return this.captures().bulkPut(captures).catch(this.logError); 205 | }) 206 | .catch(this.logError); 207 | } 208 | 209 | async deleteAllTweets() { 210 | return this.tweets().clear().catch(this.logError); 211 | } 212 | 213 | async deleteAllUsers() { 214 | return this.users().clear().catch(this.logError); 215 | } 216 | 217 | async deleteAllCaptures() { 218 | return this.captures().clear().catch(this.logError); 219 | } 220 | 221 | private filterEmptyData(data: Tweet | User) { 222 | if (!data?.legacy) { 223 | logger.warn('Empty data found in DB', data); 224 | return false; 225 | } 226 | return true; 227 | } 228 | 229 | /* 230 | |-------------------------------------------------------------------------- 231 | | Migrations 232 | |-------------------------------------------------------------------------- 233 | */ 234 | 235 | async init() { 236 | // Indexes for the "tweets" table. 237 | const tweetIndexPaths: KeyPaths[] = [ 238 | 'rest_id', 239 | 'twe_private_fields.created_at', 240 | 'twe_private_fields.updated_at', 241 | 'twe_private_fields.media_count', 242 | 'core.user_results.result.legacy.screen_name', 243 | 'legacy.favorite_count', 244 | 'legacy.retweet_count', 245 | 'legacy.bookmark_count', 246 | 'legacy.quote_count', 247 | 'legacy.reply_count', 248 | 'views.count', 249 | 'legacy.favorited', 250 | 'legacy.retweeted', 251 | 'legacy.bookmarked', 252 | ]; 253 | 254 | // Indexes for the "users" table. 255 | const userIndexPaths: KeyPaths[] = [ 256 | 'rest_id', 257 | 'twe_private_fields.created_at', 258 | 'twe_private_fields.updated_at', 259 | 'legacy.screen_name', 260 | 'legacy.followers_count', 261 | // 'legacy.friends_count', 262 | 'legacy.statuses_count', 263 | 'legacy.favourites_count', 264 | 'legacy.listed_count', 265 | 'legacy.verified_type', 266 | 'is_blue_verified', 267 | 'legacy.following', 268 | 'legacy.followed_by', 269 | ]; 270 | 271 | // Indexes for the "captures" table. 272 | const captureIndexPaths: KeyPaths[] = ['id', 'extension', 'type', 'created_at']; 273 | 274 | // Take care of database schemas and versioning. 275 | // See: https://dexie.org/docs/Tutorial/Design#database-versioning 276 | try { 277 | this.db 278 | .version(DB_VERSION) 279 | .stores({ 280 | tweets: tweetIndexPaths.join(','), 281 | users: userIndexPaths.join(','), 282 | captures: captureIndexPaths.join(','), 283 | }) 284 | .upgrade(async () => { 285 | logger.info('Database upgraded'); 286 | }); 287 | 288 | await this.db.open(); 289 | logger.info('Database connected'); 290 | } catch (error) { 291 | this.logError(error); 292 | } 293 | } 294 | 295 | /* 296 | |-------------------------------------------------------------------------- 297 | | Loggers 298 | |-------------------------------------------------------------------------- 299 | */ 300 | 301 | logError(error: unknown) { 302 | logger.error(`Database Error: ${(error as Error).message}`, error); 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /src/core/extensions/extension.ts: -------------------------------------------------------------------------------- 1 | import { ComponentType } from 'preact'; 2 | import type { ExtensionManager } from './manager'; 3 | 4 | export interface ExtensionConstructor { 5 | new (manager: ExtensionManager): Extension; 6 | } 7 | 8 | /** 9 | * Extension UI component type. 10 | */ 11 | export type ExtensionUIComponentType = ComponentType<{ extension: Extension }> | null; 12 | 13 | /** 14 | * HTTP response interceptor. 15 | */ 16 | export type Interceptor = ( 17 | request: Pick, 18 | response: XMLHttpRequest, 19 | extension: Extension, 20 | ) => void; 21 | 22 | /** 23 | * Wether the extension works on tweets or users. 24 | */ 25 | export enum ExtensionType { 26 | TWEET = 'tweet', 27 | USER = 'user', 28 | CUSTOM = 'custom', 29 | NONE = 'none', 30 | } 31 | 32 | /** 33 | * The base class for all extensions. 34 | */ 35 | export abstract class Extension { 36 | public name: string = ''; 37 | public enabled = true; 38 | public type: ExtensionType = ExtensionType.NONE; 39 | 40 | protected manager: ExtensionManager; 41 | 42 | constructor(manager: ExtensionManager) { 43 | this.manager = manager; 44 | } 45 | 46 | /** 47 | * Optionally run side effects when enabled. 48 | */ 49 | public setup(): void { 50 | // noop 51 | } 52 | 53 | /** 54 | * Optionally clear side effects when disabled. 55 | */ 56 | public dispose(): void { 57 | // noop 58 | } 59 | 60 | /** 61 | * Intercept HTTP responses. 62 | */ 63 | public intercept(): Interceptor | null { 64 | return null; 65 | } 66 | 67 | /** 68 | * Render extension UI. 69 | */ 70 | public render(): ExtensionUIComponentType { 71 | return null; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/core/extensions/index.ts: -------------------------------------------------------------------------------- 1 | import { ExtensionManager } from './manager'; 2 | 3 | export * from './manager'; 4 | export * from './extension'; 5 | 6 | /** 7 | * Global extension manager singleton instance. 8 | */ 9 | const extensionManager = new ExtensionManager(); 10 | 11 | export default extensionManager; 12 | -------------------------------------------------------------------------------- /src/core/extensions/manager.ts: -------------------------------------------------------------------------------- 1 | import { unsafeWindow } from '$'; 2 | import { options } from '@/core/options'; 3 | import logger from '@/utils/logger'; 4 | import { Signal } from '@preact/signals'; 5 | import { Extension, ExtensionConstructor } from './extension'; 6 | 7 | /** 8 | * Global object reference. In some cases, the `unsafeWindow` is not available. 9 | */ 10 | const globalObject = unsafeWindow ?? window ?? globalThis; 11 | 12 | /** 13 | * The original XHR method backup. 14 | */ 15 | const xhrOpen = globalObject.XMLHttpRequest.prototype.open; 16 | 17 | /** 18 | * The registry for all extensions. 19 | */ 20 | export class ExtensionManager { 21 | private extensions: Map = new Map(); 22 | private disabledExtensions: Set = new Set(); 23 | private debugEnabled = false; 24 | 25 | /** 26 | * Signal for subscribing to extension changes. 27 | */ 28 | public signal = new Signal(1); 29 | 30 | constructor() { 31 | this.installHttpHooks(); 32 | this.disabledExtensions = new Set(options.get('disabledExtensions', [])); 33 | 34 | // Do some extra logging when debug mode is enabled. 35 | if (options.get('debug')) { 36 | this.debugEnabled = true; 37 | logger.info('Debug mode enabled'); 38 | } 39 | } 40 | 41 | /** 42 | * Register and instantiate a new extension. 43 | * 44 | * @param ctor Extension constructor. 45 | */ 46 | public add(ctor: ExtensionConstructor) { 47 | try { 48 | logger.debug(`Register new extension: ${ctor.name}`); 49 | const instance = new ctor(this); 50 | this.extensions.set(instance.name, instance); 51 | } catch (err) { 52 | logger.error(`Failed to register extension: ${ctor.name}`, err); 53 | } 54 | } 55 | 56 | /** 57 | * Set up all enabled extensions. 58 | */ 59 | public start() { 60 | for (const ext of this.extensions.values()) { 61 | if (this.disabledExtensions.has(ext.name)) { 62 | this.disable(ext.name); 63 | } else { 64 | this.enable(ext.name); 65 | } 66 | } 67 | } 68 | 69 | public enable(name: string) { 70 | try { 71 | this.disabledExtensions.delete(name); 72 | options.set('disabledExtensions', [...this.disabledExtensions]); 73 | 74 | const ext = this.extensions.get(name)!; 75 | ext.enabled = true; 76 | ext.setup(); 77 | 78 | logger.debug(`Enabled extension: ${name}`); 79 | this.signal.value++; 80 | } catch (err) { 81 | logger.error(`Failed to enable extension: ${name}`, err); 82 | } 83 | } 84 | 85 | public disable(name: string) { 86 | try { 87 | this.disabledExtensions.add(name); 88 | options.set('disabledExtensions', [...this.disabledExtensions]); 89 | 90 | const ext = this.extensions.get(name)!; 91 | ext.enabled = false; 92 | ext.dispose(); 93 | 94 | logger.debug(`Disabled extension: ${name}`); 95 | this.signal.value++; 96 | } catch (err) { 97 | logger.error(`Failed to disable extension: ${name}`, err); 98 | } 99 | } 100 | 101 | public getExtensions() { 102 | return [...this.extensions.values()]; 103 | } 104 | 105 | /** 106 | * Here we hooks the browser's XHR method to intercept Twitter's Web API calls. 107 | * This need to be done before any XHR request is made. 108 | */ 109 | private installHttpHooks() { 110 | // eslint-disable-next-line @typescript-eslint/no-this-alias 111 | const manager = this; 112 | 113 | globalObject.XMLHttpRequest.prototype.open = function (method: string, url: string) { 114 | if (manager.debugEnabled) { 115 | logger.debug(`XHR initialized`, { method, url }); 116 | } 117 | 118 | // When the request is done, we call all registered interceptors. 119 | this.addEventListener('load', () => { 120 | if (manager.debugEnabled) { 121 | logger.debug(`XHR finished`, { method, url }); 122 | } 123 | 124 | // Run current enabled interceptors. 125 | manager 126 | .getExtensions() 127 | .filter((ext) => ext.enabled) 128 | .forEach((ext) => { 129 | const func = ext.intercept(); 130 | if (func) { 131 | func({ method, url }, this, ext); 132 | } 133 | }); 134 | }); 135 | 136 | // @ts-expect-error it's fine. 137 | // eslint-disable-next-line prefer-rest-params 138 | xhrOpen.apply(this, arguments); 139 | }; 140 | 141 | logger.info('Hooked into XMLHttpRequest'); 142 | 143 | // Check for current execution context. 144 | // The `webpackChunk_twitter_responsive_web` is injected by the Twitter website. 145 | // See: https://violentmonkey.github.io/posts/inject-into-context/ 146 | setTimeout(() => { 147 | if (!('webpackChunk_twitter_responsive_web' in globalObject)) { 148 | logger.error( 149 | 'Error: Wrong execution context detected.\n ' + 150 | 'This script needs to be injected into "page" context rather than "content" context.\n ' + 151 | 'The XMLHttpRequest hook will not work properly.\n ' + 152 | 'See: https://github.com/prinsss/twitter-web-exporter/issues/19', 153 | ); 154 | } 155 | }, 1000); 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/core/options/index.ts: -------------------------------------------------------------------------------- 1 | import { AppOptionsManager } from './manager'; 2 | 3 | export * from './manager'; 4 | 5 | /** 6 | * Global options manager singleton instance. 7 | */ 8 | const appOptionsManager = new AppOptionsManager(); 9 | 10 | export { appOptionsManager as options }; 11 | -------------------------------------------------------------------------------- /src/core/options/manager.ts: -------------------------------------------------------------------------------- 1 | import { Signal } from '@preact/signals'; 2 | import { isEqual, safeJSONParse } from '@/utils/common'; 3 | import logger from '@/utils/logger'; 4 | import packageJson from '@/../package.json'; 5 | 6 | /** 7 | * Type for global app options. 8 | */ 9 | export interface AppOptions { 10 | theme?: string; 11 | debug?: boolean; 12 | showControlPanel?: boolean; 13 | disabledExtensions?: string[]; 14 | dateTimeFormat?: string; 15 | filenamePattern?: string; 16 | language?: string; 17 | version?: string; 18 | } 19 | 20 | export const DEFAULT_APP_OPTIONS: AppOptions = { 21 | theme: 'system', 22 | debug: false, 23 | showControlPanel: true, 24 | disabledExtensions: [ 25 | 'HomeTimelineModule', 26 | 'ListTimelineModule', 27 | 'ListSubscribersModule', 28 | 'ListMembersModule', 29 | ], 30 | dateTimeFormat: 'YYYY-MM-DD HH:mm:ss Z', 31 | filenamePattern: '{screen_name}_{id}_{type}_{num}_{date}.{ext}', 32 | language: '', 33 | version: packageJson.version, 34 | }; 35 | 36 | // https://daisyui.com/docs/themes/ 37 | export const THEMES = [ 38 | 'system', 39 | 'cupcake', 40 | 'dark', 41 | 'emerald', 42 | 'cyberpunk', 43 | 'valentine', 44 | 'lofi', 45 | 'dracula', 46 | 'cmyk', 47 | 'business', 48 | 'winter', 49 | ] as const; 50 | 51 | const LOCAL_STORAGE_KEY = packageJson.name; 52 | 53 | /** 54 | * Persist app options to browser local storage. 55 | */ 56 | export class AppOptionsManager { 57 | private appOptions: AppOptions = { ...DEFAULT_APP_OPTIONS }; 58 | private previous: AppOptions = { ...DEFAULT_APP_OPTIONS }; 59 | 60 | /** 61 | * Signal for subscribing to option changes. 62 | */ 63 | public signal = new Signal(0); 64 | 65 | constructor() { 66 | this.loadAppOptions(); 67 | } 68 | 69 | public get(key: T, defaultValue?: AppOptions[T]) { 70 | return this.appOptions[key] ?? defaultValue; 71 | } 72 | 73 | public set(key: T, value: AppOptions[T]) { 74 | this.appOptions[key] = value; 75 | this.saveAppOptions(); 76 | } 77 | 78 | /** 79 | * Read app options from local storage. 80 | */ 81 | private loadAppOptions() { 82 | this.appOptions = { 83 | ...this.appOptions, 84 | ...safeJSONParse(localStorage.getItem(LOCAL_STORAGE_KEY) || '{}'), 85 | }; 86 | 87 | const oldVersion = this.appOptions.version ?? ''; 88 | const newVersion = DEFAULT_APP_OPTIONS.version ?? ''; 89 | 90 | // Migrate from v1.0 to v1.1. 91 | if (newVersion.startsWith('1.1') && oldVersion.startsWith('1.0')) { 92 | this.appOptions.disabledExtensions = [ 93 | ...(this.appOptions.disabledExtensions ?? []), 94 | 'HomeTimelineModule', 95 | 'ListTimelineModule', 96 | ]; 97 | logger.info(`App options migrated from v${oldVersion} to v${newVersion}`); 98 | setTimeout(() => this.saveAppOptions(), 0); 99 | } 100 | 101 | this.previous = { ...this.appOptions }; 102 | logger.info('App options loaded', this.appOptions); 103 | this.signal.value++; 104 | } 105 | 106 | /** 107 | * Write app options to local storage. 108 | */ 109 | private saveAppOptions() { 110 | const oldValue = this.previous; 111 | const newValue = { 112 | ...this.appOptions, 113 | version: packageJson.version, 114 | }; 115 | 116 | if (isEqual(oldValue, newValue)) { 117 | return; 118 | } 119 | 120 | this.appOptions = newValue; 121 | localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(this.appOptions)); 122 | 123 | this.previous = { ...this.appOptions }; 124 | logger.debug('App options saved', this.appOptions); 125 | this.signal.value++; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/core/settings.tsx: -------------------------------------------------------------------------------- 1 | import { Fragment } from 'preact'; 2 | import { useEffect } from 'preact/hooks'; 3 | import { useSignal } from '@preact/signals'; 4 | import { 5 | IconSettings, 6 | IconBrandGithubFilled, 7 | IconHelp, 8 | IconDatabaseExport, 9 | IconTrashX, 10 | IconReportAnalytics, 11 | } from '@tabler/icons-preact'; 12 | import { GM_registerMenuCommand } from '$'; 13 | 14 | import packageJson from '@/../package.json'; 15 | import { Modal } from '@/components/common'; 16 | import { useTranslation, detectBrowserLanguage, LANGUAGES_CONFIG, TranslationKey } from '@/i18n'; 17 | import { capitalizeFirstLetter, cx, useToggle } from '@/utils/common'; 18 | import { saveFile } from '@/utils/exporter'; 19 | 20 | import { db } from './database'; 21 | import extensionManager from './extensions'; 22 | import { DEFAULT_APP_OPTIONS, options, THEMES } from './options'; 23 | 24 | export function Settings() { 25 | const { t, i18n } = useTranslation(); 26 | 27 | const currentTheme = useSignal(options.get('theme')); 28 | const [showSettings, toggleSettings] = useToggle(false); 29 | 30 | const styles = { 31 | subtitle: 'mb-2 text-base-content ml-4 opacity-50 font-semibold text-xs', 32 | block: 33 | 'text-sm mb-2 w-full flex px-4 py-2 text-base-content bg-base-200 rounded-box justify-between', 34 | item: 'label cursor-pointer flex justify-between h-8 items-center p-0', 35 | }; 36 | 37 | useEffect(() => { 38 | GM_registerMenuCommand(`${t('Version')} ${packageJson.version}`, () => { 39 | window.open(packageJson.homepage, '_blank'); 40 | }); 41 | }, []); 42 | 43 | return ( 44 | 45 | {/* Settings button. */} 46 |
50 | 51 |
52 | {/* Settings modal. */} 53 | 54 | {/* Common settings. */} 55 |

{t('General')}

56 |
57 | 74 | 95 | 106 | 130 | {/* Database operations. */} 131 |
132 |
133 | {t('Local Database')} 134 |
135 |
136 | 158 | 170 | 181 |
182 |
183 |
184 | {/* Enable or disable modules. */} 185 |

{t('Modules (Scroll to see more)')}

186 |
187 | {extensionManager.getExtensions().map((extension) => ( 188 | 205 | ))} 206 |
207 | {/* Information about this script. */} 208 |

{t('About')}

209 |
210 | 211 | {t('Version')} {packageJson.version} 212 | 213 | 214 | 215 | GitHub 216 | 217 |
218 | 219 | 220 | ); 221 | } 222 | -------------------------------------------------------------------------------- /src/i18n/detector.ts: -------------------------------------------------------------------------------- 1 | export const LANGUAGES_CONFIG = { 2 | en: { 3 | name: 'English', 4 | nameEn: 'English', 5 | test: (code: string) => /^en/.test(code), 6 | }, 7 | 'zh-Hans': { 8 | name: '简体中文', 9 | nameEn: 'Simplified Chinese', 10 | test: (code: string) => /^zh/.test(code), 11 | }, 12 | }; 13 | 14 | /** 15 | * Detect the browser language. 16 | * 17 | * @see https://datatracker.ietf.org/doc/html/rfc4646 18 | * @returns The detected language code. 19 | */ 20 | export function detectBrowserLanguage() { 21 | const language = window.navigator.language || 'en'; 22 | 23 | for (const [langTag, langConf] of Object.entries(LANGUAGES_CONFIG)) { 24 | if (langConf.test(language)) { 25 | return langTag; 26 | } 27 | } 28 | 29 | return language; 30 | } 31 | -------------------------------------------------------------------------------- /src/i18n/i18next.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'virtual:i18next-loader' { 2 | import en_common from '@/i18n/locales/en/common.json'; 3 | import en_exporter from '@/i18n/locales/en/exporter.json'; 4 | 5 | export type LocaleResources = { 6 | en: { 7 | common: typeof en_common; 8 | exporter: typeof en_exporter; 9 | }; 10 | }; 11 | 12 | export type TranslationKey = 13 | | keyof LocaleResources['en']['common'] 14 | | keyof LocaleResources['en']['exporter']; 15 | 16 | const resources: LocaleResources; 17 | export default resources; 18 | } 19 | -------------------------------------------------------------------------------- /src/i18n/index.tsx: -------------------------------------------------------------------------------- 1 | import { Namespace } from 'i18next'; 2 | import { useEffect, useRef, useState } from 'preact/hooks'; 3 | import { TranslationKey } from 'virtual:i18next-loader'; 4 | import { initI18n } from './init'; 5 | 6 | export type { LocaleResources, TranslationKey } from 'virtual:i18next-loader'; 7 | export * from './detector'; 8 | 9 | /** 10 | * A simplified implementation of react-i18next's `useTranslation` for Preact. 11 | * 12 | * @see https://react.i18next.com/latest/usetranslation-hook 13 | * @param namespace The namespace to use for the translation. 14 | * @returns An object with the `t` function and the `i18n` instance. 15 | */ 16 | export function useTranslation(ns?: Namespace) { 17 | const i18n = initI18n(); 18 | 19 | // Bind t function to namespace (acts also as rerender trigger *when* args have changed). 20 | const [t, setT] = useState(() => i18n.getFixedT(null, ns ?? null)); 21 | 22 | // Do not update state if component is unmounted. 23 | const isMountedRef = useRef(true); 24 | const previousNamespaceRef = useRef(ns); 25 | 26 | // Reset t function when namespace changes. 27 | useEffect(() => { 28 | isMountedRef.current = true; 29 | 30 | if (previousNamespaceRef.current !== ns) { 31 | previousNamespaceRef.current = ns; 32 | setT(() => i18n.getFixedT(null, ns ?? null)); 33 | } 34 | 35 | function boundReset() { 36 | if (isMountedRef.current) { 37 | setT(() => i18n.getFixedT(null, ns ?? null)); 38 | } 39 | } 40 | 41 | // Bind events to trigger change. 42 | i18n.on('languageChanged', boundReset); 43 | 44 | // Unbind events on unmount. 45 | return () => { 46 | isMountedRef.current = false; 47 | i18n.off('languageChanged', boundReset); 48 | }; 49 | }, [ns]); 50 | 51 | return { t, i18n }; 52 | } 53 | 54 | /** 55 | * A simplified implementation of react-i18next's `Trans` component for Preact. 56 | * 57 | * @see https://react.i18next.com/latest/trans-component 58 | * @param i18nKey The translation key to use. 59 | * @param ns The namespace to use for the translation. 60 | * @returns The translated string. 61 | */ 62 | export function Trans({ i18nKey, ns = 'exporter' }: { i18nKey: TranslationKey; ns?: Namespace }) { 63 | const { t } = useTranslation(ns); 64 | return {t(i18nKey)}; 65 | } 66 | -------------------------------------------------------------------------------- /src/i18n/init.ts: -------------------------------------------------------------------------------- 1 | import i18next, { LanguageDetectorModule, i18n } from 'i18next'; 2 | import resources, { LocaleResources } from 'virtual:i18next-loader'; 3 | import { options } from '@/core/options'; 4 | import { detectBrowserLanguage } from './detector'; 5 | 6 | declare module 'i18next' { 7 | interface CustomTypeOptions { 8 | defaultNS: 'common'; 9 | resources: LocaleResources['en']; 10 | } 11 | } 12 | 13 | /** 14 | * The language detector for i18next. 15 | */ 16 | export const languageDetector: LanguageDetectorModule = { 17 | type: 'languageDetector', 18 | detect: function () { 19 | return options.get('language') || detectBrowserLanguage(); 20 | }, 21 | }; 22 | 23 | /** 24 | * Initialize i18next and return the instance. 25 | */ 26 | export function initI18n(): i18n { 27 | // We will have only one instance of i18next in the app. 28 | if (i18next.isInitialized) { 29 | return i18next; 30 | } 31 | 32 | // Persist selected language to options storage. 33 | i18next.on('languageChanged', (lng) => { 34 | if (!options.get('language')) { 35 | options.set('language', lng); 36 | } 37 | }); 38 | 39 | // Initialize i18next with the language detector. 40 | i18next.use(languageDetector).init({ 41 | initImmediate: true, 42 | defaultNS: 'common', 43 | fallbackLng: 'en', 44 | nsSeparator: '::', 45 | debug: options.get('debug'), 46 | resources, 47 | }); 48 | 49 | return i18next; 50 | } 51 | -------------------------------------------------------------------------------- /src/i18n/locales/en/common.json: -------------------------------------------------------------------------------- 1 | { 2 | "Open Control Panel": "Open Control Panel", 3 | "Browse around to capture more data.": "Browse around to capture more data.", 4 | "Settings": "Settings", 5 | "General": "General", 6 | "Theme": "Theme", 7 | "Language": "Language", 8 | "Debug": "Debug", 9 | "Date Time Format": "Date Time Format", 10 | "Click for more information. This will take effect on both previewer and exported files.": "Click for more information. This will take effect on both previewer and exported files.", 11 | "Local Database": "Local Database", 12 | "Analyze DB": "Analyze", 13 | "Export DB": "Export", 14 | "Clear DB": "Clear", 15 | "Are you sure to clear all data in the database?": "Are you sure to clear all data in the database?", 16 | "Database cleared.": "Database cleared.", 17 | "Module": "Module", 18 | "Modules (Scroll to see more)": "Modules (Scroll to see more)", 19 | "About": "About", 20 | "Version": "Version", 21 | "Search...": "Search...", 22 | "Something went wrong.": "Something went wrong.", 23 | "Error:": "Error:", 24 | "Captured:": "Captured:", 25 | "Rows per page:": "Rows per page:", 26 | "A - B of N items": "{{from}} - {{to}} of {{total}} items", 27 | "No data available.": "No data available.", 28 | "Clear": "Clear", 29 | "Export Media": "Export Media", 30 | "Export Data": "Export Data", 31 | "JSON View": "JSON View", 32 | "Media View": "Media View", 33 | 34 | "Bookmarks": "Bookmarks", 35 | "DirectMessages": "DirectMessages", 36 | "Followers": "Followers", 37 | "Following": "Following", 38 | "HomeTimeline": "HomeTimeline", 39 | "Likes": "Likes", 40 | "ListMembers": "ListMembers", 41 | "ListSubscribers": "ListSubscribers", 42 | "ListTimeline": "ListTimeline", 43 | "RuntimeLogs": "RuntimeLogs", 44 | "SearchTimeline": "SearchTimeline", 45 | "TweetDetail": "TweetDetail", 46 | "UserDetail": "UserDetail", 47 | "UserMedia": "UserMedia", 48 | "UserTweets": "UserTweets" 49 | } 50 | -------------------------------------------------------------------------------- /src/i18n/locales/en/exporter.json: -------------------------------------------------------------------------------- 1 | { 2 | "ID": "ID", 3 | "Date": "Date", 4 | "Content": "Content", 5 | "Show Full Text": "Show Full Text", 6 | "Media": "Media", 7 | "Screen Name": "Screen Name", 8 | "Profile Name": "Profile Name", 9 | "Profile Image": "Profile Image", 10 | "Replying To": "Replying To", 11 | "RT Source": "RT Source", 12 | "Quote Source": "Quote Source", 13 | "Media Tags": "Media Tags", 14 | "Favorites": "Favorites", 15 | "Retweets": "Retweets", 16 | "Bookmarks": "Bookmarks", 17 | "Quotes": "Quotes", 18 | "Replies": "Replies", 19 | "Views": "Views", 20 | "Favorited": "Favorited", 21 | "Retweeted": "Retweeted", 22 | "Bookmarked": "Bookmarked", 23 | "URL": "URL", 24 | "Actions": "Actions", 25 | "Details": "Details", 26 | "Description": "Description", 27 | "Profile Banner": "Profile Banner", 28 | "Followers": "Followers", 29 | "FollowingCount": "Following Count", 30 | "Statuses": "Statuses", 31 | "Favourites": "Favourites", 32 | "Listed": "Listed", 33 | "Location": "Location", 34 | "Website": "Website", 35 | "Birthdate": "Birthdate", 36 | "Verified Type": "Verified Type", 37 | "Blue Verified": "Blue Verified", 38 | "Following": "Following", 39 | "Follows You": "Follows You", 40 | "Can DM": "Can DM", 41 | "Protected": "Protected", 42 | "Created At": "Created At", 43 | "Sender": "Sender", 44 | "Recipient": "Recipient", 45 | "Conversation ID": "Conversation ID", 46 | "Conversation Type": "Conversation Type", 47 | 48 | "Data": "Data", 49 | "Export captured data as JSON/HTML/CSV file. This may take a while depending on the amount of data. The exported file does not include media files such as images and videos but only the URLs.": "Export captured data as JSON/HTML/CSV file. This may take a while depending on the amount of data. The exported file does not include media files such as images and videos but only the URLs.", 50 | "Data length:": "Data length:", 51 | "Include all metadata:": "Include all metadata:", 52 | "Export as:": "Export as:", 53 | "No data selected.": "No data selected.", 54 | "Cancel": "Cancel", 55 | "Start Export": "Start Export", 56 | "Download and save media files from captured data. This may take a while depending on the amount of data. Media that will be downloaded includes: profile images, profile banners (for users), images, videos (for tweets).": "Download and save media files from captured data. This may take a while depending on the amount of data. Media that will be downloaded includes: profile images, profile banners (for users), images, videos (for tweets).", 57 | "For more than 100 media or large files, it is recommended to copy the URLs and download them with an external download manager such as aria2.": "For more than 100 media or large files, it is recommended to copy the URLs and download them with an external download manager such as aria2.", 58 | "Filename template:": "Filename template:", 59 | "Use aria2 format:": "Use aria2 format:", 60 | "Click for more information. Each URL will be on a new line, with its filename on the next line. This format is compatible with aria2.": "Click for more information. Each URL will be on a new line, with its filename on the next line. This format is compatible with aria2.", 61 | "Rate limit (ms):": "Rate limit (ms):", 62 | "Media Filter:": "Media Filter:", 63 | "File Name": "File Name", 64 | "Media Type": "Media Type", 65 | "Download URL": "Download URL", 66 | "No media selected.": "No media selected.", 67 | "Copied!": "Copied!", 68 | "Copy URLs": "Copy URLs", 69 | "The tweet ID": "The tweet ID", 70 | "The username of tweet author": "The username of tweet author", 71 | "The profile name of tweet author": "The profile name of tweet author", 72 | "The media index in tweet (start from 0)": "The media index in tweet (start from 0)", 73 | "The order of media in tweet (1/2/3/4)": "The order of media in tweet (1/2/3/4)", 74 | "The post date in YYYYMMDD format": "The post date in YYYYMMDD format", 75 | "The post time in HHmmss format": "The post time in HHmmss format", 76 | "The media type (photo/video/animated_gif)": "The media type (photo/video/animated_gif)", 77 | "The file extension of media (jpg/png/mp4)": "The file extension of media (jpg/png/mp4)", 78 | "Failed to export media. Open DevTools for more details.": "Failed to export media. Open DevTools for more details.", 79 | "Failed to copy media URLs. Open DevTools for more details.": "Failed to copy media URLs. Open DevTools for more details.", 80 | 81 | "filter.photo": "Photo", 82 | "filter.video": "Video", 83 | "filter.animated_gif": "GIF", 84 | "filter.retweet": "Include retweets" 85 | } 86 | -------------------------------------------------------------------------------- /src/i18n/locales/zh-Hans/common.json: -------------------------------------------------------------------------------- 1 | { 2 | "Open Control Panel": "打开控制面板", 3 | "Browse around to capture more data.": "浏览页面以捕获更多数据。", 4 | "Settings": "设置", 5 | "General": "通用", 6 | "Theme": "主题", 7 | "Language": "语言", 8 | "Debug": "调试开关", 9 | "Date Time Format": "日期时间格式", 10 | "Click for more information. This will take effect on both previewer and exported files.": "点击查看详细信息。此选项会影响预览和导出的文件。", 11 | "Local Database": "本地数据库", 12 | "Analyze DB": "统计", 13 | "Export DB": "导出", 14 | "Clear DB": "清空", 15 | "Are you sure to clear all data in the database?": "确定要清空数据库中的所有数据吗？", 16 | "Database cleared.": "数据库已清空。", 17 | "Module": "模块", 18 | "Modules (Scroll to see more)": "模块列表（滑动以查看完整列表）", 19 | "About": "关于", 20 | "Version": "版本", 21 | "Search...": "搜索...", 22 | "Something went wrong.": "出错了。", 23 | "Error:": "错误:", 24 | "Captured:": "已捕获:", 25 | "Rows per page:": "每页显示行数:", 26 | "A - B of N items": "第 {{from}} - {{to}} 项，共 {{total}} 项", 27 | "No data available.": "没有数据。", 28 | "Clear": "清除", 29 | "Export Media": "导出媒体文件", 30 | "Export Data": "导出数据", 31 | "JSON View": "JSON 数据预览", 32 | "Media View": "媒体预览", 33 | 34 | "Bookmarks": "书签", 35 | "DirectMessages": "私信", 36 | "Followers": "关注者", 37 | "Following": "正在关注", 38 | "HomeTimeline": "主页时间线", 39 | "Likes": "喜欢", 40 | "ListMembers": "列表成员", 41 | "ListSubscribers": "列表关注者", 42 | "ListTimeline": "列表时间线", 43 | "RuntimeLogs": "运行时日志", 44 | "SearchTimeline": "搜索结果", 45 | "TweetDetail": "推文详情", 46 | "UserDetail": "用户详情", 47 | "UserMedia": "用户媒体", 48 | "UserTweets": "用户推文" 49 | } 50 | -------------------------------------------------------------------------------- /src/i18n/locales/zh-Hans/exporter.json: -------------------------------------------------------------------------------- 1 | { 2 | "ID": "ID", 3 | "Date": "日期", 4 | "Content": "内容", 5 | "Show Full Text": "显示全文", 6 | "Media": "媒体", 7 | "Screen Name": "用户名", 8 | "Profile Name": "用户昵称", 9 | "Profile Image": "用户头像", 10 | "Replying To": "回复推文", 11 | "RT Source": "转推来源", 12 | "Quote Source": "引用来源", 13 | "Media Tags": "圈人", 14 | "Favorites": "喜欢数量", 15 | "Retweets": "转推数量", 16 | "Bookmarks": "书签数量", 17 | "Quotes": "引用数量", 18 | "Replies": "回复数量", 19 | "Views": "查看次数", 20 | "Favorited": "已喜欢", 21 | "Retweeted": "已转推", 22 | "Bookmarked": "已加书签", 23 | "URL": "URL", 24 | "Actions": "操作", 25 | "Details": "查看详情", 26 | "Description": "简介", 27 | "Profile Banner": "个人资料头图", 28 | "Followers": "关注者数量", 29 | "FollowingCount": "正在关注数量", 30 | "Statuses": "推文数量", 31 | "Favourites": "喜欢数量", 32 | "Listed": "被加入列表数", 33 | "Location": "位置", 34 | "Website": "网站", 35 | "Birthdate": "出生日期", 36 | "Verified Type": "认证类型", 37 | "Blue Verified": "蓝标认证", 38 | "Following": "正在关注", 39 | "Follows You": "关注你", 40 | "Can DM": "可私信", 41 | "Protected": "受保护", 42 | "Created At": "创建时间", 43 | "Sender": "发送者", 44 | "Recipient": "接收者", 45 | "Conversation ID": "对话 ID", 46 | "Conversation Type": "对话类型", 47 | 48 | "Data": "数据", 49 | "Export captured data as JSON/HTML/CSV file. This may take a while depending on the amount of data. The exported file does not include media files such as images and videos but only the URLs.": "将捕获的数据导出为 JSON/HTML/CSV 文件。这可能需要一些时间，具体取决于数据量。导出的文件不包括图片和视频等媒体文件，只包括它们的 URL。", 50 | "Data length:": "数据长度:", 51 | "Include all metadata:": "包括所有元数据:", 52 | "Export as:": "导出为:", 53 | "No data selected.": "未选择数据。", 54 | "Cancel": "取消", 55 | "Start Export": "开始导出", 56 | "Download and save media files from captured data. This may take a while depending on the amount of data. Media that will be downloaded includes: profile images, profile banners (for users), images, videos (for tweets).": "从捕获的数据中下载并保存媒体文件。这可能需要一些时间，具体取决于数据量。将下载的媒体包括：用户的个人资料图片、个人资料头图、图片、推文中的视频。", 57 | "For more than 100 media or large files, it is recommended to copy the URLs and download them with an external download manager such as aria2.": "对于超过 100 个媒体或大文件，建议复制 URL 并使用外部下载管理器（如 aria2）下载。", 58 | "Filename template:": "文件名模板:", 59 | "Use aria2 format:": "使用 aria2 格式:", 60 | "Click for more information. Each URL will be on a new line, with its filename on the next line. This format is compatible with aria2.": "点击获取更多信息。每个 URL 将在新行中显示，其文件名在下一行。此格式与 aria2 兼容。", 61 | "Rate limit (ms):": "速率限制（毫秒）:", 62 | "Media Filter:": "媒体过滤器：", 63 | "File Name": "文件名", 64 | "Media Type": "媒体类型", 65 | "Download URL": "下载地址", 66 | "No media selected.": "未选择媒体。", 67 | "Copied!": "已复制！", 68 | "Copy URLs": "复制 URL", 69 | "The tweet ID": "推文 ID", 70 | "The username of tweet author": "推文作者的用户名", 71 | "The profile name of tweet author": "推文作者的用户昵称", 72 | "The media index in tweet (start from 0)": "推文中的媒体索引（从 0 开始）", 73 | "The order of media in tweet (1/2/3/4)": "推文中的媒体顺序（1/2/3/4）", 74 | "The post date in YYYYMMDD format": "发布日期（YYYYMMDD 格式）", 75 | "The post time in HHmmss format": "发布时间（HHmmss 格式）", 76 | "The media type (photo/video/animated_gif)": "媒体类型（photo/video/animated_gif）", 77 | "The file extension of media (jpg/png/mp4)": "媒体文件扩展名（jpg/png/mp4）", 78 | "Failed to export media. Open DevTools for more details.": "导出媒体失败。打开 DevTools 以获取更多详细信息。", 79 | "Failed to copy media URLs. Open DevTools for more details.": "复制媒体 URL 失败。打开 DevTools 以获取更多详细信息。", 80 | 81 | "filter.photo": "图片", 82 | "filter.video": "视频", 83 | "filter.animated_gif": "GIF", 84 | "filter.retweet": "包括转推" 85 | } 86 | -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | @layer utilities { 6 | /* Set border radius relative to the current theme. */ 7 | .rounded-box-half { 8 | border-radius: calc(var(--rounded-box) / 2); 9 | } 10 | 11 | /* Set a different border color for tables. */ 12 | .table-border-bc :where(thead, tbody) :where(tr:not(:last-child)), 13 | .table-border-bc :where(thead, tbody) :where(tr:first-child:last-child) { 14 | --tw-border-opacity: 20%; 15 | border-bottom-width: 1px; 16 | border-bottom-color: var(--fallback-bc, oklch(var(--bc) / var(--tw-border-opacity))); 17 | } 18 | 19 | /* Set a smaller padding for tables. */ 20 | .table-padding-sm :where(th, td) { 21 | padding-left: 0.75rem /* 12px */; 22 | padding-right: 0.75rem /* 12px */; 23 | padding-top: 0.5rem /* 8px */; 24 | padding-bottom: 0.5rem /* 8px */; 25 | } 26 | 27 | /* Hide scrollbar for Chrome, Safari and Opera. */ 28 | .no-scrollbar::-webkit-scrollbar { 29 | display: none; 30 | } 31 | 32 | /* Hide scrollbar for IE, Edge and Firefox. */ 33 | .no-scrollbar { 34 | -ms-overflow-style: none; 35 | scrollbar-width: none; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main.tsx: -------------------------------------------------------------------------------- 1 | import { render } from 'preact'; 2 | import { App } from './core/app'; 3 | import extensions from './core/extensions'; 4 | 5 | import BookmarksModule from './modules/bookmarks'; 6 | import DirectMessagesModule from './modules/direct-messages'; 7 | import FollowersModule from './modules/followers'; 8 | import FollowingModule from './modules/following'; 9 | import HomeTimelineModule from './modules/home-timeline'; 10 | import LikesModule from './modules/likes'; 11 | import ListMembersModule from './modules/list-members'; 12 | import ListSubscribersModule from './modules/list-subscribers'; 13 | import ListTimelineModule from './modules/list-timeline'; 14 | import RuntimeLogsModule from './modules/runtime-logs'; 15 | import SearchTimelineModule from './modules/search-timeline'; 16 | import TweetDetailModule from './modules/tweet-detail'; 17 | import UserDetailModule from './modules/user-detail'; 18 | import UserMediaModule from './modules/user-media'; 19 | import UserTweetsModule from './modules/user-tweets'; 20 | 21 | import './index.css'; 22 | 23 | extensions.add(FollowersModule); 24 | extensions.add(FollowingModule); 25 | extensions.add(UserDetailModule); 26 | extensions.add(ListMembersModule); 27 | extensions.add(ListSubscribersModule); 28 | extensions.add(HomeTimelineModule); 29 | extensions.add(ListTimelineModule); 30 | extensions.add(BookmarksModule); 31 | extensions.add(LikesModule); 32 | extensions.add(UserTweetsModule); 33 | extensions.add(UserMediaModule); 34 | extensions.add(TweetDetailModule); 35 | extensions.add(SearchTimelineModule); 36 | extensions.add(DirectMessagesModule); 37 | extensions.add(RuntimeLogsModule); 38 | extensions.start(); 39 | 40 | function mountApp() { 41 | const root = document.createElement('div'); 42 | root.id = 'twe-root'; 43 | document.body.append(root); 44 | 45 | render(, root); 46 | } 47 | 48 | if (document.readyState === 'loading') { 49 | document.addEventListener('DOMContentLoaded', mountApp); 50 | } else { 51 | mountApp(); 52 | } 53 | -------------------------------------------------------------------------------- /src/modules/bookmarks/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, Tweet } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineTweet } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface BookmarksResponse { 8 | data: { 9 | bookmark_timeline_v2: { 10 | timeline: { 11 | instructions: TimelineInstructions; 12 | responseObjects: unknown; 13 | }; 14 | }; 15 | }; 16 | } 17 | 18 | // https://twitter.com/i/api/graphql/j5KExFXtSWj8HjRui17ydA/Bookmarks 19 | export const BookmarksInterceptor: Interceptor = (req, res, ext) => { 20 | if (!/\/graphql\/.+\/Bookmarks/.test(req.url)) { 21 | return; 22 | } 23 | 24 | try { 25 | const newData = extractDataFromResponse( 26 | res, 27 | (json) => json.data.bookmark_timeline_v2.timeline.instructions, 28 | (entry) => extractTimelineTweet(entry.content.itemContent), 29 | ); 30 | 31 | // Add captured data to the database. 32 | db.extAddTweets(ext.name, newData); 33 | 34 | logger.info(`Bookmarks: ${newData.length} items received`); 35 | } catch (err) { 36 | logger.debug(req.method, req.url, res.status, res.responseText); 37 | logger.errorWithBanner('Bookmarks: Failed to parse API response', err as Error); 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /src/modules/bookmarks/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { BookmarksInterceptor } from './api'; 4 | 5 | export default class BookmarksModule extends Extension { 6 | name = 'BookmarksModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return BookmarksInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/direct-messages/api.ts: -------------------------------------------------------------------------------- 1 | import { signal } from '@preact/signals'; 2 | import { Interceptor } from '@/core/extensions'; 3 | import logger from '@/utils/logger'; 4 | import { 5 | Conversation, 6 | ConversationResponse, 7 | DmEntry, 8 | InboxInitialStateResponse, 9 | InboxTimelineTrustedResponse, 10 | LegacyUser, 11 | Message, 12 | } from './types'; 13 | 14 | /** 15 | * The global store for "DirectMessages". 16 | * 17 | * Still use signal here instead of storing to database since 18 | * it's a new data type and we are not ready to add new tables yet. 19 | */ 20 | export const messagesSignal = signal([]); 21 | export const conversationsCollection = new Map(); 22 | export const usersCollection = new Map(); 23 | 24 | type Strategy = { 25 | test: (url: string) => boolean; 26 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 27 | parse: (json: any) => { 28 | entries: DmEntry[]; 29 | conversations: Conversation[]; 30 | users: LegacyUser[]; 31 | }; 32 | }; 33 | 34 | // We need to intercept multiple API endpoints to get direct messages. 35 | const strategies: Strategy[] = [ 36 | { 37 | test: (url) => /\/dm\/inbox_initial_state\.json/.test(url), 38 | parse: (json: InboxInitialStateResponse) => ({ 39 | entries: json.inbox_initial_state.entries, 40 | conversations: Object.values(json.inbox_initial_state.conversations), 41 | users: Object.values(json.inbox_initial_state.users), 42 | }), 43 | }, 44 | { 45 | test: (url) => /\/dm\/inbox_timeline\/trusted\.json/.test(url), 46 | parse: (json: InboxTimelineTrustedResponse) => ({ 47 | entries: json.inbox_timeline.entries, 48 | conversations: Object.values(json.inbox_timeline.conversations), 49 | users: Object.values(json.inbox_timeline.users), 50 | }), 51 | }, 52 | { 53 | test: (url) => /\/dm\/conversation\/\d+-?\d+\.json/.test(url), 54 | parse: (json: ConversationResponse) => ({ 55 | entries: json.conversation_timeline.entries, 56 | conversations: Object.values(json.conversation_timeline.conversations), 57 | users: Object.values(json.conversation_timeline.users), 58 | }), 59 | }, 60 | ]; 61 | 62 | // https://twitter.com/i/api/1.1/dm/inbox_initial_state.json 63 | // https://twitter.com/i/api/1.1/dm/inbox_timeline/trusted.json 64 | // https://twitter.com/i/api/1.1/dm/conversation/{uid}-{uid}.json # ONE_TO_ONE 65 | // https://twitter.com/i/api/1.1/dm/conversation/{cid}.json # GROUP_DM 66 | export const DirectMessagesInterceptor: Interceptor = (req, res) => { 67 | const strategy = strategies.find((s) => s.test(req.url)); 68 | 69 | if (!strategy) { 70 | return; 71 | } 72 | 73 | try { 74 | const json = JSON.parse(res.responseText); 75 | const { entries, conversations, users } = strategy.parse(json); 76 | const messages = entries.map((entry) => entry.message).filter((message) => !!message); 77 | 78 | messagesSignal.value = [...messagesSignal.value, ...messages]; 79 | conversations.filter(Boolean).forEach((c) => conversationsCollection.set(c.conversation_id, c)); 80 | users.filter(Boolean).forEach((user) => usersCollection.set(user.id_str, user)); 81 | 82 | logger.info(`DirectMessages: ${messages.length} items received`); 83 | } catch (err) { 84 | logger.debug(req.method, req.url, res.status, res.responseText); 85 | logger.errorWithBanner('DirectMessages: Failed to parse API response', err as Error); 86 | } 87 | }; 88 | -------------------------------------------------------------------------------- /src/modules/direct-messages/columns.tsx: -------------------------------------------------------------------------------- 1 | import dayjs from 'dayjs'; 2 | 3 | import { MediaDisplayColumn } from '@/components/common'; 4 | import { options } from '@/core/options'; 5 | import { Trans } from '@/i18n'; 6 | import { formatTwitterImage, getMediaOriginalUrl } from '@/utils/api'; 7 | import { formatDateTime, strEntitiesToHTML } from '@/utils/common'; 8 | import { createColumnHelper } from '@tanstack/table-core'; 9 | 10 | import { conversationsCollection, usersCollection } from './api'; 11 | import { Message } from './types'; 12 | 13 | function getUserScreeNameFromId(id: string | undefined) { 14 | const user = id ? usersCollection.get(id) : null; 15 | return user ? user.screen_name : ''; 16 | } 17 | 18 | function getConversationTypeFromId(id: string | undefined) { 19 | const conversation = id ? conversationsCollection.get(id) : null; 20 | return conversation ? conversation.type : ''; 21 | } 22 | 23 | function extractMessageMedia(message: Message) { 24 | return [ 25 | message.message_data.attachment?.photo, 26 | message.message_data.attachment?.video, 27 | message.message_data.attachment?.animated_gif, 28 | ].filter((m) => !!m); 29 | } 30 | 31 | const columnHelper = createColumnHelper(); 32 | 33 | /** 34 | * Table columns definition for direct messages. 35 | */ 36 | export const columns = [ 37 | columnHelper.display({ 38 | id: 'select', 39 | meta: { exportable: false }, 40 | header: ({ table }) => ( 41 | 48 | ), 49 | cell: ({ row }) => ( 50 | 58 | ), 59 | }), 60 | columnHelper.accessor('id', { 61 | meta: { exportKey: 'id', exportHeader: 'ID' }, 62 | header: () => , 63 | cell: (info) =>

{info.getValue()}

, 64 | }), 65 | columnHelper.accessor('time', { 66 | meta: { 67 | exportKey: 'time', 68 | exportHeader: 'Date', 69 | exportValue: (row) => 70 | formatDateTime(dayjs(+row.original.time), options.get('dateTimeFormat')), 71 | }, 72 | header: () => , 73 | cell: (info) => ( 74 |

{formatDateTime(+info.getValue(), options.get('dateTimeFormat'))}

75 | ), 76 | }), 77 | columnHelper.accessor('message_data.text', { 78 | meta: { 79 | exportKey: 'text', 80 | exportHeader: 'Content', 81 | }, 82 | header: () => , 83 | cell: (info) => ( 84 |
85 |

94 |

95 | ), 96 | }), 97 | columnHelper.accessor((row) => extractMessageMedia(row), { 98 | id: 'media', 99 | meta: { 100 | exportKey: 'media', 101 | exportHeader: 'Media', 102 | exportValue: (row) => 103 | extractMessageMedia(row.original).map((media) => ({ 104 | type: media.type, 105 | url: media.url, 106 | thumbnail: formatTwitterImage(media.media_url_https, 'thumb'), 107 | original: getMediaOriginalUrl(media), 108 | ext_alt_text: media.ext_alt_text, 109 | })), 110 | }, 111 | header: () => , 112 | cell: (info) => ( 113 | !!m)} 115 | onClick={(media) => info.table.options.meta?.setMediaPreview(getMediaOriginalUrl(media))} 116 | /> 117 | ), 118 | }), 119 | columnHelper.accessor((row) => getUserScreeNameFromId(row.message_data.sender_id), { 120 | id: 'sender', 121 | meta: { exportKey: 'sender', exportHeader: 'Sender' }, 122 | header: () => , 123 | cell: (info) => ( 124 |

125 | 126 | @{info.getValue()} 127 | 128 |

129 | ), 130 | }), 131 | columnHelper.accessor((row) => getUserScreeNameFromId(row.message_data.recipient_id), { 132 | id: 'recipient', 133 | meta: { exportKey: 'recipient', exportHeader: 'Recipient' }, 134 | header: () => , 135 | cell: (info) => ( 136 |

137 | {info.getValue() ? ( 138 | 139 | @{info.getValue()} 140 | 141 | ) : ( 142 | 'N/A' 143 | )} 144 |

145 | ), 146 | }), 147 | columnHelper.accessor('conversation_id', { 148 | meta: { exportKey: 'conversation_id', exportHeader: 'Conversation ID' }, 149 | header: () => , 150 | cell: (info) =>

{info.getValue()}

, 151 | }), 152 | columnHelper.accessor((row) => getConversationTypeFromId(row.conversation_id), { 153 | id: 'conversation_type', 154 | meta: { exportKey: 'conversation_type', exportHeader: 'Conversation Type' }, 155 | header: () => , 156 | cell: (info) =>

{info.getValue()}

, 157 | }), 158 | columnHelper.display({ 159 | id: 'actions', 160 | meta: { exportable: false }, 161 | header: () => , 162 | cell: (info) => ( 163 |
164 | 170 |
171 | ), 172 | }), 173 | ]; 174 | -------------------------------------------------------------------------------- /src/modules/direct-messages/index.tsx: -------------------------------------------------------------------------------- 1 | import { Extension, ExtensionType } from '@/core/extensions'; 2 | import { DirectMessagesInterceptor } from './api'; 3 | import { DirectMessagesUI } from './ui'; 4 | 5 | export default class DirectMessagesModule extends Extension { 6 | name = 'DirectMessagesModule'; 7 | 8 | type = ExtensionType.CUSTOM; 9 | 10 | intercept() { 11 | return DirectMessagesInterceptor; 12 | } 13 | 14 | render() { 15 | return DirectMessagesUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/direct-messages/types.ts: -------------------------------------------------------------------------------- 1 | import { Media, TweetEntities, TweetExtendedEntities, UserEntities } from '@/types'; 2 | 3 | export interface InboxInitialStateResponse { 4 | inbox_initial_state: DmTimeline; 5 | } 6 | 7 | export interface InboxTimelineTrustedResponse { 8 | inbox_timeline: DmTimeline & { 9 | status: DmTimelineStatus; 10 | min_entry_id: string; 11 | }; 12 | } 13 | 14 | export interface ConversationResponse { 15 | conversation_timeline: DmTimeline & { 16 | status: DmTimelineStatus; 17 | min_entry_id: string; 18 | max_entry_id: string; 19 | }; 20 | } 21 | 22 | type DmTimelineStatus = 'HAS_MORE' | 'AT_END'; 23 | 24 | interface DmTimeline { 25 | entries: DmEntry[]; 26 | users: { 27 | [key: string]: U; 28 | }; 29 | conversations: { 30 | [key: string]: Conversation; 31 | }; 32 | } 33 | 34 | export interface DmEntry { 35 | message?: Message; 36 | conversation_create?: ConversationCreate; 37 | trust_conversation?: TrustConversation; 38 | join_conversation?: JoinConversation; 39 | participants_join?: ParticipantsJoin; 40 | participants_leave?: ParticipantsLeave; 41 | } 42 | 43 | export interface Message { 44 | id: string; 45 | time: string; 46 | affects_sort: boolean; 47 | request_id?: string; 48 | conversation_id: string; 49 | message_data: MessageData; 50 | message_reactions?: MessageReaction[]; 51 | } 52 | 53 | interface MessageData { 54 | id: string; 55 | time: string; 56 | recipient_id?: string; 57 | sender_id: string; 58 | text: string; 59 | edit_count?: number; 60 | conversation_id?: string; 61 | entities?: TweetEntities; 62 | attachment?: { 63 | photo?: Media; 64 | video?: Media; 65 | animated_gif?: Media; 66 | card?: AttachmentCard; 67 | tweet?: AttachmentTweet; 68 | }; 69 | } 70 | 71 | interface AttachmentCard { 72 | name: string; 73 | url: string; 74 | card_type_url: string; 75 | binding_values: unknown; 76 | users?: { 77 | [key: string]: LegacyUserExtended; 78 | }; 79 | } 80 | 81 | interface AttachmentTweet { 82 | id: string; 83 | url: string; 84 | display_url: string; 85 | expanded_url: string; 86 | indices: number[]; 87 | status: DmTweet; 88 | } 89 | 90 | interface MessageReaction { 91 | id: string; 92 | time: string; 93 | conversation_id: string; 94 | message_id: string; 95 | reaction_key: 'agree' | 'emoji'; 96 | emoji_reaction?: string; 97 | sender_id: string; 98 | } 99 | 100 | interface ConversationCreate { 101 | id: string; 102 | time: string; 103 | conversation_id: string; 104 | request_id?: string; 105 | } 106 | 107 | interface TrustConversation { 108 | id: string; 109 | time: string; 110 | affects_sort: boolean; 111 | conversation_id: string; 112 | reason: string; 113 | request_id?: string; 114 | } 115 | 116 | interface JoinConversation { 117 | id: string; 118 | time: string; 119 | affects_sort: boolean; 120 | conversation_id: string; 121 | sender_id: string; 122 | participants: { 123 | user_id: string; 124 | }[]; 125 | } 126 | 127 | interface ParticipantsJoin { 128 | id: string; 129 | time: string; 130 | affects_sort: boolean; 131 | conversation_id: string; 132 | sender_id: string; 133 | participants: { 134 | user_id: string; 135 | join_time: string; 136 | }[]; 137 | } 138 | 139 | interface ParticipantsLeave { 140 | id: string; 141 | time: string; 142 | affects_sort: boolean; 143 | conversation_id: string; 144 | participants: { 145 | user_id: string; 146 | }[]; 147 | } 148 | 149 | interface DmTweet { 150 | created_at: string; 151 | id: number; 152 | id_str: string; 153 | full_text: string; 154 | truncated: boolean; 155 | display_text_range: number[]; 156 | entities: TweetEntities; 157 | extended_entities?: TweetExtendedEntities; 158 | source: string; 159 | in_reply_to_status_id: number | null; 160 | in_reply_to_status_id_str: string | null; 161 | in_reply_to_user_id: number | null; 162 | in_reply_to_user_id_str: string | null; 163 | in_reply_to_screen_name: string | null; 164 | user: LegacyUserExtended; 165 | geo: null; 166 | coordinates: null; 167 | place: null; 168 | contributors: null; 169 | is_quote_status: boolean; 170 | quoted_status_id?: number; 171 | quoted_status_id_str?: string; 172 | quoted_status_permalink?: { 173 | url: string; 174 | expanded: string; 175 | display: string; 176 | }; 177 | retweet_count: number; 178 | favorite_count: number; 179 | reply_count: number; 180 | quote_count: number; 181 | favorited: boolean; 182 | retweeted: boolean; 183 | possibly_sensitive: boolean; 184 | possibly_sensitive_editable: boolean; 185 | lang: string; 186 | supplemental_language: null; 187 | self_thread?: { 188 | id: number; 189 | id_str: string; 190 | }; 191 | ext: unknown; 192 | } 193 | 194 | export interface LegacyUser { 195 | id: number; 196 | id_str: string; 197 | name: string; 198 | screen_name: string; 199 | profile_image_url: string; 200 | profile_image_url_https: string; 201 | following: boolean; 202 | follow_request_sent: boolean; 203 | description: string | null; 204 | entities: UserEntities; 205 | verified: boolean; 206 | is_blue_verified: boolean; // This is actually not present in LegacyUserExtended. 207 | protected: boolean; 208 | blocking: boolean; 209 | subscribed_by: boolean; 210 | can_media_tag: boolean; 211 | dm_blocked_by: boolean; 212 | dm_blocking: boolean; 213 | created_at: string; 214 | friends_count: number; 215 | followers_count: number; 216 | } 217 | 218 | interface LegacyUserExtended extends LegacyUser { 219 | location: string | null; 220 | url: string | null; 221 | listed_count: number; 222 | favourites_count: number; 223 | utc_offset: null; 224 | time_zone: null; 225 | geo_enabled: boolean; 226 | statuses_count: number; 227 | lang: null; 228 | contributors_enabled: boolean; 229 | is_translator: boolean; 230 | is_translation_enabled: boolean; 231 | profile_background_color: string; 232 | profile_background_image_url: string; 233 | profile_background_image_url_https: string; 234 | profile_background_tile: boolean; 235 | profile_banner_url: string; 236 | profile_link_color: string; 237 | profile_sidebar_border_color: string; 238 | profile_sidebar_fill_color: string; 239 | profile_text_color: string; 240 | profile_use_background_image: boolean; 241 | default_profile: boolean; 242 | default_profile_image: boolean; 243 | can_dm: null; 244 | can_secret_dm: null; 245 | notifications: boolean; 246 | blocked_by: boolean; 247 | want_retweets: boolean; 248 | business_profile_state: string; 249 | translator_type: string; 250 | withheld_in_countries: unknown[]; 251 | followed_by: boolean; 252 | ext?: unknown; 253 | } 254 | 255 | export interface Conversation { 256 | conversation_id: string; 257 | type: 'ONE_TO_ONE' | 'GROUP_DM'; 258 | sort_event_id: string; 259 | sort_timestamp: string; 260 | participants: { 261 | user_id: string; 262 | last_read_event_id?: string; 263 | }[]; 264 | nsfw: boolean; 265 | notifications_disabled: boolean; 266 | mention_notifications_disabled: boolean; 267 | last_read_event_id: string; 268 | read_only: boolean; 269 | trusted: boolean; 270 | muted: boolean; 271 | status: DmTimelineStatus; 272 | min_entry_id: string; 273 | max_entry_id: string; 274 | } 275 | -------------------------------------------------------------------------------- /src/modules/direct-messages/ui.tsx: -------------------------------------------------------------------------------- 1 | import { ColumnDef } from '@tanstack/table-core'; 2 | 3 | import { ExtensionPanel, Modal } from '@/components/common'; 4 | import { BaseTableView } from '@/components/table/base'; 5 | import { useTranslation } from '@/i18n'; 6 | import { useToggle } from '@/utils/common'; 7 | 8 | import { messagesSignal } from './api'; 9 | import { columns } from './columns'; 10 | import { Message } from './types'; 11 | 12 | export function DirectMessagesUI() { 13 | const { t } = useTranslation(); 14 | const [showModal, toggleShowModal] = useToggle(); 15 | 16 | const title = t('DirectMessages'); 17 | const count = messagesSignal.value.length; 18 | 19 | return ( 20 | 0} 24 | onClick={toggleShowModal} 25 | indicatorColor="bg-accent" 26 | > 27 | 33 | 34 | title={title} 35 | records={messagesSignal.value} 36 | columns={columns as ColumnDef[]} 37 | clear={() => (messagesSignal.value = [])} 38 | /> 39 | 40 | 41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /src/modules/followers/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, User } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineUser } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface FollowersResponse { 8 | data: { 9 | user: { 10 | result: { 11 | timeline: { 12 | timeline: { 13 | instructions: TimelineInstructions; 14 | }; 15 | }; 16 | __typename: 'User'; 17 | }; 18 | }; 19 | }; 20 | } 21 | 22 | // https://twitter.com/i/api/graphql/rRXFSG5vR6drKr5M37YOTw/Followers 23 | // https://twitter.com/i/api/graphql/kXi37EbqWokFUNypPHhQDQ/BlueVerifiedFollowers 24 | export const FollowersInterceptor: Interceptor = (req, res, ext) => { 25 | if (!/\/graphql\/.+\/(BlueVerified)*Followers/.test(req.url)) { 26 | return; 27 | } 28 | 29 | try { 30 | const newData = extractDataFromResponse( 31 | res, 32 | (json) => json.data.user.result.timeline.timeline.instructions, 33 | (entry) => extractTimelineUser(entry.content.itemContent), 34 | ); 35 | 36 | // Add captured data to the database. 37 | db.extAddUsers(ext.name, newData); 38 | 39 | logger.info(`Followers: ${newData.length} items received`); 40 | } catch (err) { 41 | logger.debug(req.method, req.url, res.status, res.responseText); 42 | logger.errorWithBanner('Followers: Failed to parse API response', err as Error); 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /src/modules/followers/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { FollowersInterceptor } from './api'; 4 | 5 | export default class FollowersModule extends Extension { 6 | name = 'FollowersModule'; 7 | 8 | type = ExtensionType.USER; 9 | 10 | intercept() { 11 | return FollowersInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/following/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, User } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineUser } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface FollowingResponse { 8 | data: { 9 | user: { 10 | result: { 11 | timeline: { 12 | timeline: { 13 | instructions: TimelineInstructions; 14 | }; 15 | }; 16 | __typename: 'User'; 17 | }; 18 | }; 19 | }; 20 | } 21 | 22 | // https://twitter.com/i/api/graphql/iSicc7LrzWGBgDPL0tM_TQ/Following 23 | export const FollowingInterceptor: Interceptor = (req, res, ext) => { 24 | if (!/\/graphql\/.+\/Following/.test(req.url)) { 25 | return; 26 | } 27 | 28 | try { 29 | const newData = extractDataFromResponse( 30 | res, 31 | (json) => json.data.user.result.timeline.timeline.instructions, 32 | (entry) => extractTimelineUser(entry.content.itemContent), 33 | ); 34 | 35 | // Add captured data to the database. 36 | db.extAddUsers(ext.name, newData); 37 | 38 | logger.info(`Following: ${newData.length} items received`); 39 | } catch (err) { 40 | logger.debug(req.method, req.url, res.status, res.responseText); 41 | logger.errorWithBanner('Following: Failed to parse API response', err as Error); 42 | } 43 | }; 44 | -------------------------------------------------------------------------------- /src/modules/following/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { FollowingInterceptor } from './api'; 4 | 5 | export default class FollowingModule extends Extension { 6 | name = 'FollowingModule'; 7 | 8 | type = ExtensionType.USER; 9 | 10 | intercept() { 11 | return FollowingInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/home-timeline/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, Tweet } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineTweet } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface HomeTimelineResponse { 8 | data: { 9 | home: { 10 | home_timeline_urt: { 11 | instructions: TimelineInstructions; 12 | metadata: unknown; 13 | responseObjects: unknown; 14 | }; 15 | }; 16 | }; 17 | } 18 | 19 | // https://twitter.com/i/api/graphql/uPv755D929tshj6KsxkSZg/HomeTimeline 20 | // https://twitter.com/i/api/graphql/70b_oNkcK9IEN13WNZv8xA/HomeLatestTimeline 21 | export const HomeTimelineInterceptor: Interceptor = (req, res, ext) => { 22 | if (!/\/graphql\/.+\/Home(Latest)?Timeline/.test(req.url)) { 23 | return; 24 | } 25 | 26 | try { 27 | const newData = extractDataFromResponse( 28 | res, 29 | (json) => json.data.home.home_timeline_urt.instructions, 30 | (entry) => extractTimelineTweet(entry.content.itemContent), 31 | ); 32 | 33 | // Add captured data to the database. 34 | db.extAddTweets(ext.name, newData); 35 | 36 | logger.info(`HomeTimeline: ${newData.length} items received`); 37 | } catch (err) { 38 | logger.debug(req.method, req.url, res.status, res.responseText); 39 | logger.errorWithBanner('HomeTimeline: Failed to parse API response', err as Error); 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /src/modules/home-timeline/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { HomeTimelineInterceptor } from './api'; 4 | 5 | export default class HomeTimelineModule extends Extension { 6 | name = 'HomeTimelineModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return HomeTimelineInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/likes/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, Tweet } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineTweet } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface LikesResponse { 8 | data: { 9 | user: { 10 | result: { 11 | timeline: { 12 | timeline: { 13 | instructions: TimelineInstructions; 14 | responseObjects: unknown; 15 | }; 16 | }; 17 | __typename: 'User'; 18 | }; 19 | }; 20 | }; 21 | } 22 | 23 | // https://twitter.com/i/api/graphql/lVf2NuhLoYVrpN4nO7uw0Q/Likes 24 | export const LikesInterceptor: Interceptor = (req, res, ext) => { 25 | if (!/\/graphql\/.+\/Likes/.test(req.url)) { 26 | return; 27 | } 28 | 29 | try { 30 | const newData = extractDataFromResponse( 31 | res, 32 | (json) => json.data.user.result.timeline.timeline.instructions, 33 | (entry) => extractTimelineTweet(entry.content.itemContent), 34 | ); 35 | 36 | // Add captured data to the database. 37 | db.extAddTweets(ext.name, newData); 38 | 39 | logger.info(`Likes: ${newData.length} items received`); 40 | } catch (err) { 41 | logger.debug(req.method, req.url, res.status, res.responseText); 42 | logger.errorWithBanner('Likes: Failed to parse API response', err as Error); 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /src/modules/likes/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { LikesInterceptor } from './api'; 4 | 5 | export default class LikesModule extends Extension { 6 | name = 'LikesModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return LikesInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/list-members/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, User } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineUser } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface ListMembersResponse { 8 | data: { 9 | list: { 10 | members_timeline: { 11 | timeline: { 12 | instructions: TimelineInstructions; 13 | }; 14 | }; 15 | }; 16 | }; 17 | } 18 | 19 | // https://twitter.com/i/api/graphql/-5VwQkb7axZIxFkFS44iWw/ListMembers 20 | export const ListMembersInterceptor: Interceptor = (req, res, ext) => { 21 | if (!/\/graphql\/.+\/ListMembers/.test(req.url)) { 22 | return; 23 | } 24 | 25 | try { 26 | const newData = extractDataFromResponse( 27 | res, 28 | (json) => json.data.list.members_timeline.timeline.instructions, 29 | (entry) => extractTimelineUser(entry.content.itemContent), 30 | ); 31 | 32 | // Add captured data to the database. 33 | db.extAddUsers(ext.name, newData); 34 | 35 | logger.info(`ListMembers: ${newData.length} items received`); 36 | } catch (err) { 37 | logger.debug(req.method, req.url, res.status, res.responseText); 38 | logger.errorWithBanner('ListMembers: Failed to parse API response', err as Error); 39 | } 40 | }; 41 | -------------------------------------------------------------------------------- /src/modules/list-members/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { ListMembersInterceptor } from './api'; 4 | 5 | export default class ListMembersModule extends Extension { 6 | name = 'ListMembersModule'; 7 | 8 | type = ExtensionType.USER; 9 | 10 | intercept() { 11 | return ListMembersInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/list-subscribers/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, User } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineUser } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface ListSubscribersResponse { 8 | data: { 9 | list: { 10 | subscribers_timeline: { 11 | timeline: { 12 | instructions: TimelineInstructions; 13 | }; 14 | }; 15 | }; 16 | }; 17 | } 18 | 19 | // https://twitter.com/i/api/graphql/B9F2680qyuI6keStbcgv6w/ListSubscribers 20 | export const ListSubscribersInterceptor: Interceptor = (req, res, ext) => { 21 | if (!/\/graphql\/.+\/ListSubscribers/.test(req.url)) { 22 | return; 23 | } 24 | 25 | try { 26 | const newData = extractDataFromResponse( 27 | res, 28 | (json) => json.data.list.subscribers_timeline.timeline.instructions, 29 | (entry) => extractTimelineUser(entry.content.itemContent), 30 | ); 31 | 32 | // Add captured data to the database. 33 | db.extAddUsers(ext.name, newData); 34 | 35 | logger.info(`ListSubscribers: ${newData.length} items received`); 36 | } catch (err) { 37 | logger.debug(req.method, req.url, res.status, res.responseText); 38 | logger.errorWithBanner('ListSubscribers: Failed to parse API response', err as Error); 39 | } 40 | }; 41 | -------------------------------------------------------------------------------- /src/modules/list-subscribers/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { ListSubscribersInterceptor } from './api'; 4 | 5 | export default class ListSubscribersModule extends Extension { 6 | name = 'ListSubscribersModule'; 7 | 8 | type = ExtensionType.USER; 9 | 10 | intercept() { 11 | return ListSubscribersInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/list-timeline/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { TimelineInstructions, Tweet } from '@/types'; 4 | import { extractDataFromResponse, extractTimelineTweet } from '@/utils/api'; 5 | import logger from '@/utils/logger'; 6 | 7 | interface ListTimelineResponse { 8 | data: { 9 | list: { 10 | tweets_timeline: { 11 | timeline: { 12 | instructions: TimelineInstructions; 13 | metadata: unknown; 14 | }; 15 | }; 16 | }; 17 | }; 18 | } 19 | 20 | // https://twitter.com/i/api/graphql/asz3yj2ZCgJt3pdZEY2zgA/ListLatestTweetsTimeline 21 | export const ListTimelineInterceptor: Interceptor = (req, res, ext) => { 22 | if (!/\/graphql\/.+\/ListLatestTweetsTimeline/.test(req.url)) { 23 | return; 24 | } 25 | 26 | try { 27 | const newData = extractDataFromResponse( 28 | res, 29 | (json) => json.data.list.tweets_timeline.timeline.instructions, 30 | (entry) => extractTimelineTweet(entry.content.itemContent), 31 | ); 32 | 33 | // Add captured data to the database. 34 | db.extAddTweets(ext.name, newData); 35 | 36 | logger.info(`ListTimeline: ${newData.length} items received`); 37 | } catch (err) { 38 | logger.debug(req.method, req.url, res.status, res.responseText); 39 | logger.errorWithBanner('ListTimeline: Failed to parse API response', err as Error); 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /src/modules/list-timeline/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { ListTimelineInterceptor } from './api'; 4 | 5 | export default class ListTimelineModule extends Extension { 6 | name = 'ListTimelineModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return ListTimelineInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/runtime-logs/index.ts: -------------------------------------------------------------------------------- 1 | import { Extension } from '@/core/extensions'; 2 | import { RuntimeLogsPanel } from '@/modules/runtime-logs/ui'; 3 | 4 | export default class RuntimeLogsModule extends Extension { 5 | name = 'RuntimeLogsModule'; 6 | 7 | render() { 8 | return RuntimeLogsPanel; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/modules/runtime-logs/ui.tsx: -------------------------------------------------------------------------------- 1 | import { Fragment } from 'preact'; 2 | import { Signal } from '@preact/signals'; 3 | import { LogLine, logLinesSignal } from '@/utils/logger'; 4 | 5 | const colors = { 6 | info: 'text-base-content', 7 | warn: 'text-warning', 8 | error: 'text-error', 9 | }; 10 | 11 | type LogsProps = { 12 | lines: Signal; 13 | }; 14 | 15 | function Logs({ lines }: LogsProps) { 16 | const reversed = lines.value.slice().reverse(); 17 | 18 | return ( 19 | 
20 |       {reversed.map((line) => (
21 |         
22 |           #{line.index} {line.line}
23 |           {'\n'}
24 |         
25 |       ))}
26 |
27 | ); 28 | } 29 | 30 | export function RuntimeLogsPanel() { 31 | return ( 32 | 33 |
34 | 35 | 36 | ); 37 | } 38 | -------------------------------------------------------------------------------- /src/modules/search-timeline/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { 4 | ItemContentUnion, 5 | List, 6 | TimelineAddEntriesInstruction, 7 | TimelineAddToModuleInstruction, 8 | TimelineInstructions, 9 | TimelineTweet, 10 | TimelineTwitterList, 11 | Tweet, 12 | User, 13 | } from '@/types'; 14 | import { 15 | extractTimelineTweet, 16 | extractTimelineUser, 17 | isTimelineEntryListSearch, 18 | isTimelineEntrySearchGrid, 19 | isTimelineEntryTweet, 20 | isTimelineEntryUser, 21 | } from '@/utils/api'; 22 | import logger from '@/utils/logger'; 23 | 24 | interface SearchTimelineResponse { 25 | data: { 26 | search_by_raw_query: { 27 | search_timeline: { 28 | timeline: { 29 | instructions: TimelineInstructions; 30 | responseObjects: unknown; 31 | }; 32 | }; 33 | }; 34 | }; 35 | } 36 | 37 | // https://twitter.com/i/api/graphql/Aj1nGkALq99Xg3XI0OZBtw/SearchTimeline 38 | export const SearchTimelineInterceptor: Interceptor = (req, res, ext) => { 39 | if (!/\/graphql\/.+\/SearchTimeline/.test(req.url)) { 40 | return; 41 | } 42 | 43 | try { 44 | const json: SearchTimelineResponse = JSON.parse(res.responseText); 45 | const instructions = json.data.search_by_raw_query.search_timeline.timeline.instructions; 46 | 47 | // Parse tweets in search results. 48 | // Currently, only "Top", "Latest" and "Media" are supported. "People" and "Lists" are ignored. 49 | const newTweets: Tweet[] = []; 50 | const newUsers: User[] = []; 51 | const newLists: List[] = []; 52 | 53 | // The most complicated part starts here. 54 | // 55 | // For "Top" and "Latest", the "TimelineAddEntries" instruction contains normal tweets. 56 | // For "People", the "TimelineAddEntries" instruction contains normal users. 57 | // For "Media", the "TimelineAddEntries" instruction initializes "search-grid" module. 58 | // For "Lists", the "TimelineAddToModule" instruction initializes "list-search" module. 59 | const timelineAddEntriesInstruction = instructions.find( 60 | (i) => i.type === 'TimelineAddEntries', 61 | ) as TimelineAddEntriesInstruction; 62 | 63 | // There will be two requests for "Media" and "Lists" search results. 64 | // The "TimelineAddToModule" instruction then prepends items to existing module. 65 | const timelineAddToModuleInstruction = instructions.find( 66 | (i) => i.type === 'TimelineAddToModule', 67 | ) as TimelineAddToModuleInstruction; 68 | 69 | // The "TimelineAddEntries" instruction may not exist in some cases. 70 | const timelineAddEntriesInstructionEntries = timelineAddEntriesInstruction?.entries ?? []; 71 | 72 | // First, parse "TimelineAddEntries" instruction. 73 | for (const entry of timelineAddEntriesInstructionEntries) { 74 | // Extract normal tweets. 75 | if (isTimelineEntryTweet(entry)) { 76 | const tweet = extractTimelineTweet(entry.content.itemContent); 77 | if (tweet) { 78 | newTweets.push(tweet); 79 | } 80 | } 81 | 82 | // Extract media tweets. 83 | if (isTimelineEntrySearchGrid(entry)) { 84 | const tweetsInSearchGrid = entry.content.items 85 | .map((i) => extractTimelineTweet(i.item.itemContent)) 86 | .filter((t): t is Tweet => !!t); 87 | 88 | newTweets.push(...tweetsInSearchGrid); 89 | } 90 | 91 | // Extract users. 92 | if (isTimelineEntryUser(entry)) { 93 | const user = extractTimelineUser(entry.content.itemContent); 94 | if (user) { 95 | newUsers.push(user); 96 | } 97 | } 98 | 99 | // Extract lists. 100 | if (isTimelineEntryListSearch(entry)) { 101 | const lists = entry.content.items.map((i) => i.item.itemContent.list); 102 | newLists.push(...lists); 103 | } 104 | } 105 | 106 | // Second, parse "TimelineAddToModule" instruction. 107 | if (timelineAddToModuleInstruction) { 108 | const items = timelineAddToModuleInstruction.moduleItems.map((i) => i.item.itemContent); 109 | 110 | const tweets = items 111 | .filter((i): i is TimelineTweet => i.__typename === 'TimelineTweet') 112 | .map((t) => extractTimelineTweet(t)) 113 | .filter((t): t is Tweet => !!t); 114 | 115 | newTweets.push(...tweets); 116 | 117 | const lists = items 118 | .filter((i): i is TimelineTwitterList => i.__typename === 'TimelineTwitterList') 119 | .map((i) => i.list); 120 | 121 | newLists.push(...lists); 122 | } 123 | 124 | // Finally, add captured tweets to the database. 125 | db.extAddTweets(ext.name, newTweets); 126 | logger.info(`SearchTimeline: ${newTweets.length} items received`); 127 | 128 | // TODO: Implement "People" and "Lists" search results. 129 | if (newLists.length > 0) { 130 | logger.warn( 131 | `SearchList: ${newLists.length} lists received but ignored (Reason: not implemented)`, 132 | newLists, 133 | ); 134 | } 135 | 136 | if (newUsers.length > 0) { 137 | logger.warn( 138 | `SearchUser: ${newUsers.length} users received but ignored (Reason: not implemented)`, 139 | newUsers, 140 | ); 141 | } 142 | } catch (err) { 143 | logger.debug(req.method, req.url, res.status, res.responseText); 144 | logger.errorWithBanner('SearchTimeline: Failed to parse API response', err as Error); 145 | } 146 | }; 147 | -------------------------------------------------------------------------------- /src/modules/search-timeline/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { SearchTimelineInterceptor } from './api'; 4 | 5 | export default class SearchTimelineModule extends Extension { 6 | name = 'SearchTimelineModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return SearchTimelineInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/tweet-detail/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { 4 | TimelineAddEntriesInstruction, 5 | TimelineAddToModuleInstruction, 6 | TimelineInstructions, 7 | TimelineTweet, 8 | Tweet, 9 | } from '@/types'; 10 | import { 11 | extractTimelineTweet, 12 | isTimelineEntryConversationThread, 13 | isTimelineEntryTweet, 14 | } from '@/utils/api'; 15 | import logger from '@/utils/logger'; 16 | 17 | interface TweetDetailResponse { 18 | data: { 19 | threaded_conversation_with_injections_v2: { 20 | instructions: TimelineInstructions; 21 | }; 22 | }; 23 | } 24 | 25 | interface ModeratedTimelineResponse { 26 | data: { 27 | tweet: { 28 | result: { 29 | timeline_response: { 30 | timeline: { 31 | instructions: TimelineInstructions; 32 | }; 33 | }; 34 | }; 35 | }; 36 | }; 37 | } 38 | 39 | // https://twitter.com/i/api/graphql/8sK2MBRZY9z-fgmdNpR3LA/TweetDetail 40 | // https://twitter.com/i/api/graphql/a8M2LqEB5TwbW_eDrsmcDA/ModeratedTimeline 41 | export const TweetDetailInterceptor: Interceptor = (req, res, ext) => { 42 | const isTweetDetail = /\/graphql\/.+\/TweetDetail/.test(req.url); 43 | const isModeratedTimeline = /\/graphql\/.+\/ModeratedTimeline/.test(req.url); 44 | 45 | if (!isTweetDetail && !isModeratedTimeline) { 46 | return; 47 | } 48 | 49 | try { 50 | const json: TweetDetailResponse | ModeratedTimelineResponse = JSON.parse(res.responseText); 51 | let instructions: TimelineInstructions = []; 52 | 53 | // Determine the endpoint and extract instructions accordingly. 54 | if (isTweetDetail) { 55 | instructions = (json as TweetDetailResponse).data.threaded_conversation_with_injections_v2 56 | .instructions; 57 | } else if (isModeratedTimeline) { 58 | instructions = (json as ModeratedTimelineResponse).data.tweet.result.timeline_response 59 | .timeline.instructions; 60 | } 61 | 62 | const newData: Tweet[] = []; 63 | 64 | const timelineAddEntriesInstruction = instructions.find( 65 | (i) => i.type === 'TimelineAddEntries', 66 | ) as TimelineAddEntriesInstruction; 67 | 68 | // When loading more tweets in conversation, the "TimelineAddEntries" instruction may not exist. 69 | const timelineAddEntriesInstructionEntries = timelineAddEntriesInstruction?.entries ?? []; 70 | 71 | for (const entry of timelineAddEntriesInstructionEntries) { 72 | // The main tweet. 73 | if (isTimelineEntryTweet(entry)) { 74 | const tweet = extractTimelineTweet(entry.content.itemContent); 75 | if (tweet) { 76 | newData.push(tweet); 77 | } 78 | } 79 | 80 | // The conversation thread (only for TweetDetail). 81 | if (isTweetDetail && isTimelineEntryConversationThread(entry)) { 82 | // Be careful about the "conversationthread-{id}-cursor-showmore-{cid}" item. 83 | const tweetsInConversation = entry.content.items.map((i) => { 84 | if (i.entryId.includes('-tweet-')) { 85 | return extractTimelineTweet(i.item.itemContent); 86 | } 87 | }); 88 | 89 | newData.push(...tweetsInConversation.filter((t): t is Tweet => !!t)); 90 | } 91 | } 92 | 93 | // Lazy-loaded conversations. 94 | const timelineAddToModuleInstruction = instructions.find( 95 | (i) => i.type === 'TimelineAddToModule', 96 | ) as TimelineAddToModuleInstruction; 97 | 98 | if (timelineAddToModuleInstruction) { 99 | const tweetsInConversation = timelineAddToModuleInstruction.moduleItems 100 | .map((i) => extractTimelineTweet(i.item.itemContent)) 101 | .filter((t): t is Tweet => !!t); 102 | 103 | newData.push(...tweetsInConversation); 104 | } 105 | 106 | // Add captured tweets to the database. 107 | db.extAddTweets(ext.name, newData); 108 | 109 | logger.info(`TweetDetail: ${newData.length} items received`); 110 | } catch (err) { 111 | logger.debug(req.method, req.url, res.status, res.responseText); 112 | logger.errorWithBanner('TweetDetail: Failed to parse API response', err as Error); 113 | } 114 | }; 115 | -------------------------------------------------------------------------------- /src/modules/tweet-detail/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { TweetDetailInterceptor } from './api'; 4 | 5 | export default class TweetDetailModule extends Extension { 6 | name = 'TweetDetailModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return TweetDetailInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/user-detail/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { User } from '@/types'; 4 | import logger from '@/utils/logger'; 5 | 6 | interface UserDetailResponse { 7 | data: { 8 | user: { 9 | result: User; 10 | }; 11 | }; 12 | } 13 | 14 | // https://twitter.com/i/api/graphql/BQ6xjFU6Mgm-WhEP3OiT9w/UserByScreenName 15 | export const UserDetailInterceptor: Interceptor = (req, res, ext) => { 16 | if (!/\/graphql\/.+\/UserByScreenName/.test(req.url)) { 17 | return; 18 | } 19 | 20 | try { 21 | const json: UserDetailResponse = JSON.parse(res.responseText); 22 | const newData = [json.data.user.result]; 23 | 24 | // Add captured data to the database. 25 | db.extAddUsers(ext.name, newData); 26 | 27 | logger.info(`UserDetail: ${newData.length} items received`); 28 | } catch (err) { 29 | logger.debug(req.method, req.url, res.status, res.responseText); 30 | logger.errorWithBanner('UserDetail: Failed to parse API response', err as Error); 31 | } 32 | }; 33 | -------------------------------------------------------------------------------- /src/modules/user-detail/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { UserDetailInterceptor } from './api'; 4 | 5 | export default class UserDetailModule extends Extension { 6 | name = 'UserDetailModule'; 7 | 8 | type = ExtensionType.USER; 9 | 10 | intercept() { 11 | return UserDetailInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/user-media/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { 4 | TimelineAddEntriesInstruction, 5 | TimelineAddToModuleInstruction, 6 | TimelineInstructions, 7 | TimelineTweet, 8 | Tweet, 9 | } from '@/types'; 10 | import { extractTimelineTweet, isTimelineEntryProfileGrid } from '@/utils/api'; 11 | import logger from '@/utils/logger'; 12 | 13 | interface UserMediaResponse { 14 | data: { 15 | user: { 16 | result: { 17 | timeline: { 18 | timeline: { 19 | instructions: TimelineInstructions; 20 | metadata: unknown; 21 | }; 22 | }; 23 | __typename: 'User'; 24 | }; 25 | }; 26 | }; 27 | } 28 | 29 | // https://twitter.com/i/api/graphql/oMVVrI5kt3kOpyHHTTKf5Q/UserMedia 30 | export const UserMediaInterceptor: Interceptor = (req, res, ext) => { 31 | if (!/\/graphql\/.+\/UserMedia/.test(req.url)) { 32 | return; 33 | } 34 | 35 | try { 36 | const json: UserMediaResponse = JSON.parse(res.responseText); 37 | const instructions = json.data.user.result.timeline.timeline.instructions; 38 | 39 | const newData: Tweet[] = []; 40 | 41 | // There are two types of instructions: "TimelineAddEntries" and "TimelineAddToModule". 42 | // For "Media", the "TimelineAddEntries" instruction initializes "profile-grid" module. 43 | const timelineAddEntriesInstruction = instructions.find( 44 | (i) => i.type === 'TimelineAddEntries', 45 | ) as TimelineAddEntriesInstruction; 46 | 47 | // The "TimelineAddEntries" instruction may not exist in some cases. 48 | const timelineAddEntriesInstructionEntries = timelineAddEntriesInstruction?.entries ?? []; 49 | 50 | for (const entry of timelineAddEntriesInstructionEntries) { 51 | if (isTimelineEntryProfileGrid(entry)) { 52 | const tweetsInSearchGrid = entry.content.items 53 | .map((i) => extractTimelineTweet(i.item.itemContent)) 54 | .filter((t): t is Tweet => !!t); 55 | 56 | newData.push(...tweetsInSearchGrid); 57 | } 58 | } 59 | 60 | // The "TimelineAddToModule" instruction then prepends items to existing "profile-grid" module. 61 | const timelineAddToModuleInstruction = instructions.find( 62 | (i) => i.type === 'TimelineAddToModule', 63 | ) as TimelineAddToModuleInstruction; 64 | 65 | if (timelineAddToModuleInstruction) { 66 | const tweetsInProfileGrid = timelineAddToModuleInstruction.moduleItems 67 | .map((i) => extractTimelineTweet(i.item.itemContent)) 68 | .filter((t): t is Tweet => !!t); 69 | 70 | newData.push(...tweetsInProfileGrid); 71 | } 72 | 73 | // Add captured tweets to the database. 74 | db.extAddTweets(ext.name, newData); 75 | 76 | logger.info(`UserMedia: ${newData.length} items received`); 77 | } catch (err) { 78 | logger.debug(req.method, req.url, res.status, res.responseText); 79 | logger.errorWithBanner('UserMedia: Failed to parse API response', err as Error); 80 | } 81 | }; 82 | -------------------------------------------------------------------------------- /src/modules/user-media/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { UserMediaInterceptor } from './api'; 4 | 5 | export default class UserMediaModule extends Extension { 6 | name = 'UserMediaModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return UserMediaInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/modules/user-tweets/api.ts: -------------------------------------------------------------------------------- 1 | import { Interceptor } from '@/core/extensions'; 2 | import { db } from '@/core/database'; 3 | import { 4 | TimelineAddEntriesInstruction, 5 | TimelineInstructions, 6 | TimelinePinEntryInstruction, 7 | TimelineTweet, 8 | Tweet, 9 | } from '@/types'; 10 | import { 11 | extractTimelineTweet, 12 | isTimelineEntryProfileConversation, 13 | isTimelineEntryTweet, 14 | } from '@/utils/api'; 15 | import logger from '@/utils/logger'; 16 | 17 | interface UserTweetsResponse { 18 | data: { 19 | user: { 20 | result: { 21 | timeline: { 22 | timeline: { 23 | instructions: TimelineInstructions; 24 | metadata: unknown; 25 | }; 26 | }; 27 | __typename: 'User'; 28 | }; 29 | }; 30 | }; 31 | } 32 | 33 | // https://twitter.com/i/api/graphql/H8OOoI-5ZE4NxgRr8lfyWg/UserTweets 34 | // https://twitter.com/i/api/graphql/Q6aAvPw7azXZbqXzuqTALA/UserTweetsAndReplies 35 | export const UserTweetsInterceptor: Interceptor = (req, res, ext) => { 36 | if (!/\/graphql\/.+\/UserTweets/.test(req.url)) { 37 | return; 38 | } 39 | 40 | try { 41 | const json: UserTweetsResponse = JSON.parse(res.responseText); 42 | const instructions = json.data.user.result.timeline.timeline.instructions; 43 | 44 | const newData: Tweet[] = []; 45 | 46 | // The pinned tweet. 47 | const timelinePinEntryInstruction = instructions.find( 48 | (i) => i.type === 'TimelinePinEntry', 49 | ) as TimelinePinEntryInstruction; 50 | 51 | if (timelinePinEntryInstruction) { 52 | const tweet = extractTimelineTweet(timelinePinEntryInstruction.entry.content.itemContent); 53 | if (tweet) { 54 | newData.push(tweet); 55 | } 56 | } 57 | 58 | // Normal tweets. 59 | const timelineAddEntriesInstruction = instructions.find( 60 | (i) => i.type === 'TimelineAddEntries', 61 | ) as TimelineAddEntriesInstruction; 62 | 63 | // The "TimelineAddEntries" instruction may not exist in some cases. 64 | const timelineAddEntriesInstructionEntries = timelineAddEntriesInstruction?.entries ?? []; 65 | 66 | for (const entry of timelineAddEntriesInstructionEntries) { 67 | // Extract normal tweets. 68 | if (isTimelineEntryTweet(entry)) { 69 | const tweet = extractTimelineTweet(entry.content.itemContent); 70 | if (tweet) { 71 | newData.push(tweet); 72 | } 73 | } 74 | 75 | // Extract conversations. 76 | if (isTimelineEntryProfileConversation(entry)) { 77 | const tweetsInConversation = entry.content.items 78 | .map((i) => extractTimelineTweet(i.item.itemContent)) 79 | .filter((t): t is Tweet => !!t); 80 | 81 | newData.push(...tweetsInConversation); 82 | } 83 | } 84 | 85 | // Add captured tweets to the database. 86 | db.extAddTweets(ext.name, newData); 87 | 88 | logger.info(`UserTweets: ${newData.length} items received`); 89 | } catch (err) { 90 | logger.debug(req.method, req.url, res.status, res.responseText); 91 | logger.errorWithBanner('UserTweets: Failed to parse API response', err as Error); 92 | } 93 | }; 94 | -------------------------------------------------------------------------------- /src/modules/user-tweets/index.tsx: -------------------------------------------------------------------------------- 1 | import { CommonModuleUI } from '@/components/module-ui'; 2 | import { Extension, ExtensionType } from '@/core/extensions'; 3 | import { UserTweetsInterceptor } from './api'; 4 | 5 | export default class UserTweetsModule extends Extension { 6 | name = 'UserTweetsModule'; 7 | 8 | type = ExtensionType.TWEET; 9 | 10 | intercept() { 11 | return UserTweetsInterceptor; 12 | } 13 | 14 | render() { 15 | return CommonModuleUI; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/types/index.ts: -------------------------------------------------------------------------------- 1 | import { TimelineTwitterList } from './list'; 2 | import { TimelineTweet } from './tweet'; 3 | import { TimelineUser } from './user'; 4 | 5 | export * from './list'; 6 | export * from './tweet'; 7 | export * from './user'; 8 | 9 | export interface EntityURL { 10 | display_url: string; 11 | expanded_url: string; 12 | url: string; 13 | indices: number[]; 14 | } 15 | 16 | export type TimelineInstructions = Array< 17 | | TimelineClearCacheInstruction 18 | | TimelineTerminateTimelineInstruction 19 | | TimelinePinEntryInstruction 20 | | TimelineAddEntriesInstruction 21 | | TimelineAddToModuleInstruction 22 | >; 23 | 24 | export interface TimelineClearCacheInstruction { 25 | type: 'TimelineClearCache'; 26 | } 27 | 28 | export interface TimelineTerminateTimelineInstruction { 29 | type: 'TimelineTerminateTimeline'; 30 | direction: 'Top' | 'Bottom'; 31 | } 32 | 33 | export interface TimelineEntry< 34 | T = ItemContentUnion, 35 | C = TimelineTimelineItem | TimelineTimelineModule | TimelineTimelineCursor, 36 | > { 37 | content: C; 38 | entryId: string; 39 | sortIndex: string; 40 | } 41 | 42 | export interface TimelinePinEntryInstruction { 43 | type: 'TimelinePinEntry'; 44 | entry: TimelineEntry>; 45 | } 46 | 47 | export interface TimelineAddEntriesInstruction { 48 | type: 'TimelineAddEntries'; 49 | entries: TimelineEntry[]; 50 | } 51 | 52 | export interface TimelineAddToModuleInstruction { 53 | type: 'TimelineAddToModule'; 54 | // "conversationthread-{id}" 55 | // "profile-grid-{id}" 56 | moduleEntryId: string; 57 | prepend: boolean; 58 | moduleItems: { 59 | // "conversationthread-{id}-tweet-{tid}" 60 | // "profile-grid-{id}-tweet-{tid}" 61 | entryId: string; 62 | item: { 63 | clientEventInfo: unknown; 64 | itemContent: T; 65 | }; 66 | }[]; 67 | } 68 | 69 | // TimelineEntry.entryId: "tweet-{id}" 70 | // TimelineEntry.entryId: "user-{id}" 71 | export interface TimelineTimelineItem { 72 | entryType: 'TimelineTimelineItem'; 73 | __typename: 'TimelineTimelineItem'; 74 | itemContent: T; 75 | clientEventInfo: unknown; 76 | } 77 | 78 | // TimelineEntry.entryId: "cursor-top-{id}" 79 | // TimelineEntry.entryId: "cursor-bottom-{id}" 80 | export interface TimelineTimelineCursor { 81 | entryType: 'TimelineTimelineCursor'; 82 | __typename: 'TimelineTimelineCursor'; 83 | value: string; 84 | cursorType: 'Top' | 'Bottom' | 'ShowMore' | 'ShowMoreThreads'; 85 | } 86 | 87 | export interface TimelinePrompt { 88 | itemType: 'TimelinePrompt'; 89 | __typename: 'TimelinePrompt'; 90 | } 91 | 92 | export interface TimelineMessagePrompt { 93 | itemType: 'TimelineMessagePrompt'; 94 | __typename: 'TimelineMessagePrompt'; 95 | } 96 | 97 | export type ItemContentUnion = 98 | | TimelineTweet 99 | | TimelineUser 100 | | TimelinePrompt 101 | | TimelineMessagePrompt 102 | | TimelineTimelineCursor 103 | | TimelineTwitterList; 104 | 105 | // TimelineEntry.entryId: "who-to-follow-{id}" 106 | // TimelineEntry.entryId: "profile-conversation-{id}" 107 | // TimelineEntry.entryId: "conversationthread-{id}" 108 | // TimelineEntry.entryId: "tweetdetailrelatedtweets-{id}" 109 | // TimelineEntry.entryId: "profile-grid-{id}" 110 | // TimelineEntry.entryId: "search-grid-{id}" 111 | // TimelineEntry.entryId: "list-search-{id}" 112 | export interface TimelineTimelineModule { 113 | entryType: 'TimelineTimelineModule'; 114 | __typename: 'TimelineTimelineModule'; 115 | clientEventInfo: unknown; 116 | displayType: 117 | | 'Vertical' 118 | | 'VerticalConversation' 119 | | 'VerticalGrid' 120 | | 'ListWithPin' 121 | | 'ListWithSubscribe' 122 | | string; 123 | items: { 124 | // "who-to-follow-{id}-user-{uid}" 125 | // "profile-conversation-{id}-tweet-{tid}" 126 | // "conversationthread-{id}-tweet-{tid}" 127 | // "conversationthread-{id}-cursor-showmore-{cid}" 128 | // "tweetdetailrelatedtweets-{id}-tweet-{tid}" 129 | // "profile-grid-{id}-tweet-{tid}" 130 | // "search-grid-{id}-tweet-{tid}" 131 | // "list-search-{id}-list-{lid}" 132 | entryId: string; 133 | item: { 134 | clientEventInfo: unknown; 135 | feedbackInfo: unknown; 136 | itemContent: T; 137 | }; 138 | }[]; 139 | header?: unknown; 140 | metadata?: { 141 | conversationMetadata: { 142 | allTweetIds: string[]; 143 | enableDeduplication: boolean; 144 | }; 145 | }; 146 | } 147 | 148 | /** 149 | * Represents a piece of data captured by an extension. 150 | */ 151 | export interface Capture { 152 | /** Unique identifier for the capture. */ 153 | id: string; 154 | /** Name of extension that captured the data. */ 155 | extension: string; 156 | /** Type of data captured. Possible values: `tweet`, `user`. */ 157 | type: string; 158 | /** The index of the captured item. Use this to query actual data from the database. */ 159 | data_key: string; 160 | /** Timestamp when the data was captured. */ 161 | created_at: number; 162 | } 163 | -------------------------------------------------------------------------------- /src/types/list.ts: -------------------------------------------------------------------------------- 1 | import { User } from './user'; 2 | 3 | export interface TimelineTwitterList { 4 | __typename: 'TimelineTwitterList'; 5 | itemType: 'TimelineTwitterList'; 6 | displayType: 'ListWithSubscribe'; 7 | list: List; 8 | } 9 | 10 | export interface List { 11 | created_at: number; 12 | default_banner_media: { 13 | media_info: MediaInfo; 14 | }; 15 | default_banner_media_results: { 16 | result: { 17 | id: string; 18 | media_key: string; 19 | media_id: string; 20 | media_info: MediaInfo; 21 | __typename: 'ApiMedia'; 22 | }; 23 | }; 24 | description: string; 25 | facepile_urls: string[]; 26 | followers_context: string; 27 | following: boolean; 28 | id: string; 29 | id_str: string; 30 | is_member: boolean; 31 | member_count: number; 32 | members_context: string; 33 | mode: 'Public' | 'Private'; 34 | muting: boolean; 35 | name: string; 36 | pinning: boolean; 37 | subscriber_count: number; 38 | user_results: { 39 | result: User; 40 | }; 41 | } 42 | 43 | interface MediaInfo { 44 | __typename?: 'ApiImage'; 45 | original_img_height: number; 46 | original_img_width: number; 47 | original_img_url: string; 48 | salient_rect: SalientRect; 49 | } 50 | 51 | interface SalientRect { 52 | left: number; 53 | top: number; 54 | width: number; 55 | height: number; 56 | } 57 | -------------------------------------------------------------------------------- /src/types/tweet.ts: -------------------------------------------------------------------------------- 1 | import { EntityURL } from './index'; 2 | import { User } from './user'; 3 | 4 | export interface TimelineTweet { 5 | itemType: 'TimelineTweet'; 6 | __typename: 'TimelineTweet'; 7 | tweet_results: { 8 | // In rare cases, for example when a tweet has its visibility limited by Twitter, 9 | // this field may not be present. 10 | result?: TweetUnion; 11 | }; 12 | tweetDisplayType: 'Tweet' | 'SelfThread' | 'MediaGrid'; 13 | hasModeratedReplies?: boolean; 14 | socialContext?: unknown; 15 | highlights?: { 16 | textHighlights: Array<{ 17 | startIndex: number; 18 | endIndex: number; 19 | }>; 20 | }; 21 | } 22 | 23 | export type TweetUnion = Tweet | TweetWithVisibilityResults | TweetTombstone | TweetUnavailable; 24 | 25 | export interface TweetWithVisibilityResults { 26 | __typename: 'TweetWithVisibilityResults'; 27 | limitedActionResults: { 28 | limited_actions: { 29 | action: string; 30 | prompt: unknown; 31 | }[]; 32 | }; 33 | tweet: Tweet; 34 | } 35 | 36 | // Deleted tweets or tweets from protected accounts. 37 | // See: https://github.com/JustAnotherArchivist/snscrape/issues/392 38 | export interface TweetTombstone { 39 | __typename: 'TweetTombstone'; 40 | tombstone: { 41 | __typename: 'TextTombstone'; 42 | text: { 43 | rtl: boolean; 44 | // "You’re unable to view this Post because this account owner limits who can view their Posts. Learn more" 45 | // "This Post is from an account that no longer exists. Learn more" 46 | // "This Post is from a suspended account. Learn more" 47 | // "This Post was deleted by the Post author. Learn more" 48 | text: string; 49 | entities: unknown[]; 50 | }; 51 | }; 52 | } 53 | 54 | // Tweets that are unavailable for some reason. Maybe NSFW. 55 | // See: https://github.com/JustAnotherArchivist/snscrape/issues/433 56 | export interface TweetUnavailable { 57 | __typename: 'TweetUnavailable'; 58 | } 59 | 60 | export interface Tweet { 61 | __typename: 'Tweet'; 62 | rest_id: string; 63 | core: { 64 | user_results: { 65 | result: User; 66 | }; 67 | }; 68 | has_birdwatch_notes?: boolean; 69 | // Usually used by advertisers. 70 | card?: unknown; 71 | unified_card?: unknown; 72 | edit_control: { 73 | edit_tweet_ids: string[]; 74 | editable_until_msecs: string; 75 | is_edit_eligible: boolean; 76 | edits_remaining: string; 77 | }; 78 | is_translatable: boolean; 79 | quoted_status_result?: { 80 | result: TweetUnion; 81 | }; 82 | quotedRefResult?: { 83 | result: Partial; 84 | }; 85 | views: { 86 | count?: string; 87 | state: 'Enabled' | 'EnabledWithCount'; 88 | }; 89 | source: string; 90 | // Used for long tweets. 91 | note_tweet?: { 92 | is_expandable: boolean; 93 | note_tweet_results: { 94 | result: NoteTweet; 95 | }; 96 | }; 97 | legacy: { 98 | bookmark_count: number; 99 | bookmarked: boolean; 100 | created_at: string; 101 | conversation_control?: unknown; 102 | conversation_id_str: string; 103 | display_text_range: number[]; 104 | entities: TweetEntities; 105 | extended_entities?: TweetExtendedEntities; 106 | favorite_count: number; 107 | favorited: boolean; 108 | full_text: string; 109 | in_reply_to_screen_name?: string; 110 | in_reply_to_status_id_str?: string; 111 | in_reply_to_user_id_str?: string; 112 | is_quote_status: boolean; 113 | lang: string; 114 | limited_actions?: string; 115 | possibly_sensitive: boolean; 116 | possibly_sensitive_editable: boolean; 117 | quote_count: number; 118 | quoted_status_id_str?: string; 119 | quoted_status_permalink?: { 120 | url: string; 121 | expanded: string; 122 | display: string; 123 | }; 124 | reply_count: number; 125 | retweet_count: number; 126 | retweeted: boolean; 127 | scopes?: unknown; 128 | user_id_str: string; 129 | id_str: string; 130 | retweeted_status_result?: { 131 | result: TweetUnion; 132 | }; 133 | }; 134 | /** 135 | * Some extra properties added by the script when inserting to local database. 136 | * These are not present in the original tweet object and are used for internal purposes only. 137 | */ 138 | twe_private_fields: { 139 | /** The UNIX timestamp representation of `legacy.created_at` in milliseconds. */ 140 | created_at: number; 141 | /** The UNIX timestamp in ms when inserted or updated to local database. */ 142 | updated_at: number; 143 | /** The number of media items in the tweet. */ 144 | media_count: number; 145 | }; 146 | } 147 | 148 | export interface RichTextTag { 149 | from_index: number; 150 | to_index: number; 151 | richtext_types: ('Bold' | 'Italic')[]; 152 | } 153 | 154 | export interface NoteTweet { 155 | id: string; 156 | text: string; 157 | entity_set: TweetEntities; 158 | richtext: { 159 | richtext_tags: RichTextTag[]; 160 | }; 161 | media: { 162 | inline_media: unknown[]; 163 | }; 164 | } 165 | 166 | export interface TweetEntities { 167 | media?: Media[]; 168 | user_mentions: UserMention[]; 169 | urls: EntityURL[]; 170 | hashtags: Hashtag[]; 171 | symbols: unknown[]; 172 | } 173 | 174 | export interface Hashtag { 175 | indices: number[]; 176 | text: string; 177 | } 178 | 179 | export interface Media { 180 | display_url: string; 181 | expanded_url: string; 182 | id_str: string; 183 | indices: number[]; 184 | media_url_https: string; 185 | type: 'video' | 'photo' | 'animated_gif'; 186 | additional_media_info?: { 187 | title: string; 188 | description: string; 189 | }; 190 | mediaStats?: { 191 | viewCount: number; 192 | }; 193 | url: string; 194 | features: { 195 | all?: Feature; 196 | large: Feature; 197 | medium: Feature; 198 | small: Feature; 199 | orig: Feature; 200 | }; 201 | sizes: { 202 | large: Size; 203 | medium: Size; 204 | small: Size; 205 | thumb: Size; 206 | }; 207 | original_info: { 208 | height: number; 209 | width: number; 210 | focus_rects?: FocusRect[]; 211 | }; 212 | video_info?: { 213 | aspect_ratio: number[]; 214 | duration_millis: number; 215 | variants: Variant[]; 216 | }; 217 | ext_alt_text?: string; 218 | media_key?: string; 219 | ext_media_availability?: { 220 | status: string; 221 | }; 222 | } 223 | 224 | export interface Variant { 225 | bitrate?: number; 226 | content_type: string; 227 | url: string; 228 | } 229 | 230 | interface Feature { 231 | faces: FocusRect[]; 232 | tags?: Tag[]; 233 | } 234 | 235 | interface FocusRect { 236 | x: number; 237 | y: number; 238 | h: number; 239 | w: number; 240 | } 241 | 242 | export interface Tag { 243 | user_id: string; 244 | name: string; 245 | screen_name: string; 246 | type: string; 247 | } 248 | 249 | interface Size { 250 | h: number; 251 | w: number; 252 | resize: string; 253 | } 254 | 255 | interface UserMention { 256 | id_str: string; 257 | name: string; 258 | screen_name: string; 259 | indices: number[]; 260 | } 261 | 262 | export interface TweetExtendedEntities { 263 | media: Media[]; 264 | } 265 | -------------------------------------------------------------------------------- /src/types/user.ts: -------------------------------------------------------------------------------- 1 | import { EntityURL } from './index'; 2 | 3 | export interface TimelineUser { 4 | itemType: 'TimelineUser'; 5 | __typename: 'TimelineUser'; 6 | user_results: { 7 | result: User | UserUnavailable; 8 | }; 9 | userDisplayType: string; 10 | } 11 | 12 | export interface UserUnavailable { 13 | __typename: 'UserUnavailable'; 14 | message: string; 15 | reason: string; 16 | } 17 | 18 | export interface User { 19 | __typename: 'User'; 20 | id: string; 21 | rest_id: string; 22 | affiliates_highlighted_label: unknown; 23 | has_graduated_access: boolean; 24 | is_blue_verified: boolean; 25 | profile_image_shape: 'Square' | 'Circle'; 26 | legacy: { 27 | followed_by: boolean; 28 | following: boolean; 29 | can_dm: boolean; 30 | can_media_tag: boolean; 31 | created_at: string; 32 | default_profile: boolean; 33 | default_profile_image: boolean; 34 | description: string; 35 | entities: UserEntities; 36 | fast_followers_count: number; 37 | favourites_count: number; 38 | followers_count: number; 39 | friends_count: number; 40 | has_custom_timelines: boolean; 41 | is_translator: boolean; 42 | listed_count: number; 43 | location: string; 44 | media_count: number; 45 | name: string; 46 | normal_followers_count: number; 47 | pinned_tweet_ids_str: string[]; 48 | possibly_sensitive: boolean; 49 | profile_banner_url?: string; 50 | profile_image_url_https: string; 51 | profile_interstitial_type: string; 52 | protected?: boolean; 53 | screen_name: string; 54 | statuses_count: number; 55 | translator_type: string; 56 | url: string; 57 | verified: boolean; 58 | verified_type: string; 59 | want_retweets: boolean; 60 | withheld_in_countries: unknown[]; 61 | }; 62 | legacy_extended_profile?: { 63 | birthdate?: { 64 | day: number; 65 | month: number; 66 | year?: number; 67 | visibility: string; 68 | year_visibility: string; 69 | }; 70 | }; 71 | professional?: { 72 | rest_id: string; 73 | professional_type: string; 74 | category: { 75 | id: number; 76 | name: string; 77 | icon_name: string; 78 | }[]; 79 | }; 80 | /** 81 | * Some extra properties added by the script when inserting to local database. 82 | * These are not present in the original tweet object and are used for internal purposes only. 83 | */ 84 | twe_private_fields: { 85 | /** The UNIX timestamp representation of `legacy.created_at` in milliseconds. */ 86 | created_at: number; 87 | /** The UNIX timestamp in ms when inserted or updated to local database. */ 88 | updated_at: number; 89 | }; 90 | } 91 | 92 | export interface UserEntities { 93 | description: { 94 | urls: EntityURL[]; 95 | }; 96 | url?: { 97 | urls: EntityURL[]; 98 | }; 99 | } 100 | -------------------------------------------------------------------------------- /src/utils/common.ts: -------------------------------------------------------------------------------- 1 | import dayjs from 'dayjs'; 2 | import { useSignal } from '@preact/signals'; 3 | import { EntityURL } from '@/types'; 4 | import logger from './logger'; 5 | 6 | /** 7 | * JSON.parse with error handling. 8 | */ 9 | export function safeJSONParse(text: string) { 10 | try { 11 | return JSON.parse(text); 12 | } catch (e) { 13 | logger.error((e as Error).message); 14 | return null; 15 | } 16 | } 17 | 18 | /** 19 | * Use signal to mimic React's `useState` hook. 20 | */ 21 | export function useSignalState(value: T) { 22 | const signal = useSignal(value); 23 | 24 | const updateSignal = (newValue: T) => { 25 | signal.value = newValue; 26 | }; 27 | 28 | return [signal.value, updateSignal, signal] as const; 29 | } 30 | 31 | /** 32 | * A signal representing a boolean value. 33 | */ 34 | export function useToggle(defaultValue = false) { 35 | const signal = useSignal(defaultValue); 36 | 37 | const toggle = () => { 38 | signal.value = !signal.value; 39 | }; 40 | 41 | return [signal.value, toggle, signal] as const; 42 | } 43 | 44 | /** 45 | * Merge CSS class names. 46 | * Avoid using `tailwind-merge` here since it increases bundle size. 47 | * 48 | * @example 49 | * cx('foo', 'bar', false && 'baz') // => 'foo bar' 50 | */ 51 | export function cx(...classNames: (string | boolean | undefined)[]) { 52 | return classNames.filter(Boolean).join(' '); 53 | } 54 | 55 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 56 | export function isEqual(obj1: any, obj2: any) { 57 | return JSON.stringify(obj1) === JSON.stringify(obj2); 58 | } 59 | 60 | export function capitalizeFirstLetter(str: string) { 61 | return str.charAt(0).toUpperCase() + str.slice(1); 62 | } 63 | 64 | export function xssFilter(str: string) { 65 | return str.replace(/"/g, '"').replace(//g, '>'); 66 | } 67 | 68 | /** 69 | * Replace t.co URLs in a string with real HTML links. 70 | * 71 | * @example 72 | * ```jsx 73 | * // Input: 74 | * strEntitiesToHtml('Verification: https://t.co/hHSWmpjfbA NASA Hubble Space Telescope', [ 75 | * { 76 | * "display_url": "nasa.gov/socialmedia", 77 | * "expanded_url": "http://nasa.gov/socialmedia", 78 | * "url": "https://t.co/hHSWmpjfbA", 79 | * "indices": [140, 163] 80 | * } 81 | * ]); 82 | * 83 | * // Output: 84 | *

Verification: nasa.gov/socialmedia NASA Hubble Space Telescope

85 | * ``` 86 | */ 87 | export function strEntitiesToHTML(str: string, urls?: EntityURL[]) { 88 | let temp = str; 89 | 90 | if (!urls?.length) { 91 | return temp; 92 | } 93 | 94 | for (const { url, display_url, expanded_url } of urls) { 95 | temp = temp.replaceAll( 96 | url, 97 | `${xssFilter( 98 | display_url ?? url, 99 | )}`, 100 | ); 101 | } 102 | 103 | return temp; 104 | } 105 | 106 | export function parseTwitterDateTime(str: string) { 107 | // "Thu Sep 28 11:07:25 +0000 2023" 108 | // const regex = /^\w+ (\w+) (\d+) ([\d:]+) \+0000 (\d+)$/; 109 | const trimmed = str.replace(/^\w+ (.*)$/, '$1'); 110 | return dayjs(trimmed, 'MMM DD HH:mm:ss ZZ YYYY', 'en'); 111 | } 112 | 113 | export function formatDateTime(date: string | number | dayjs.Dayjs, format?: string) { 114 | if (typeof date === 'number' || typeof date === 'string') { 115 | date = dayjs(date); 116 | } 117 | 118 | // Display in local time zone. 119 | return date.format(format); 120 | } 121 | 122 | export function formatTwitterBirthdate(arg?: { day: number; month: number; year?: number }) { 123 | if (!arg) { 124 | return null; 125 | } 126 | 127 | const { day, month, year } = arg; 128 | const date = dayjs() 129 | .set('year', year ?? 0) 130 | .set('month', month - 1) 131 | .set('date', day); 132 | 133 | return year ? date.format('MMM DD, YYYY') : date.format('MMM DD'); 134 | } 135 | 136 | export function formatVideoDuration(durationInMs?: number): string { 137 | if (typeof durationInMs !== 'number' || Number.isNaN(durationInMs)) { 138 | return 'N/A'; 139 | } 140 | 141 | const durationInSeconds = Math.floor(durationInMs / 1000); 142 | const minutes = Math.floor(durationInSeconds / 60); 143 | const seconds = durationInSeconds % 60; 144 | 145 | return `${minutes}:${seconds.toString().padStart(2, '0')}`; 146 | } 147 | -------------------------------------------------------------------------------- /src/utils/download.ts: -------------------------------------------------------------------------------- 1 | import { saveAs } from 'file-saver-es'; 2 | import createWriter from './zip-stream'; 3 | import logger from './logger'; 4 | 5 | export type FileLike = { filename: string; url: string; type?: string }; 6 | 7 | export type ProgressCallback = (current: number, total: number, value?: T) => void; 8 | 9 | /** 10 | * Download multiple files from URL and save as a zip archive. 11 | * 12 | * @see https://github.com/jimmywarting/StreamSaver.js/issues/106 13 | * @param zipFilename Name of the zip archive file 14 | * @param files List of files to download 15 | * @param onProgress Callback function to track progress 16 | * @param rateLimit The minimum time gap between two downloads (in milliseconds) 17 | */ 18 | export async function zipStreamDownload( 19 | zipFilename: string, 20 | files: FileLike[], 21 | onProgress?: ProgressCallback, 22 | rateLimit = 1000, 23 | ) { 24 | // NOTE: StreamSaver.js fails on sites with strict Content-Security-Policy (CSP) such as Twitter, 25 | // since it uses iframe and service worker to download files. Use file-saver instead here. 26 | // See: https://github.com/jimmywarting/StreamSaver.js/issues/203 27 | 28 | // The data written to this stream will be streamed to the user's browser as a file download. 29 | // const writableOutputStream = streamSaver.createWriteStream(zipFilename); 30 | 31 | let current = 0; 32 | const total = files.length; 33 | const fileIterator = files.values(); 34 | 35 | // Add files to zip archive stream. 36 | const readableZipStream: ReadableStream = createWriter({ 37 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 38 | async pull(ctrl: any) { 39 | const fileInfo = fileIterator.next(); 40 | if (fileInfo.done) { 41 | // All files have been downloaded. 42 | ctrl.close(); 43 | } else { 44 | // Download file and add to zip. 45 | const { filename, url } = fileInfo.value; 46 | 47 | const start = Date.now(); 48 | logger.debug(`Start downloading ${filename} from ${url}`); 49 | return fetch(url) 50 | .then((res) => { 51 | ctrl.enqueue({ 52 | name: filename, 53 | stream: () => res.body, 54 | }); 55 | 56 | // Update progress. 57 | onProgress?.(++current, total, fileInfo.value); 58 | logger.debug(`Finished downloading ${filename} in ${Date.now() - start}ms`); 59 | }) 60 | .then(() => { 61 | // Wait for a while to prevent rate limit. 62 | return new Promise((resolve) => setTimeout(resolve, rateLimit)); 63 | }); 64 | } 65 | }, 66 | }); 67 | 68 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 69 | const chunks: any[] = []; 70 | const writableOutputStream = new WritableStream({ 71 | write(chunk) { 72 | chunks.push(chunk); 73 | }, 74 | close() { 75 | logger.info('Zip stream closed.'); 76 | }, 77 | }); 78 | 79 | // Download the zip archive. 80 | logger.info(`Exporting to ZIP file: ${zipFilename}`); 81 | await readableZipStream.pipeTo(writableOutputStream); 82 | 83 | const arrayBuffer = await new Blob(chunks).arrayBuffer(); 84 | const blob = new Blob([arrayBuffer]); 85 | saveAs(blob, zipFilename); 86 | } 87 | -------------------------------------------------------------------------------- /src/utils/exporter.ts: -------------------------------------------------------------------------------- 1 | import logger from './logger'; 2 | 3 | /** 4 | * Supported formats of exporting. 5 | */ 6 | export const EXPORT_FORMAT = { 7 | JSON: 'JSON', 8 | HTML: 'HTML', 9 | CSV: 'CSV', 10 | } as const; 11 | 12 | export type ExportFormatType = (typeof EXPORT_FORMAT)[keyof typeof EXPORT_FORMAT]; 13 | 14 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 15 | export type DataType = Record; 16 | 17 | /** 18 | * Escape characters for CSV file. 19 | */ 20 | export function csvEscapeStr(str: string) { 21 | return `"${str.replace(/"/g, '""').replace(/\n/g, '\\n').replace(/\r/g, '\\r')}"`; 22 | } 23 | 24 | /** 25 | * Save a text file to disk. 26 | */ 27 | export function saveFile(filename: string, content: string | Blob, prependBOM: boolean = false) { 28 | const link = document.createElement('a'); 29 | let blob: Blob; 30 | 31 | if (content instanceof Blob) { 32 | blob = content; 33 | } else { 34 | blob = new Blob(prependBOM ? [new Uint8Array([0xef, 0xbb, 0xbf]), content] : [content], { 35 | type: 'text/plain;charset=utf-8', 36 | }); 37 | } 38 | 39 | const url = URL.createObjectURL(blob); 40 | 41 | link.href = url; 42 | link.download = filename; 43 | link.click(); 44 | URL.revokeObjectURL(url); 45 | } 46 | 47 | /** 48 | * Export data and download as a file. 49 | * 50 | * @param data Data list to export. 51 | * @param format Export format. (JSON, HTML, CSV) 52 | * @param filename Filename to save. 53 | * @param translations Translations for headers. 54 | */ 55 | export async function exportData( 56 | data: DataType[], 57 | format: ExportFormatType, 58 | filename: string, 59 | translations: Record, 60 | ) { 61 | try { 62 | let content = ''; 63 | let prependBOM = false; 64 | logger.info(`Exporting to ${format} file: ${filename}`); 65 | 66 | switch (format) { 67 | case EXPORT_FORMAT.JSON: 68 | content = await jsonExporter(data); 69 | break; 70 | case EXPORT_FORMAT.HTML: 71 | content = await htmlExporter(data, translations); 72 | break; 73 | case EXPORT_FORMAT.CSV: 74 | prependBOM = true; 75 | content = await csvExporter(data); 76 | break; 77 | } 78 | saveFile(filename, content, prependBOM); 79 | } catch (err) { 80 | logger.errorWithBanner('Failed to export file', err as Error); 81 | } 82 | } 83 | 84 | export async function jsonExporter(data: DataType[]) { 85 | return JSON.stringify(data, undefined, ' '); 86 | } 87 | 88 | export async function htmlExporter(data: DataType[], translations: Record) { 89 | const table = document.createElement('table'); 90 | const thead = document.createElement('thead'); 91 | const tbody = document.createElement('tbody'); 92 | 93 | // The keys of the first row are translated and used as headers. 94 | const exportKeys = Object.keys(data[0] ?? {}); 95 | const headerRow = document.createElement('tr'); 96 | for (const exportKey of exportKeys) { 97 | const th = document.createElement('th'); 98 | th.textContent = translations[exportKey] ?? exportKey; 99 | headerRow.appendChild(th); 100 | } 101 | 102 | thead.appendChild(headerRow); 103 | table.appendChild(thead); 104 | table.className = 'table table-striped'; 105 | 106 | for (const row of data) { 107 | const tr = document.createElement('tr'); 108 | for (const exportKey of exportKeys) { 109 | const td = document.createElement('td'); 110 | const value = row[exportKey]; 111 | 112 | if (exportKey === 'profile_image_url' || exportKey === 'profile_banner_url') { 113 | const img = document.createElement('img'); 114 | img.src = value; 115 | img.width = 50; 116 | td.innerHTML = ''; 117 | td.appendChild(img); 118 | } else if (exportKey === 'media') { 119 | if (value?.length > 0) { 120 | for (const media of value) { 121 | const img = document.createElement('img'); 122 | img.src = media.thumbnail; 123 | img.width = 50; 124 | img.alt = media.ext_alt_text || ''; 125 | img.title = media.ext_alt_text || ''; 126 | const link = document.createElement('a'); 127 | link.href = media.original; 128 | link.target = '_blank'; 129 | link.style.marginRight = '0.5em'; 130 | link.appendChild(img); 131 | td.appendChild(link); 132 | } 133 | } 134 | } else if (exportKey === 'full_text' || exportKey === 'description') { 135 | const p = document.createElement('p'); 136 | p.innerHTML = value; 137 | p.style.whiteSpace = 'pre-wrap'; 138 | p.style.maxWidth = '640px'; 139 | td.appendChild(p); 140 | } else if (exportKey === 'metadata') { 141 | const details = document.createElement('details'); 142 | const summary = document.createElement('summary'); 143 | summary.textContent = 'Expand'; 144 | details.appendChild(summary); 145 | const pre = document.createElement('pre'); 146 | pre.textContent = JSON.stringify(value, undefined, ' '); 147 | details.appendChild(pre); 148 | td.appendChild(details); 149 | } else if (exportKey === 'url') { 150 | const link = document.createElement('a'); 151 | link.href = value; 152 | link.target = '_blank'; 153 | link.textContent = value; 154 | td.appendChild(link); 155 | } else { 156 | td.textContent = typeof value === 'string' ? value : JSON.stringify(row[exportKey]); 157 | } 158 | 159 | tr.appendChild(td); 160 | } 161 | tbody.appendChild(tr); 162 | } 163 | 164 | table.appendChild(tbody); 165 | 166 | return ` 167 | 168 | 169 | 170 | Exported Data ${new Date().toISOString()} 171 | 172 | 173 | 174 | ${table.outerHTML} 175 | 176 | 177 | `; 178 | } 179 | 180 | export async function csvExporter(data: DataType[]) { 181 | const headers = Object.keys(data[0] ?? {}); 182 | let content = headers.join(',') + '\n'; 183 | 184 | for (const row of data) { 185 | const values = headers.map((header) => { 186 | const value = row[header]; 187 | if (typeof value === 'string') { 188 | return csvEscapeStr(value); 189 | } 190 | 191 | if (typeof value === 'object') { 192 | return csvEscapeStr(JSON.stringify(value)); 193 | } 194 | 195 | return value; 196 | }); 197 | content += values.join(','); 198 | content += '\n'; 199 | } 200 | 201 | return content; 202 | } 203 | -------------------------------------------------------------------------------- /src/utils/logger.ts: -------------------------------------------------------------------------------- 1 | import { signal } from '@preact/signals'; 2 | 3 | export interface LogLine { 4 | type: 'info' | 'warn' | 'error'; 5 | line: string; 6 | index: number; 7 | } 8 | 9 | export const logLinesSignal = signal([]); 10 | 11 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 12 | type LogExtraArgs = any[]; 13 | 14 | /** 15 | * Global logger that writes logs to both screen and console. 16 | */ 17 | class Logger { 18 | private index = 0; 19 | private buffer: LogLine[] = []; 20 | private bufferTimer: number | null = null; 21 | 22 | public info(line: string, ...args: LogExtraArgs) { 23 | console.info('[twitter-web-exporter]', line, ...args); 24 | this.writeBuffer({ type: 'info', line, index: this.index++ }); 25 | } 26 | 27 | public warn(line: string, ...args: LogExtraArgs) { 28 | console.warn('[twitter-web-exporter]', line, ...args); 29 | this.writeBuffer({ type: 'warn', line, index: this.index++ }); 30 | } 31 | 32 | public error(line: string, ...args: LogExtraArgs) { 33 | console.error('[twitter-web-exporter]', line, ...args); 34 | this.writeBuffer({ type: 'error', line, index: this.index++ }); 35 | } 36 | 37 | public errorWithBanner(msg: string, err?: Error, ...args: LogExtraArgs) { 38 | this.error( 39 | `${msg} (Message: ${err?.message ?? 'none'})\n` + 40 | ' This may be a problem caused by Twitter updates.\n Please file an issue on GitHub:\n' + 41 | ' https://github.com/prinsss/twitter-web-exporter/issues', 42 | ...args, 43 | ); 44 | } 45 | 46 | public debug(...args: LogExtraArgs) { 47 | console.debug('[twitter-web-exporter]', ...args); 48 | } 49 | 50 | /** 51 | * Buffer log lines to reduce the number of signal and DOM updates. 52 | */ 53 | private writeBuffer(log: LogLine) { 54 | this.buffer.push(log); 55 | 56 | if (this.bufferTimer) { 57 | clearTimeout(this.bufferTimer); 58 | } 59 | 60 | this.bufferTimer = window.setTimeout(() => { 61 | this.bufferTimer = null; 62 | this.flushBuffer(); 63 | }, 0); 64 | } 65 | 66 | /** 67 | * Flush buffered log lines and update the UI. 68 | */ 69 | private flushBuffer() { 70 | logLinesSignal.value = [...logLinesSignal.value, ...this.buffer]; 71 | this.buffer = []; 72 | } 73 | } 74 | 75 | /** 76 | * Global logger singleton instance. 77 | */ 78 | const logger = new Logger(); 79 | 80 | export default logger; 81 | -------------------------------------------------------------------------------- /src/utils/media.ts: -------------------------------------------------------------------------------- 1 | import { Media, Tweet, User } from '@/types'; 2 | import { 3 | extractTweetMedia, 4 | getFileExtensionFromUrl, 5 | getMediaIndex, 6 | getMediaOriginalUrl, 7 | getProfileImageOriginalUrl, 8 | } from './api'; 9 | import { parseTwitterDateTime } from './common'; 10 | import { FileLike } from './download'; 11 | 12 | export type PatternExtractor = (tweet: Tweet, media: Media) => string; 13 | 14 | /** 15 | * All available patterns for customizing filenames when downloading media files. 16 | */ 17 | export const patterns: Record = { 18 | id: { 19 | description: 'The tweet ID', 20 | extractor: (tweet) => tweet.rest_id, 21 | }, 22 | screen_name: { 23 | description: 'The username of tweet author', 24 | extractor: (tweet) => tweet.core.user_results.result.legacy.screen_name, 25 | }, 26 | name: { 27 | description: 'The profile name of tweet author', 28 | extractor: (tweet) => tweet.core.user_results.result.legacy.name, 29 | }, 30 | index: { 31 | description: 'The media index in tweet (start from 0)', 32 | extractor: (tweet, media) => String(getMediaIndex(tweet, media)), 33 | }, 34 | num: { 35 | description: 'The order of media in tweet (1/2/3/4)', 36 | extractor: (tweet, media) => String(getMediaIndex(tweet, media) + 1), 37 | }, 38 | date: { 39 | description: 'The post date in YYYYMMDD format', 40 | extractor: (tweet) => parseTwitterDateTime(tweet.legacy.created_at).format('YYYYMMDD'), 41 | }, 42 | time: { 43 | description: 'The post time in HHmmss format', 44 | extractor: (tweet) => parseTwitterDateTime(tweet.legacy.created_at).format('HHmmss'), 45 | }, 46 | type: { 47 | description: 'The media type (photo/video/animated_gif)', 48 | extractor: (tweet, media) => media.type, 49 | }, 50 | ext: { 51 | description: 'The file extension of media (jpg/png/mp4)', 52 | extractor: (tweet, media) => getFileExtensionFromUrl(getMediaOriginalUrl(media)), 53 | }, 54 | }; 55 | 56 | export const DEFAULT_MEDIA_TYPES = ['photo', 'video', 'animated_gif'] as const; 57 | 58 | /** 59 | * Extract media from tweets and users. 60 | */ 61 | export function extractMedia( 62 | data: Tweet[] | User[], 63 | includeRetweets: boolean, 64 | filenamePattern: string, 65 | ): FileLike[] { 66 | const gallery = new Map(); 67 | 68 | for (const item of data) { 69 | // For tweets, download media files with custom filenames. 70 | // NOTE: __typename is undefined in TweetWithVisibilityResults. 71 | if (item.__typename === 'Tweet' || (typeof item.__typename === 'undefined' && 'core' in item)) { 72 | if (!includeRetweets && item.legacy.retweeted_status_result) { 73 | continue; 74 | } 75 | 76 | const tweetMedia = extractTweetMedia(item).map((media) => { 77 | // Parse and apply custom filename pattern. 78 | let filename = filenamePattern; 79 | for (const [key, value] of Object.entries(patterns)) { 80 | filename = filename.replace(`{${key}}`, value.extractor(item, media)); 81 | } 82 | 83 | return { filename, type: media.type, url: getMediaOriginalUrl(media) }; 84 | }); 85 | 86 | for (const media of tweetMedia) { 87 | gallery.set(media.filename, media); 88 | } 89 | } 90 | 91 | // For users, download their profile images and banners. 92 | if (item.__typename === 'User') { 93 | if (item.legacy.profile_image_url_https) { 94 | const ext = getFileExtensionFromUrl(item.legacy.profile_image_url_https); 95 | const filename = `${item.legacy.screen_name}_profile_image.${ext}`; 96 | gallery.set(filename, { 97 | filename, 98 | type: 'photo', 99 | url: getProfileImageOriginalUrl(item.legacy.profile_image_url_https), 100 | }); 101 | } 102 | 103 | if (item.legacy.profile_banner_url) { 104 | const ext = getFileExtensionFromUrl(item.legacy.profile_banner_url); 105 | const filename = `${item.legacy.screen_name}_profile_banner.${ext}`; 106 | gallery.set(filename, { 107 | filename, 108 | type: 'photo', 109 | url: item.legacy.profile_banner_url, 110 | }); 111 | } 112 | } 113 | } 114 | 115 | return Array.from(gallery.values()); 116 | } 117 | -------------------------------------------------------------------------------- /src/utils/observable.ts: -------------------------------------------------------------------------------- 1 | import { liveQuery, Observable } from 'dexie'; 2 | import { useEffect, useMemo, useReducer, useRef } from 'preact/hooks'; 3 | 4 | /** 5 | * Modified from `dexie-react-hooks` with some lines removed. The modified version 6 | * is specifically designed for `Observable` and `liveQuery` from Dexie.js. 7 | * 8 | * @license Apache-2.0 9 | * @see https://dexie.org/docs/dexie-react-hooks/useObservable() 10 | * @see https://github.com/dexie/Dexie.js/blob/v4.0.4/libs/dexie-react-hooks/src/useObservable.ts 11 | * @param observableFactory Function that returns an observable. 12 | * @param deps The observableFactory function will be re-executed if deps change. 13 | * @param defaultResult Result returned on initial render. 14 | * @returns The current result of the observable. 15 | */ 16 | export function useObservable( 17 | observableFactory: () => Observable, 18 | deps: unknown[], 19 | defaultResult: TDefault, 20 | ): T | TDefault { 21 | // Create a ref that keeps the state we need. 22 | const monitor = useRef({ 23 | hasResult: false, 24 | result: defaultResult as T | TDefault, 25 | error: null, 26 | }); 27 | 28 | // We control when component should rerender. 29 | const [, triggerUpdate] = useReducer((x) => x + 1, 0); 30 | 31 | // Memoize the observable based on deps. 32 | const observable = useMemo(() => { 33 | const observable = observableFactory(); 34 | if (!observable || typeof observable.subscribe !== 'function') { 35 | throw new TypeError( 36 | `Observable factory given to useObservable() did not return a valid observable.`, 37 | ); 38 | } 39 | 40 | if (monitor.current.hasResult) { 41 | return observable; 42 | } 43 | 44 | if (typeof observable.hasValue !== 'function' || observable.hasValue()) { 45 | if (typeof observable.getValue === 'function') { 46 | monitor.current.result = observable.getValue(); 47 | monitor.current.hasResult = true; 48 | } 49 | } 50 | 51 | return observable; 52 | }, deps); 53 | 54 | // Subscribe to the observable. 55 | useEffect(() => { 56 | const subscription = observable.subscribe( 57 | (val) => { 58 | const state = monitor.current; 59 | if (state.error !== null || state.result !== val) { 60 | state.error = null; 61 | state.result = val; 62 | state.hasResult = true; 63 | triggerUpdate(1); 64 | } 65 | }, 66 | (err) => { 67 | if (monitor.current.error !== err) { 68 | monitor.current.error = err; 69 | triggerUpdate(1); 70 | } 71 | }, 72 | ); 73 | 74 | return subscription.unsubscribe.bind(subscription); 75 | }, deps); 76 | 77 | // Throw if observable has emitted error so that an ErrorBoundary can catch it. 78 | if (monitor.current.error) { 79 | throw monitor.current.error; 80 | } 81 | 82 | // Return the current result. 83 | return monitor.current.result; 84 | } 85 | 86 | /** 87 | * A hook that subscribes to a live query and returns the current result. 88 | * Copied from `dexie-react-hooks` with some function overloads removed. 89 | * 90 | * @license Apache-2.0 91 | * @see https://dexie.org/docs/dexie-react-hooks/useLiveQuery() 92 | * @see https://github.com/dexie/Dexie.js/blob/v4.0.4/libs/dexie-react-hooks/src/useLiveQuery.ts 93 | * @see https://github.com/dexie/Dexie.js/blob/v4.0.4/src/live-query/live-query.ts 94 | * @param querier Function that returns a final result (Promise). 95 | * @param deps Variables that querier is dependent on. 96 | * @param defaultResult Result returned on initial render. 97 | * @returns The current result of the live query. 98 | */ 99 | export function useLiveQuery( 100 | querier: () => Promise | T, 101 | deps?: unknown[], 102 | defaultResult?: TDefault, 103 | ): T | TDefault { 104 | return useObservable(() => liveQuery(querier), deps || [], defaultResult as TDefault); 105 | } 106 | -------------------------------------------------------------------------------- /src/utils/react-table.tsx: -------------------------------------------------------------------------------- 1 | import { ComponentType, JSX } from 'preact'; 2 | import { useState } from 'preact/hooks'; 3 | import { createTable, RowData, TableOptions, TableOptionsResolved } from '@tanstack/table-core'; 4 | 5 | export type Renderable = ComponentType | null | undefined | string; 6 | 7 | export function flexRender( 8 | Comp: Renderable, 9 | props: TProps, 10 | ): JSX.Element | string | null { 11 | return !Comp ? null : isComponent(Comp) ? : Comp; 12 | } 13 | 14 | function isComponent(component: unknown): component is ComponentType { 15 | return typeof component === 'function'; 16 | } 17 | 18 | /** 19 | * @license MIT 20 | * https://github.com/TanStack/table/blob/main/packages/react-table/src/index.tsx 21 | */ 22 | export function useReactTable(options: TableOptions) { 23 | // Compose in the generic options to the user options 24 | const resolvedOptions: TableOptionsResolved = { 25 | state: {}, // Dummy state 26 | onStateChange: () => {}, // noop 27 | renderFallbackValue: null, 28 | ...options, 29 | }; 30 | 31 | // Create a new table and store it in state 32 | const [tableRef] = useState(() => ({ 33 | current: createTable(resolvedOptions), 34 | })); 35 | 36 | // By default, manage table state here using the table's initial state 37 | const [state, setState] = useState(() => tableRef.current.initialState); 38 | 39 | // Compose the default state above with any user state. This will allow the user 40 | // to only control a subset of the state if desired. 41 | tableRef.current.setOptions((prev) => ({ 42 | ...prev, 43 | ...options, 44 | state: { 45 | ...state, 46 | ...options.state, 47 | }, 48 | // Similarly, we'll maintain both our internal state and any user-provided 49 | // state. 50 | onStateChange: (updater) => { 51 | setState(updater); 52 | options.onStateChange?.(updater); 53 | }, 54 | })); 55 | 56 | return tableRef.current; 57 | } 58 | -------------------------------------------------------------------------------- /src/utils/zip-stream.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-var, @typescript-eslint/ban-ts-comment */ 2 | // @ts-nocheck 3 | /** 4 | * This file zip-stream.ts is copied from StreamSaver.js. 5 | * 6 | * @see https://github.com/jimmywarting/StreamSaver.js/blob/master/examples/zip-stream.js 7 | * @license MIT 8 | */ 9 | 10 | class Crc32 { 11 | constructor() { 12 | this.crc = -1; 13 | } 14 | 15 | append(data) { 16 | var crc = this.crc | 0; 17 | var table = this.table; 18 | for (var offset = 0, len = data.length | 0; offset < len; offset++) { 19 | crc = (crc >>> 8) ^ table[(crc ^ data[offset]) & 0xff]; 20 | } 21 | this.crc = crc; 22 | } 23 | 24 | get() { 25 | return ~this.crc; 26 | } 27 | } 28 | Crc32.prototype.table = (() => { 29 | var i; 30 | var j; 31 | var t; 32 | var table = []; 33 | for (i = 0; i < 256; i++) { 34 | t = i; 35 | for (j = 0; j < 8; j++) { 36 | t = t & 1 ? (t >>> 1) ^ 0xedb88320 : t >>> 1; 37 | } 38 | table[i] = t; 39 | } 40 | return table; 41 | })(); 42 | 43 | const getDataHelper = (byteLength) => { 44 | var uint8 = new Uint8Array(byteLength); 45 | return { 46 | array: uint8, 47 | view: new DataView(uint8.buffer), 48 | }; 49 | }; 50 | 51 | const pump = (zipObj) => 52 | zipObj.reader.read().then((chunk) => { 53 | if (chunk.done) return zipObj.writeFooter(); 54 | const outputData = chunk.value; 55 | zipObj.crc.append(outputData); 56 | zipObj.uncompressedLength += outputData.length; 57 | zipObj.compressedLength += outputData.length; 58 | zipObj.ctrl.enqueue(outputData); 59 | }); 60 | 61 | /** 62 | * [createWriter description] 63 | * @param {Object} underlyingSource [description] 64 | * @return {Boolean} [description] 65 | */ 66 | function createWriter(underlyingSource) { 67 | const files = Object.create(null); 68 | const filenames = []; 69 | const encoder = new TextEncoder(); 70 | let offset = 0; 71 | let activeZipIndex = 0; 72 | let ctrl; 73 | let activeZipObject, closed; 74 | 75 | function next() { 76 | activeZipIndex++; 77 | activeZipObject = files[filenames[activeZipIndex]]; 78 | if (activeZipObject) processNextChunk(); 79 | else if (closed) closeZip(); 80 | } 81 | 82 | var zipWriter = { 83 | enqueue(fileLike) { 84 | if (closed) 85 | throw new TypeError( 86 | 'Cannot enqueue a chunk into a readable stream that is closed or has been requested to be closed', 87 | ); 88 | 89 | let name = fileLike.name.trim(); 90 | const date = new Date( 91 | typeof fileLike.lastModified === 'undefined' ? Date.now() : fileLike.lastModified, 92 | ); 93 | 94 | if (fileLike.directory && !name.endsWith('/')) name += '/'; 95 | if (files[name]) throw new Error('File already exists.'); 96 | 97 | const nameBuf = encoder.encode(name); 98 | filenames.push(name); 99 | 100 | const zipObject = (files[name] = { 101 | level: 0, 102 | ctrl, 103 | directory: !!fileLike.directory, 104 | nameBuf, 105 | comment: encoder.encode(fileLike.comment || ''), 106 | compressedLength: 0, 107 | uncompressedLength: 0, 108 | writeHeader() { 109 | var header = getDataHelper(26); 110 | var data = getDataHelper(30 + nameBuf.length); 111 | 112 | zipObject.offset = offset; 113 | zipObject.header = header; 114 | if (zipObject.level !== 0 && !zipObject.directory) { 115 | header.view.setUint16(4, 0x0800); 116 | } 117 | header.view.setUint32(0, 0x14000808); 118 | header.view.setUint16( 119 | 6, 120 | (((date.getHours() << 6) | date.getMinutes()) << 5) | (date.getSeconds() / 2), 121 | true, 122 | ); 123 | header.view.setUint16( 124 | 8, 125 | ((((date.getFullYear() - 1980) << 4) | (date.getMonth() + 1)) << 5) | date.getDate(), 126 | true, 127 | ); 128 | header.view.setUint16(22, nameBuf.length, true); 129 | data.view.setUint32(0, 0x504b0304); 130 | data.array.set(header.array, 4); 131 | data.array.set(nameBuf, 30); 132 | offset += data.array.length; 133 | ctrl.enqueue(data.array); 134 | }, 135 | writeFooter() { 136 | var footer = getDataHelper(16); 137 | footer.view.setUint32(0, 0x504b0708); 138 | 139 | if (zipObject.crc) { 140 | zipObject.header.view.setUint32(10, zipObject.crc.get(), true); 141 | zipObject.header.view.setUint32(14, zipObject.compressedLength, true); 142 | zipObject.header.view.setUint32(18, zipObject.uncompressedLength, true); 143 | footer.view.setUint32(4, zipObject.crc.get(), true); 144 | footer.view.setUint32(8, zipObject.compressedLength, true); 145 | footer.view.setUint32(12, zipObject.uncompressedLength, true); 146 | } 147 | 148 | ctrl.enqueue(footer.array); 149 | offset += zipObject.compressedLength + 16; 150 | next(); 151 | }, 152 | fileLike, 153 | }); 154 | 155 | if (!activeZipObject) { 156 | activeZipObject = zipObject; 157 | processNextChunk(); 158 | } 159 | }, 160 | close() { 161 | if (closed) 162 | throw new TypeError( 163 | 'Cannot close a readable stream that has already been requested to be closed', 164 | ); 165 | if (!activeZipObject) closeZip(); 166 | closed = true; 167 | }, 168 | }; 169 | 170 | function closeZip() { 171 | var length = 0; 172 | var index = 0; 173 | var indexFilename, file; 174 | for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) { 175 | file = files[filenames[indexFilename]]; 176 | length += 46 + file.nameBuf.length + file.comment.length; 177 | } 178 | const data = getDataHelper(length + 22); 179 | for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) { 180 | file = files[filenames[indexFilename]]; 181 | data.view.setUint32(index, 0x504b0102); 182 | data.view.setUint16(index + 4, 0x1400); 183 | data.array.set(file.header.array, index + 6); 184 | data.view.setUint16(index + 32, file.comment.length, true); 185 | if (file.directory) { 186 | data.view.setUint8(index + 38, 0x10); 187 | } 188 | data.view.setUint32(index + 42, file.offset, true); 189 | data.array.set(file.nameBuf, index + 46); 190 | data.array.set(file.comment, index + 46 + file.nameBuf.length); 191 | index += 46 + file.nameBuf.length + file.comment.length; 192 | } 193 | data.view.setUint32(index, 0x504b0506); 194 | data.view.setUint16(index + 8, filenames.length, true); 195 | data.view.setUint16(index + 10, filenames.length, true); 196 | data.view.setUint32(index + 12, length, true); 197 | data.view.setUint32(index + 16, offset, true); 198 | ctrl.enqueue(data.array); 199 | ctrl.close(); 200 | } 201 | 202 | function processNextChunk() { 203 | if (!activeZipObject) return; 204 | if (activeZipObject.directory) 205 | return activeZipObject.writeFooter(activeZipObject.writeHeader()); 206 | if (activeZipObject.reader) return pump(activeZipObject); 207 | if (activeZipObject.fileLike.stream) { 208 | activeZipObject.crc = new Crc32(); 209 | activeZipObject.reader = activeZipObject.fileLike.stream().getReader(); 210 | activeZipObject.writeHeader(); 211 | } else next(); 212 | } 213 | return new ReadableStream({ 214 | start: (c) => { 215 | ctrl = c; 216 | if (underlyingSource.start) Promise.resolve(underlyingSource.start(zipWriter)); 217 | }, 218 | pull() { 219 | return ( 220 | processNextChunk() || 221 | (underlyingSource.pull && Promise.resolve(underlyingSource.pull(zipWriter))) 222 | ); 223 | }, 224 | }); 225 | } 226 | 227 | // window.ZIP = createWriter 228 | export default createWriter; 229 | -------------------------------------------------------------------------------- /src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | //// 4 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | import daisyui from 'daisyui'; 2 | 3 | /** @type {import('tailwindcss').Config} */ 4 | export default { 5 | content: ['./src/**/*.{js,ts,jsx,tsx}'], 6 | theme: { 7 | extend: {}, 8 | }, 9 | plugins: [daisyui], 10 | daisyui: { 11 | themes: [ 12 | 'cupcake', 13 | 'dark', 14 | 'emerald', 15 | 'cyberpunk', 16 | 'valentine', 17 | 'lofi', 18 | 'dracula', 19 | 'cmyk', 20 | 'business', 21 | 'winter', 22 | ], 23 | darkTheme: 'dracula', 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": false, 4 | "esModuleInterop": false, 5 | "isolatedModules": true, 6 | "jsx": "react-jsx", 7 | "jsxImportSource": "preact", 8 | "lib": ["ES2022", "DOM", "DOM.Iterable"], 9 | "module": "ESNext", 10 | "moduleDetection": "force", 11 | "moduleResolution": "Bundler", 12 | "noEmit": true, 13 | "noUncheckedIndexedAccess": true, 14 | "paths": { 15 | "@/*": ["./src/*"] 16 | }, 17 | "resolveJsonModule": true, 18 | "skipLibCheck": true, 19 | "strict": true, 20 | "target": "ES2022" 21 | }, 22 | "include": ["src"] 23 | } 24 | -------------------------------------------------------------------------------- /vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite'; 2 | import { fileURLToPath, URL } from 'node:url'; 3 | 4 | import preact from '@preact/preset-vite'; 5 | import monkey from 'vite-plugin-monkey'; 6 | import i18nextLoader from 'vite-plugin-i18next-loader'; 7 | 8 | import tailwindcss from 'tailwindcss'; 9 | import autoprefixer from 'autoprefixer'; 10 | import prefixSelector from 'postcss-prefix-selector'; 11 | import remToPx from 'postcss-rem-to-pixel-next'; 12 | 13 | // https://vitejs.dev/config/ 14 | export default defineConfig({ 15 | resolve: { 16 | alias: { 17 | '@': fileURLToPath(new URL('./src', import.meta.url)), 18 | }, 19 | }, 20 | build: { 21 | minify: false, 22 | }, 23 | css: { 24 | postcss: { 25 | plugins: [ 26 | tailwindcss(), 27 | autoprefixer(), 28 | remToPx({ propList: ['*'] }), 29 | // Use scoped CSS. 30 | prefixSelector({ 31 | prefix: '#twe-root', 32 | exclude: [/^#twe-root/], // This may be a bug. 33 | }), 34 | ], 35 | }, 36 | }, 37 | plugins: [ 38 | preact(), 39 | i18nextLoader({ paths: ['./src/i18n/locales'], namespaceResolution: 'basename' }), 40 | monkey({ 41 | entry: 'src/main.tsx', 42 | userscript: { 43 | name: { 44 | '': 'Twitter Web Exporter', 45 | 'zh-CN': 'Twitter 数据导出工具', 46 | 'zh-TW': 'Twitter 資料匯出工具', 47 | ja: 'Twitter データエクスポーター', 48 | }, 49 | description: { 50 | '': 'Export tweets, bookmarks, lists and much more to JSON/CSV/HTML from Twitter(X) web app.', 51 | 'zh-CN': '从 Twitter(X) 网页版导出推文、书签、列表等各种数据，支持导出 JSON/CSV/HTML。', 52 | 'zh-TW': '從 Twitter(X) 網頁版匯出推文、書籤、列表等各種資料，支援匯出 JSON/CSV/HTML。', 53 | ja: 'Twitter(X) ブラウザ版からツイート、ブックマーク、リストなどを取得し JSON/CSV/HTML に出力します。', 54 | }, 55 | namespace: 'https://github.com/prinsss', 56 | icon: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAABmklEQVR4Ae3XA4wcARSA4dq2bUQ1g9pRbVtBzai2otpug9pxUttn2753/3m9Ozq/5NsdvvfGM6VKoshE8/ORFbAMbxCGWHzDHjS2sXxPlM0eKYclGoq3w1eIHVGYikaYg6e4ZppgAgQrVBSvDw+IEylIhSAATUyTHIYgFdsUNnAGosAfDMccLMtOchli4g7quFC8FhIhCsRD8Bk1sxMdgVjwxRyUdtDABIgKH9DQNNEkiB1fMB9VbDSwEKLQJ1S1TFQRXhAHYnADy9ETdTEeotAze7tzNJIhCiRBFLpnq/hmzMR65UkVO2WrgaOQPLLW3u6XPDLAVgOl8R5isEhUtHcSdkEoxEBXnN3ZuuMbxCDDnTVQF52xBcEQHX1BaWcNtDLwMpzg6tNtN0RnD5U8XsviGkQnYWih9CWjNBbDHaJBMsZqec8rjV54B1EoFXO0Fh+DrxCFEjBTTdFy6IvNGu4Hf9FXSdGheAUvjZdgLPajqtp3+jl4jVSIAgHYjRZ6fWC0wSpcwScEQZCMUPzEfezEYJQrVRKFOdIAZGq1QBG8EiYAAAAASUVORK5CYII=', 57 | match: ['*://twitter.com/*', '*://x.com/*'], 58 | grant: ['unsafeWindow'], 59 | 'run-at': 'document-start', 60 | updateURL: 61 | 'https://github.com/prinsss/twitter-web-exporter/releases/latest/download/twitter-web-exporter.user.js', 62 | downloadURL: 63 | 'https://github.com/prinsss/twitter-web-exporter/releases/latest/download/twitter-web-exporter.user.js', 64 | require: [ 65 | 'https://cdn.jsdelivr.net/npm/dayjs@1.11.13/dayjs.min.js', 66 | 'https://cdn.jsdelivr.net/npm/dexie@4.0.11/dist/dexie.min.js', 67 | 'https://cdn.jsdelivr.net/npm/dexie-export-import@4.1.4/dist/dexie-export-import.js', 68 | 'https://cdn.jsdelivr.net/npm/file-saver-es@2.0.5/dist/FileSaver.min.js', 69 | 'https://cdn.jsdelivr.net/npm/i18next@24.2.3/i18next.min.js', 70 | 'https://cdn.jsdelivr.net/npm/preact@10.26.4/dist/preact.min.js', 71 | 'https://cdn.jsdelivr.net/npm/preact@10.26.4/hooks/dist/hooks.umd.js', 72 | 'https://cdn.jsdelivr.net/npm/@preact/signals-core@1.8.0/dist/signals-core.min.js', 73 | 'https://cdn.jsdelivr.net/npm/@preact/signals@2.0.0/dist/signals.min.js', 74 | 'https://cdn.jsdelivr.net/npm/@tanstack/table-core@8.21.2/build/umd/index.production.js', 75 | ], 76 | }, 77 | build: { 78 | externalGlobals: { 79 | dayjs: 'dayjs', 80 | dexie: 'Dexie', 81 | 'dexie-export-import': 'DexieExportImport', 82 | 'file-saver-es': 'FileSaver', 83 | i18next: 'i18next', 84 | preact: 'preact', 85 | 'preact/hooks': 'preactHooks', 86 | '@preact/signals': 'preactSignals', 87 | '@tanstack/table-core': 'TableCore', 88 | }, 89 | }, 90 | }), 91 | ], 92 | }); 93 | --------------------------------------------------------------------------------