├── .github └── FUNDING.yml ├── .gitignore ├── .prettierrc ├── LICENSE ├── README.md ├── __fixtures__ ├── markdown │ ├── cjk-html-entities.md │ ├── condensed-thread.md │ ├── image-tweet.md │ ├── multiple-mentions.md │ ├── non-mangled-urls.md │ ├── partial-thread.md │ ├── textonly.md │ └── tweet-thread.md └── tweets │ ├── cashtag_tweet.ts │ ├── cjk_tweets.ts │ ├── emoji_tweets.ts │ ├── image_tweet.ts │ ├── index.ts │ ├── mentions_tweet.ts │ ├── poll_tweet.ts │ ├── profile_pic_tweets.ts │ ├── tweet_thread.ts │ └── url_tweet.ts ├── __tests__ ├── consts.ts ├── main.test.ts └── util.test.ts ├── images ├── repo-open-graph.png ├── ttm_demo.gif ├── tweet-markdown-screenshot.png └── tweet-to-markdown-logo.svg ├── package.json ├── rollup.config.js ├── src ├── .gitattributes ├── main.ts ├── mocks │ ├── handlers.ts │ └── server.ts ├── models.ts ├── process.ts ├── unicodeSubstring.ts └── util.ts ├── tsconfig.json └── yarn.lock /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: kbravh 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "semi": false, 4 | "singleQuote": true, 5 | "bracketSpacing": false, 6 | "arrowParens": "avoid" 7 | } 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Karey Higuera 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | [![Stargazers][stars-shield]][stars-url] 6 | [![Issues][issues-shield]][issues-url] 7 | [![MIT License][license-shield]][license-url] 8 | 9 |
10 |

11 | 12 | Logo 13 | 14 | 15 |

Tweet to Markdown

16 | 17 |

18 | A command line tool to quickly save tweets as Markdown. 19 |
20 |
21 | Report a Bug 22 | · 23 | Request a Feature 24 |

25 |

26 | 27 | ## ⚠️ Heads up! ⚠️ 28 | 29 | If your API key is not working, you will need to apply for a Twitter bearer token. 30 | 31 | Due to recent changes to the Twitter API, the free access method listed below has stopped working as of April 27, 2023. You must sign up for your own Twitter bearer token to use this application. 32 | 33 | 34 | 35 | ## About The Project 36 | 37 | This command line tool allows you to quickly save a tweet in Markdown format. This is great for Zettelkasten note-taking or any other commonplace notebook, vade mecum, Obsidian, Roam, Foam, &c. It is built on the new Twitter v2 API. 38 | 39 | ![Demo of `ttm` in the command line](./images/ttm_demo.gif) 40 | 41 | ## Installing 42 | 43 | ⚠ **You'll need to have Node.js of at least `v12.x` to use this tool.** 44 | 45 | You can install this CLI tool by running 46 | 47 | ```bash 48 | yarn global add tweet-to-markdown 49 | ``` 50 | 51 | or 52 | 53 | ```bash 54 | npm install --global tweet-to-markdown 55 | ``` 56 | 57 | You can also run it without installing: 58 | 59 | ```bash 60 | npx tweet-to-markdown 61 | ``` 62 | 63 | 64 | 65 | ## Setup 66 | 67 | To use this tool, you have ~~two~~ options: 68 | 69 | - ~~Sign up for a free API key from https://ttm.kbravh.dev (new in v2.0.0)~~ 70 | - Sign up for a bearer token through the Twitter Developer dashboard 71 | 72 | ### Free TTM API key (❌ disabled) 73 | 74 | ~~You can sign up for a free API key at https://ttm.kbravh.dev by signing in with either your GitHub or Twitter account and heading to your account page. Once you sign in and retrieve your API key from your account page, either store it in the environment variable `TTM_API_KEY` or pass it to the command line tool with the `-b` (`--bearer`) flag with each call.~~ 75 | 76 | ### Twitter Developer bearer token 77 | 78 | Nota bene: You need at least a **Basic** plan in order to look up tweets. The **Free** plan is not sufficient. 79 | 80 | To get a bearer token from Twitter, you'll need to set up an application on the [Twitter developer dashboard](https://developer.twitter.com/en/portal/dashboard). For a guide on doing so, see [Getting a bearer token](https://github.com/kbravh/obsidian-tweet-to-markdown/blob/main/BearerTokenGuide.md). Once you have the bearer token, either store it in the environment variable `TWITTER_BEARER_TOKEN` or pass it to the command line tool with the `-b` (`--bearer`) flag with each call. 81 | 82 | ## Usage 83 | 84 | Grabbing a tweet is as easy as calling the `ttm` command and passing in the tweet URL. 85 | 86 | ```bash 87 | ttm -b "" https://twitter.com/JoshWComeau/status/1213870628895428611 88 | # Tweet saved as JoshWComeau - 1213870628895428611.md 89 | ``` 90 | 91 | Nota bene: If passing your bearer token as an argument instead of an environment variable, wrap your bearer token or API key in quotes to prevent any special symbols from being misinterpreted by the command line. Also, if using a TTM API key, be sure to include the entire key, from the `TTM` at the beginning all the way to the end with the alphanumeric characters (e.g. `"TTM>asdf1123"` or `"TTM_ghjk4567"`). 92 | 93 | The tweet will be saved to a Markdown file in the current directory. Here's how the tweet will look: 94 | 95 | ![Screenshot of the rendered Markdown file](images/tweet-markdown-screenshot.png) 96 | 97 | Any attached images, polls, and links will also be linked and displayed in the file. 98 | 99 | ## Options 100 | 101 | There are _many_ options to customize how this tool works. It is highly recommended to find the options you need and set up an alias in your terminal. 102 | 103 | For example on Mac or Linux, you can define an alias with your options like so: 104 | 105 | ```bash 106 | alias ttm="ttm -p $HOME/notes --assets --quoted" 107 | ``` 108 | 109 | For Windows, have a look at [DOSKEY](https://superuser.com/a/560558). 110 | 111 | ### Copy to Clipboard 112 | 113 | What if you want to just copy the Markdown to the clipboard instead of saving to a file? Just pass the `-c` (`--clipboard`) flag. 114 | 115 | ```bash 116 | ttm -c https://twitter.com/JoshWComeau/status/1213870628895428611 117 | #Tweet copied to the clipboard. 118 | ``` 119 | 120 | ### Quoted tweets 121 | 122 | If you would like to include quoted tweets, pass the `-q` (`--quoted`) flag. This is disabled by default because a separate request has to be made to fetch the quoted tweet. 123 | 124 | ```bash 125 | ttm -q 126 | ``` 127 | 128 | ### Tweet threads 129 | 130 | To capture an entire tweet thread, use the `-t` (`--thread`) flag and pass the URL of the **last** tweet in the thread. 131 | 132 | Nota bene: this will make a separate network request for each tweet. 133 | 134 | ```bash 135 | ttm -t 136 | ``` 137 | 138 | #### Condensed threads 139 | 140 | Instead of showing complete, individual tweets with profile picture, date, etc. when downloading a thread, this option will show the header once and then only show the tweet bodies, representing tweet threads as a cohesive body of text. A header will be shown if a different author appears in the thread, for example if you're downloading a conversation between various authors. 141 | 142 | ```bash 143 | ttm -T 144 | ``` 145 | 146 | #### Semicondensed threads 147 | 148 | These follow the same rules as condensed threads, but each tweet will still be separated by `---`. 149 | 150 | ```bash 151 | ttm -s 152 | ``` 153 | 154 | ### Text only 155 | 156 | With this flag, only the text of the tweet itself will be included. No author tags, frontmatter, or other information will be attached. 157 | 158 | Nota bene: This has not been tested well with threads. Please use at your own risk. Condensed threads may be a better fit for you. 159 | 160 | ### Custom File Name 161 | 162 | In order to save the tweet with a custom filename, pass the desired name to the `--filename` flag. You can use the variables `[[name]]`, `[[handle]]`, `[[text]]`, and `[[id]]` in your filename, which will be replaced according to the following chart. The file extension `.md` will also be added automatically. 163 | 164 | | Variable | Replacement | 165 | | :--------: | ------------------------------------------------------------------------------- | --- | 166 | | [[handle]] | The user's handle (the part that follows the @ symbol) | 167 | | [[name]] | The user's name | 168 | | [[id]] | The unique ID assigned to the tweet | 169 | | [[text]] | The entire text of the tweet (truncated to fit OS filename length restrictions) | . | 170 | 171 | ```bash 172 | ttm --filename "[[handle]] - Favicon versioning". 173 | # Tweet saved to JoshWComeau - Favicon versioning.md 174 | ``` 175 | 176 | If the file already exists, an error will be thrown unless you pass the `-f` (`--force`) flag to overwrite the file. 177 | 178 | ### Custom File Path 179 | 180 | To save the tweet to a place other than the current directory, pass the location to the `-p` (`--path`) flag. If this path doesn't exist, it will be recursively created. 181 | 182 | ```bash 183 | ttm -p "./tweets/" 184 | # Tweet saved to ./tweets/JoshWComeau - 1213870628895428611.md 185 | ``` 186 | 187 | ### Tweet Metrics 188 | 189 | If you'd also like to record the number of likes, retweets, and replies the tweet has, pass the `-m` (`--metrics`) flag. This will save those numbers in the frontmatter of the file. 190 | 191 | ```bash 192 | ttm -m 193 | ``` 194 | 195 | ```yaml 196 | --- 197 | author: Josh ✨ 198 | handle: @JoshWComeau 199 | likes: 993 200 | retweets: 163 201 | replies: 24 202 | --- 203 | ``` 204 | 205 | ### Save Images Locally 206 | 207 | Want to really capture the entire tweet locally? You can pass the `-a` (`--assets`) flag to download all the tweet images as well, instead of just linking to the images on the web. If the tweet is ever deleted or Twitter is unavailable, you'll still have your note. 208 | 209 | ```bash 210 | ttm -a 211 | ``` 212 | 213 | Tweet images will be automatically saved to `./tweet-assets`. If you'd like to save the assets to a custom directory, use the `--assets-path` flag and pass in the directory. 214 | 215 | ```bash 216 | ttm -a --assets-path "./images" 217 | ``` 218 | 219 | Nota bene: Unfortunately, there is currently not a way to save gifs or videos from tweets using the v2 API. 220 | 221 | ### Date locale 222 | 223 | Using the `--date_locale` option, you can pass a BCP 47 language code to have the date displayed in a specific locale. If you don't pass this option (or if you're using lower than Node 13), your computer's default locale will be used. 224 | 225 | Finding a complete list of these language codes is difficult, but here's a short list of examples: 226 | | Tag | Language | 227 | | --- | --- | 228 | | en | English | 229 | | en-US | American English | 230 | | es-MX | Mexican Spanish | 231 | | ta-IN | Indian Tamil | 232 | 233 | A more complete list can be found here: https://gist.github.com/typpo/b2b828a35e683b9bf8db91b5404f1bd1 234 | 235 | 236 | 237 | ## Contributing 238 | 239 | Contributions are what make the open source community such an amazing place to be learn, inspire, and create. Any contributions you make are **greatly appreciated**. 240 | 241 | 1. Fork the Project 242 | 2. Create your Feature Branch ( `git checkout -b feature` ) 243 | 3. Commit your Changes ( `git commit -m "Add a cool feature"` ) 244 | 4. Push to the Branch ( `git push origin feature` ) 245 | 5. Open a Pull Request 246 | 247 | ## License 248 | 249 | This project is licensed under the MIT License - see the [ `LICENSE` ](LICENSE) file for details 250 | 251 | 252 | 253 | ## Contact 254 | 255 | Karey Higuera - [@kbravh](https://twitter.com/kbravh) - karey.higuera@gmail.com 256 | 257 | Project Link: [https://github.com/kbravh/tweet-to-markdown](https://github.com/kbravh/tweet-to-markdown) 258 | 259 | 260 | 261 | [issues-shield]: https://img.shields.io/github/issues/kbravh/tweet-to-markdown.svg?style=flat-square 262 | [issues-url]: https://github.com/kbravh/tweet-to-markdown/issues 263 | [license-shield]: https://img.shields.io/github/license/kbravh/tweet-to-markdown.svg?style=flat-square 264 | [license-url]: https://github.com/kbravh/tweet-to-markdown/blob/master/LICENSE 265 | [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=flat-square&logo=linkedin&colorB=555 266 | [linkedin-url]: https://linkedin.com/in/kbravh 267 | [stars-shield]: https://img.shields.io/github/stars/kbravh/tweet-to-markdown.svg?style=flat-square 268 | [stars-url]: https://github.com/kbravh/tweet-to-markdown/stargazers 269 | -------------------------------------------------------------------------------- /__fixtures__/markdown/cjk-html-entities.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: "Karey Higuera 🦈" 3 | handle: "@kbravh" 4 | source: "https://twitter.com/kbravh/status/1566771957646757889" 5 | date: 2022-09-05T12:55:18.000Z 6 | --- 7 | ![kbravh](https://pbs.twimg.com/profile_images/1539402405506334721/1V5Xt64P_normal.jpg) 8 | Karey Higuera 🦈 ([@kbravh](https://twitter.com/kbravh)) - September 5, 2022 at 8:55 AM 9 | 10 | 11 | I've been working through some bugs on Tweet to Markdown regarding text parsing and [#hashtags](https://twitter.com/hashtag/hashtags), HTML entities (&,%, etc.), and CJK characters in hashtags ([#9月5日](https://twitter.com/hashtag/9月5日)). 12 | 13 | I never knew how poorly programming languages handled non-Latin characters 😕 14 | 15 | 16 | [Tweet link](https://twitter.com/kbravh/status/1566771957646757889) 17 | -------------------------------------------------------------------------------- /__fixtures__/markdown/condensed-thread.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: "Geoffrey Litt" 3 | handle: "@geoffreylitt" 4 | source: "https://twitter.com/geoffreylitt/status/1277645969975377923" 5 | date: 2020-06-29T16:51:51.000Z 6 | --- 7 | ![geoffreylitt](https://pbs.twimg.com/profile_images/722626068293763072/4erM-SPN_normal.jpg) 8 | Geoffrey Litt ([@geoffreylitt](https://twitter.com/geoffreylitt)) - June 29, 2020 at 12:51 PM 9 | 10 | 11 | A theory about why tools like Airtable and Notion are so compelling: they provide a much-needed synthesis between the design philosophies of UNIX and Apple. 12 | 13 | Short thread: [pic.twitter.com/YjOLsIGVRD](https://twitter.com/geoffreylitt/status/1277645969975377923/photo/1) 14 | 15 | ![](https://pbs.twimg.com/media/EbsMfE8XkAc9TiK.png) 16 | 17 | ![](https://pbs.twimg.com/media/EbsMiqGX0AAwi9R.jpg) 18 | 19 | UNIX is still the best working example of "tools not apps": small sharp tools that the user can flexibly compose to meet their needs. 20 | 21 | Once you've written a few bash pipelines, it's hard to be satisfied with disconnected, siloed "apps" 22 | 23 | The problem is, while the roots are solid, the terminal as UI is extremely hostile to users, esp beginners. No discoverability, cryptic flags, lots of cruft and chaos. 24 | 25 | [twitter.com/geoffreylitt/s…](https://twitter.com/geoffreylitt/status/1187357294415302657) [pic.twitter.com/TjOL7PXU2y](https://twitter.com/geoffreylitt/status/1277645972529647616/photo/1) 26 | 27 | ![](https://pbs.twimg.com/media/EbsNx5_XkAEncJW.png) 28 | 29 | 30 | 31 | [Thread link](https://twitter.com/geoffreylitt/status/1277645969975377923) 32 | -------------------------------------------------------------------------------- /__fixtures__/markdown/image-tweet.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: "Maggie Appleton 🧭" 3 | handle: "@Mappletons" 4 | source: "https://twitter.com/Mappletons/status/1292845757297557505" 5 | date: 2020-08-10T15:30:23.000Z 6 | --- 7 | ![Mappletons](https://pbs.twimg.com/profile_images/1079304561892966406/1AHsGSnz_normal.jpg) 8 | Maggie Appleton 🧭 ([@Mappletons](https://twitter.com/Mappletons)) - August 10, 2020 at 11:30 AM 9 | 10 | 11 | "Dirt is matter out of place" - the loveliest definition of dirt you could hope for from anthropologist Mary Douglas in her classic 1966 book Purity and Danger 12 | 13 | Hair on my head? Clean. Hair on the table? Dirty! 14 | 15 | Illustrating & expanding on her main ideas: [maggieappleton.com/dirt](http://maggieappleton.com/dirt) [pic.twitter.com/PSk7lHiv7z](https://twitter.com/Mappletons/status/1292845757297557505/photo/1) 16 | 17 | ![](https://pbs.twimg.com/media/EfEcPs8XoAIXwvH.jpg) 18 | 19 | ![](https://pbs.twimg.com/media/EfEcQ5HX0AA2EvY.jpg) 20 | 21 | 22 | [Tweet link](https://twitter.com/Mappletons/status/1292845757297557505) 23 | -------------------------------------------------------------------------------- /__fixtures__/markdown/multiple-mentions.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: "Edil Medeiros 🚢 ✏️🏴‍☠️ 🧩" 3 | handle: "@jose_edil" 4 | source: "https://twitter.com/jose_edil/status/1538271708918034433" 5 | date: 2022-06-18T21:25:29.000Z 6 | --- 7 | ![jose_edil](https://pbs.twimg.com/profile_images/669315274353561600/eHheoab4_normal.jpg) 8 | Edil Medeiros 🚢 ✏️🏴‍☠️ 🧩 ([@jose_edil](https://twitter.com/jose_edil)) - June 18, 2022 at 5:25 PM 9 | 10 | 11 | [@visualizevalue](https://twitter.com/visualizevalue) [@EvansNifty](https://twitter.com/EvansNifty) [@OzolinsJanis](https://twitter.com/OzolinsJanis) [@milanicreative](https://twitter.com/milanicreative) [@design_by_kp](https://twitter.com/design_by_kp) [@victor_bigfield](https://twitter.com/victor_bigfield) [@StartupIllustr](https://twitter.com/StartupIllustr) [@tracytangtt](https://twitter.com/tracytangtt) [@AlexMaeseJ](https://twitter.com/AlexMaeseJ) [@ash_lmb](https://twitter.com/ash_lmb) [@moina_abdul](https://twitter.com/moina_abdul) @Its_Prasa [@elliottaleksndr](https://twitter.com/elliottaleksndr) [@aaraalto](https://twitter.com/aaraalto) [@tanoseihito](https://twitter.com/tanoseihito) [@jeffkortenbosch](https://twitter.com/jeffkortenbosch) [@FerraroRoberto](https://twitter.com/FerraroRoberto) [@eneskartall](https://twitter.com/eneskartall) [@SachinRamje](https://twitter.com/SachinRamje) [@AidanYeep](https://twitter.com/AidanYeep) [@jozzua](https://twitter.com/jozzua) Here they are: 12 | 13 | [@EvansNifty](https://twitter.com/EvansNifty) 14 | [@OzolinsJanis](https://twitter.com/OzolinsJanis) 15 | [@milanicreative](https://twitter.com/milanicreative) 16 | [@design_by_kp](https://twitter.com/design_by_kp) 17 | [@victor_bigfield](https://twitter.com/victor_bigfield) 18 | [@StartupIllustr](https://twitter.com/StartupIllustr) 19 | [@tracytangtt](https://twitter.com/tracytangtt) 20 | [@AlexMaeseJ](https://twitter.com/AlexMaeseJ) 21 | [@ash_lmb](https://twitter.com/ash_lmb) 22 | [@moina_abdul](https://twitter.com/moina_abdul) 23 | @Its_Prasa 24 | [@elliottaleksndr](https://twitter.com/elliottaleksndr) 25 | [@aaraalto](https://twitter.com/aaraalto) 26 | [@tanoseihito](https://twitter.com/tanoseihito) 27 | [@jeffkortenbosch](https://twitter.com/jeffkortenbosch) 28 | [@FerraroRoberto](https://twitter.com/FerraroRoberto) 29 | [@eneskartall](https://twitter.com/eneskartall) 30 | [@SachinRamje](https://twitter.com/SachinRamje) 31 | [@AidanYeep](https://twitter.com/AidanYeep) 32 | [@jozzua](https://twitter.com/jozzua) 33 | 34 | 35 | [Tweet link](https://twitter.com/jose_edil/status/1538271708918034433) 36 | -------------------------------------------------------------------------------- /__fixtures__/markdown/non-mangled-urls.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: "Tauri" 3 | handle: "@TauriApps" 4 | source: "https://twitter.com/TauriApps/status/1537347873565773824" 5 | date: 2022-06-16T08:14:30.000Z 6 | --- 7 | ![TauriApps](https://pbs.twimg.com/profile_images/1427375984475578389/jWzgho1b_normal.png) 8 | Tauri ([@TauriApps](https://twitter.com/TauriApps)) - June 16, 2022 at 4:14 AM 9 | 10 | 11 | After 4 months of release candidates we're proud to release version 1.0 of Tauri! 🎉 Windows, Menus, System Trays, Auto Updater and much more are now at your fingertips! 12 | 13 | Check it out! ✨ 14 | [tauri.app](https://tauri.app/) 15 | 16 | 17 | [Tweet link](https://twitter.com/TauriApps/status/1537347873565773824) 18 | -------------------------------------------------------------------------------- /__fixtures__/markdown/partial-thread.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: "Rasmus Andersson" 3 | handle: "@rsms" 4 | source: "https://twitter.com/rsms/status/1543295680088662016" 5 | date: 2022-07-02T18:08:57.000Z 6 | --- 7 | ![rsms](https://pbs.twimg.com/profile_images/1432033002880520193/nuDFioj3_normal.jpg) 8 | Rasmus Andersson ([@rsms](https://twitter.com/rsms)) - July 2, 2022 at 2:08 PM 9 | 10 | 11 | [@nevyn](https://twitter.com/nevyn) Flash was awesome in so many ways! Being proprietary closed-source software was not awesome though. 12 | 13 | 14 | [Tweet link](https://twitter.com/rsms/status/1543295680088662016) 15 | -------------------------------------------------------------------------------- /__fixtures__/markdown/textonly.md: -------------------------------------------------------------------------------- 1 | After 4 months of release candidates we're proud to release version 1.0 of Tauri! 🎉 Windows, Menus, System Trays, Auto Updater and much more are now at your fingertips! 2 | 3 | Check it out! ✨ 4 | [tauri.app](https://tauri.app/) 5 | -------------------------------------------------------------------------------- /__fixtures__/markdown/tweet-thread.md: -------------------------------------------------------------------------------- 1 | --- 2 | author: "Geoffrey Litt" 3 | handle: "@geoffreylitt" 4 | source: "https://twitter.com/geoffreylitt/status/1277645969975377923" 5 | date: 2020-06-29T16:51:51.000Z 6 | --- 7 | ![geoffreylitt](https://pbs.twimg.com/profile_images/722626068293763072/4erM-SPN_normal.jpg) 8 | Geoffrey Litt ([@geoffreylitt](https://twitter.com/geoffreylitt)) - June 29, 2020 at 12:51 PM 9 | 10 | 11 | A theory about why tools like Airtable and Notion are so compelling: they provide a much-needed synthesis between the design philosophies of UNIX and Apple. 12 | 13 | Short thread: [pic.twitter.com/YjOLsIGVRD](https://twitter.com/geoffreylitt/status/1277645969975377923/photo/1) 14 | 15 | ![](https://pbs.twimg.com/media/EbsMfE8XkAc9TiK.png) 16 | 17 | ![](https://pbs.twimg.com/media/EbsMiqGX0AAwi9R.jpg) 18 | 19 | 20 | [Tweet link](https://twitter.com/geoffreylitt/status/1277645969975377923) 21 | 22 | --- 23 | 24 | ![geoffreylitt](https://pbs.twimg.com/profile_images/722626068293763072/4erM-SPN_normal.jpg) 25 | Geoffrey Litt ([@geoffreylitt](https://twitter.com/geoffreylitt)) - June 29, 2020 at 12:51 PM 26 | 27 | 28 | UNIX is still the best working example of "tools not apps": small sharp tools that the user can flexibly compose to meet their needs. 29 | 30 | Once you've written a few bash pipelines, it's hard to be satisfied with disconnected, siloed "apps" 31 | 32 | 33 | [Tweet link](https://twitter.com/geoffreylitt/status/1277645971401433090) 34 | 35 | --- 36 | 37 | ![geoffreylitt](https://pbs.twimg.com/profile_images/722626068293763072/4erM-SPN_normal.jpg) 38 | Geoffrey Litt ([@geoffreylitt](https://twitter.com/geoffreylitt)) - June 29, 2020 at 12:51 PM 39 | 40 | 41 | The problem is, while the roots are solid, the terminal as UI is extremely hostile to users, esp beginners. No discoverability, cryptic flags, lots of cruft and chaos. 42 | 43 | [twitter.com/geoffreylitt/s…](https://twitter.com/geoffreylitt/status/1187357294415302657) [pic.twitter.com/TjOL7PXU2y](https://twitter.com/geoffreylitt/status/1277645972529647616/photo/1) 44 | 45 | ![](https://pbs.twimg.com/media/EbsNx5_XkAEncJW.png) 46 | 47 | 48 | [Tweet link](https://twitter.com/geoffreylitt/status/1277645972529647616) 49 | -------------------------------------------------------------------------------- /__fixtures__/tweets/cashtag_tweet.ts: -------------------------------------------------------------------------------- 1 | import type {Tweet} from 'src/models' 2 | 3 | export const cashtagTweet: Tweet = { 4 | data: { 5 | public_metrics: { 6 | retweet_count: 0, 7 | reply_count: 0, 8 | like_count: 1, 9 | quote_count: 0, 10 | }, 11 | created_at: '2020-09-02T16:15:47.000Z', 12 | id: '1301192107143561219', 13 | entities: { 14 | urls: [ 15 | { 16 | start: 102, 17 | end: 125, 18 | url: 'https://t.co/qnyDphmJm2', 19 | expanded_url: 20 | 'https://twitter.com/BTheriot2014/status/1301180406226513921', 21 | display_url: 'twitter.com/BTheriot2014/s…', 22 | }, 23 | ], 24 | hashtags: [ 25 | { 26 | start: 22, 27 | end: 31, 28 | tag: 'cashtags', 29 | }, 30 | { 31 | start: 88, 32 | end: 95, 33 | tag: 'coffee', 34 | }, 35 | ], 36 | cashtags: [ 37 | { 38 | start: 52, 39 | end: 57, 40 | tag: 'SBUX', 41 | }, 42 | ], 43 | }, 44 | conversation_id: '1301192107143561219', 45 | text: 'Today I learned about #cashtags - and found out my $SBUX is in current tweet! Must be #coffee time! https://t.co/qnyDphmJm2', 46 | referenced_tweets: [ 47 | { 48 | type: 'quoted', 49 | id: '1301180406226513921', 50 | }, 51 | ], 52 | author_id: '1058876047465209856', 53 | }, 54 | includes: { 55 | users: [ 56 | { 57 | id: '1058876047465209856', 58 | username: 'Ceascape_ca', 59 | name: 'ceascape.business.solutions', 60 | profile_image_url: 61 | 'https://pbs.twimg.com/profile_images/1058877044015038464/u68hN9LW_normal.jpg', 62 | }, 63 | ], 64 | }, 65 | } 66 | -------------------------------------------------------------------------------- /__fixtures__/tweets/cjk_tweets.ts: -------------------------------------------------------------------------------- 1 | import {Tweet} from 'src/models' 2 | 3 | export const koreanTweet: Tweet = { 4 | data: { 5 | id: '1564281755531722755', 6 | created_at: '2022-08-29T16:00:08.000Z', 7 | text: 'A Minji a day keeps the bad vibes away~\n\nPlaying with our money \n\n#JiU #지유 #Dreamcatcher #드림캐쳐 https://t.co/IRpqINac5X', 8 | attachments: { 9 | media_keys: ['3_1564281751844884482'], 10 | }, 11 | conversation_id: '1564281755531722755', 12 | entities: { 13 | hashtags: [ 14 | { 15 | start: 66, 16 | end: 70, 17 | tag: 'JiU', 18 | }, 19 | { 20 | start: 71, 21 | end: 74, 22 | tag: '지유', 23 | }, 24 | { 25 | start: 75, 26 | end: 88, 27 | tag: 'Dreamcatcher', 28 | }, 29 | { 30 | start: 89, 31 | end: 94, 32 | tag: '드림캐쳐', 33 | }, 34 | ], 35 | urls: [ 36 | { 37 | start: 95, 38 | end: 118, 39 | url: 'https://t.co/IRpqINac5X', 40 | expanded_url: 41 | 'https://twitter.com/PaniclnTheCity/status/1564281755531722755/photo/1', 42 | display_url: 'pic.twitter.com/IRpqINac5X', 43 | media_key: '3_1564281751844884482', 44 | }, 45 | ], 46 | }, 47 | author_id: '2539875322', 48 | public_metrics: { 49 | retweet_count: 160, 50 | reply_count: 4, 51 | like_count: 967, 52 | quote_count: 9, 53 | }, 54 | }, 55 | includes: { 56 | media: [ 57 | { 58 | media_key: '3_1564281751844884482', 59 | type: 'photo', 60 | url: 'https://pbs.twimg.com/media/FbVx9yNXEAIhA9O.jpg', 61 | }, 62 | ], 63 | users: [ 64 | { 65 | profile_image_url: 66 | 'https://pbs.twimg.com/profile_images/1534928249217851396/Mn95uof8_normal.jpg', 67 | name: 'tm ツ', 68 | username: 'PaniclnTheCity', 69 | id: '2539875322', 70 | }, 71 | ], 72 | }, 73 | } 74 | 75 | export const japaneseWithHTMLEntitiesTweet: Tweet = { 76 | data: { 77 | public_metrics: { 78 | retweet_count: 0, 79 | reply_count: 0, 80 | like_count: 0, 81 | quote_count: 0, 82 | }, 83 | id: '1566771957646757889', 84 | author_id: '1143604512999034881', 85 | conversation_id: '1566771957646757889', 86 | text: "I've been working through some bugs on Tweet to Markdown regarding text parsing and #hashtags, HTML entities (&,%, etc.), and CJK characters in hashtags (#9月5日).\n\nI never knew how poorly programming languages handled non-Latin characters 😕", 87 | created_at: '2022-09-05T12:55:18.000Z', 88 | entities: { 89 | hashtags: [ 90 | { 91 | start: 84, 92 | end: 93, 93 | tag: 'hashtags', 94 | }, 95 | { 96 | start: 158, 97 | end: 163, 98 | tag: '9月5日', 99 | }, 100 | ], 101 | }, 102 | }, 103 | includes: { 104 | users: [ 105 | { 106 | profile_image_url: 107 | 'https://pbs.twimg.com/profile_images/1539402405506334721/1V5Xt64P_normal.jpg', 108 | id: '1143604512999034881', 109 | username: 'kbravh', 110 | name: 'Karey Higuera 🦈', 111 | }, 112 | ], 113 | }, 114 | } 115 | -------------------------------------------------------------------------------- /__fixtures__/tweets/emoji_tweets.ts: -------------------------------------------------------------------------------- 1 | import {Tweet} from 'src/models' 2 | 3 | export const emojiTweet: Tweet = { 4 | data: { 5 | conversation_id: '1565545829942845440', 6 | author_id: '1371963673103728642', 7 | text: 'FOLLOW YOUR INTEREST 💯\n\nDrop an emoji and #followback everyone who likes it ✨\n\n 💛 For everyone \n🏒 For #NHL\n🏀 For #NBA\n🏈 For #NFL\n⛏️ For Fortnite\n⚽ For Soccer\n🫰 For #KPOP\n🌊 For #Bluewaves\n🎶 For Music \n🎨 For Art \n📚 For #Anitwt\n📷 For Photography\n\nAdd 🔥 for a ShoutOut', 8 | id: '1565545829942845440', 9 | public_metrics: { 10 | retweet_count: 69, 11 | reply_count: 54, 12 | like_count: 71, 13 | quote_count: 1, 14 | }, 15 | created_at: '2022-09-02T03:43:06.000Z', 16 | entities: { 17 | hashtags: [ 18 | { 19 | start: 42, 20 | end: 53, 21 | tag: 'followback', 22 | }, 23 | { 24 | start: 102, 25 | end: 106, 26 | tag: 'NHL', 27 | }, 28 | { 29 | start: 113, 30 | end: 117, 31 | tag: 'NBA', 32 | }, 33 | { 34 | start: 124, 35 | end: 128, 36 | tag: 'NFL', 37 | }, 38 | { 39 | start: 164, 40 | end: 169, 41 | tag: 'KPOP', 42 | }, 43 | { 44 | start: 176, 45 | end: 186, 46 | tag: 'Bluewaves', 47 | }, 48 | { 49 | start: 217, 50 | end: 224, 51 | tag: 'Anitwt', 52 | }, 53 | ], 54 | }, 55 | }, 56 | includes: { 57 | users: [ 58 | { 59 | username: 'FollowBackAddic', 60 | id: '1371963673103728642', 61 | name: 'FollowBackAddict', 62 | profile_image_url: 63 | 'https://pbs.twimg.com/profile_images/1547021534119723009/X2GTbeAa_normal.jpg', 64 | }, 65 | ], 66 | }, 67 | } 68 | -------------------------------------------------------------------------------- /__fixtures__/tweets/image_tweet.ts: -------------------------------------------------------------------------------- 1 | import type {Tweet} from 'src/models' 2 | 3 | export const imageTweet: Tweet = { 4 | data: { 5 | created_at: '2020-08-10T15:30:23.000Z', 6 | conversation_id: '1292845757297557505', 7 | id: '1292845757297557505', 8 | attachments: { 9 | media_keys: ['3_1292845624120025090', '3_1292845644567269376'], 10 | }, 11 | entities: { 12 | annotations: [ 13 | { 14 | start: 104, 15 | end: 115, 16 | probability: 0.9801, 17 | type: 'Person', 18 | normalized_text: 'Mary Douglas', 19 | }, 20 | ], 21 | urls: [ 22 | { 23 | start: 260, 24 | end: 283, 25 | url: 'https://t.co/O2P7WRO1XL', 26 | expanded_url: 'http://maggieappleton.com/dirt', 27 | display_url: 'maggieappleton.com/dirt', 28 | }, 29 | { 30 | start: 284, 31 | end: 307, 32 | url: 'https://t.co/PSk7lHiv7z', 33 | expanded_url: 34 | 'https://twitter.com/Mappletons/status/1292845757297557505/photo/1', 35 | display_url: 'pic.twitter.com/PSk7lHiv7z', 36 | media_key: '3_1292845624120025090', 37 | }, 38 | { 39 | start: 284, 40 | end: 307, 41 | url: 'https://t.co/PSk7lHiv7z', 42 | expanded_url: 43 | 'https://twitter.com/Mappletons/status/1292845757297557505/photo/1', 44 | display_url: 'pic.twitter.com/PSk7lHiv7z', 45 | media_key: '3_1292845644567269376', 46 | }, 47 | ], 48 | }, 49 | author_id: '1343443016', 50 | text: '"Dirt is matter out of place" - the loveliest definition of dirt you could hope for from anthropologist Mary Douglas in her classic 1966 book Purity and Danger\n\nHair on my head? Clean. Hair on the table? Dirty!\n\nIllustrating & expanding on her main ideas: https://t.co/O2P7WRO1XL https://t.co/PSk7lHiv7z', 51 | public_metrics: { 52 | retweet_count: 29, 53 | reply_count: 11, 54 | like_count: 191, 55 | quote_count: 2, 56 | }, 57 | }, 58 | includes: { 59 | media: [ 60 | { 61 | media_key: '3_1292845624120025090', 62 | type: 'photo', 63 | url: 'https://pbs.twimg.com/media/EfEcPs8XoAIXwvH.jpg', 64 | }, 65 | { 66 | media_key: '3_1292845644567269376', 67 | type: 'photo', 68 | url: 'https://pbs.twimg.com/media/EfEcQ5HX0AA2EvY.jpg', 69 | }, 70 | ], 71 | users: [ 72 | { 73 | username: 'Mappletons', 74 | id: '1343443016', 75 | name: 'Maggie Appleton 🧭', 76 | profile_image_url: 77 | 'https://pbs.twimg.com/profile_images/1079304561892966406/1AHsGSnz_normal.jpg', 78 | }, 79 | ], 80 | }, 81 | } 82 | 83 | export const imageTweetWithAnnotations: Tweet = { 84 | data: { 85 | author_id: '15668072', 86 | text: 'This variation on the smashburger with white onions smashed right into the patty is *solid*. @jhooks puttin’ that Smashula to work. 😍🍔 https://t.co/5xdUezbxfd', 87 | entities: { 88 | urls: [ 89 | { 90 | start: 135, 91 | end: 158, 92 | url: 'https://t.co/5xdUezbxfd', 93 | expanded_url: 94 | 'https://twitter.com/jlengstorf/status/1556457532037554176/photo/1', 95 | display_url: 'pic.twitter.com/5xdUezbxfd', 96 | media_key: '3_1556457395634577409', 97 | }, 98 | { 99 | start: 135, 100 | end: 158, 101 | url: 'https://t.co/5xdUezbxfd', 102 | expanded_url: 103 | 'https://twitter.com/jlengstorf/status/1556457532037554176/photo/1', 104 | display_url: 'pic.twitter.com/5xdUezbxfd', 105 | media_key: '3_1556457395638792192', 106 | }, 107 | { 108 | start: 135, 109 | end: 158, 110 | url: 'https://t.co/5xdUezbxfd', 111 | expanded_url: 112 | 'https://twitter.com/jlengstorf/status/1556457532037554176/photo/1', 113 | display_url: 'pic.twitter.com/5xdUezbxfd', 114 | media_key: '3_1556457395714269184', 115 | }, 116 | ], 117 | mentions: [ 118 | { 119 | start: 93, 120 | end: 100, 121 | username: 'jhooks', 122 | id: '12087242', 123 | }, 124 | ], 125 | }, 126 | attachments: { 127 | media_keys: [ 128 | '3_1556457395634577409', 129 | '3_1556457395638792192', 130 | '3_1556457395714269184', 131 | ], 132 | }, 133 | conversation_id: '1556457532037554176', 134 | id: '1556457532037554176', 135 | public_metrics: { 136 | retweet_count: 0, 137 | reply_count: 4, 138 | like_count: 13, 139 | quote_count: 0, 140 | }, 141 | created_at: '2022-08-08T01:49:27.000Z', 142 | }, 143 | includes: { 144 | media: [ 145 | { 146 | alt_text: 'Joel smashing burgers.', 147 | media_key: '3_1556457395634577409', 148 | type: 'photo', 149 | url: 'https://pbs.twimg.com/media/FZmlwT7UIAEql9P.jpg', 150 | }, 151 | { 152 | alt_text: 'Smash burgers with white onions smashed into the top side.', 153 | media_key: '3_1556457395638792192', 154 | type: 'photo', 155 | url: 'https://pbs.twimg.com/media/FZmlwT8UcAAotNc.jpg', 156 | }, 157 | { 158 | alt_text: 159 | 'Smash burgers in the process of being flipped. There are onions on the griddle side of the patties and a serious crust on the top.', 160 | media_key: '3_1556457395714269184', 161 | type: 'photo', 162 | url: 'https://pbs.twimg.com/media/FZmlwUOUIAAl41A.jpg', 163 | }, 164 | ], 165 | users: [ 166 | { 167 | username: 'jlengstorf', 168 | name: 'Jason Lengstorf', 169 | profile_image_url: 170 | 'https://pbs.twimg.com/profile_images/1524064394157576193/tB5HL_ES_normal.jpg', 171 | id: '15668072', 172 | }, 173 | ], 174 | }, 175 | } 176 | export const imageTweetWithAnnotationsAndNewlines: Tweet = { 177 | data: { 178 | id: '1555175849569263618', 179 | text: 'Which of the following function argument styles do you prefer?\n\nMultiple arguments (extra values destructured from 3rd arg)? \n\nOr single argument (all values destructured)?\n\nOr something else? https://t.co/cfRvBKXafI', 180 | attachments: { 181 | media_keys: ['3_1555174879342825479'], 182 | }, 183 | public_metrics: { 184 | retweet_count: 2, 185 | reply_count: 76, 186 | like_count: 106, 187 | quote_count: 1, 188 | }, 189 | entities: { 190 | urls: [ 191 | { 192 | start: 193, 193 | end: 216, 194 | url: 'https://t.co/cfRvBKXafI', 195 | expanded_url: 196 | 'https://twitter.com/DavidKPiano/status/1555175849569263618/photo/1', 197 | display_url: 'pic.twitter.com/cfRvBKXafI', 198 | media_key: '3_1555174879342825479', 199 | }, 200 | ], 201 | }, 202 | created_at: '2022-08-04T12:56:30.000Z', 203 | conversation_id: '1555175849569263618', 204 | author_id: '992126114', 205 | }, 206 | includes: { 207 | media: [ 208 | { 209 | alt_text: 210 | ' // Multiple arguments\n entry: (context, event, { action }) => {/* ... */},\n\n // Unused arguments\n entry: (_context, _event, { action }) => {/* ... */},\n\n \n // Single argument (destructuring)\n entry: ({ context, event, action }) => {/* ... */},\n\n // Unused properties\n entry: ({ action }) => {/* ... */}', 211 | media_key: '3_1555174879342825479', 212 | type: 'photo', 213 | url: 'https://pbs.twimg.com/media/FZUXUCbXEAcAj_L.jpg', 214 | }, 215 | ], 216 | users: [ 217 | { 218 | profile_image_url: 219 | 'https://pbs.twimg.com/profile_images/619677584805208064/RwwbnNpi_normal.jpg', 220 | name: 'David K. 🎹', 221 | id: '992126114', 222 | username: 'DavidKPiano', 223 | }, 224 | ], 225 | }, 226 | } 227 | -------------------------------------------------------------------------------- /__fixtures__/tweets/index.ts: -------------------------------------------------------------------------------- 1 | export * from './cashtag_tweet' 2 | export * from './cjk_tweets' 3 | export * from './emoji_tweets' 4 | export * from './image_tweet' 5 | export * from './mentions_tweet' 6 | export * from './poll_tweet' 7 | export * from './profile_pic_tweets' 8 | export * from './tweet_thread' 9 | export * from './url_tweet' 10 | 11 | import {Tweet} from 'src/models' 12 | import {cashtagTweet} from './cashtag_tweet' 13 | import {japaneseWithHTMLEntitiesTweet, koreanTweet} from './cjk_tweets' 14 | import { 15 | imageTweet, 16 | imageTweetWithAnnotations, 17 | imageTweetWithAnnotationsAndNewlines, 18 | } from './image_tweet' 19 | import {mentionsTweet, multipleMentionsTweet} from './mentions_tweet' 20 | import {pollTweet} from './poll_tweet' 21 | import {oldProfileTweet, newProfileTweet} from './profile_pic_tweets' 22 | import {tweetThread, tweetWithMissingParent} from './tweet_thread' 23 | import {singleUrlTweet} from './url_tweet' 24 | export const tweets: Record = { 25 | [cashtagTweet.data.id]: cashtagTweet, 26 | [koreanTweet.data.id]: koreanTweet, 27 | [imageTweet.data.id]: imageTweet, 28 | [imageTweetWithAnnotations.data.id]: imageTweetWithAnnotations, 29 | [imageTweetWithAnnotationsAndNewlines.data.id]: 30 | imageTweetWithAnnotationsAndNewlines, 31 | [japaneseWithHTMLEntitiesTweet.data.id]: japaneseWithHTMLEntitiesTweet, 32 | [mentionsTweet.data.id]: mentionsTweet, 33 | [multipleMentionsTweet.data.id]: multipleMentionsTweet, 34 | [pollTweet.data.id]: pollTweet, 35 | [oldProfileTweet.data.id]: oldProfileTweet, 36 | [newProfileTweet.data.id]: newProfileTweet, 37 | ...tweetThread.reduce( 38 | (tweets, tweet) => ({ 39 | ...tweets, 40 | [tweet.data.id]: tweet, 41 | }), 42 | {} 43 | ), 44 | [singleUrlTweet.data.id]: singleUrlTweet, 45 | [tweetWithMissingParent.data.id]: tweetWithMissingParent, 46 | } 47 | -------------------------------------------------------------------------------- /__fixtures__/tweets/mentions_tweet.ts: -------------------------------------------------------------------------------- 1 | import type {Tweet} from 'src/models' 2 | 3 | export const mentionsTweet: Tweet = { 4 | data: { 5 | text: "I've just created a Node.js CLI tool to save tweets as Markdown, great for @NotionHQ, @RoamResearch, @obsdmd, and other Markdown based note-taking systems! https://t.co/9qzNhz5cmN", 6 | public_metrics: { 7 | retweet_count: 0, 8 | reply_count: 0, 9 | like_count: 0, 10 | quote_count: 0, 11 | }, 12 | created_at: '2020-09-09T17:55:42.000Z', 13 | entities: { 14 | urls: [ 15 | { 16 | start: 156, 17 | end: 179, 18 | url: 'https://t.co/9qzNhz5cmN', 19 | expanded_url: 'https://github.com/kbravh/tweet-to-markdown', 20 | display_url: 'github.com/kbravh/tweet-t…', 21 | images: [ 22 | { 23 | url: 'https://pbs.twimg.com/news_img/1501819606129950722/opZrrpCT?format=jpg&name=orig', 24 | width: 1280, 25 | height: 640, 26 | }, 27 | { 28 | url: 'https://pbs.twimg.com/news_img/1501819606129950722/opZrrpCT?format=jpg&name=150x150', 29 | width: 150, 30 | height: 150, 31 | }, 32 | ], 33 | status: 200, 34 | title: 35 | 'GitHub - kbravh/tweet-to-markdown: A command line tool to convert Tweets to Markdown.', 36 | description: 37 | 'A command line tool to convert Tweets to Markdown. - GitHub - kbravh/tweet-to-markdown: A command line tool to convert Tweets to Markdown.', 38 | unwound_url: 'https://github.com/kbravh/tweet-to-markdown', 39 | }, 40 | ], 41 | mentions: [ 42 | { 43 | start: 75, 44 | end: 84, 45 | username: 'NotionHQ', 46 | id: '708915428454576128', 47 | }, 48 | { 49 | start: 86, 50 | end: 99, 51 | username: 'RoamResearch', 52 | id: '1190410678273626113', 53 | }, 54 | { 55 | start: 101, 56 | end: 108, 57 | username: 'obsdmd', 58 | id: '1239876481951596545', 59 | }, 60 | ], 61 | }, 62 | id: '1303753964291338240', 63 | author_id: '1143604512999034881', 64 | conversation_id: '1303753964291338240', 65 | }, 66 | includes: { 67 | users: [ 68 | { 69 | username: 'kbravh', 70 | name: 'Karey Higuera 🦈', 71 | id: '1143604512999034881', 72 | profile_image_url: 73 | 'https://pbs.twimg.com/profile_images/1163169960505610240/R8BoDqiT_normal.jpg', 74 | }, 75 | ], 76 | }, 77 | } 78 | 79 | export const multipleMentionsTweet: Tweet = { 80 | data: { 81 | created_at: '2022-06-18T21:25:29.000Z', 82 | referenced_tweets: [ 83 | { 84 | type: 'replied_to', 85 | id: '1538271706682384385', 86 | }, 87 | ], 88 | conversation_id: '1538271613770158082', 89 | public_metrics: { 90 | retweet_count: 0, 91 | reply_count: 4, 92 | like_count: 14, 93 | quote_count: 0, 94 | }, 95 | author_id: '55014825', 96 | text: '@visualizevalue @EvansNifty @OzolinsJanis @milanicreative @design_by_kp @victor_bigfield @StartupIllustr @tracytangtt @AlexMaeseJ @ash_lmb @moina_abdul @Its_Prasa @elliottaleksndr @aaraalto @tanoseihito @jeffkortenbosch @FerraroRoberto @eneskartall @SachinRamje @AidanYeep @jozzua Here they are:\n\n@EvansNifty\n@OzolinsJanis\n@milanicreative\n@design_by_kp\n@victor_bigfield\n@StartupIllustr\n@tracytangtt\n@AlexMaeseJ\n@ash_lmb\n@moina_abdul\n@Its_Prasa\n@elliottaleksndr\n@aaraalto\n@tanoseihito\n@jeffkortenbosch\n@FerraroRoberto\n@eneskartall\n@SachinRamje\n@AidanYeep\n@jozzua', 97 | id: '1538271708918034433', 98 | entities: { 99 | mentions: [ 100 | { 101 | start: 0, 102 | end: 15, 103 | username: 'visualizevalue', 104 | id: '1086480695428567041', 105 | }, 106 | { 107 | start: 16, 108 | end: 27, 109 | username: 'EvansNifty', 110 | id: '1318516465725788164', 111 | }, 112 | { 113 | start: 28, 114 | end: 41, 115 | username: 'OzolinsJanis', 116 | id: '1326123605709688832', 117 | }, 118 | { 119 | start: 42, 120 | end: 57, 121 | username: 'milanicreative', 122 | id: '1462211068625924100', 123 | }, 124 | { 125 | start: 58, 126 | end: 71, 127 | username: 'design_by_kp', 128 | id: '1444046159782158337', 129 | }, 130 | { 131 | start: 72, 132 | end: 88, 133 | username: 'victor_bigfield', 134 | id: '1353447414624116736', 135 | }, 136 | { 137 | start: 89, 138 | end: 104, 139 | username: 'StartupIllustr', 140 | id: '1334449652549083137', 141 | }, 142 | { 143 | start: 105, 144 | end: 117, 145 | username: 'tracytangtt', 146 | id: '1330784657160294401', 147 | }, 148 | { 149 | start: 118, 150 | end: 129, 151 | username: 'AlexMaeseJ', 152 | id: '1324290451831201792', 153 | }, 154 | { 155 | start: 130, 156 | end: 138, 157 | username: 'ash_lmb', 158 | id: '1311813178003730432', 159 | }, 160 | { 161 | start: 139, 162 | end: 151, 163 | username: 'moina_abdul', 164 | id: '1306429743739236352', 165 | }, 166 | { 167 | start: 163, 168 | end: 179, 169 | username: 'elliottaleksndr', 170 | id: '1266118755005935621', 171 | }, 172 | { 173 | start: 180, 174 | end: 189, 175 | username: 'aaraalto', 176 | id: '856260757033451522', 177 | }, 178 | { 179 | start: 190, 180 | end: 202, 181 | username: 'tanoseihito', 182 | id: '2987250326', 183 | }, 184 | { 185 | start: 203, 186 | end: 219, 187 | username: 'jeffkortenbosch', 188 | id: '974181336', 189 | }, 190 | { 191 | start: 220, 192 | end: 235, 193 | username: 'FerraroRoberto', 194 | id: '349202256', 195 | }, 196 | { 197 | start: 236, 198 | end: 248, 199 | username: 'eneskartall', 200 | id: '96602730', 201 | }, 202 | { 203 | start: 249, 204 | end: 261, 205 | username: 'SachinRamje', 206 | id: '38111262', 207 | }, 208 | { 209 | start: 262, 210 | end: 272, 211 | username: 'AidanYeep', 212 | id: '25479191', 213 | }, 214 | { 215 | start: 273, 216 | end: 280, 217 | username: 'jozzua', 218 | id: '4311721', 219 | }, 220 | { 221 | start: 297, 222 | end: 308, 223 | username: 'EvansNifty', 224 | id: '1318516465725788164', 225 | }, 226 | { 227 | start: 309, 228 | end: 322, 229 | username: 'OzolinsJanis', 230 | id: '1326123605709688832', 231 | }, 232 | { 233 | start: 323, 234 | end: 338, 235 | username: 'milanicreative', 236 | id: '1462211068625924100', 237 | }, 238 | { 239 | start: 339, 240 | end: 352, 241 | username: 'design_by_kp', 242 | id: '1444046159782158337', 243 | }, 244 | { 245 | start: 353, 246 | end: 369, 247 | username: 'victor_bigfield', 248 | id: '1353447414624116736', 249 | }, 250 | { 251 | start: 370, 252 | end: 385, 253 | username: 'StartupIllustr', 254 | id: '1334449652549083137', 255 | }, 256 | { 257 | start: 386, 258 | end: 398, 259 | username: 'tracytangtt', 260 | id: '1330784657160294401', 261 | }, 262 | { 263 | start: 399, 264 | end: 410, 265 | username: 'AlexMaeseJ', 266 | id: '1324290451831201792', 267 | }, 268 | { 269 | start: 411, 270 | end: 419, 271 | username: 'ash_lmb', 272 | id: '1311813178003730432', 273 | }, 274 | { 275 | start: 420, 276 | end: 432, 277 | username: 'moina_abdul', 278 | id: '1306429743739236352', 279 | }, 280 | { 281 | start: 444, 282 | end: 460, 283 | username: 'elliottaleksndr', 284 | id: '1266118755005935621', 285 | }, 286 | { 287 | start: 461, 288 | end: 470, 289 | username: 'aaraalto', 290 | id: '856260757033451522', 291 | }, 292 | { 293 | start: 471, 294 | end: 483, 295 | username: 'tanoseihito', 296 | id: '2987250326', 297 | }, 298 | { 299 | start: 484, 300 | end: 500, 301 | username: 'jeffkortenbosch', 302 | id: '974181336', 303 | }, 304 | { 305 | start: 501, 306 | end: 516, 307 | username: 'FerraroRoberto', 308 | id: '349202256', 309 | }, 310 | { 311 | start: 517, 312 | end: 529, 313 | username: 'eneskartall', 314 | id: '96602730', 315 | }, 316 | { 317 | start: 530, 318 | end: 542, 319 | username: 'SachinRamje', 320 | id: '38111262', 321 | }, 322 | { 323 | start: 543, 324 | end: 553, 325 | username: 'AidanYeep', 326 | id: '25479191', 327 | }, 328 | { 329 | start: 554, 330 | end: 561, 331 | username: 'jozzua', 332 | id: '4311721', 333 | }, 334 | ], 335 | }, 336 | }, 337 | includes: { 338 | users: [ 339 | { 340 | name: 'Edil Medeiros 🚢 ✏️🏴‍☠️ 🧩', 341 | profile_image_url: 342 | 'https://pbs.twimg.com/profile_images/669315274353561600/eHheoab4_normal.jpg', 343 | username: 'jose_edil', 344 | id: '55014825', 345 | }, 346 | ], 347 | }, 348 | } 349 | -------------------------------------------------------------------------------- /__fixtures__/tweets/poll_tweet.ts: -------------------------------------------------------------------------------- 1 | import type {Tweet} from 'src/models' 2 | 3 | export const pollTweet: Tweet = { 4 | data: { 5 | conversation_id: '1029121914260860929', 6 | id: '1029121914260860929', 7 | created_at: '2018-08-13T21:45:59.000Z', 8 | text: 'Which is Better?', 9 | attachments: { 10 | poll_ids: ['1029121913858269191'], 11 | }, 12 | public_metrics: { 13 | retweet_count: 7, 14 | reply_count: 11, 15 | like_count: 47, 16 | quote_count: 2, 17 | }, 18 | author_id: '4071934995', 19 | }, 20 | includes: { 21 | polls: [ 22 | { 23 | id: '1029121913858269191', 24 | options: [ 25 | { 26 | position: 1, 27 | label: 'Spring', 28 | votes: 1373, 29 | }, 30 | { 31 | position: 2, 32 | label: 'Fall', 33 | votes: 3054, 34 | }, 35 | ], 36 | }, 37 | ], 38 | users: [ 39 | { 40 | id: '4071934995', 41 | name: 'polls', 42 | username: 'polls', 43 | profile_image_url: 44 | 'https://pbs.twimg.com/profile_images/660160253913382913/qgvYqknJ_normal.jpg', 45 | }, 46 | ], 47 | }, 48 | } 49 | -------------------------------------------------------------------------------- /__fixtures__/tweets/profile_pic_tweets.ts: -------------------------------------------------------------------------------- 1 | import type {Tweet} from 'src/models' 2 | 3 | export const oldProfileTweet: Tweet = { 4 | data: { 5 | author_id: '1143604512999034881', 6 | text: 'Another valuable reference was Edition 16 of the Jamaica Philatelist magazine from 1942. https://t.co/PDFacV5JfY', 7 | public_metrics: { 8 | retweet_count: 0, 9 | reply_count: 0, 10 | like_count: 1, 11 | quote_count: 0, 12 | }, 13 | referenced_tweets: [ 14 | { 15 | type: 'replied_to', 16 | id: '1511740298682109953', 17 | }, 18 | ], 19 | id: '1511740301118951431', 20 | entities: { 21 | annotations: [ 22 | { 23 | start: 49, 24 | end: 76, 25 | probability: 0.1709, 26 | type: 'Other', 27 | normalized_text: 'Jamaica Philatelist magazine', 28 | }, 29 | ], 30 | urls: [ 31 | { 32 | start: 89, 33 | end: 112, 34 | url: 'https://t.co/PDFacV5JfY', 35 | expanded_url: 'http://jamaicaphilately.info/jamaica-philatelist', 36 | display_url: 'jamaicaphilately.info/jamaica-philat…', 37 | status: 200, 38 | title: 'Jamaica Philatelist – EJP', 39 | unwound_url: 'http://jamaicaphilately.info/jamaica-philatelist', 40 | }, 41 | ], 42 | }, 43 | conversation_id: '1511740254000226316', 44 | created_at: '2022-04-06T16:19:09.000Z', 45 | }, 46 | includes: { 47 | users: [ 48 | { 49 | id: '1143604512999034881', 50 | profile_image_url: 51 | 'https://pbs.twimg.com/profile_images/1163169960505610240/R8BoDqiT_normal.jpg', 52 | name: 'Karey Higuera 🦈', 53 | username: 'kbravh', 54 | }, 55 | ], 56 | }, 57 | } 58 | 59 | export const newProfileTweet: Tweet = { 60 | data: { 61 | author_id: '1143604512999034881', 62 | text: 'Another valuable reference was Edition 16 of the Jamaica Philatelist magazine from 1942. https://t.co/PDFacV5JfY', 63 | public_metrics: { 64 | retweet_count: 0, 65 | reply_count: 0, 66 | like_count: 1, 67 | quote_count: 0, 68 | }, 69 | referenced_tweets: [ 70 | { 71 | type: 'replied_to', 72 | id: '1511740298682109953', 73 | }, 74 | ], 75 | id: '1511740301118951431', 76 | entities: { 77 | annotations: [ 78 | { 79 | start: 49, 80 | end: 76, 81 | probability: 0.1709, 82 | type: 'Other', 83 | normalized_text: 'Jamaica Philatelist magazine', 84 | }, 85 | ], 86 | urls: [ 87 | { 88 | start: 89, 89 | end: 112, 90 | url: 'https://t.co/PDFacV5JfY', 91 | expanded_url: 'http://jamaicaphilately.info/jamaica-philatelist', 92 | display_url: 'jamaicaphilately.info/jamaica-philat…', 93 | status: 200, 94 | title: 'Jamaica Philatelist – EJP', 95 | unwound_url: 'http://jamaicaphilately.info/jamaica-philatelist', 96 | }, 97 | ], 98 | }, 99 | conversation_id: '1511740254000226316', 100 | created_at: '2022-04-06T16:19:09.000Z', 101 | }, 102 | includes: { 103 | users: [ 104 | { 105 | name: 'Karey Higuera 🦈', 106 | profile_image_url: 107 | 'https://pbs.twimg.com/profile_images/1539402405506334721/1V5Xt64P_normal.jpg', 108 | id: '1143604512999034881', 109 | username: 'kbravh', 110 | }, 111 | ], 112 | }, 113 | } 114 | -------------------------------------------------------------------------------- /__fixtures__/tweets/tweet_thread.ts: -------------------------------------------------------------------------------- 1 | import type {Tweet} from 'src/models' 2 | 3 | export const tweetThread: Tweet[] = [ 4 | { 5 | data: { 6 | author_id: '221658618', 7 | conversation_id: '1277645969975377923', 8 | entities: { 9 | annotations: [ 10 | { 11 | start: 150, 12 | end: 154, 13 | probability: 0.5227, 14 | type: 'Product', 15 | normalized_text: 'Apple', 16 | }, 17 | ], 18 | urls: [ 19 | { 20 | start: 172, 21 | end: 195, 22 | url: 'https://t.co/YjOLsIGVRD', 23 | expanded_url: 24 | 'https://twitter.com/geoffreylitt/status/1277645969975377923/photo/1', 25 | display_url: 'pic.twitter.com/YjOLsIGVRD', 26 | media_key: '3_1277628647332089863', 27 | }, 28 | { 29 | start: 172, 30 | end: 195, 31 | url: 'https://t.co/YjOLsIGVRD', 32 | expanded_url: 33 | 'https://twitter.com/geoffreylitt/status/1277645969975377923/photo/1', 34 | display_url: 'pic.twitter.com/YjOLsIGVRD', 35 | media_key: '3_1277628708845768704', 36 | }, 37 | ], 38 | }, 39 | public_metrics: { 40 | retweet_count: 13, 41 | reply_count: 5, 42 | like_count: 119, 43 | quote_count: 1, 44 | }, 45 | id: '1277645969975377923', 46 | text: 'A theory about why tools like Airtable and Notion are so compelling: they provide a much-needed synthesis between the design philosophies of UNIX and Apple.\n\nShort thread: https://t.co/YjOLsIGVRD', 47 | attachments: { 48 | media_keys: ['3_1277628647332089863', '3_1277628708845768704'], 49 | }, 50 | created_at: '2020-06-29T16:51:51.000Z', 51 | }, 52 | includes: { 53 | media: [ 54 | { 55 | media_key: '3_1277628647332089863', 56 | type: 'photo', 57 | url: 'https://pbs.twimg.com/media/EbsMfE8XkAc9TiK.png', 58 | }, 59 | { 60 | media_key: '3_1277628708845768704', 61 | type: 'photo', 62 | url: 'https://pbs.twimg.com/media/EbsMiqGX0AAwi9R.jpg', 63 | }, 64 | ], 65 | users: [ 66 | { 67 | name: 'Geoffrey Litt', 68 | profile_image_url: 69 | 'https://pbs.twimg.com/profile_images/722626068293763072/4erM-SPN_normal.jpg', 70 | id: '221658618', 71 | username: 'geoffreylitt', 72 | }, 73 | ], 74 | }, 75 | }, 76 | { 77 | data: { 78 | public_metrics: { 79 | retweet_count: 2, 80 | reply_count: 1, 81 | like_count: 29, 82 | quote_count: 0, 83 | }, 84 | text: 'UNIX is still the best working example of "tools not apps": small sharp tools that the user can flexibly compose to meet their needs.\n\nOnce you\'ve written a few bash pipelines, it\'s hard to be satisfied with disconnected, siloed "apps"', 85 | conversation_id: '1277645969975377923', 86 | created_at: '2020-06-29T16:51:51.000Z', 87 | entities: { 88 | annotations: [ 89 | { 90 | start: 0, 91 | end: 3, 92 | probability: 0.4213, 93 | type: 'Product', 94 | normalized_text: 'UNIX', 95 | }, 96 | ], 97 | }, 98 | id: '1277645971401433090', 99 | author_id: '221658618', 100 | referenced_tweets: [ 101 | { 102 | type: 'replied_to', 103 | id: '1277645969975377923', 104 | }, 105 | ], 106 | }, 107 | includes: { 108 | users: [ 109 | { 110 | id: '221658618', 111 | profile_image_url: 112 | 'https://pbs.twimg.com/profile_images/722626068293763072/4erM-SPN_normal.jpg', 113 | name: 'Geoffrey Litt', 114 | username: 'geoffreylitt', 115 | }, 116 | ], 117 | }, 118 | }, 119 | { 120 | data: { 121 | attachments: { 122 | media_keys: ['3_1277630070321025025'], 123 | }, 124 | id: '1277645972529647616', 125 | text: 'The problem is, while the roots are solid, the terminal as UI is extremely hostile to users, esp beginners. No discoverability, cryptic flags, lots of cruft and chaos.\n\nhttps://t.co/JOVVRw3iWU https://t.co/TjOL7PXU2y', 126 | conversation_id: '1277645969975377923', 127 | referenced_tweets: [ 128 | { 129 | type: 'quoted', 130 | id: '1187357294415302657', 131 | }, 132 | { 133 | type: 'replied_to', 134 | id: '1277645971401433090', 135 | }, 136 | ], 137 | entities: { 138 | urls: [ 139 | { 140 | start: 169, 141 | end: 192, 142 | url: 'https://t.co/JOVVRw3iWU', 143 | expanded_url: 144 | 'https://twitter.com/geoffreylitt/status/1187357294415302657', 145 | display_url: 'twitter.com/geoffreylitt/s…', 146 | }, 147 | { 148 | start: 193, 149 | end: 216, 150 | url: 'https://t.co/TjOL7PXU2y', 151 | expanded_url: 152 | 'https://twitter.com/geoffreylitt/status/1277645972529647616/photo/1', 153 | display_url: 'pic.twitter.com/TjOL7PXU2y', 154 | media_key: '3_1277630070321025025', 155 | }, 156 | ], 157 | }, 158 | created_at: '2020-06-29T16:51:52.000Z', 159 | author_id: '221658618', 160 | public_metrics: { 161 | retweet_count: 0, 162 | reply_count: 1, 163 | like_count: 19, 164 | quote_count: 0, 165 | }, 166 | }, 167 | includes: { 168 | media: [ 169 | { 170 | media_key: '3_1277630070321025025', 171 | type: 'photo', 172 | url: 'https://pbs.twimg.com/media/EbsNx5_XkAEncJW.png', 173 | }, 174 | ], 175 | users: [ 176 | { 177 | username: 'geoffreylitt', 178 | id: '221658618', 179 | profile_image_url: 180 | 'https://pbs.twimg.com/profile_images/722626068293763072/4erM-SPN_normal.jpg', 181 | name: 'Geoffrey Litt', 182 | }, 183 | ], 184 | }, 185 | }, 186 | ] 187 | 188 | export const tweetWithMissingParent: Tweet = { 189 | data: { 190 | created_at: '2022-07-02T18:08:57.000Z', 191 | text: '@nevyn Flash was awesome in so many ways! Being proprietary closed-source software was not awesome though.', 192 | public_metrics: { 193 | retweet_count: 0, 194 | reply_count: 0, 195 | like_count: 13, 196 | quote_count: 1, 197 | }, 198 | conversation_id: '1543284859572867072', 199 | referenced_tweets: [ 200 | { 201 | type: 'replied_to', 202 | id: '1543286531431059456', 203 | }, 204 | ], 205 | id: '1543295680088662016', 206 | author_id: '2475171', 207 | entities: { 208 | mentions: [ 209 | { 210 | start: 0, 211 | end: 6, 212 | username: 'nevyn', 213 | id: '40593', 214 | }, 215 | ], 216 | }, 217 | }, 218 | includes: { 219 | users: [ 220 | { 221 | profile_image_url: 222 | 'https://pbs.twimg.com/profile_images/1432033002880520193/nuDFioj3_normal.jpg', 223 | id: '2475171', 224 | name: 'Rasmus Andersson', 225 | username: 'rsms', 226 | }, 227 | ], 228 | }, 229 | } 230 | -------------------------------------------------------------------------------- /__fixtures__/tweets/url_tweet.ts: -------------------------------------------------------------------------------- 1 | import {Tweet} from 'src/models' 2 | 3 | export const singleUrlTweet: Tweet = { 4 | data: { 5 | public_metrics: { 6 | retweet_count: 375, 7 | reply_count: 30, 8 | like_count: 1263, 9 | quote_count: 82, 10 | }, 11 | id: '1537347873565773824', 12 | author_id: '1168568340492750849', 13 | conversation_id: '1537347873565773824', 14 | entities: { 15 | urls: [ 16 | { 17 | start: 186, 18 | end: 209, 19 | url: 'https://t.co/NEt3knFTIs', 20 | expanded_url: 'https://tauri.app/', 21 | display_url: 'tauri.app', 22 | images: [ 23 | { 24 | url: 'https://pbs.twimg.com/news_img/1565311456815370241/YEpYALAz?format=jpg&name=orig', 25 | width: 1280, 26 | height: 640, 27 | }, 28 | { 29 | url: 'https://pbs.twimg.com/news_img/1565311456815370241/YEpYALAz?format=jpg&name=150x150', 30 | width: 150, 31 | height: 150, 32 | }, 33 | ], 34 | status: 200, 35 | title: 36 | 'Build smaller, faster, and more secure desktop applications with a web frontend | Tauri Apps', 37 | description: 38 | 'Tauri is a framework for building tiny, blazing fast binaries for all major desktop platforms. Developers can integrate any front-end framework that compiles to HTML, JS and CSS for building their user interface.', 39 | unwound_url: 'https://tauri.app/', 40 | }, 41 | ], 42 | annotations: [ 43 | { 44 | start: 115, 45 | end: 126, 46 | probability: 0.4185, 47 | type: 'Product', 48 | normalized_text: 'Auto Updater', 49 | }, 50 | ], 51 | }, 52 | text: "After 4 months of release candidates we're proud to release version 1.0 of Tauri! 🎉 Windows, Menus, System Trays, Auto Updater and much more are now at your fingertips!\n\nCheck it out! ✨\nhttps://t.co/NEt3knFTIs", 53 | created_at: '2022-06-16T08:14:30.000Z', 54 | }, 55 | includes: { 56 | users: [ 57 | { 58 | profile_image_url: 59 | 'https://pbs.twimg.com/profile_images/1427375984475578389/jWzgho1b_normal.png', 60 | id: '1168568340492750849', 61 | username: 'TauriApps', 62 | name: 'Tauri', 63 | }, 64 | ], 65 | }, 66 | } 67 | -------------------------------------------------------------------------------- /__tests__/consts.ts: -------------------------------------------------------------------------------- 1 | export const BEARER_TOKEN = 'TTM_b34r3r' 2 | -------------------------------------------------------------------------------- /__tests__/main.test.ts: -------------------------------------------------------------------------------- 1 | import {server} from 'src/mocks/server' 2 | import {log, writeTweet} from '../src/util' 3 | import { 4 | afterAll, 5 | afterEach, 6 | beforeAll, 7 | beforeEach, 8 | describe, 9 | expect, 10 | it, 11 | vi, 12 | } from 'vitest' 13 | import {processTweetRequest} from 'src/process' 14 | import { 15 | imageTweet, 16 | multipleMentionsTweet, 17 | singleUrlTweet, 18 | tweetThread, 19 | tweetWithMissingParent, 20 | } from '__fixtures__/tweets' 21 | import {BEARER_TOKEN} from './consts' 22 | import {japaneseWithHTMLEntitiesTweet} from '__fixtures__/tweets/cjk_tweets' 23 | import {readFile} from 'fs/promises' 24 | 25 | vi.mock('../src/util', async () => { 26 | const original = await vi.importActual( 27 | '../src/util' 28 | ) 29 | return { 30 | ...original, 31 | log: vi.fn(), 32 | logErrorToFile: vi.fn(), 33 | writeTweet: vi.fn(), 34 | } 35 | }) 36 | 37 | describe('End to end tests', () => { 38 | beforeEach(() => { 39 | vi.clearAllMocks() 40 | }) 41 | beforeAll(() => server.listen({onUnhandledRequest: 'error'})) 42 | afterEach(() => server.resetHandlers()) 43 | afterAll(() => server.close()) 44 | 45 | it('Downloads a tweet and generates markdown', async () => { 46 | const options = { 47 | src: imageTweet.data.id, 48 | bearer: BEARER_TOKEN, 49 | } 50 | await processTweetRequest(options) 51 | const text = await readFile('__fixtures__/markdown/image-tweet.md', 'utf-8') 52 | expect(writeTweet).toBeCalledWith( 53 | imageTweet, 54 | text.trim(), 55 | options 56 | ) 57 | }) 58 | it('Downloads a tweet thread and generates markdown', async () => { 59 | const options = { 60 | bearer: BEARER_TOKEN, 61 | src: tweetThread[tweetThread.length - 1].data.id, 62 | thread: true, 63 | } 64 | await processTweetRequest(options) 65 | const text = await readFile('__fixtures__/markdown/tweet-thread.md', 'utf-8') 66 | expect(writeTweet).toBeCalledWith( 67 | tweetThread[0], 68 | text.trim(), 69 | options 70 | ) 71 | }) 72 | it('Downloads a condensed tweet thread and generates markdown', async () => { 73 | const options = { 74 | bearer: BEARER_TOKEN, 75 | src: tweetThread[tweetThread.length - 1].data.id, 76 | condensedThread: true, 77 | } 78 | await processTweetRequest(options) 79 | const text = await readFile('__fixtures__/markdown/condensed-thread.md', 'utf-8') 80 | expect(writeTweet).toBeCalledWith( 81 | tweetThread[0], 82 | text.trim(), 83 | options 84 | ) 85 | }) 86 | it('Creates a partial markdown file even if thread download fails', async () => { 87 | const options = { 88 | bearer: BEARER_TOKEN, 89 | src: tweetWithMissingParent.data.id, 90 | thread: true, 91 | } 92 | await processTweetRequest(options) 93 | const text = await readFile('__fixtures__/markdown/partial-thread.md', 'utf-8') 94 | expect(writeTweet).toBeCalledWith( 95 | tweetWithMissingParent, 96 | text.trim(), 97 | options 98 | ) 99 | expect(log).toBeCalled() 100 | }) 101 | 102 | it('Properly replaces multiple occurrences of the same mention', async () => { 103 | const options = { 104 | bearer: BEARER_TOKEN, 105 | src: multipleMentionsTweet.data.id, 106 | } 107 | await processTweetRequest(options) 108 | const text = await readFile('__fixtures__/markdown/multiple-mentions.md', 'utf-8') 109 | expect(writeTweet).toBeCalledWith( 110 | multipleMentionsTweet, 111 | text.trim(), 112 | options 113 | ) 114 | }) 115 | it('Properly renders tweet with CJK and HTML entities', async () => { 116 | const options = { 117 | bearer: BEARER_TOKEN, 118 | src: japaneseWithHTMLEntitiesTweet.data.id, 119 | } 120 | await processTweetRequest(options) 121 | const text = await readFile('__fixtures__/markdown/cjk-html-entities.md', 'utf-8') 122 | expect(writeTweet).toBeCalledWith( 123 | japaneseWithHTMLEntitiesTweet, 124 | text.trim(), 125 | options 126 | ) 127 | }) 128 | it('Does not mangle urls with decoding', async () => { 129 | const options = { 130 | bearer: BEARER_TOKEN, 131 | src: singleUrlTweet.data.id, 132 | } 133 | await processTweetRequest(options) 134 | const text = await readFile('__fixtures__/markdown/non-mangled-urls.md', 'utf-8') 135 | expect(writeTweet).toBeCalledWith( 136 | singleUrlTweet, 137 | text.trim(), 138 | options 139 | ) 140 | }) 141 | it('Provides just the tweet text with textOnly option', async () => { 142 | const options = { 143 | bearer: BEARER_TOKEN, 144 | src: singleUrlTweet.data.id, 145 | textOnly: true, 146 | } 147 | await processTweetRequest(options) 148 | const text = await readFile('__fixtures__/markdown/textonly.md', 'utf-8') 149 | expect(writeTweet).toBeCalledWith(singleUrlTweet, text.trim(), options) 150 | }) 151 | }) 152 | -------------------------------------------------------------------------------- /__tests__/util.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | createFilename, 3 | createMediaElements, 4 | createPollTable, 5 | getLocalAssetPath, 6 | getTweet, 7 | getTweetID, 8 | replaceEntities, 9 | sanitizeFilename, 10 | truncateBytewise, 11 | } from 'src/util' 12 | import {afterAll, afterEach, beforeAll, describe, expect, it} from 'vitest' 13 | import { 14 | imageTweet, 15 | imageTweetWithAnnotations, 16 | pollTweet, 17 | } from '../__fixtures__/tweets' 18 | import {BEARER_TOKEN} from './consts' 19 | import {server} from 'src/mocks/server' 20 | import {platform} from 'os' 21 | import {emojiTweet} from '__fixtures__/tweets/emoji_tweets' 22 | 23 | describe('Tweet download functions', () => { 24 | it('Extracts tweet Id from regular URL', async () => { 25 | expect( 26 | getTweetID({ 27 | src: 'https://twitter.com/whataweekhuh/status/1511656964538916868', 28 | }) 29 | ).toBe('1511656964538916868') 30 | }) 31 | it('Extracts tweet Id from URL with query params', async () => { 32 | expect( 33 | getTweetID({ 34 | src: 'https://twitter.com/whataweekhuh/status/1511656964538916868?s=20&t=tbYKVygf0nKOlvn4CpyKYw', 35 | }) 36 | ).toBe('1511656964538916868') 37 | }) 38 | it('returns the id if not a url', () => { 39 | expect(getTweetID({src: '1556128943861899264'})).toBe('1556128943861899264') 40 | }) 41 | 42 | describe('Downloads tweets', () => { 43 | beforeAll(() => server.listen({onUnhandledRequest: 'error'})) 44 | afterEach(() => server.resetHandlers()) 45 | afterAll(() => server.close()) 46 | it('Downloads tweet from ttm', async () => { 47 | await expect( 48 | getTweet(imageTweet.data.id, BEARER_TOKEN) 49 | ).resolves.toStrictEqual(imageTweet) 50 | }) 51 | }) 52 | }) 53 | 54 | describe('Tweet construction functions', () => { 55 | it('Creates a poll table', () => { 56 | expect(createPollTable(pollTweet.includes.polls)).toStrictEqual([ 57 | ` 58 | |Option|Votes| 59 | |---|:---:| 60 | |Spring|1373| 61 | |Fall|3054|`, 62 | ]) 63 | }) 64 | 65 | it('Creates photo media elements for linked assets', () => { 66 | expect(createMediaElements(imageTweet.includes.media, {})).toStrictEqual([ 67 | '\n![](https://pbs.twimg.com/media/EfEcPs8XoAIXwvH.jpg)', 68 | '\n![](https://pbs.twimg.com/media/EfEcQ5HX0AA2EvY.jpg)', 69 | ]) 70 | }) 71 | it('Creates photo media elements for local assets', () => { 72 | expect( 73 | createMediaElements(imageTweet.includes.media, {assets: true}) 74 | ).toStrictEqual( 75 | platform() === 'win32' 76 | ? [ 77 | '\n![](tweet-assets\\3_1292845624120025090.jpg)', 78 | '\n![](tweet-assets\\3_1292845644567269376.jpg)', 79 | ] 80 | : [ 81 | '\n![](tweet-assets/3_1292845624120025090.jpg)', 82 | '\n![](tweet-assets/3_1292845644567269376.jpg)', 83 | ] 84 | ) 85 | }) 86 | it('Creates photo media elements with alt text', () => { 87 | expect( 88 | createMediaElements(imageTweetWithAnnotations.includes.media, {}) 89 | ).toStrictEqual([ 90 | '\n![Joel smashing burgers.](https://pbs.twimg.com/media/FZmlwT7UIAEql9P.jpg)', 91 | '\n![Smash burgers with white onions smashed into the top side.](https://pbs.twimg.com/media/FZmlwT8UcAAotNc.jpg)', 92 | '\n![Smash burgers in the process of being flipped. There are onions on the griddle side of the patties and a serious crust on the top.](https://pbs.twimg.com/media/FZmlwUOUIAAl41A.jpg)', 93 | ]) 94 | }) 95 | }) 96 | 97 | describe('Filename functions', () => { 98 | it('Truncates a string to a specified byte length', () => { 99 | expect(truncateBytewise('😎 truncate me down', 10)).toBe('😎 trunc') 100 | }) 101 | it('Sanitizes filename', () => { 102 | expect(sanitizeFilename('Illegal:<> Filename*')).toBe('Illegal Filename') 103 | }) 104 | 105 | it('Defaults to "handle - id" if no pattern provided', () => { 106 | expect(createFilename(imageTweet, {})).toBe( 107 | 'Mappletons - 1292845757297557505.md' 108 | ) 109 | }) 110 | it('Sanitizes unsafe filename characters', () => { 111 | expect(createFilename(imageTweet, {filename: '?<>hello:*|"'})).toBe( 112 | 'hello.md' 113 | ) 114 | }) 115 | it('Sanitizes slashes from filename', () => { 116 | expect(createFilename(imageTweet, {filename: 'this/is/a/file'})).toBe( 117 | 'thisisafile.md' 118 | ) 119 | }) 120 | it('Replaces handle, id, and name', () => { 121 | expect( 122 | createFilename(imageTweet, {filename: '[[handle]] - [[id]] - [[name]]'}) 123 | ).toBe('Mappletons - 1292845757297557505 - Maggie Appleton 🧭.md') 124 | }) 125 | it('Does not double extension if .md is present', () => { 126 | expect( 127 | createFilename(imageTweet, { 128 | filename: '[[handle]] - [[id]] - [[name]].md', 129 | }) 130 | ).toBe('Mappletons - 1292845757297557505 - Maggie Appleton 🧭.md') 131 | }) 132 | it('Replaces multiple occurrences of placeholders', () => { 133 | expect(createFilename(imageTweet, {filename: '[[id]] - [[id]]'})).toBe( 134 | '1292845757297557505 - 1292845757297557505.md' 135 | ) 136 | }) 137 | it('Replaces text and truncates', () => { 138 | expect(createFilename(imageTweet, {filename: '[[text]]'})).toBe( 139 | 'Dirt is matter out of place - the loveliest definition of dirt you could hope for from anthropologist Mary Douglas in her classic 1966 book Purity and DangerHair on my head Clean. Hair on the table Dirty!Illustrating & expanding on her main ideas h.md' 140 | ) 141 | }) 142 | }) 143 | 144 | describe('File writing and path creation functions', () => { 145 | it('Returns default asset path', () => { 146 | expect(getLocalAssetPath({})).toBe('tweet-assets') 147 | }) 148 | it('Return custom asset path', () => { 149 | expect(getLocalAssetPath({assetsPath: 'assets'})).toBe('assets') 150 | }) 151 | }) 152 | 153 | describe('Entity replacements', () => { 154 | it('replaces hashtags without infixing substrings', () => { 155 | expect( 156 | replaceEntities( 157 | { 158 | hashtags: [ 159 | {start: 20, end: 23, tag: 'ab'}, 160 | {start: 39, end: 43, tag: 'abc'}, 161 | ], 162 | }, 163 | "I'm a tweet with an #ab hashtag and an #abc hashtag." 164 | ) 165 | ).toBe( 166 | "I'm a tweet with an [#ab](https://twitter.com/hashtag/ab) hashtag and an [#abc](https://twitter.com/hashtag/abc) hashtag." 167 | ) 168 | }) 169 | it('replaces cashtags without infixing substrings', () => { 170 | expect( 171 | replaceEntities( 172 | { 173 | cashtags: [ 174 | {start: 20, end: 23, tag: 'ab'}, 175 | {start: 39, end: 43, tag: 'abc'}, 176 | ], 177 | }, 178 | "I'm a tweet with an $ab cashtag and an $abc cashtag." 179 | ) 180 | ).toBe( 181 | "I'm a tweet with an [$ab](https://twitter.com/search?q=%24ab) cashtag and an [$abc](https://twitter.com/search?q=%24abc) cashtag." 182 | ) 183 | }) 184 | it('replaces mentions without infixing substrings', () => { 185 | expect( 186 | replaceEntities( 187 | { 188 | mentions: [ 189 | {start: 20, end: 23, username: 'ab'}, 190 | {start: 39, end: 43, username: 'abc'}, 191 | ], 192 | }, 193 | "I'm a tweet with an @ab mention and an @abc mention." 194 | ) 195 | ).toBe( 196 | "I'm a tweet with an [@ab](https://twitter.com/ab) mention and an [@abc](https://twitter.com/abc) mention." 197 | ) 198 | }) 199 | it('Correctly replaces entities with CJK characters', () => { 200 | expect( 201 | replaceEntities( 202 | { 203 | hashtags: [ 204 | {start: 66, end: 70, tag: 'JiU'}, 205 | {start: 71, end: 74, tag: '지유'}, 206 | {start: 75, end: 88, tag: 'Dreamcatcher'}, 207 | {start: 89, end: 94, tag: '드림캐쳐'}, 208 | ], 209 | urls: [ 210 | { 211 | start: 95, 212 | end: 118, 213 | url: 'https://t.co/IRpqINac5X', 214 | expanded_url: 215 | 'https://twitter.com/PaniclnTheCity/status/1564281755531722755/photo/1', 216 | display_url: 'pic.twitter.com/IRpqINac5X', 217 | media_key: '3_1564281751844884482', 218 | }, 219 | ], 220 | }, 221 | 'A Minji a day keeps the bad vibes away~\n\nPlaying with our money \n\n#JiU #지유 #Dreamcatcher #드림캐쳐 https://t.co/IRpqINac5X' 222 | ) 223 | ).toBe( 224 | 'A Minji a day keeps the bad vibes away~\n\nPlaying with our money \n\n[#JiU](https://twitter.com/hashtag/JiU) [#지유](https://twitter.com/hashtag/지유) [#Dreamcatcher](https://twitter.com/hashtag/Dreamcatcher) [#드림캐쳐](https://twitter.com/hashtag/드림캐쳐) [pic.twitter.com/IRpqINac5X](https://twitter.com/PaniclnTheCity/status/1564281755531722755/photo/1)' 225 | ) 226 | }) 227 | it('Does not incorrectly split hashtags with CJK characters', () => { 228 | expect( 229 | replaceEntities( 230 | { 231 | hashtags: [ 232 | {start: 1, end: 2, tag: '드림'}, 233 | {start: 3, end: 6, tag: '드림캐쳐'}, 234 | ], 235 | }, 236 | 'A tweet with a #드림 hashtag and a #드림캐쳐 hashtag.' 237 | ) 238 | ) 239 | }) 240 | }) 241 | -------------------------------------------------------------------------------- /images/repo-open-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kbravh/tweet-to-markdown/0f179a4b4553822407d22a8bb95b2fcc0ea724f9/images/repo-open-graph.png -------------------------------------------------------------------------------- /images/ttm_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kbravh/tweet-to-markdown/0f179a4b4553822407d22a8bb95b2fcc0ea724f9/images/ttm_demo.gif -------------------------------------------------------------------------------- /images/tweet-markdown-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kbravh/tweet-to-markdown/0f179a4b4553822407d22a8bb95b2fcc0ea724f9/images/tweet-markdown-screenshot.png -------------------------------------------------------------------------------- /images/tweet-to-markdown-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tweet-to-markdown", 3 | "version": "2.4.4", 4 | "description": "Quickly save Tweets as Markdown files.", 5 | "main": "dist/main.js", 6 | "repository": "https://github.com/kbravh/tweet-to-markdown.git", 7 | "author": "Karey Higuera ", 8 | "license": "MIT", 9 | "private": false, 10 | "files": [ 11 | "dist/main.js", 12 | "src/models.ts" 13 | ], 14 | "keywords": [ 15 | "tweet", 16 | "markdown", 17 | "roam", 18 | "md" 19 | ], 20 | "bin": { 21 | "ttm": "dist/main.js" 22 | }, 23 | "scripts": { 24 | "test": "vitest", 25 | "coverage": "vitest run --coverage", 26 | "dev": "rollup --config rollup.config.js -w", 27 | "build": "rollup --config rollup.config.js --environment BUILD:production" 28 | }, 29 | "dependencies": { 30 | "array.prototype.flatmap": "^1.2.5", 31 | "axios": "^0.21.1", 32 | "axios-retry": "^3.1.8", 33 | "chalk": "^4.1.0", 34 | "clipboardy": "^2.3.0", 35 | "command-line-args": "^5.1.1", 36 | "command-line-usage": "^6.1.0", 37 | "html-entities": "^2.3.2" 38 | }, 39 | "devDependencies": { 40 | "@rollup/plugin-commonjs": "^18.0.0", 41 | "@rollup/plugin-json": "^4.1.0", 42 | "@rollup/plugin-node-resolve": "^13.0.6", 43 | "@rollup/plugin-typescript": "^8.2.1", 44 | "@types/array.prototype.flatmap": "^1.2.2", 45 | "@types/command-line-args": "^5.2.0", 46 | "@types/command-line-usage": "^5.0.2", 47 | "@types/node": "^14.14.37", 48 | "@typescript-eslint/eslint-plugin": "^5.4.0", 49 | "@typescript-eslint/parser": "^5.4.0", 50 | "c8": "^7.12.0", 51 | "eslint": "^7.32.0", 52 | "msw": "^0.44.2", 53 | "rollup": "^2.32.1", 54 | "tslib": "^2.2.0", 55 | "typescript": "^4.2.4", 56 | "vite": "^3.0.4", 57 | "vitest": "^0.21.0" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import typescript from '@rollup/plugin-typescript' 2 | import { nodeResolve } from '@rollup/plugin-node-resolve' 3 | import commonjs from '@rollup/plugin-commonjs' 4 | import json from '@rollup/plugin-json' 5 | 6 | const isProd = process.env.BUILD === 'production' 7 | 8 | const banner = `#!/usr/bin/env node` 9 | 10 | export default { 11 | input: 'src/main.ts', 12 | output: { 13 | dir: 'dist', 14 | sourcemap: 'inline', 15 | sourcemapExcludeSources: isProd, 16 | format: 'cjs', 17 | banner, 18 | }, 19 | plugins: [nodeResolve({ preferBuiltins: true }), commonjs(), json(), typescript()], 20 | } 21 | -------------------------------------------------------------------------------- /src/.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import commandLineArgs from 'command-line-args' 2 | import commandLineUsage, {OptionDefinition} from 'command-line-usage' 3 | import chalk from 'chalk' 4 | import {processTweetRequest} from './process' 5 | import {log, panic} from './util' 6 | 7 | /** 8 | * The definitions of the command line flags 9 | */ 10 | const optionDefinitions: OptionDefinition[] = [ 11 | {name: 'src', defaultOption: true}, 12 | { 13 | name: 'help', 14 | alias: 'h', 15 | type: Boolean, 16 | description: 'Display this usage guide.', 17 | }, 18 | { 19 | name: 'bearer', 20 | alias: 'b', 21 | description: 22 | 'The bearer token from the Twitter developer account to authenticate requests.', 23 | }, 24 | { 25 | name: 'clipboard', 26 | alias: 'c', 27 | type: Boolean, 28 | description: 29 | 'Copy the generated markdown to the clipboard instead of saving a Markdown file.', 30 | }, 31 | { 32 | name: 'path', 33 | alias: 'p', 34 | description: 35 | 'The path to save the file. This path must {italic already exist}. Defaults to the current directory.', 36 | }, 37 | { 38 | name: 'assets', 39 | alias: 'a', 40 | type: Boolean, 41 | description: 'Save tweet images locally instead of just a link.', 42 | }, 43 | {name: 'assets-path', description: 'The path to store the tweet images.'}, 44 | { 45 | name: 'filename', 46 | description: 47 | 'The name of the markdown file to be saved. The .md extension will be automatically added. You can use the variables [[name]], [[handle]], [[text]], and [[id]].', 48 | }, 49 | { 50 | name: 'force', 51 | alias: 'f', 52 | type: Boolean, 53 | description: 'Overwrite the file if it already exists.', 54 | }, 55 | { 56 | name: 'metrics', 57 | alias: 'm', 58 | type: Boolean, 59 | description: 60 | 'Store the number of likes, tweets, and replies in the frontmatter of the document.', 61 | }, 62 | { 63 | name: 'quoted', 64 | alias: 'q', 65 | type: Boolean, 66 | description: 67 | 'Fetch and store quoted tweets in the document instead of just a link.', 68 | }, 69 | { 70 | name: 'thread', 71 | alias: 't', 72 | type: Boolean, 73 | description: 74 | 'Save an entire tweet thread in a single document. Use the link of the last tweet.', 75 | }, 76 | { 77 | name: 'condensed_thread', 78 | alias: 'T', 79 | type: Boolean, 80 | description: 81 | 'Save an entire tweet thread in a single document, but only show the author on the first tweet or on author changes. Use the link of the last tweet.', 82 | }, 83 | { 84 | name: 'semicondensed_thread', 85 | alias: 's', 86 | type: Boolean, 87 | description: 88 | 'Save an entire tweet thread in a single document, but only show the author on the first tweet or on author changes. Add a --- divider between tweets. Use the link of the last tweet.', 89 | }, 90 | { 91 | name: 'text_only', 92 | alias: 'x', 93 | type: Boolean, 94 | description: 95 | 'Only save the text of the tweet, without author or any other metadata.', 96 | }, 97 | { 98 | name: 'date_locale', 99 | description: 100 | "The BCP 47 locale to use when displaying the date next to the author's name. E.g.: 'en-US'. Defaults to your computer's locale.", 101 | }, 102 | ] 103 | 104 | /** 105 | * The definition of the help page 106 | */ 107 | const help = [ 108 | { 109 | header: 'Tweet to Markdown', 110 | content: 'Quickly and easily save Tweets as Markdown files.', 111 | }, 112 | { 113 | header: 'Usage', 114 | content: 'ttm [flags] ', 115 | }, 116 | { 117 | header: 'Options', 118 | optionList: optionDefinitions, 119 | hide: 'src', // src is the default option so we won't show it in the main list 120 | }, 121 | ] 122 | 123 | // Parse the command line options and generate the help page 124 | let options: commandLineArgs.CommandLineOptions 125 | try { 126 | options = commandLineArgs(optionDefinitions, {camelCase: true}) 127 | } catch (error) { 128 | panic(chalk`{red ${error.message}}`) 129 | } 130 | const helpPage = commandLineUsage(help) 131 | 132 | /** 133 | * Show the help page if requested and immediately end execution 134 | */ 135 | if (options.help) { 136 | log(helpPage) 137 | process.exit(0) 138 | } 139 | 140 | /** 141 | * If no tweet source was provided, panic 142 | */ 143 | if (!options.src) { 144 | panic( 145 | chalk`{red A tweet url or id was not provided.}\n{bold {underline Usage}}: ttm [options] ` 146 | ) 147 | } 148 | 149 | // pull 🐻 token first from options, then from environment variable 150 | if (!options.bearer) { 151 | options.bearer = process.env.TTM_API_KEY ?? process.env.TWITTER_BEARER_TOKEN 152 | } 153 | 154 | // if no 🐻 token provided, panic 155 | if (!options.bearer) { 156 | panic( 157 | chalk`{red No authorization provided.} You must provide your bearer token.` 158 | ) 159 | } 160 | 161 | const main = async () => { 162 | await processTweetRequest(options) 163 | } 164 | 165 | main() 166 | -------------------------------------------------------------------------------- /src/mocks/handlers.ts: -------------------------------------------------------------------------------- 1 | import {rest} from 'msw' 2 | import {tweets} from '__fixtures__/tweets' 3 | import { BEARER_TOKEN } from '__tests__/consts' 4 | 5 | export const handlers = [ 6 | rest.get('https://ttm.kbravh.dev/api/tweet', (req, res, ctx) => { 7 | const auth = req.headers.get('Authorization') 8 | const key = auth.split('Bearer ')?.[1] ?? '' 9 | if (key !== BEARER_TOKEN) { 10 | return res(ctx.status(401)) 11 | } 12 | 13 | const id = req.url.searchParams.get('tweet') 14 | const tweet = tweets[id] 15 | 16 | if (tweet) { 17 | return res(ctx.status(200), ctx.json(tweet)) 18 | } 19 | return res(ctx.status(400), ctx.text(`Could not find tweet with id: [${id}].`)) 20 | }), 21 | ] 22 | -------------------------------------------------------------------------------- /src/mocks/server.ts: -------------------------------------------------------------------------------- 1 | import {setupServer} from 'msw/node' 2 | import {handlers} from './handlers' 3 | 4 | export const server = setupServer(...handlers) 5 | -------------------------------------------------------------------------------- /src/models.ts: -------------------------------------------------------------------------------- 1 | export interface Annotation { 2 | start: number 3 | end: number 4 | probability: number 5 | type: 'Person' | 'Place' | 'Product' | 'Organization' | 'Other' 6 | normalized_text: string 7 | } 8 | 9 | export interface Attachment { 10 | poll_ids?: string[] 11 | media_keys?: string[] 12 | } 13 | 14 | export interface Data { 15 | id: string 16 | text: string 17 | created_at: string 18 | author_id: string 19 | public_metrics: Metrics 20 | entities?: Entities 21 | conversation_id?: string 22 | attachments?: Attachment 23 | referenced_tweets?: ReferencedTweet[] 24 | } 25 | 26 | export interface Entities { 27 | urls?: TweetURL[] 28 | mentions?: Mention[] 29 | annotations?: Annotation[] 30 | hashtags?: Tag[] 31 | cashtags?: Tag[] 32 | } 33 | 34 | export interface Error { 35 | detail: string 36 | } 37 | 38 | export interface Includes { 39 | polls?: Poll[] 40 | users: User[] 41 | media?: Media[] 42 | } 43 | 44 | export interface Media { 45 | media_key: string 46 | type: 'photo' | 'gif' | 'video' 47 | url?: string 48 | alt_text?: string 49 | } 50 | 51 | export interface Mention { 52 | start: number 53 | end: number 54 | username: string 55 | id?: string 56 | } 57 | 58 | export interface Metrics { 59 | retweet_count: number 60 | reply_count: number 61 | like_count: number 62 | quote_count: number 63 | } 64 | 65 | export interface OpenGraphImage { 66 | url: string 67 | width: number 68 | height: number 69 | } 70 | 71 | export interface Poll { 72 | id: string 73 | options: PollOption[] 74 | } 75 | 76 | export interface PollOption { 77 | position: number 78 | label: string 79 | votes: number 80 | } 81 | 82 | export interface ReferencedTweet { 83 | type: 'quoted' | 'replied_to' 84 | id: string 85 | } 86 | 87 | export interface Tag { 88 | start: number 89 | end: number 90 | tag: string 91 | } 92 | 93 | export interface Tweet { 94 | includes: Includes 95 | data: Data 96 | errors?: Error[] 97 | // other error fields 98 | reason?: string 99 | status?: number 100 | } 101 | 102 | export interface TweetURL { 103 | start: number 104 | end: number 105 | url: string 106 | expanded_url: string 107 | display_url: string 108 | media_key?: string 109 | images?: OpenGraphImage[] 110 | status?: number 111 | title?: string 112 | description?: string 113 | unwound_url?: string 114 | } 115 | 116 | export interface User { 117 | name: string 118 | id: string 119 | username: string 120 | profile_image_url: string 121 | } 122 | 123 | export type TimestampFormat = { 124 | locale: string 125 | format?: Intl.DateTimeFormatOptions 126 | } 127 | -------------------------------------------------------------------------------- /src/process.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import chalk from 'chalk' 3 | import commandLineArgs from 'command-line-args' 4 | import type {ReferencedTweet, Tweet} from './models' 5 | import { 6 | buildMarkdown, 7 | copyToClipboard, 8 | generateErrorMessageFromError, 9 | getTweet, 10 | getTweetID, 11 | log, 12 | logErrorToFile, 13 | panic, 14 | writeTweet, 15 | } from './util' 16 | 17 | export const processTweetRequest = async ( 18 | options: commandLineArgs.CommandLineOptions 19 | ) => { 20 | const tweets: Tweet[] = [] 21 | let error: Error 22 | let currentTweet: Tweet 23 | 24 | // extract tweet ID from URL 25 | const id = getTweetID(options) 26 | 27 | try { 28 | currentTweet = await getTweet(id, options.bearer) 29 | } catch (err) { 30 | error = err 31 | } 32 | 33 | !error && tweets.push(currentTweet) 34 | process.stdout.write(`Tweets downloaded: ${tweets.length}\r`) 35 | // special handling for threads 36 | if ( 37 | options.thread || 38 | options.condensedThread || 39 | options.semicondensedThread 40 | ) { 41 | // check if this is the head tweet 42 | while (currentTweet?.data?.conversation_id !== currentTweet?.data?.id) { 43 | // load in parent tweet 44 | const [parent_tweet] = currentTweet.data.referenced_tweets.filter( 45 | (ref_tweet: ReferencedTweet) => ref_tweet.type === 'replied_to' 46 | ) 47 | try { 48 | currentTweet = await getTweet(parent_tweet.id, options.bearer) 49 | } catch (err) { 50 | error = err 51 | break 52 | } 53 | tweets.push(currentTweet) 54 | process.stdout.write(`Tweets downloaded: ${tweets.length}\r`) 55 | } 56 | } 57 | if (error) { 58 | const message = generateErrorMessageFromError(error) 59 | await logErrorToFile(message) 60 | } 61 | 62 | if (error && !tweets.length) { 63 | if (axios.isAxiosError(error)) { 64 | if (error.response) { 65 | panic(chalk.red(error.response.statusText)) 66 | } else if (error.request) { 67 | panic(chalk.red('There seems to be a connection issue.')) 68 | } else { 69 | panic(chalk.red('An error occurred.')) 70 | } 71 | } 72 | panic( 73 | chalk`{red Unfortunately, an error occurred.} See ttm.log in this directory file for details.` 74 | ) 75 | } 76 | 77 | if (error && tweets.length) { 78 | log( 79 | chalk`{red An error occurred while downloading tweets.} I'll generate your markdown file with the information I did fetch.\nSee ttm.log in this directory file for details.` 80 | ) 81 | } 82 | // reverse the thread so the tweets are in chronological order 83 | tweets.reverse() 84 | const markdowns = await Promise.all( 85 | tweets.map(async (tweet, index) => { 86 | return await buildMarkdown( 87 | tweet, 88 | options, 89 | index === 0 ? 'normal' : 'thread', 90 | index === 0 ? null : tweets[index - 1].includes.users[0] 91 | ) 92 | }) 93 | ) 94 | const firstTweet = tweets[0] 95 | if ( 96 | (options.condensedThread || options.semicondensedThread) && 97 | !options.textOnly 98 | ) { 99 | markdowns.push( 100 | `\n\n[Thread link](https://twitter.com/${firstTweet.includes.users[0].username}/status/${firstTweet.data.id})` 101 | ) 102 | } 103 | 104 | const final = 105 | options.condensedThread 106 | ? markdowns.join('\n\n') 107 | : markdowns.join('\n\n---\n\n') 108 | 109 | if (options.clipboard) { 110 | copyToClipboard(final) 111 | } else { 112 | writeTweet(firstTweet, final, options) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/unicodeSubstring.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Credit: lautis/unicode-substring 3 | * Rewritten for Obsidian mobile functionality. 4 | */ 5 | 6 | const charAt = (string: string, index: number): string => { 7 | const first = string.charCodeAt(index) 8 | let second 9 | if (first >= 0xd800 && first <= 0xdbff && string.length > index + 1) { 10 | second = string.charCodeAt(index + 1) 11 | if (second >= 0xdc00 && second <= 0xdfff) { 12 | return string.substring(index, index + 2) 13 | } 14 | } 15 | return string[index] 16 | } 17 | 18 | const slice = (string: string, start: number, end: number): string => { 19 | let accumulator = '' 20 | let character 21 | let stringIndex = 0 22 | let unicodeIndex = 0 23 | const length = string.length 24 | 25 | while (stringIndex < length) { 26 | character = charAt(string, stringIndex) 27 | if (unicodeIndex >= start && unicodeIndex < end) { 28 | accumulator += character 29 | } 30 | stringIndex += character.length 31 | unicodeIndex += 1 32 | } 33 | return accumulator 34 | } 35 | 36 | const toNumber = (value: string | number, fallback: number): number => { 37 | if (value === undefined) { 38 | return fallback 39 | } else { 40 | return Number(value) 41 | } 42 | } 43 | 44 | export const unicodeSubstring = ( 45 | string: string, 46 | start: number, 47 | end: number 48 | ): string => { 49 | const realStart = toNumber(start, 0) 50 | const realEnd = toNumber(end, string.length) 51 | if (realEnd === realStart) { 52 | return '' 53 | } else if (realEnd > realStart) { 54 | return slice(string, realStart, realEnd) 55 | } else { 56 | return slice(string, realEnd, realStart) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/util.ts: -------------------------------------------------------------------------------- 1 | import {AxiosError, default as Axios} from 'axios' 2 | import axiosRetry from 'axios-retry' 3 | import clipboard from 'clipboardy' 4 | import flatMap from 'array.prototype.flatmap' 5 | import fs from 'fs' 6 | import path from 'path' 7 | const fsp = fs.promises 8 | import chalk from 'chalk' 9 | import { 10 | Entities, 11 | Media, 12 | Mention, 13 | Poll, 14 | Tag, 15 | TimestampFormat, 16 | Tweet, 17 | TweetURL, 18 | User, 19 | } from './models' 20 | import {CommandLineOptions} from 'command-line-args' 21 | import {URL, URLSearchParams} from 'url' 22 | import {unicodeSubstring} from './unicodeSubstring' 23 | import {decode} from 'html-entities' 24 | 25 | export const log = console.info 26 | 27 | axiosRetry(Axios, {retries: 3}) 28 | 29 | /** 30 | * Displays an error message to the user, then exits the program with a failure code. 31 | * @param {string} message - The error message to be displayed to the user 32 | */ 33 | export const panic = (message: string): void => { 34 | log(message) 35 | process.exit(1) 36 | } 37 | 38 | /** 39 | * Download the remote image url to the local path. 40 | * @param {string} url - The remote image URL to download 41 | * @param {string} image_path - The local path to save the image 42 | */ 43 | export const downloadImage = (url: string, image_path: string): Promise => 44 | Axios({ 45 | url, 46 | responseType: 'stream', 47 | }).then( 48 | response => 49 | new Promise((resolve, reject) => { 50 | response.data 51 | .pipe(fs.createWriteStream(image_path)) 52 | .on('finish', () => resolve()) 53 | .on('error', (e: Error) => reject(e)) 54 | }) 55 | ) 56 | 57 | /** 58 | * Parses out the tweet ID from the URL or ID that the user provided 59 | * @param {CommandLineOptions} options - The parsed command line arguments 60 | */ 61 | export const getTweetID = ({src}: CommandLineOptions): string => { 62 | let id 63 | try { 64 | // Create a URL object with the source. If it fails, it's not a URL. 65 | const url = new URL(src) 66 | id = url.pathname 67 | .split('/') 68 | .filter((piece: string) => !!piece) // remove enpty strings from array 69 | .slice(-1)[0] 70 | } catch (error) { 71 | id = src 72 | if (typeof id !== 'string') { 73 | panic(chalk`{red Could not determine tweet ID.}`) 74 | } 75 | } 76 | return id 77 | } 78 | 79 | /** 80 | * Fetches a tweet object 81 | * @param {string} id - The ID of the tweet to fetch from the API 82 | * @param {string} bearer - The bearer token 83 | * @returns {Promise} - The tweet from the Twitter API 84 | */ 85 | export const getTweet = async (id: string, bearer: string): Promise => { 86 | if (bearer.startsWith('TTM')) { 87 | return getTweetFromTTM(id, bearer) 88 | } 89 | return getTweetFromTwitter(id, bearer) 90 | } 91 | 92 | /** 93 | * Fetches a tweet object from the Twitter v2 API 94 | * @param {string} id - The ID of the tweet to fetch from the API 95 | * @param {string} bearer - The bearer token 96 | * @returns {Promise} - The tweet from the Twitter API 97 | */ 98 | const getTweetFromTwitter = async ( 99 | id: string, 100 | bearer: string 101 | ): Promise => { 102 | const twitterUrl = new URL(`https://api.twitter.com/2/tweets/${id}`) 103 | const params = new URLSearchParams({ 104 | expansions: 'author_id,attachments.poll_ids,attachments.media_keys', 105 | 'user.fields': 'name,username,profile_image_url', 106 | 'tweet.fields': 107 | 'attachments,public_metrics,entities,conversation_id,referenced_tweets,created_at', 108 | 'media.fields': 'url,alt_text', 109 | 'poll.fields': 'options', 110 | }) 111 | 112 | return await Axios({ 113 | method: 'GET', 114 | url: `${twitterUrl.href}?${params.toString()}`, 115 | headers: {Authorization: `Bearer ${bearer}`}, 116 | }) 117 | .then(response => response.data) 118 | .then(tweet => { 119 | if (tweet.errors) { 120 | panic(chalk`{red ${tweet.errors[0].detail}}`) 121 | } else { 122 | return tweet 123 | } 124 | }) 125 | } 126 | 127 | /** 128 | * Fetches a tweet object from the TTM service API 129 | * @param {string} id - The ID of the tweet to fetch from the API 130 | * @param {string} bearer - The bearer token 131 | * @returns {Promise} - The tweet from the Twitter API 132 | */ 133 | const getTweetFromTTM = async (id: string, bearer: string): Promise => { 134 | const ttmUrl = new URL(`https://ttm.kbravh.dev/api/tweet`) 135 | const params = new URLSearchParams({ 136 | tweet: id, 137 | source: 'cli', 138 | }) 139 | return await Axios({ 140 | method: 'GET', 141 | url: `${ttmUrl.href}?${params.toString()}`, 142 | headers: {Authorization: `Bearer ${bearer}`}, 143 | }).then(response => response.data) 144 | } 145 | 146 | export const generateErrorMessageFromError = ( 147 | error: Error | AxiosError 148 | ): string => { 149 | let errorMessage: string 150 | const stack = error.stack 151 | 152 | // if Axios error, dig in a bit more 153 | if (Axios.isAxiosError(error)) { 154 | if (error.response) { 155 | errorMessage = `${error.response.statusText}; ${error.response.data}` 156 | } else if (error.request) { 157 | errorMessage = `There may be a connection error: ${error.message}` 158 | } 159 | } 160 | return `${errorMessage ?? error.message}\n${stack}` 161 | } 162 | 163 | export const logErrorToFile = async (error: string): Promise => { 164 | try { 165 | await fsp.appendFile('ttm.log', `${new Date().toISOString()}: ${error}`) 166 | } catch (error) { 167 | log(`There was an error writing to the TTM log file: ${error.message}`) 168 | } 169 | } 170 | 171 | /** 172 | * Writes the tweet to a markdown file. 173 | * @param {Tweet} tweet - The entire tweet object from the Twitter v2 API 174 | * @param {string} markdown - The markdown string to be written to the file 175 | * @param {CommandLineOptions} options - The parsed command line options 176 | */ 177 | export const writeTweet = async ( 178 | tweet: Tweet, 179 | markdown: string, 180 | options: CommandLineOptions 181 | ): Promise => { 182 | let filepath = '' 183 | // check if path provided by user is valid and writeable 184 | if (options.path) { 185 | await testPath(options.path) 186 | filepath = options.path 187 | } 188 | 189 | // create filename 190 | const filename = createFilename(tweet, options) 191 | // combine name and path 192 | filepath = path.join(filepath, filename) 193 | 194 | //check if file already exists 195 | await fsp 196 | .access(filepath, fs.constants.F_OK) 197 | .then(_ => { 198 | if (!options.force) { 199 | panic( 200 | chalk`{red File already exists.} Use {bold --force (-f)} to overwrite.` 201 | ) 202 | } 203 | }) 204 | .catch((error: Error) => { 205 | //file does not exist so we can write to it 206 | }) 207 | 208 | // clean up excessive newlines 209 | markdown = markdown.replace(/\n{2,}/g, '\n\n') 210 | 211 | // write the tweet to the file 212 | await fsp.writeFile(filepath, markdown).catch(error => { 213 | panic(error) 214 | }) 215 | log(chalk`Tweet saved to {bold {underline ${filepath}}}`) 216 | } 217 | 218 | /** 219 | * Copies the provided string to the clipboard. 220 | * @param {string} markdown - The markdown to be copied to the clipboard 221 | */ 222 | export const copyToClipboard = async (markdown: string): Promise => { 223 | await clipboard.write(markdown).catch(error => { 224 | panic(chalk`{red There was a problem writing to the clipboard.}`) 225 | }) 226 | log('Tweet copied to the clipboard.') 227 | } 228 | 229 | /** 230 | * Creates markdown table to capture poll options and votes 231 | * @param {Poll[]} polls - The polls array provided by the Twitter v2 API 232 | * @returns {string[]} - Array of Markdown tables as string of the poll 233 | */ 234 | export const createPollTable = (polls: Poll[]): string[] => { 235 | return polls.map(poll => { 236 | const table = ['\n|Option|Votes|', '|---|:---:|'] 237 | const options = poll.options.map( 238 | option => `|${option.label}|${option.votes}|` 239 | ) 240 | return table.concat(options).join('\n') 241 | }) 242 | } 243 | /** 244 | * Truncate a string to a specified number of bytes 245 | * @param string the string to truncate 246 | * @param length the maximum length in bytes of the trimmed string 247 | * @returns string 248 | */ 249 | export const truncateBytewise = (string: string, length: number): string => { 250 | const originalLength = length 251 | while (new TextEncoder().encode(string).length > originalLength) { 252 | string = unicodeSubstring(string, 0, length--) 253 | } 254 | return string 255 | } 256 | 257 | /** 258 | * Filename sanitization. Credit: parshap/node-sanitize-filename 259 | * Rewrite to allow functionality on Obsidian mobile. 260 | */ 261 | const illegalRe = /[/?<>\\:*|"]/g 262 | // eslint-disable-next-line no-control-regex 263 | const controlRe = /[\x00-\x1f\x80-\x9f]/g 264 | const reservedRe = /^\.+$/ 265 | const windowsReservedRe = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$/i 266 | const windowsTrailingRe = /[. ]+$/ 267 | 268 | /** 269 | * Sanitize a filename to remove any illegal characters. 270 | * Also keeps the filename to 255 bytes or below. 271 | * @param filename string 272 | * @returns string 273 | */ 274 | export const sanitizeFilename = (filename: string): string => { 275 | filename = filename 276 | .replace(illegalRe, '') 277 | .replace(controlRe, '') 278 | .replace(reservedRe, '') 279 | .replace(windowsReservedRe, '') 280 | .replace(windowsTrailingRe, '') 281 | return truncateBytewise(filename, 252) 282 | } 283 | 284 | /** 285 | * Creates a filename based on the tweet and the user defined options. 286 | * @param {Tweet} tweet - The entire tweet object from the Twitter v2 API 287 | * @param {CommandLineOptions} options - The parsed command line arguments 288 | * @returns {string} - The filename based on tweet and options 289 | */ 290 | export const createFilename = ( 291 | tweet: Tweet, 292 | options: CommandLineOptions 293 | ): string => { 294 | let filename: string = options.filename 295 | ? options.filename 296 | : '[[handle]] - [[id]]' 297 | filename = filename.replace(/\.md$/, '') // remove md extension if provided 298 | filename = filename.replace(/\[\[name\]\]/gi, tweet.includes.users[0].name) 299 | filename = filename.replace( 300 | /\[\[handle\]\]/gi, 301 | tweet.includes.users[0].username 302 | ) 303 | filename = filename.replace(/\[\[id\]\]/gi, tweet.data.id) 304 | filename = filename.replace(/\[\[text\]\]/gi, tweet.data.text) 305 | return sanitizeFilename(filename) + '.md' 306 | } 307 | 308 | /** 309 | * Returns the local path to the asset, taking into account the path 310 | * for the tweet itself so that the asset path is relative. 311 | * @param {CommandLineOptions} options - The parsed command line arguments 312 | * @returns {string} - The local asset path 313 | */ 314 | export const getLocalAssetPath = (options: CommandLineOptions): string => { 315 | // If the user wants to download assets locally, we'll need to define the path 316 | const localAssetPath = options.assetsPath 317 | ? options.assetsPath 318 | : './tweet-assets' 319 | // we need the relative path to the assets from the notes 320 | return path.relative(options.path ? options.path : '.', localAssetPath) 321 | } 322 | 323 | /** 324 | * Creates media links to embed media into the markdown file 325 | * @param {Media[]} media - The tweet media objects provided by the Twitter v2 API 326 | * @param {CommandLineOptions} options - The parsed command line arguments 327 | * @returns {string[]} - An array of markdown image links 328 | */ 329 | export const createMediaElements = ( 330 | media: Media[], 331 | options: CommandLineOptions 332 | ): string[] => { 333 | const localAssetPath = getLocalAssetPath(options) 334 | return media.map(medium => { 335 | switch (medium.type) { 336 | case 'photo': 337 | const alt_text = medium.alt_text 338 | ? medium.alt_text.replace(/\n/g, ' ') 339 | : '' 340 | return options.assets 341 | ? `\n![${alt_text ?? medium.media_key}](${path.join( 342 | localAssetPath, 343 | `${medium.media_key}.jpg` 344 | )})` 345 | : `\n![${alt_text ?? medium.media_key}](${medium.url})` 346 | default: 347 | break 348 | } 349 | }) 350 | } 351 | type GenericEntity = Pick< 352 | Mention & Tag & TweetURL & {replacement: string}, 353 | 'start' | 'end' | 'replacement' 354 | > 355 | /** 356 | * replace any mentions, hashtags, cashtags, urls with links 357 | */ 358 | export const replaceEntities = (entities: Entities, text: string): string => { 359 | /** 360 | * Each entity comes with start and end indices. However, if we were to replace 361 | * them in the order they occur, the indices further down the line would be shifted 362 | * and inaccurate. So we sort them in reverse order and work up from the end of the tweet. 363 | */ 364 | const allEntities: GenericEntity[] = [ 365 | ...(entities?.mentions ?? []).map(mention => ({ 366 | ...mention, 367 | replacement: `[@${mention.username}](https://twitter.com/${mention.username})`, 368 | })), 369 | ...(entities?.hashtags ?? []).map(hashtag => ({ 370 | ...hashtag, 371 | replacement: `[#${hashtag.tag}](https://twitter.com/hashtag/${hashtag.tag})`, 372 | })), 373 | ...(entities?.cashtags ?? []).map(cashtag => ({ 374 | ...cashtag, 375 | replacement: `[$${cashtag.tag}](https://twitter.com/search?q=%24${cashtag.tag})`, 376 | })), 377 | // Sort in reverse order 378 | ].sort((a, b) => b.start - a.start) 379 | 380 | const urlSet = new Set() 381 | const urls = (entities?.urls ?? []).filter(url => { 382 | if (urlSet.has(url.expanded_url)) { 383 | return false 384 | } else { 385 | urlSet.add(url.expanded_url) 386 | return true 387 | } 388 | }) 389 | 390 | for (const entity of allEntities) { 391 | let chars = [...text] 392 | text = 393 | chars.slice(0, entity.start).join('') + 394 | entity.replacement + 395 | chars.slice(entity.end).join('') 396 | } 397 | 398 | urls.forEach(url => { 399 | text = text.replace( 400 | new RegExp(url.url, 'g'), 401 | `[${url.display_url}](${url.expanded_url})` 402 | ) 403 | }) 404 | 405 | return text 406 | } 407 | 408 | /** 409 | * Tests if a path exists and if the user has write permission. 410 | * @param {string} path - the path to test for access 411 | */ 412 | export const testPath = async (path: string): Promise => 413 | fsp.mkdir(path, {recursive: true}).catch(error => { 414 | panic( 415 | chalk`{red Unable to write to the path {bold {underline ${path}}}. Do you have write permission?}` 416 | ) 417 | }) 418 | 419 | export const formatTimestamp = ( 420 | timestamp: string, 421 | timestampFormat: TimestampFormat 422 | ): string => 423 | new Date(timestamp).toLocaleDateString( 424 | timestampFormat.locale, 425 | timestampFormat.format ?? { 426 | day: 'numeric', 427 | year: 'numeric', 428 | month: 'long', 429 | hour: 'numeric', 430 | minute: 'numeric', 431 | timeZoneName: 'short', 432 | } 433 | ) 434 | 435 | /** 436 | * Creates the entire Markdown string of the provided tweet 437 | * @param {Tweet} tweet - The entire tweet object provided by the Twitter v2 API 438 | * @param {CommandLineOptions} options - The parsed command line arguments 439 | * @param {("normal" | "thread" | "quoted")} type - Whether this is a normal, thread, or quoted tweet 440 | * @returns {Promise} - The Markdown string of the tweet 441 | */ 442 | export const buildMarkdown = async ( 443 | tweet: Tweet, 444 | options: CommandLineOptions, 445 | type: 'normal' | 'thread' | 'quoted' = 'normal', 446 | previousAuthor?: User 447 | ): Promise => { 448 | if (type === 'thread' && !previousAuthor) { 449 | panic('A thread tweet must have a previous author') 450 | } 451 | 452 | let text = tweet.data.text 453 | 454 | /** 455 | * replace entities with markdown links 456 | */ 457 | if (tweet.data.entities) { 458 | text = replaceEntities(tweet.data.entities, text) 459 | } 460 | 461 | // Decode HTML entities 462 | text = decode(text) 463 | const user = tweet.includes.users[0] 464 | 465 | const iscondensedThreadTweet = !( 466 | type !== 'thread' || 467 | (type === 'thread' && 468 | !options.condensedThread && 469 | !options.semicondensedThread) 470 | ) 471 | 472 | const showAuthor = 473 | ((iscondensedThreadTweet && user.id !== previousAuthor.id) || 474 | !iscondensedThreadTweet) && 475 | !options.textOnly 476 | 477 | let metrics: string[] = [] 478 | if (options.metrics) { 479 | metrics = [ 480 | `likes: ${tweet.data.public_metrics.like_count}`, 481 | `retweets: ${tweet.data.public_metrics.retweet_count}`, 482 | `replies: ${tweet.data.public_metrics.reply_count}`, 483 | ] 484 | } 485 | 486 | const isoDate = new Date(tweet.data.created_at).toISOString() 487 | 488 | const date = formatTimestamp(tweet.data.created_at, { 489 | locale: options.dateLocale, 490 | format: { 491 | day: 'numeric', 492 | year: 'numeric', 493 | month: 'long', 494 | hour: 'numeric', 495 | minute: 'numeric', 496 | }, 497 | }) 498 | 499 | /** 500 | * Define the frontmatter as the name, handle, and source url 501 | */ 502 | const frontmatter = options.textOnly 503 | ? [] 504 | : [ 505 | '---', 506 | `author: "${user.name}"`, 507 | `handle: "@${user.username}"`, 508 | `source: "https://twitter.com/${user.username}/status/${tweet.data.id}"`, 509 | `date: ${isoDate}`, 510 | ...metrics, 511 | '---', 512 | ] 513 | 514 | // if the user wants local assets, download them 515 | if (options.assets) { 516 | await downloadAssets(tweet, options) 517 | } 518 | 519 | let markdown = [] 520 | if (showAuthor) { 521 | markdown.push( 522 | `![${user.username}](${ 523 | options.assets 524 | ? path.join( 525 | getLocalAssetPath(options), 526 | `${user.username}-${user.id}.jpg` 527 | ) 528 | : user.profile_image_url 529 | })`, // profile image 530 | `${user.name} ([@${user.username}](https://twitter.com/${user.username})) - ${date}`, // name and handle 531 | '\n', 532 | text 533 | ) 534 | } else { 535 | markdown.push(text) 536 | } 537 | 538 | // remove newlines from within tweet text to avoid breaking our formatting 539 | markdown = flatMap(markdown, line => line.split('\n')) 540 | 541 | // Add in other tweet elements 542 | if (tweet.includes.polls) { 543 | markdown = markdown.concat(createPollTable(tweet.includes.polls)) 544 | } 545 | 546 | if (tweet.includes.media) { 547 | markdown = markdown.concat( 548 | createMediaElements(tweet.includes.media, options) 549 | ) 550 | } 551 | 552 | // check for quoted tweets to be included 553 | if (options.quoted && tweet.data && tweet.data.referenced_tweets) { 554 | for (const subtweet_ref of tweet.data.referenced_tweets) { 555 | if (subtweet_ref && subtweet_ref.type === 'quoted') { 556 | const subtweet = await getTweet(subtweet_ref.id, options.bearer) 557 | const subtweet_text = await buildMarkdown(subtweet, options, 'quoted') 558 | markdown.push('\n\n' + subtweet_text) 559 | } 560 | } 561 | } 562 | 563 | // indent all lines for a quoted tweet 564 | if (type === 'quoted') { 565 | markdown = markdown.map(line => '> ' + line) 566 | } 567 | 568 | // add original tweet link to end of tweet if not a condensed thread 569 | if ( 570 | !(options.condensedThread || options.semicondensedThread) && 571 | !options.textOnly 572 | ) { 573 | markdown.push( 574 | '\n\n' + 575 | `[Tweet link](https://twitter.com/${user.username}/status/${tweet.data.id})` 576 | ) 577 | } 578 | 579 | switch (type) { 580 | case 'normal': 581 | return frontmatter.concat(markdown).join('\n') 582 | case 'thread': 583 | return markdown.join('\n') 584 | case 'quoted': 585 | return '\n\n' + markdown.join('\n') 586 | default: 587 | return '\n\n' + markdown.join('\n') 588 | } 589 | } 590 | 591 | /** 592 | * Downloads all tweet images locally if they do not yet exist 593 | * @param {Tweet} tweet - The entire tweet object from the twitter API 594 | * @param {CommandLineOptions} options - The command line options 595 | */ 596 | export const downloadAssets = async ( 597 | tweet: Tweet, 598 | options: CommandLineOptions 599 | ): Promise => { 600 | const user = tweet.includes.users[0] 601 | // determine path to download local assets 602 | const localAssetPath = options.assetsPath 603 | ? options.assetsPath 604 | : './tweet-assets' 605 | // create this directory if it doesn't yet exist 606 | await testPath(localAssetPath) 607 | 608 | // grab a list of all files to download and their paths 609 | let files = [] 610 | // add profile image to download list 611 | files.push({ 612 | url: user.profile_image_url, 613 | path: path.join(localAssetPath, `${user.username}-${user.id}.jpg`), 614 | }) 615 | 616 | // add tweet images to download list 617 | if (tweet.includes.media) { 618 | tweet.includes.media.forEach(medium => { 619 | switch (medium.type) { 620 | case 'photo': 621 | files.push({ 622 | url: medium.url, 623 | path: path.join(localAssetPath, `${medium.media_key}.jpg`), 624 | }) 625 | break 626 | default: 627 | break 628 | } 629 | }) 630 | } 631 | 632 | /** 633 | * Filter out tweet assets that already exist locally. 634 | * Array.filter() is only synchronous, so we can't use it here. 635 | */ 636 | // Determine which assets do exist 637 | let assetTests = await asyncMap(files, ({path}: {path: string}) => 638 | doesFileExist(path) 639 | ) 640 | // Invert the test results to know which don't exist 641 | assetTests = assetTests.map(result => !result) 642 | // filter the list of assets to download 643 | files = files.filter((_, index) => assetTests[index]) 644 | 645 | // Download missing assets 646 | return Promise.all(files.map(file => downloadImage(file.url, file.path))) 647 | } 648 | 649 | /** 650 | * An async version of the Array.map() function. 651 | * @param {*[]} array - The array to be mapped over 652 | * @param {Function} mutator - The function to apply to every array element 653 | * @returns {Promise} - A Promise that resolves to the mapped array values 654 | */ 655 | export const asyncMap = async ( 656 | array: any[], 657 | mutator: Function 658 | ): Promise => Promise.all(array.map(element => mutator(element))) 659 | 660 | /** 661 | * Determines if a file exists locally. 662 | * @param {String} filepath - The filepath to test 663 | * @returns {Promise} - True if file exists, false otherwise 664 | */ 665 | export const doesFileExist = (filepath: string): Promise => 666 | fsp 667 | .access(filepath, fs.constants.F_OK) 668 | .then(_ => true) 669 | .catch(_ => false) 670 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "inlineSourceMap": true, 5 | "inlineSources": true, 6 | "module": "ESNext", 7 | "target": "es6", 8 | "allowJs": true, 9 | "noImplicitAny": true, 10 | "moduleResolution": "node", 11 | "importHelpers": true, 12 | "allowSyntheticDefaultImports": true, 13 | "resolveJsonModule": true, 14 | "useUnknownInCatchVariables": false, 15 | "esModuleInterop": true, 16 | "lib": ["dom", "es5", "scripthost", "es2015"] 17 | }, 18 | "include": ["**/*.ts", "src/test.ts", "mock/image_tweet.ts", "mock/mentions_tweet.ts", "src/test.mjs", "src/test.js"] 19 | } 20 | --------------------------------------------------------------------------------