├── .env.template ├── .gitignore ├── .yarnrc.yml ├── README.md ├── index.js ├── package.json └── yarn.lock /.env.template: -------------------------------------------------------------------------------- 1 | TWITTER_CONSUMER_KEY="" 2 | TWITTER_CONSUMER_SECRET="" 3 | TWITTER_ACCESS_TOKEN="" 4 | TWITTER_ACCESS_TOKEN_SECRET="" 5 | HTTP_PROXY="" 6 | MEMOS_OPEN_ID="" 7 | MEMOS_HOST="" 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | node_modules 3 | .yarn 4 | db.json 5 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ice-Hazymoon/memos-twitter-bot/f0e97c4fa5fdae4730dec1211724ace7cce5b78f/README.md -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | const fetch = require('isomorphic-fetch'); 3 | const HttpsProxyAgent = require("https-proxy-agent"); 4 | const { 5 | TwitterApi, 6 | EDirectMessageEventTypeV1 7 | } = require("twitter-api-v2"); 8 | const fs = require('fs'); 9 | const FormData = require('form-data'); 10 | 11 | const dbfile = './db.json'; 12 | if (!fs.existsSync(dbfile)) { 13 | fs.writeFileSync(dbfile, JSON.stringify({ 14 | saved: [] 15 | })); 16 | } 17 | 18 | const httpAgent = process.env.HTTP_PROXY ? new HttpsProxyAgent(process.env.HTTP_PROXY) : undefined; 19 | 20 | const client = new TwitterApi({ 21 | appKey: process.env.TWITTER_CONSUMER_KEY, 22 | appSecret: process.env.TWITTER_CONSUMER_SECRET, 23 | accessToken: process.env.TWITTER_ACCESS_TOKEN, 24 | accessSecret: process.env.TWITTER_ACCESS_TOKEN_SECRET 25 | }, { 26 | httpAgent 27 | }); 28 | 29 | async function uploadMedia(media_url, filename) { 30 | const _media = await fetch(media_url, { 31 | agent: httpAgent 32 | }); 33 | const media = await _media.buffer(); 34 | const formData = new FormData(); 35 | formData.append('file', media, { 36 | filename: filename, 37 | }); 38 | const _result = await fetch(`${MEMOS_HOST}/api/resource/blob?openId=${process.env.MEMOS_OPEN_ID}`, { 39 | method: 'post', 40 | body: formData, 41 | agent: httpAgent 42 | }) 43 | const result = await _result.json(); 44 | if (result.error) { 45 | throw new Error(result.error); 46 | } else { 47 | return result; 48 | } 49 | } 50 | 51 | async function newMemos(memos, resourceList) { 52 | const _result = await fetch(`${MEMOS_HOST}/api/memo?openId=${process.env.MEMOS_OPEN_ID}`, { 53 | method: 'post', 54 | body: JSON.stringify({ 55 | content: memos, 56 | visibility: '', 57 | resourceIdList: resourceList 58 | }), 59 | headers: { 60 | 'Content-Type': 'application/json' 61 | }, 62 | agent: httpAgent 63 | }); 64 | const result = await _result.json(); 65 | if (result.error) { 66 | throw new Error(result.error); 67 | } else { 68 | return result; 69 | } 70 | } 71 | 72 | async function getTweet(tweet_id) { 73 | const status = await client.v1.singleTweet(tweet_id, { 74 | include_entities: true, 75 | }); 76 | const status_entities = status.entities; 77 | const status_urls = status_entities.urls; 78 | let status_text = status.full_text; 79 | if (status_urls.length) { 80 | status_urls.forEach((url) => { 81 | status_text = status_text.replace(url.url, url.expanded_url); 82 | }) 83 | status.extended_entities?.media?.forEach(media => { 84 | status_text = status_text.replace(media.url, ''); 85 | }) 86 | }; 87 | const tweet_date = new Date(status.created_at).toLocaleString(); 88 | const tweet = { 89 | id: status.id_str, 90 | text: status_text.replace(/#([^\s#]+)/g, '*$1*').trim(), 91 | user_name: status.user.name, 92 | user_screen_name: status.user.screen_name, 93 | date: tweet_date, 94 | media: status.extended_entities?.media?.map((media) => { 95 | return { 96 | url: media.media_url_https, 97 | video_url: media.video_info?.variants?.find(variant => variant.content_type === 'video/mp4')?.url, 98 | type: media.type, 99 | width: media.sizes.large.w, 100 | height: media.sizes.large.h, 101 | } 102 | }), 103 | } 104 | return tweet; 105 | } 106 | 107 | (async (start) => { 108 | try { 109 | const db = fs.readFileSync('db.json', 'utf8'); 110 | const dbJson = JSON.parse(db); 111 | 112 | const loggedUser = await client.v1.verifyCredentials(); 113 | const id = loggedUser.id; 114 | 115 | console.log('bot info:', { 116 | name: loggedUser.name, 117 | screen_name: loggedUser.screen_name, 118 | }); 119 | 120 | const mentionTimeline = await client.v1.mentionTimeline({ trim_user: true }); 121 | const fetchedTweets = mentionTimeline.tweets; 122 | const memos_tweets = fetchedTweets.filter(tweet => { 123 | return tweet.full_text.trim().endsWith(`@${loggedUser.screen_name} memo`); 124 | }); 125 | for (let index = 0; index < memos_tweets.length; index++) { 126 | const tweet_id = memos_tweets[index].in_reply_to_status_id_str; 127 | const tweet = await getTweet(tweet_id); 128 | memos_tweets[index] = tweet; 129 | } 130 | 131 | const eventsPaginator = await client.v1.listDmEvents(); 132 | const dm_tweets = []; 133 | for await (const event of eventsPaginator) { 134 | if (event.type === EDirectMessageEventTypeV1.Create && event[EDirectMessageEventTypeV1.Create].sender_id !== id) { 135 | const message = event[EDirectMessageEventTypeV1.Create]; 136 | const message_data = message.message_data; 137 | const entities = message_data.entities; 138 | const urls = entities.urls; 139 | if (!urls.length) { 140 | continue; 141 | } 142 | const url = urls[0]; 143 | const expanded_url = url.expanded_url; 144 | const status_id = expanded_url.split('/').pop(); 145 | // check is tweet id 146 | if (!status_id.match(/^\d+$/)) { 147 | continue; 148 | } 149 | 150 | const tweet = await getTweet(status_id); 151 | dm_tweets.push(tweet); 152 | } 153 | } 154 | 155 | const tweets = [...memos_tweets, ...dm_tweets]; 156 | console.log('tweets:', tweets); 157 | for (let index = 0; index < tweets.length; index++) { 158 | const tweet = tweets[index]; 159 | const saved = dbJson.saved.find(saved => saved.tweet.id === tweet.id); 160 | if (saved) { 161 | continue; 162 | } 163 | 164 | const resourceList = []; 165 | if (!tweet.media) { 166 | tweet.media = []; 167 | } 168 | for (let index = 0; index < tweet.media.length; index++) { 169 | const media = tweet.media[index]; 170 | const media_url = media.video_url || media.url; 171 | const media_ext = new URL(media_url).pathname.split('.').pop(); 172 | const result = await uploadMedia(media_url, `${tweet.id}_${index+1}.${media_ext}`); 173 | if (result.data.id) { 174 | resourceList.push(result.data.id); 175 | } 176 | console.log('upload media success:', result.data.id, media_url); 177 | } 178 | const tweet_url = `https://twitter.com/${tweet.user_screen_name}/status/${tweet.id}`; 179 | const tweet_user_url = `https://twitter.com/${tweet.user_screen_name}`; 180 | const markdown = `${tweet.text}\n\n---\n${tweet_url}\n[@${tweet.user_name}](${tweet_user_url})\n${tweet.date}\n\n#tweet`; 181 | const newMemosResult = await newMemos(markdown, resourceList); 182 | dbJson.saved.push({ 183 | resourceList, 184 | tweet, 185 | }); 186 | fs.writeFileSync('db.json', JSON.stringify(dbJson)); 187 | console.log('new memos success:', newMemosResult?.id); 188 | } 189 | } catch (e) { 190 | // Display the error and quit 191 | console.error(e.message); 192 | // process.exit(1); 193 | } 194 | })(); 195 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "memos-bot", 3 | "scripts": { 4 | "start": "node index.js" 5 | }, 6 | "dependencies": { 7 | "dotenv": "^16.0.3", 8 | "form-data": "^4.0.0", 9 | "https-proxy-agent": "^5.0.1", 10 | "isomorphic-fetch": "^3.0.0", 11 | "twitter-api-v2": "^1.14.2" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # This file is generated by running "yarn install" inside your project. 2 | # Manual changes might be lost - proceed with caution! 3 | 4 | __metadata: 5 | version: 6 6 | cacheKey: 8 7 | 8 | "agent-base@npm:6": 9 | version: 6.0.2 10 | resolution: "agent-base@npm:6.0.2" 11 | dependencies: 12 | debug: 4 13 | checksum: f52b6872cc96fd5f622071b71ef200e01c7c4c454ee68bc9accca90c98cfb39f2810e3e9aa330435835eedc8c23f4f8a15267f67c6e245d2b33757575bdac49d 14 | languageName: node 15 | linkType: hard 16 | 17 | "asynckit@npm:^0.4.0": 18 | version: 0.4.0 19 | resolution: "asynckit@npm:0.4.0" 20 | checksum: 7b78c451df768adba04e2d02e63e2d0bf3b07adcd6e42b4cf665cb7ce899bedd344c69a1dcbce355b5f972d597b25aaa1c1742b52cffd9caccb22f348114f6be 21 | languageName: node 22 | linkType: hard 23 | 24 | "combined-stream@npm:^1.0.8": 25 | version: 1.0.8 26 | resolution: "combined-stream@npm:1.0.8" 27 | dependencies: 28 | delayed-stream: ~1.0.0 29 | checksum: 49fa4aeb4916567e33ea81d088f6584749fc90c7abec76fd516bf1c5aa5c79f3584b5ba3de6b86d26ddd64bae5329c4c7479343250cfe71c75bb366eae53bb7c 30 | languageName: node 31 | linkType: hard 32 | 33 | "debug@npm:4": 34 | version: 4.3.4 35 | resolution: "debug@npm:4.3.4" 36 | dependencies: 37 | ms: 2.1.2 38 | peerDependenciesMeta: 39 | supports-color: 40 | optional: true 41 | checksum: 3dbad3f94ea64f34431a9cbf0bafb61853eda57bff2880036153438f50fb5a84f27683ba0d8e5426bf41a8c6ff03879488120cf5b3a761e77953169c0600a708 42 | languageName: node 43 | linkType: hard 44 | 45 | "delayed-stream@npm:~1.0.0": 46 | version: 1.0.0 47 | resolution: "delayed-stream@npm:1.0.0" 48 | checksum: 46fe6e83e2cb1d85ba50bd52803c68be9bd953282fa7096f51fc29edd5d67ff84ff753c51966061e5ba7cb5e47ef6d36a91924eddb7f3f3483b1c560f77a0020 49 | languageName: node 50 | linkType: hard 51 | 52 | "dotenv@npm:^16.0.3": 53 | version: 16.0.3 54 | resolution: "dotenv@npm:16.0.3" 55 | checksum: afcf03f373d7a6d62c7e9afea6328e62851d627a4e73f2e12d0a8deae1cd375892004f3021883f8aec85932cd2834b091f568ced92b4774625b321db83b827f8 56 | languageName: node 57 | linkType: hard 58 | 59 | "form-data@npm:^4.0.0": 60 | version: 4.0.0 61 | resolution: "form-data@npm:4.0.0" 62 | dependencies: 63 | asynckit: ^0.4.0 64 | combined-stream: ^1.0.8 65 | mime-types: ^2.1.12 66 | checksum: 01135bf8675f9d5c61ff18e2e2932f719ca4de964e3be90ef4c36aacfc7b9cb2fceb5eca0b7e0190e3383fe51c5b37f4cb80b62ca06a99aaabfcfd6ac7c9328c 67 | languageName: node 68 | linkType: hard 69 | 70 | "https-proxy-agent@npm:^5.0.1": 71 | version: 5.0.1 72 | resolution: "https-proxy-agent@npm:5.0.1" 73 | dependencies: 74 | agent-base: 6 75 | debug: 4 76 | checksum: 571fccdf38184f05943e12d37d6ce38197becdd69e58d03f43637f7fa1269cf303a7d228aa27e5b27bbd3af8f09fd938e1c91dcfefff2df7ba77c20ed8dfc765 77 | languageName: node 78 | linkType: hard 79 | 80 | "isomorphic-fetch@npm:^3.0.0": 81 | version: 3.0.0 82 | resolution: "isomorphic-fetch@npm:3.0.0" 83 | dependencies: 84 | node-fetch: ^2.6.1 85 | whatwg-fetch: ^3.4.1 86 | checksum: e5ab79a56ce5af6ddd21265f59312ad9a4bc5a72cebc98b54797b42cb30441d5c5f8d17c5cd84a99e18101c8af6f90c081ecb8d12fd79e332be1778d58486d75 87 | languageName: node 88 | linkType: hard 89 | 90 | "memos-bot@workspace:.": 91 | version: 0.0.0-use.local 92 | resolution: "memos-bot@workspace:." 93 | dependencies: 94 | dotenv: ^16.0.3 95 | form-data: ^4.0.0 96 | https-proxy-agent: ^5.0.1 97 | isomorphic-fetch: ^3.0.0 98 | twitter-api-v2: ^1.14.2 99 | languageName: unknown 100 | linkType: soft 101 | 102 | "mime-db@npm:1.52.0": 103 | version: 1.52.0 104 | resolution: "mime-db@npm:1.52.0" 105 | checksum: 0d99a03585f8b39d68182803b12ac601d9c01abfa28ec56204fa330bc9f3d1c5e14beb049bafadb3dbdf646dfb94b87e24d4ec7b31b7279ef906a8ea9b6a513f 106 | languageName: node 107 | linkType: hard 108 | 109 | "mime-types@npm:^2.1.12": 110 | version: 2.1.35 111 | resolution: "mime-types@npm:2.1.35" 112 | dependencies: 113 | mime-db: 1.52.0 114 | checksum: 89a5b7f1def9f3af5dad6496c5ed50191ae4331cc5389d7c521c8ad28d5fdad2d06fd81baf38fed813dc4e46bb55c8145bb0ff406330818c9cf712fb2e9b3836 115 | languageName: node 116 | linkType: hard 117 | 118 | "ms@npm:2.1.2": 119 | version: 2.1.2 120 | resolution: "ms@npm:2.1.2" 121 | checksum: 673cdb2c3133eb050c745908d8ce632ed2c02d85640e2edb3ace856a2266a813b30c613569bf3354fdf4ea7d1a1494add3bfa95e2713baa27d0c2c71fc44f58f 122 | languageName: node 123 | linkType: hard 124 | 125 | "node-fetch@npm:^2.6.1": 126 | version: 2.6.9 127 | resolution: "node-fetch@npm:2.6.9" 128 | dependencies: 129 | whatwg-url: ^5.0.0 130 | peerDependencies: 131 | encoding: ^0.1.0 132 | peerDependenciesMeta: 133 | encoding: 134 | optional: true 135 | checksum: acb04f9ce7224965b2b59e71b33c639794d8991efd73855b0b250921382b38331ffc9d61bce502571f6cc6e11a8905ca9b1b6d4aeb586ab093e2756a1fd190d0 136 | languageName: node 137 | linkType: hard 138 | 139 | "tr46@npm:~0.0.3": 140 | version: 0.0.3 141 | resolution: "tr46@npm:0.0.3" 142 | checksum: 726321c5eaf41b5002e17ffbd1fb7245999a073e8979085dacd47c4b4e8068ff5777142fc6726d6ca1fd2ff16921b48788b87225cbc57c72636f6efa8efbffe3 143 | languageName: node 144 | linkType: hard 145 | 146 | "twitter-api-v2@npm:^1.14.2": 147 | version: 1.14.2 148 | resolution: "twitter-api-v2@npm:1.14.2" 149 | checksum: e9bd9cc58f2b834df32f971b60fa338c8e38b2db2e4b4d3eb31dc34f92782e89a5f530bfa369aa38ae88da9d66ed0e754bd47a7e4a1b0393075980fd12d31eb7 150 | languageName: node 151 | linkType: hard 152 | 153 | "webidl-conversions@npm:^3.0.0": 154 | version: 3.0.1 155 | resolution: "webidl-conversions@npm:3.0.1" 156 | checksum: c92a0a6ab95314bde9c32e1d0a6dfac83b578f8fa5f21e675bc2706ed6981bc26b7eb7e6a1fab158e5ce4adf9caa4a0aee49a52505d4d13c7be545f15021b17c 157 | languageName: node 158 | linkType: hard 159 | 160 | "whatwg-fetch@npm:^3.4.1": 161 | version: 3.6.2 162 | resolution: "whatwg-fetch@npm:3.6.2" 163 | checksum: ee976b7249e7791edb0d0a62cd806b29006ad7ec3a3d89145921ad8c00a3a67e4be8f3fb3ec6bc7b58498724fd568d11aeeeea1f7827e7e1e5eae6c8a275afed 164 | languageName: node 165 | linkType: hard 166 | 167 | "whatwg-url@npm:^5.0.0": 168 | version: 5.0.0 169 | resolution: "whatwg-url@npm:5.0.0" 170 | dependencies: 171 | tr46: ~0.0.3 172 | webidl-conversions: ^3.0.0 173 | checksum: b8daed4ad3356cc4899048a15b2c143a9aed0dfae1f611ebd55073310c7b910f522ad75d727346ad64203d7e6c79ef25eafd465f4d12775ca44b90fa82ed9e2c 174 | languageName: node 175 | linkType: hard 176 | --------------------------------------------------------------------------------