├── .gitignore ├── package.json ├── tsconfig.json ├── LICENSE.md ├── README.md ├── logstash.conf └── src └── index.ts /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | package-lock.json 4 | .vscode 5 | src/data -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "discord-logs", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "ts-node src/index.ts", 8 | "build": "tsc" 9 | }, 10 | "author": "", 11 | "license": "MIT", 12 | "dependencies": { 13 | "request": "^2.88.0" 14 | }, 15 | "devDependencies": { 16 | "@types/node": "^10.7.1", 17 | "@types/request": "^2.47.1", 18 | "ts-node": "^5.0.1", 19 | "typescript": "^3.0.1" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es5", 5 | "lib": [ 6 | "es6", 7 | "es2015" 8 | ], 9 | "moduleResolution": "node", 10 | "rootDir": "src", 11 | "outDir": "dist", 12 | "alwaysStrict": true, 13 | "allowJs": false, 14 | "noImplicitAny": true, 15 | "noUnusedLocals": true, 16 | "noImplicitThis": true, 17 | "strictNullChecks": true, 18 | "noImplicitReturns": true, 19 | "preserveConstEnums": true, 20 | "suppressImplicitAnyIndexErrors": true, 21 | "forceConsistentCasingInFileNames": true, 22 | "strict": true 23 | }, 24 | "include": [ 25 | "src" 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Mihail Cristian Dumitru 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Discord logs 2 | 3 | Download all your Discord messages locally in `json` files. 4 | 5 | ## Installation 6 | 7 | - Install the dependencies 8 | 9 | ```bash 10 | $ npm install 11 | ``` 12 | 13 | ## Running the project 14 | 15 | ```bash 16 | $ DATA_PATH= TOKEN= npm start 17 | ``` 18 | 19 | | Variable | Description | Required | 20 | | --- | --- | --- | 21 | | `DATA_PATH` | Path to the directory where the messages will be stored. | `true` | 22 | | `TOKEN` | User's authorization token. Used for the API calls when getting the messages. | `true` | 23 | | `SYNC_GUILDS` | `Optional`. If you also want to synchronize the messages of a guild, you can include the ids separated by comma, e.g. `id1,id2,id3`. | `false` | 24 | | `INTERVAL` | `Optional`. You can specity the interval, in miliseconds, when the messages should be synchronized, e.g. every 10 minutes. If not specified, the script will exit after synchronizing once. | `false` | 25 | 26 | You can also build the project if you don't want to rely on `ts-node`. 27 | 28 | ```bash 29 | $ npm run build 30 | $ DATA_PATH= TOKEN= node dist/index.js 31 | ``` 32 | 33 | ## Elasticsearch 34 | 35 | If you want to index the messages in `Elasticsearch`, you can find an example `Logstash` config in `logstash.conf`. 36 | 37 | **Note**: The example config uses the `Prune` plugin which doesn't come installed by default. To install it, run from the Logstash installation folder: 38 | 39 | ```bash 40 | ./bin/logstash-plugin install logstash-filter-prune 41 | ``` 42 | 43 | More details [here](https://www.elastic.co/guide/en/logstash/current/plugins-filters-prune.html#_installation_59). 44 | 45 | ## License 46 | 47 | Open sourced under the [MIT license](./LICENSE.md). 48 | -------------------------------------------------------------------------------- /logstash.conf: -------------------------------------------------------------------------------- 1 | # Example logstash config which you can use to index all messages in elasticsearch 2 | # Replace below to the path where you downloaded the Discord messages 3 | 4 | input { 5 | # Channels 6 | file { 7 | id => "channels" 8 | codec => json 9 | path => "/channels/*.json" 10 | start_position => "beginning" 11 | close_older => 1 12 | tags => ["channel"] 13 | } 14 | 15 | # Messages 16 | file { 17 | id => "messages" 18 | codec => json 19 | path => "/messages/*.json" 20 | start_position => "beginning" 21 | close_older => 1 22 | tags => ["message"] 23 | } 24 | } 25 | 26 | filter { 27 | # ignore invalid json 28 | if "_jsonparsefailure" in [tags] { 29 | drop { } 30 | } 31 | 32 | if "message" in [tags] { 33 | # replace default timestamp with the message timestamp 34 | date { 35 | match => ["timestamp", "ISO8601"] 36 | target => "@timestamp" 37 | } 38 | 39 | # set @type 40 | mutate { 41 | add_field => { "@type" => "message" } 42 | } 43 | 44 | prune { 45 | whitelist_names => ["id", "@timestamp", "@type", "content", "channel_id", "author", "attachments"] 46 | } 47 | 48 | # add the channel name to the message (slow and depends on order, e.g. channel must be indexed before messages) 49 | # elasticsearch { 50 | # hosts => ["localhost:9200"] 51 | # enable_sort => false 52 | # query => "id:%{channel_id}" 53 | # fields => { "name" => "channel_name" } 54 | # } 55 | } 56 | 57 | if "channel" in [tags] { 58 | # set @type 59 | mutate { 60 | add_field => { "@type" => "channel" } 61 | } 62 | 63 | prune { 64 | whitelist_names => ["id", "@timestamp", "@type", "name", "parent_id", "guild_id", "recipients", "owner_id", "last_message_id"] 65 | } 66 | } 67 | } 68 | 69 | output { 70 | elasticsearch { 71 | hosts => ["localhost:9200"] 72 | 73 | # replace the default _id with the message/channel id 74 | document_id => "%{[id]}" 75 | } 76 | stdout { codec => rubydebug } 77 | } 78 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import * as request from "request"; 3 | 4 | export interface DiscordOptions { 5 | token?: string; 6 | settings?: DiscordSettings; 7 | } 8 | 9 | export interface DiscordSettings { 10 | baseUrl: string; 11 | } 12 | 13 | export interface Filter { 14 | around?: string; 15 | before?: string; 16 | after?: string; 17 | limit?: number; 18 | } 19 | 20 | export interface LoginRequest { 21 | email: string; 22 | password: string; 23 | } 24 | 25 | export interface Credentials { 26 | token: string; 27 | } 28 | 29 | const defaultSettings: DiscordSettings = { 30 | baseUrl: "https://discordapp.com/api/v6", 31 | }; 32 | 33 | export class DiscordAPI { 34 | protected token: string = ""; 35 | protected settings: DiscordSettings = defaultSettings; 36 | 37 | constructor(options?: DiscordOptions) { 38 | if (options) { 39 | if (options.token) { 40 | this.token = options.token; 41 | } 42 | if (options.settings) { 43 | this.settings = options.settings; 44 | } 45 | } 46 | } 47 | 48 | public request(options: (request.UriOptions & request.CoreOptions)): Promise { 49 | return new Promise((fulfill, reject) => { 50 | return request(options, (err, res, body) => { 51 | if (err) return reject(err); 52 | if (res.statusCode !== 200) { 53 | return reject(new Error(`Unexpected status code ${res.statusCode}`)); 54 | } 55 | return fulfill(body); 56 | }); 57 | }); 58 | } 59 | 60 | public setToken(token: string) { 61 | this.token = token; 62 | } 63 | 64 | public login(req: LoginRequest): Promise { 65 | return this.request({ 66 | baseUrl: this.settings.baseUrl, 67 | uri: `/auth/login`, 68 | method: "POST", 69 | gzip: true, 70 | json: true, 71 | body: req, 72 | }); 73 | } 74 | 75 | public getDMChannels(): Promise { 76 | return this.request({ 77 | baseUrl: this.settings.baseUrl, 78 | uri: `/users/@me/channels`, 79 | method: "GET", 80 | headers: { 81 | "Authorization": this.token, 82 | }, 83 | gzip: true, 84 | json: true, 85 | }); 86 | } 87 | 88 | public getChannelMessages(id: string, filter?: Filter): Promise { 89 | return this.request({ 90 | baseUrl: this.settings.baseUrl, 91 | uri: `/channels/${id}/messages`, 92 | method: "GET", 93 | headers: { 94 | "Authorization": this.token, 95 | }, 96 | gzip: true, 97 | json: true, 98 | qs: filter, 99 | }); 100 | } 101 | 102 | public getGuilds(): Promise { 103 | return this.request({ 104 | baseUrl: this.settings.baseUrl, 105 | uri: `/users/@me/guilds`, 106 | method: "GET", 107 | headers: { 108 | "Authorization": this.token, 109 | }, 110 | gzip: true, 111 | json: true, 112 | }); 113 | } 114 | 115 | public getGuildChannels(id: string): Promise { 116 | return this.request({ 117 | baseUrl: this.settings.baseUrl, 118 | uri: `/guilds/${id}/channels`, 119 | method: "GET", 120 | headers: { 121 | "Authorization": this.token, 122 | }, 123 | gzip: true, 124 | json: true, 125 | }); 126 | } 127 | 128 | } 129 | 130 | if (!process.env.DATA_PATH) { 131 | throw new Error("DATA_PATH environment variable not set"); 132 | } 133 | 134 | if (!process.env.TOKEN) { 135 | throw new Error("TOKEN environment variable not set"); 136 | } 137 | 138 | const client = new DiscordAPI({ 139 | token: process.env.TOKEN, 140 | }); 141 | 142 | async function main() { 143 | const limit = 100; 144 | 145 | const messagesPath = `${process.env.DATA_PATH}/messages`; 146 | const channelsPath = `${process.env.DATA_PATH}/channels`; 147 | 148 | console.log(`Synchronizing messages...`); 149 | 150 | // get the user's DM channels 151 | let channels = await client.getDMChannels(); 152 | 153 | // if we also need to sync guilds 154 | if (process.env.SYNC_GUILDS) { 155 | const guildIds = process.env.SYNC_GUILDS.split(","); 156 | 157 | // get the user's guilds 158 | const guilds = await client.getGuilds(); 159 | 160 | // search for whitelisted guilds 161 | for (const guild of guilds) { 162 | for (const guildId of guildIds) { 163 | if (guild.id === guildId) { 164 | // get the guild's channels and add them to the DMs 165 | const guildChannels = await client.getGuildChannels(guild.id); 166 | channels = channels.concat(guildChannels); 167 | } 168 | } 169 | } 170 | } 171 | 172 | console.log(`Got ${channels.length} channels`); 173 | 174 | for (const channel of channels) { 175 | console.log(`Processing ${channel.id} (${channel.name}) channel`); 176 | 177 | // check if we already had the channel 178 | if (fs.existsSync(`${channelsPath}/${channel.id}.json`)) { 179 | // if the channel already existed, we need to check if there are any missing messages 180 | const existingChannel = JSON.parse(fs.readFileSync(`${channelsPath}/${channel.id}.json`, "utf-8")); 181 | 182 | // if the last_message_id matches 183 | if (existingChannel.last_message_id === channel.last_message_id) { 184 | console.log(`Channel ${channel.id} (${channel.name}) is already synchronized`); 185 | // messages for this channel are already synced 186 | continue; 187 | } 188 | 189 | console.log(`Channel ${channel.id} (${channel.name}) exists but is missing messages. Synchronizing...`); 190 | 191 | // otherwise we need to download the missing ones 192 | // start at existingChannel.last_message_id 193 | let after = existingChannel.last_message_id; 194 | do { 195 | // get the messages 196 | const messages = await client.getChannelMessages(channel.id, { 197 | limit, 198 | after, 199 | }); 200 | 201 | console.log(`Got ${messages.length} messages for channel ${channel.id} (${channel.name})`); 202 | 203 | // save them to disk 204 | for (const message of messages) { 205 | fs.writeFileSync(`${messagesPath}/${message.id}.json`, `${JSON.stringify(message)}${"\n"}`); 206 | // the \n is necessary for logstash to work correctly, also don't add formatting to the json 207 | } 208 | 209 | // check if we need to get more messages for this channel 210 | if (messages.length === limit) { 211 | after = messages[0].id; 212 | } else { 213 | break; 214 | } 215 | } while (true); 216 | } else { 217 | console.log(`New channel ${channel.id} (${channel.name}). Synchronizing...`); 218 | 219 | // if we don't have it, then we need to synchronize all messages 220 | let before: string | undefined; 221 | do { 222 | // get the messages 223 | const messages = await client.getChannelMessages(channel.id, { 224 | limit, 225 | before, 226 | }); 227 | 228 | console.log(`Got ${messages.length} messages for channel ${channel.id} (${channel.name})`); 229 | 230 | // save them to disk 231 | for (const message of messages) { 232 | fs.writeFileSync(`${messagesPath}/${message.id}.json`, `${JSON.stringify(message)}${"\n"}`); 233 | } 234 | 235 | // check if we need to get more messages for this channel 236 | if (messages.length === limit) { 237 | before = messages[messages.length - 1].id; 238 | } else { 239 | break; 240 | } 241 | } while (true); 242 | } 243 | 244 | // when we're done synchronizing the messages, also save the channel 245 | fs.writeFileSync(`${channelsPath}/${channel.id}.json`, `${JSON.stringify(channel)}${"\n"}`); 246 | } 247 | 248 | console.log("All messages synchronized"); 249 | } 250 | 251 | let interval: number; 252 | let isProcessing = false; 253 | 254 | async function loop() { 255 | if (!isProcessing) { 256 | isProcessing = true; 257 | await main(); 258 | isProcessing = false; 259 | } 260 | 261 | setTimeout(loop, interval); 262 | } 263 | 264 | if (process.env.INTERVAL) { 265 | interval = parseInt(process.env.INTERVAL); 266 | 267 | if (isNaN(interval)) { 268 | throw new Error(`Invalid INTERVAL: ${process.env.INTERVAL}`); 269 | } 270 | 271 | loop(); 272 | } else { 273 | main(); 274 | } 275 | --------------------------------------------------------------------------------