├── .gitignore ├── README.md ├── package-lock.json ├── package.json ├── src ├── bot.ts ├── bot_tokens.ts ├── datatypes.d.ts ├── encode.ts ├── index.ts ├── read_messages.ts ├── scraperbot.ts └── utils.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Local folders 2 | encoded_data 3 | logging 4 | dist 5 | 6 | # Dont leak your tokens :P 7 | src/bot_tokens.ts 8 | 9 | # Server-data 10 | messages.json 11 | encode_test.txt 12 | encode_test_channel.txt 13 | 14 | .vscode 15 | 16 | # Logs 17 | logs 18 | *.log 19 | npm-debug.log* 20 | yarn-debug.log* 21 | yarn-error.log* 22 | lerna-debug.log* 23 | 24 | # Diagnostic reports (https://nodejs.org/api/report.html) 25 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 26 | 27 | # Runtime data 28 | pids 29 | *.pid 30 | *.seed 31 | *.pid.lock 32 | 33 | # Directory for instrumented libs generated by jscoverage/JSCover 34 | lib-cov 35 | 36 | # Coverage directory used by tools like istanbul 37 | coverage 38 | *.lcov 39 | 40 | # nyc test coverage 41 | .nyc_output 42 | 43 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 44 | .grunt 45 | 46 | # Bower dependency directory (https://bower.io/) 47 | bower_components 48 | 49 | # node-waf configuration 50 | .lock-wscript 51 | 52 | # Compiled binary addons (https://nodejs.org/api/addons.html) 53 | build/Release 54 | 55 | # Dependency directories 56 | node_modules/ 57 | jspm_packages/ 58 | 59 | # TypeScript v1 declaration files 60 | typings/ 61 | 62 | # TypeScript cache 63 | *.tsbuildinfo 64 | 65 | # Optional npm cache directory 66 | .npm 67 | 68 | # Optional eslint cache 69 | .eslintcache 70 | 71 | # Optional REPL history 72 | .node_repl_history 73 | 74 | # Output of 'npm pack' 75 | *.tgz 76 | 77 | # Yarn Integrity file 78 | .yarn-integrity 79 | 80 | # dotenv environment variables file 81 | .env 82 | .env.test 83 | 84 | # parcel-bundler cache (https://parceljs.org/) 85 | .cache 86 | 87 | # next.js build output 88 | .next 89 | 90 | # nuxt.js build output 91 | .nuxt 92 | 93 | # vuepress build output 94 | .vuepress/dist 95 | 96 | # Serverless directories 97 | .serverless/ 98 | 99 | # FuseBox cache 100 | .fusebox/ 101 | 102 | # DynamoDB Local files 103 | .dynamodb/ 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Discord-GPT2-Bot 2 | 3 | Originally made as an April Fool's joke in the Hopson Community discord. 4 | 5 | This Discord bot is capable of scraping messages from a server and encode them for GPT-2 finetuning. It then controls a series of bots that can react to a conversation in a given server. 6 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scrapin", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@discordjs/collection": { 8 | "version": "0.1.6", 9 | "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-0.1.6.tgz", 10 | "integrity": "sha512-utRNxnd9kSS2qhyivo9lMlt5qgAUasH2gb7BEOn6p0efFh24gjGomHzWKMAPn2hEReOPQZCJaRKoURwRotKucQ==" 11 | }, 12 | "@discordjs/form-data": { 13 | "version": "3.0.1", 14 | "resolved": "https://registry.npmjs.org/@discordjs/form-data/-/form-data-3.0.1.tgz", 15 | "integrity": "sha512-ZfFsbgEXW71Rw/6EtBdrP5VxBJy4dthyC0tpQKGKmYFImlmmrykO14Za+BiIVduwjte0jXEBlhSKf0MWbFp9Eg==", 16 | "requires": { 17 | "asynckit": "^0.4.0", 18 | "combined-stream": "^1.0.8", 19 | "mime-types": "^2.1.12" 20 | } 21 | }, 22 | "@types/node": { 23 | "version": "14.14.37", 24 | "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.37.tgz", 25 | "integrity": "sha512-XYmBiy+ohOR4Lh5jE379fV2IU+6Jn4g5qASinhitfyO71b/sCo6MKsMLF5tc7Zf2CE8hViVQyYSobJNke8OvUw==", 26 | "dev": true 27 | }, 28 | "abort-controller": { 29 | "version": "3.0.0", 30 | "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", 31 | "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", 32 | "requires": { 33 | "event-target-shim": "^5.0.0" 34 | } 35 | }, 36 | "asynckit": { 37 | "version": "0.4.0", 38 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 39 | "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" 40 | }, 41 | "axios": { 42 | "version": "0.21.1", 43 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", 44 | "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", 45 | "requires": { 46 | "follow-redirects": "^1.10.0" 47 | } 48 | }, 49 | "combined-stream": { 50 | "version": "1.0.8", 51 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 52 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 53 | "requires": { 54 | "delayed-stream": "~1.0.0" 55 | } 56 | }, 57 | "delayed-stream": { 58 | "version": "1.0.0", 59 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 60 | "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" 61 | }, 62 | "discord.js": { 63 | "version": "12.5.1", 64 | "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-12.5.1.tgz", 65 | "integrity": "sha512-VwZkVaUAIOB9mKdca0I5MefPMTQJTNg0qdgi1huF3iwsFwJ0L5s/Y69AQe+iPmjuV6j9rtKoG0Ta0n9vgEIL6w==", 66 | "requires": { 67 | "@discordjs/collection": "^0.1.6", 68 | "@discordjs/form-data": "^3.0.1", 69 | "abort-controller": "^3.0.0", 70 | "node-fetch": "^2.6.1", 71 | "prism-media": "^1.2.2", 72 | "setimmediate": "^1.0.5", 73 | "tweetnacl": "^1.0.3", 74 | "ws": "^7.3.1" 75 | } 76 | }, 77 | "event-target-shim": { 78 | "version": "5.0.1", 79 | "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", 80 | "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" 81 | }, 82 | "follow-redirects": { 83 | "version": "1.13.3", 84 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz", 85 | "integrity": "sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==" 86 | }, 87 | "mime-db": { 88 | "version": "1.46.0", 89 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", 90 | "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==" 91 | }, 92 | "mime-types": { 93 | "version": "2.1.29", 94 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", 95 | "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", 96 | "requires": { 97 | "mime-db": "1.46.0" 98 | } 99 | }, 100 | "node-fetch": { 101 | "version": "2.6.1", 102 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", 103 | "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" 104 | }, 105 | "prism-media": { 106 | "version": "1.2.9", 107 | "resolved": "https://registry.npmjs.org/prism-media/-/prism-media-1.2.9.tgz", 108 | "integrity": "sha512-UHCYuqHipbTR1ZsXr5eg4JUmHER8Ss4YEb9Azn+9zzJ7/jlTtD1h0lc4g6tNx3eMlB8Mp6bfll0LPMAV4R6r3Q==" 109 | }, 110 | "setimmediate": { 111 | "version": "1.0.5", 112 | "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", 113 | "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" 114 | }, 115 | "tweetnacl": { 116 | "version": "1.0.3", 117 | "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", 118 | "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==" 119 | }, 120 | "typescript": { 121 | "version": "4.2.3", 122 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.3.tgz", 123 | "integrity": "sha512-qOcYwxaByStAWrBf4x0fibwZvMRG+r4cQoTjbPtUlrWjBHbmCAww1i448U0GJ+3cNNEtebDteo/cHOR3xJ4wEw==", 124 | "dev": true 125 | }, 126 | "typescript-map": { 127 | "version": "0.1.0", 128 | "resolved": "https://registry.npmjs.org/typescript-map/-/typescript-map-0.1.0.tgz", 129 | "integrity": "sha512-py1csqjGZR7LoENWvr5aqN4jYFDkthQZ1sCuMBr0/NuRJ5sSs5RqopqZhE2gFbLesKDec+RE0sraYV2ebrDexQ==" 130 | }, 131 | "ws": { 132 | "version": "7.4.4", 133 | "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.4.tgz", 134 | "integrity": "sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==" 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scrapin", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "./dist/index.js", 6 | "scripts": { 7 | "run": "node ./dist/index.js" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "axios": "^0.21.1", 13 | "discord.js": "^12.5.1", 14 | "typescript-map": "^0.1.0" 15 | }, 16 | "devDependencies": { 17 | "@types/node": "^14.14.37", 18 | "typescript": "^4.2.3" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/bot.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Channel, 3 | Client, 4 | DMChannel, 5 | MessageManager, 6 | NewsChannel, 7 | TextChannel, 8 | } from 'discord.js'; 9 | import { getContext, convertOutputToJSon, isTextChannel } from './utils'; 10 | import { respond } from './index'; 11 | 12 | import axios from 'axios'; 13 | 14 | class Bot { 15 | username: string; 16 | client: Client; 17 | 18 | constructor(username: string, token: string) { 19 | this.username = username; 20 | 21 | this.client = new Client(); 22 | this.client.login(token); 23 | 24 | this.client.on('message', (msg) => { 25 | if (this.client.user && msg.mentions.has(this.client.user?.id)) { 26 | const prompt = 27 | msg.content.substr(msg.content.indexOf('>')) + 28 | ' <|EOL|>\n' + 29 | this.username; 30 | console.log(prompt); 31 | axios 32 | .post('http://192.168.0.67:5000/generate', { 33 | text: prompt, 34 | length: 120, 35 | }) 36 | .then((res) => { 37 | let data = JSON.stringify(res.data).substr(2); 38 | data = data.substr(0, data.length - 2); 39 | 40 | const obj = convertOutputToJSon(data, ''); 41 | 42 | if (isTextChannel(msg.channel)) { 43 | //msg.reply(JSON.stringify(obj)); 44 | respond(msg.channel, obj); 45 | } 46 | }) 47 | .catch(console.error); 48 | } 49 | }); 50 | } 51 | 52 | say(channel: TextChannel, msg: string) { 53 | this.client.channels.fetch(channel.id).then((c_channel) => { 54 | if (isTextChannel(c_channel)) { 55 | c_channel.send(msg); 56 | } 57 | }); 58 | } 59 | } 60 | 61 | export default Bot; 62 | -------------------------------------------------------------------------------- /src/bot_tokens.ts: -------------------------------------------------------------------------------- 1 | import Bot from './bot'; 2 | 3 | export function getAllUserBots(): Map { 4 | const bots = new Map(); 5 | bots.set( 6 | 'INSERT_NAME', // This corresponds to the name that appears in the encoded message 7 | new Bot( 8 | 'INSERT_NAME', 9 | 'BOT_TOKEN' // Bot token 10 | ) 11 | ); 12 | // Repeat for all the bots 13 | 14 | return bots; 15 | } 16 | 17 | export const masterBotToken = 'INSERT_MASTER_BOT_TOKEN'; 18 | -------------------------------------------------------------------------------- /src/datatypes.d.ts: -------------------------------------------------------------------------------- 1 | import { TSMap } from 'typescript-map'; 2 | 3 | export interface MessageData { 4 | id: string; 5 | content: string; 6 | author: string; 7 | timestamp: Date; 8 | } 9 | 10 | interface BotMessage { 11 | username: string; 12 | content: string; 13 | } 14 | 15 | export type MessageChannelData = Array; 16 | export type ServerMessageData = TSMap; 17 | -------------------------------------------------------------------------------- /src/encode.ts: -------------------------------------------------------------------------------- 1 | import { TSMap } from 'typescript-map'; 2 | import fs from 'fs'; 3 | import { 4 | MessageChannelData, 5 | MessageData, 6 | ServerMessageData, 7 | } from './datatypes'; 8 | 9 | const channeldivider = '=============================='; 10 | 11 | export function encode(data: ServerMessageData, fileName: string) { 12 | let filedata = channeldivider + '\n'; 13 | for (const [name, channeldata] of data.entries()) { 14 | filedata += '<|channelstart|> ' + name + '\n'; 15 | for (let msgIndex = channeldata.length - 1; msgIndex >= 0; msgIndex--) { 16 | const msg = channeldata[msgIndex] as MessageData; 17 | if (msg.content === '') { 18 | continue; 19 | } 20 | filedata += 21 | msg.author.replace(' ', '') + ': ' + msg.content + ' <|EOL|>\n'; 22 | } 23 | filedata += channeldivider + '\n'; 24 | } 25 | fs.writeFile(fileName, filedata, (err) => { 26 | if (err) console.error(`Error Encoding: ${err}`); 27 | console.log('Finished encoding'); 28 | }); 29 | return; 30 | } 31 | 32 | export function encodeWithChannelContext( 33 | data: ServerMessageData, 34 | fileName: string 35 | ) { 36 | let filedata = channeldivider + '\n'; 37 | for (const [name, channeldata] of data.entries()) { 38 | let channelNameToken = name as string; 39 | channelNameToken = '<|' + channelNameToken.toUpperCase() + '|> '; 40 | 41 | for (let msgIndex = channeldata.length - 1; msgIndex >= 0; msgIndex--) { 42 | const msg = channeldata[msgIndex] as MessageData; 43 | if (msg.content === '') { 44 | continue; 45 | } 46 | filedata += 47 | channelNameToken + 48 | msg.author.replace(' ', '') + 49 | ': ' + 50 | msg.content + 51 | ' <|EOL|>\n'; 52 | } 53 | filedata += channeldivider + '\n'; 54 | } 55 | fs.writeFile(fileName, filedata, (err) => { 56 | if (err) console.error(`Error Encoding: ${err}`); 57 | console.log('Finished encoding'); 58 | }); 59 | return; 60 | } 61 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Channel, 3 | Client, 4 | DMChannel, 5 | MessageManager, 6 | NewsChannel, 7 | TextChannel, 8 | } from 'discord.js'; 9 | import Bot from './bot'; 10 | import { getContext, convertOutputToJSon, delay, isTextChannel } from './utils'; 11 | 12 | import axios from 'axios'; 13 | import { BotMessage, MessageData } from './datatypes'; 14 | import { getAllUserBots, masterBotToken } from './bot_tokens'; 15 | import { loadScraperCommands } from './scraperbot'; 16 | 17 | const client = new Client(); 18 | loadScraperCommands(client); 19 | 20 | client.on('ready', () => { 21 | console.log('Hello im the master'); 22 | }); 23 | 24 | const bots = getAllUserBots(); 25 | 26 | export async function respond(channel: TextChannel, bot_msgs: BotMessage[]) { 27 | console.log('hey dude'); 28 | const should_others_talk = Math.random() * 100 > 50; 29 | let name_of_respondent = ''; 30 | 31 | // thinking wait 🤔 32 | await delay(Math.floor(Math.random() * 1000) + 1000); 33 | 34 | for (const msg of bot_msgs) { 35 | if (msg.username == '') continue; 36 | if (name_of_respondent == '') name_of_respondent = msg.username; 37 | if (!should_others_talk && msg.username != name_of_respondent) break; 38 | 39 | const user_bot = bots.get(msg.username); 40 | if (user_bot) { 41 | await delay(Math.floor(Math.random() * 2000) + 500); 42 | user_bot.say(channel, msg.content.substr(2)); 43 | } else { 44 | const other_bot = bots.get('Other'); 45 | await delay(Math.floor(Math.random() * 2000) + 500); 46 | if (other_bot) { 47 | other_bot.say(channel, `**${msg.username}**\n${msg.content.substr(2)}`); 48 | } 49 | } 50 | } 51 | } 52 | 53 | client.on('message', (msg) => { 54 | if (msg.author.id === '150049709698973696') { 55 | //if (msg.channel.id === '825416382560600124') { // Beware of Jack 56 | if (msg.content == '!get') { 57 | axios 58 | .get('http://192.168.0.67:5000/gen') 59 | .then((res) => { 60 | msg.reply(res.data); 61 | }) 62 | .catch(console.error); 63 | } 64 | if (msg.content.startsWith('!post')) { 65 | const prompt = msg.content.substr(6) + ' <|EOL|>'; 66 | console.log(prompt); 67 | axios 68 | .post('http://192.168.0.67:5000/generate', { 69 | text: prompt, 70 | length: 100, 71 | }) 72 | .then((res) => { 73 | let data = JSON.stringify(res.data).substr(2); 74 | data = data.substr(0, data.length - 2); 75 | 76 | msg.reply(JSON.stringify(convertOutputToJSon(data, prompt))); 77 | }) 78 | .catch(console.error); 79 | } 80 | if (msg.content.startsWith('!convert')) { 81 | if (!isTextChannel(msg.channel)) return; 82 | 83 | getContext(msg.channel, 3) 84 | .then((prompt) => { 85 | msg.channel.send(prompt); 86 | }) 87 | .catch(console.error); 88 | } 89 | if (msg.content.startsWith('!ask')) { 90 | } 91 | if (msg.content.startsWith('!discuss')) { 92 | const args = msg.content.split(' '); 93 | const channelName = args[1]; 94 | 95 | if (channelName === undefined) { 96 | msg.delete(); 97 | return; 98 | } 99 | 100 | const channel = msg.guild?.channels.cache.find( 101 | (channel) => channel.name == channelName 102 | ); 103 | if (channel && isTextChannel(channel)) { 104 | getContext(channel, 3) 105 | .then((prompt) => { 106 | axios 107 | .post('http://192.168.0.67:5000/generate', { 108 | text: prompt, 109 | length: 120, 110 | }) 111 | .then((res) => { 112 | let data = JSON.stringify(res.data).substr(2); 113 | data = data.substr(0, data.length - 2); 114 | 115 | const obj = convertOutputToJSon(data, prompt); 116 | 117 | if (isTextChannel(channel)) { 118 | //msg.reply(JSON.stringify(obj)); 119 | obj[0].username = obj[0].username.substr(1); 120 | respond(channel, obj); 121 | } 122 | }) 123 | .catch(console.error); 124 | }) 125 | .catch(console.error); 126 | } 127 | } 128 | } 129 | }); 130 | 131 | client 132 | .login(masterBotToken) 133 | .then(() => console.log('Logged in')) 134 | .catch((err) => console.error('Error logging in')); 135 | -------------------------------------------------------------------------------- /src/read_messages.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Channel, 3 | ChannelLogsQueryOptions, 4 | Collection, 5 | Message, 6 | TextChannel, 7 | } from 'discord.js'; 8 | import { MessageChannelData } from './datatypes'; 9 | import { isTextChannel } from './utils'; 10 | 11 | const maxlimit = 100; 12 | 13 | export async function readMessages( 14 | channel: TextChannel, 15 | channeldata: MessageChannelData, 16 | chunkNumber: number 17 | ) { 18 | // Get latest message gathered from this place 19 | let options: ChannelLogsQueryOptions = { limit: maxlimit }; 20 | if (channeldata.length > 0) { 21 | options = { ...options, before: channeldata[channeldata.length - 1].id }; 22 | } 23 | 24 | return channel.messages 25 | .fetch(options) 26 | .then(async (messages) => { 27 | await getMessageChunk(messages, chunkNumber, channeldata, true); 28 | }) 29 | .catch((err) => console.error(`[Error] ${err}`)); 30 | } 31 | 32 | export async function getMessageChunk( 33 | messages: Collection, 34 | chunksLeft: number, 35 | channeldata: MessageChannelData, 36 | log?: boolean 37 | ) { 38 | console.log(`Received ${messages.size} messages`); 39 | 40 | const earliest_msg = messages.last(); 41 | 42 | // TODO: Write all messages to JSON 43 | //const enddate = new Date(2021, 3, 8); 44 | for (const msg of messages.array()) { 45 | //if (msg.createdAt > enddate) { 46 | channeldata.push({ 47 | id: msg.id, 48 | author: msg.author.username, // Unsure of this (id?) 49 | content: msg.content, 50 | timestamp: msg.createdAt, 51 | }); 52 | //} 53 | } 54 | 55 | if (earliest_msg !== undefined) { 56 | if (log && isTextChannel(earliest_msg.channel)) { 57 | console.log( 58 | `Extracted chunk in ${earliest_msg.channel.name} Earliest message "${ 59 | earliest_msg.content 60 | }" from ${earliest_msg.createdAt.toISOString()}` 61 | ); 62 | } 63 | 64 | if (chunksLeft > 1) { 65 | const newMessages = await earliest_msg.channel.messages.fetch({ 66 | limit: maxlimit, 67 | before: earliest_msg.id, 68 | }); 69 | 70 | chunksLeft = chunksLeft - 1; 71 | await getMessageChunk(newMessages, chunksLeft, channeldata, log); 72 | } 73 | return; 74 | } 75 | 76 | throw new Error('Earliest message didnt work :/'); 77 | } 78 | -------------------------------------------------------------------------------- /src/scraperbot.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Channel, 3 | Client, 4 | DMChannel, 5 | MessageManager, 6 | NewsChannel, 7 | TextChannel, 8 | } from 'discord.js'; 9 | import { readMessages, getMessageChunk } from './read_messages'; 10 | import { 11 | ServerMessageData, 12 | MessageChannelData, 13 | MessageData, 14 | } from './datatypes'; 15 | import fs from 'fs'; 16 | import { TSMap } from 'typescript-map'; 17 | import { encode, encodeWithChannelContext } from './encode'; 18 | import { isTextChannel, tempMessage } from './utils'; 19 | 20 | const data: ServerMessageData = new TSMap(); 21 | 22 | export function loadScraperCommands(client: Client) { 23 | // Read json file in sync 24 | try { 25 | const msg_data = fs.readFileSync('./messages.json'); 26 | data.fromJSON(JSON.parse(msg_data.toString())); 27 | } catch (err) { 28 | console.log("Couldn't load messages.json, use !save to generate a new one"); 29 | } 30 | 31 | client.on('ready', () => { 32 | console.log('Scraper commands loaded'); 33 | }); 34 | 35 | client.on('message', (msg) => { 36 | //check its me 👀 37 | if (msg.author.id === '150049709698973696') { 38 | if (msg.content.startsWith('!gather')) { 39 | const args = msg.content.split(' '); 40 | const channelName = args[1]; 41 | const chunkNumber = parseInt(args[2]); 42 | 43 | console.log(`channel: ${channelName}`); 44 | 45 | // If invalid, show "Invalid command" for 3 seconds 46 | if (channelName === undefined || isNaN(chunkNumber)) { 47 | tempMessage('Invalid command', msg.channel, 3000); 48 | msg.delete(); 49 | return; 50 | } 51 | 52 | const channel = msg.guild?.channels.cache.find( 53 | (channel) => channel.name == channelName 54 | ); 55 | const now = new Date(); 56 | 57 | if (channel && isTextChannel(channel)) { 58 | // check server message data if channel is already saved 59 | let channeldata = data.get(channel.name); 60 | if (channeldata === undefined) { 61 | channeldata = new Array(); 62 | } 63 | 64 | readMessages(channel, channeldata, chunkNumber) 65 | .catch((err) => 66 | console.error(`[Error] Extracting messages: ${err}`) 67 | ) 68 | .then(() => { 69 | if (isTextChannel(channel) && channeldata !== undefined) { 70 | data.set(channel.name, channeldata); 71 | } 72 | /*const finishtime = new Date(); 73 | const diff = finishtime.getDate() - now.getDate(); 74 | const mindiff = diff / (1000 * 60);*/ 75 | msg.reply( 76 | `Finished gathering ${chunkNumber} chunk(s) in #${channel.name}.` 77 | ); 78 | msg.delete(); 79 | }); 80 | } else { 81 | tempMessage("Couldn't find that channel :/", msg.channel, 5000); 82 | } 83 | } 84 | if (msg.content === '!save') { 85 | fs.writeFile( 86 | './messages.json', 87 | JSON.stringify(data.toJSON()), 88 | (err) => { 89 | if (err) msg.channel.send(`Error saving: ${err}`); 90 | else tempMessage('Saved JSON', msg.channel, 3000); 91 | } 92 | ); 93 | msg.delete(); 94 | } 95 | if (msg.content == '!check') { 96 | tempMessage(JSON.stringify(data.toJSON()), msg.channel, 30000); 97 | msg.delete(); 98 | } 99 | if (msg.content == '!stats') { 100 | let finalmsg = ''; 101 | let totalmsg = 0; 102 | for (const [name, channeldata] of data.entries()) { 103 | const channellength = channeldata.length; 104 | finalmsg += `#${name} has ${channellength} messages gathered\n`; 105 | totalmsg += channellength; 106 | } 107 | finalmsg += `That's ${totalmsg} in total`; 108 | msg.channel.send(finalmsg); 109 | } 110 | if (msg.content == '!encode') { 111 | encode(data, './encode_test.txt'); 112 | encodeWithChannelContext(data, './encode_test_channel.txt'); 113 | } 114 | // (fuck jack) 115 | if (msg.content == 'peng') { 116 | msg.channel.send( 117 | 'https://tenor.com/view/jack-pc-drop-poonani-aired-gif-19837340' 118 | ); 119 | msg.delete(); 120 | } 121 | } 122 | 123 | // Easter eggs xddddddddddddddd 124 | if (msg.content.startsWith('owo')) { 125 | msg.reply('uwu'); 126 | } else if (msg.content.startsWith('!help')) { 127 | msg.reply('fuck you'); 128 | } else if (msg.content.startsWith('bleep')) { 129 | msg.reply('bloop'); 130 | } else if (msg.content.startsWith('pain')) { 131 | msg.channel.send('AHHHHHHHHHHHHHHHHHH'); 132 | msg.delete(); 133 | } 134 | }); 135 | } 136 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import { Channel, DMChannel, NewsChannel, TextChannel } from 'discord.js'; 2 | 3 | interface BotMessage { 4 | username: string; 5 | content: string; 6 | } 7 | 8 | // Gets the last couple of messages and encodes them in a way the bot understands 9 | export function getContext( 10 | channel: TextChannel, 11 | size: number 12 | ): Promise { 13 | return channel.messages.fetch({ limit: 3 }).then((msgs) => { 14 | let prompt = ''; 15 | msgs 16 | .array() 17 | .reverse() 18 | .map((msg) => { 19 | prompt += 20 | msg.author.username + 21 | ': ' + 22 | msg.content.split('\\n').join('\n') + 23 | ' <|EOL|>\n'; 24 | }); 25 | return prompt; 26 | }); 27 | } 28 | 29 | export function convertOutputToJSon( 30 | output_str: string, 31 | prompt: string 32 | ): BotMessage[] { 33 | const trimmed_str = output_str.substr(prompt.length + 2); 34 | const msgs = trimmed_str.split('<|EOL|>\\n'); 35 | const bot_msgs: BotMessage[] = []; 36 | for (const msg of msgs) { 37 | const bot_msg: BotMessage = { 38 | username: msg.substr(0, msg.indexOf(':')), 39 | content: msg.substr(msg.indexOf(':')), 40 | }; 41 | bot_msgs.push(bot_msg); 42 | } 43 | bot_msgs.pop(); 44 | return bot_msgs; 45 | } 46 | 47 | export function tempMessage( 48 | message: string, 49 | channel: TextChannel | DMChannel | NewsChannel, 50 | length: number 51 | ) { 52 | const imsg = channel.send(message); 53 | imsg.then(async (imsg) => { 54 | await delay(length); 55 | imsg.delete(); 56 | }); 57 | } 58 | 59 | export function isTextChannel(channel: Channel): channel is TextChannel { 60 | return channel.type === 'text'; 61 | } 62 | 63 | export function delay(ms: number) { 64 | return new Promise((resolve) => setTimeout(resolve, ms)); 65 | } 66 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 4 | 5 | /* Basic Options */ 6 | // "incremental": true, /* Enable incremental compilation */ 7 | "target": "es5" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, 8 | "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, 9 | // "lib": [], /* Specify library files to be included in the compilation. */ 10 | // "allowJs": true, /* Allow javascript files to be compiled. */ 11 | // "checkJs": true, /* Report errors in .js files. */ 12 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ 13 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 14 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 15 | "sourceMap": true /* Generates corresponding '.map' file. */, 16 | // "outFile": "./", /* Concatenate and emit output to single file. */ 17 | "outDir": "./dist/" /* Redirect output structure to the directory. */, 18 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 19 | // "composite": true, /* Enable project compilation */ 20 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 21 | // "removeComments": true, /* Do not emit comments to output. */ 22 | // "noEmit": true, /* Do not emit outputs. */ 23 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 24 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 25 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 26 | 27 | /* Strict Type-Checking Options */ 28 | "strict": true /* Enable all strict type-checking options. */, 29 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 30 | // "strictNullChecks": true, /* Enable strict null checks. */ 31 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 32 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 33 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 34 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 35 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 36 | 37 | /* Additional Checks */ 38 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 39 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 40 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 41 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 42 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 43 | // "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */ 44 | 45 | /* Module Resolution Options */ 46 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 47 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 48 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 49 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 50 | // "typeRoots": [], /* List of folders to include type definitions from. */ 51 | // "types": [], /* Type declaration files to be included in compilation. */ 52 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 53 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, 54 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 55 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 56 | 57 | /* Source Map Options */ 58 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 59 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 60 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 61 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 62 | 63 | /* Experimental Options */ 64 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 65 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 66 | 67 | /* Advanced Options */ 68 | "skipLibCheck": true /* Skip type checking of declaration files. */, 69 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 70 | } 71 | } 72 | --------------------------------------------------------------------------------