├── .node-version ├── bin └── slack-archive.js ├── .gitignore ├── .npmignore ├── static ├── fonts │ ├── Lato-Bold.ttf │ └── Lato-Regular.ttf ├── scroll.js ├── style.css └── search.html ├── src ├── ambient.d.ts ├── threads.ts ├── reactions.ts ├── timestamp.ts ├── web-client.ts ├── retry.ts ├── archive-data.ts ├── data-write.ts ├── interfaces.ts ├── data-load.ts ├── channels.ts ├── users.ts ├── backup.ts ├── config.ts ├── download-files.ts ├── emoji.ts ├── search.ts ├── messages.ts ├── cli.ts └── create-html.tsx ├── package.json ├── README.md ├── tsconfig.json └── yarn.lock /.node-version: -------------------------------------------------------------------------------- 1 | 16.4.0 2 | -------------------------------------------------------------------------------- /bin/slack-archive.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import('../lib/cli.js') 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | out 3 | slack-archive 4 | .DS_Store 5 | *.log 6 | .token 7 | lib -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | out 3 | slack-archive 4 | .DS_Store 5 | *.log 6 | .token 7 | src -------------------------------------------------------------------------------- /static/fonts/Lato-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixrieseberg/slack-archive/HEAD/static/fonts/Lato-Bold.ttf -------------------------------------------------------------------------------- /src/ambient.d.ts: -------------------------------------------------------------------------------- 1 | declare module "slack-markdown"; 2 | declare module "es-main"; 3 | declare module "emoji-datasource"; 4 | -------------------------------------------------------------------------------- /static/fonts/Lato-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felixrieseberg/slack-archive/HEAD/static/fonts/Lato-Regular.ttf -------------------------------------------------------------------------------- /static/scroll.js: -------------------------------------------------------------------------------- 1 | if (window.location.hash) { 2 | document.getElementById(window.location.hash).scrollTo(); 3 | } else { 4 | scrollBy({ top: 99999999 }); 5 | } 6 | -------------------------------------------------------------------------------- /src/threads.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "./interfaces"; 2 | 3 | export function isThread(message: Message) { 4 | return message.reply_count && message.reply_count > 0; 5 | } 6 | -------------------------------------------------------------------------------- /src/reactions.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "./interfaces"; 2 | 3 | export function hasReactions(message: Message) { 4 | return message.reactions && message.reactions.length > 0; 5 | } 6 | -------------------------------------------------------------------------------- /src/timestamp.ts: -------------------------------------------------------------------------------- 1 | export function slackTimestampToJavaScriptTimestamp(ts?: string) { 2 | if (!ts) { 3 | return 0; 4 | } 5 | 6 | const splitTs = ts.split(".") || []; 7 | const jsTs = parseInt(`${splitTs[0]}${splitTs[1].slice(0, 3)}`, 10); 8 | 9 | return jsTs; 10 | } 11 | -------------------------------------------------------------------------------- /src/web-client.ts: -------------------------------------------------------------------------------- 1 | import { WebClient } from "@slack/web-api"; 2 | 3 | import { config } from "./config.js"; 4 | 5 | let _webClient: WebClient; 6 | export function getWebClient() { 7 | if (_webClient) return _webClient; 8 | 9 | const { token } = config; 10 | return (_webClient = new WebClient(token)); 11 | } 12 | 13 | export async function authTest() { 14 | return getWebClient().auth.test(); 15 | } 16 | -------------------------------------------------------------------------------- /src/retry.ts: -------------------------------------------------------------------------------- 1 | export interface RetryOptions { 2 | retries: number; 3 | name?: string; 4 | } 5 | 6 | const defaultOptions: RetryOptions = { 7 | retries: 3, 8 | }; 9 | 10 | export async function retry( 11 | options: Partial, 12 | operation: () => T, 13 | attempt = 0 14 | ): Promise { 15 | let mergedOptions = { ...defaultOptions, ...options }; 16 | 17 | try { 18 | return operation(); 19 | } catch (error) { 20 | if (attempt >= mergedOptions.retries) { 21 | throw error; 22 | } 23 | 24 | const ms = 250 + attempt * 250; 25 | 26 | if (mergedOptions.name) { 27 | console.warn(`Operation "${options.name}" failed, retrying in ${ms}`); 28 | } 29 | 30 | await wait(ms); 31 | 32 | return retry(options, operation, attempt + 1); 33 | } 34 | } 35 | 36 | function wait(ms = 250) { 37 | return new Promise((resolve) => { 38 | setTimeout(resolve, ms); 39 | }); 40 | } 41 | -------------------------------------------------------------------------------- /src/archive-data.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs-extra"; 2 | 3 | import { SLACK_ARCHIVE_DATA_PATH } from "./config.js"; 4 | import { readJSON } from "./data-load.js"; 5 | import { write } from "./data-write.js"; 6 | import { SlackArchiveData, User } from "./interfaces.js"; 7 | 8 | export async function getSlackArchiveData(): Promise { 9 | const returnIfEmpty: SlackArchiveData = { channels: {} }; 10 | 11 | if (!fs.existsSync(SLACK_ARCHIVE_DATA_PATH)) { 12 | return returnIfEmpty; 13 | } 14 | 15 | const result = await readJSON(SLACK_ARCHIVE_DATA_PATH); 16 | const merged = { channels: result.channels || {}, auth: result.auth }; 17 | 18 | return merged; 19 | } 20 | 21 | export async function setSlackArchiveData( 22 | newData: SlackArchiveData 23 | ): Promise { 24 | const oldData = await getSlackArchiveData(); 25 | const dataToWrite = { 26 | channels: { ...oldData.channels, ...newData.channels }, 27 | auth: newData.auth, 28 | }; 29 | 30 | return write( 31 | SLACK_ARCHIVE_DATA_PATH, 32 | JSON.stringify(dataToWrite, undefined, 2) 33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /src/data-write.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs-extra"; 2 | import { differenceBy } from "lodash-es"; 3 | 4 | import { retry } from "./retry.js"; 5 | 6 | export async function write(filePath: string, data: any) { 7 | await retry({ name: `Writing ${filePath}` }, () => { 8 | fs.outputFileSync(filePath, data); 9 | }); 10 | } 11 | 12 | export async function writeAndMerge(filePath: string, newData: any) { 13 | await retry({ name: `Writing ${filePath}` }, () => { 14 | let dataToWrite = newData; 15 | 16 | if (fs.existsSync(filePath)) { 17 | const oldData = fs.readJSONSync(filePath); 18 | 19 | if (Array.isArray(oldData)) { 20 | if (newData && newData[0] && newData[0].id) { 21 | // Take the old data, exclude aything that is in the new data, 22 | // and then add the new data 23 | dataToWrite = [ 24 | ...differenceBy(oldData, newData, (v: any) => v.id), 25 | ...newData, 26 | ]; 27 | } else { 28 | dataToWrite = [...oldData, ...newData]; 29 | } 30 | } else if (typeof newData === "object") { 31 | dataToWrite = { ...oldData, ...newData }; 32 | } else { 33 | console.error(`writeAndMerge: Did not understand type of data`, { 34 | filePath, 35 | newData, 36 | }); 37 | } 38 | } 39 | 40 | fs.outputFileSync(filePath, JSON.stringify(dataToWrite, undefined, 2)); 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "slack-archive", 3 | "version": "1.6.1", 4 | "description": "Create static HTML archives for your Slack workspaces", 5 | "scripts": { 6 | "prettier": "npx prettier --write src/*", 7 | "cli": "ts-node src/cli.ts", 8 | "html": "ts-node src/create-html.tsx", 9 | "test": "echo \"Error: no test specified\" && exit 1", 10 | "compile": "tsc", 11 | "watch": "tsc -w", 12 | "prepublishOnly": "npm run compile" 13 | }, 14 | "bin": { 15 | "slack-archive": "./bin/slack-archive.js" 16 | }, 17 | "type": "module", 18 | "keywords": [ 19 | "slack", 20 | "export", 21 | "download" 22 | ], 23 | "repository": { 24 | "type": "git", 25 | "url": "https://github.com/felixrieseberg/slack-archive.git" 26 | }, 27 | "author": "Felix Rieseberg ", 28 | "license": "MIT", 29 | "dependencies": { 30 | "@slack/web-api": "^6.7.2", 31 | "date-fns": "^2.28.0", 32 | "emoji-datasource": "^14.0.0", 33 | "es-main": "^1.0.2", 34 | "fs-extra": "^10.1.0", 35 | "inquirer": "^8.2.0", 36 | "lodash-es": "^4.17.21", 37 | "node-fetch": "^2.6.7", 38 | "ora": "^6.1.0", 39 | "react": "^17.0.2", 40 | "react-dom": "^17.0.2", 41 | "rimraf": "^5.0.5", 42 | "slack-markdown": "^0.2.0", 43 | "trash": "^8.1.0" 44 | }, 45 | "devDependencies": { 46 | "@types/date-fns": "^2.6.0", 47 | "@types/fs-extra": "^9.0.13", 48 | "@types/inquirer": "^8.1.3", 49 | "@types/lodash-es": "^4.17.5", 50 | "@types/node": "^17.0.5", 51 | "@types/node-fetch": "^2.5.12", 52 | "@types/react": "^17.0.38", 53 | "@types/react-dom": "^17.0.11", 54 | "ts-node": "^10.8.1", 55 | "tslib": "^2.4.0", 56 | "typescript": "^4.7.4" 57 | }, 58 | "ts-node": { 59 | "files": true 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/interfaces.ts: -------------------------------------------------------------------------------- 1 | import { Message as SlackMessage } from "@slack/web-api/dist/response/ConversationsHistoryResponse"; 2 | import { Channel as SlackChannel } from "@slack/web-api/dist/response/ConversationsListResponse"; 3 | import { User as SlackUser } from "@slack/web-api/dist/response/UsersInfoResponse"; 4 | import { File as SlackFile } from "@slack/web-api/dist/response/FilesInfoResponse"; 5 | import { Reaction as SlackReaction } from "@slack/web-api/dist/response/ReactionsGetResponse"; 6 | import { AuthTestResponse } from "@slack/web-api"; 7 | 8 | export type User = SlackUser; 9 | 10 | export type Users = Record; 11 | 12 | export type Emojis = Record; 13 | 14 | export interface ArchiveMessage extends SlackMessage { 15 | replies?: Array; 16 | } 17 | 18 | export type Reaction = SlackReaction; 19 | 20 | export type Message = SlackMessage; 21 | 22 | export type Channel = SlackChannel; 23 | 24 | export type File = SlackFile; 25 | 26 | export type SearchPageIndex = Record>; 27 | 28 | export type SearchFile = { 29 | users: Record; // userId -> userName 30 | channels: Record; // channelId -> channelName 31 | messages: Record>; 32 | pages: SearchPageIndex; 33 | }; 34 | 35 | export type SearchMessage = { 36 | m?: string; // Message 37 | u?: string; // User 38 | t?: string; // Timestamp 39 | c?: string; // Channel 40 | }; 41 | 42 | export interface SlackArchiveChannelData { 43 | messages: number; 44 | fullyDownloaded: boolean; 45 | } 46 | 47 | export interface SlackArchiveData { 48 | channels: Record; 49 | auth?: AuthTestResponse; 50 | } 51 | 52 | export interface ChunkInfo { 53 | oldest?: string; 54 | newest?: string; 55 | count: number; 56 | } 57 | 58 | export type ChunksInfo = Array; 59 | -------------------------------------------------------------------------------- /src/data-load.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs-extra"; 2 | 3 | import { 4 | ArchiveMessage, 5 | Channel, 6 | Emojis, 7 | SearchFile, 8 | Users, 9 | } from "./interfaces.js"; 10 | import { 11 | CHANNELS_DATA_PATH, 12 | EMOJIS_DATA_PATH, 13 | getChannelDataFilePath, 14 | SEARCH_DATA_PATH, 15 | USERS_DATA_PATH, 16 | } from "./config.js"; 17 | import { retry } from "./retry.js"; 18 | 19 | async function getFile(filePath: string, returnIfEmpty: T): Promise { 20 | if (!fs.existsSync(filePath)) { 21 | return returnIfEmpty; 22 | } 23 | 24 | const data: T = await readJSON(filePath); 25 | 26 | return data; 27 | } 28 | 29 | export const messagesCache: Record> = {}; 30 | 31 | export async function getMessages( 32 | channelId: string, 33 | cachedOk: boolean = false 34 | ): Promise> { 35 | if (cachedOk && messagesCache[channelId]) { 36 | return messagesCache[channelId]; 37 | } 38 | 39 | const filePath = getChannelDataFilePath(channelId); 40 | messagesCache[channelId] = await getFile>(filePath, []); 41 | 42 | return messagesCache[channelId]; 43 | } 44 | 45 | export async function getUsers(): Promise { 46 | return getFile(USERS_DATA_PATH, {}); 47 | } 48 | 49 | export async function getEmoji(): Promise { 50 | return getFile(EMOJIS_DATA_PATH, {}); 51 | } 52 | 53 | export async function getChannels(): Promise> { 54 | return getFile>(CHANNELS_DATA_PATH, []); 55 | } 56 | 57 | export async function getSearchFile(): Promise { 58 | const returnIfEmpty = { users: {}, channels: {}, messages: {}, pages: {} }; 59 | 60 | if (!fs.existsSync(SEARCH_DATA_PATH)) { 61 | return returnIfEmpty; 62 | } 63 | 64 | const contents = await readFile(SEARCH_DATA_PATH, "utf8"); 65 | 66 | // See search.ts, the file is actually JS (not JSON) 67 | return JSON.parse(contents.slice(21, contents.length - 1)); 68 | } 69 | 70 | export async function readFile(filePath: string, encoding = "utf8") { 71 | return retry({ name: `Reading ${filePath}` }, () => { 72 | return fs.readFileSync(SEARCH_DATA_PATH, "utf8"); 73 | }); 74 | } 75 | 76 | export async function readJSON(filePath: string) { 77 | return retry({ name: `Loading JSON from ${filePath}` }, () => { 78 | return fs.readJSONSync(filePath); 79 | }); 80 | } 81 | -------------------------------------------------------------------------------- /src/channels.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ConversationsListArguments, 3 | ConversationsListResponse, 4 | } from "@slack/web-api"; 5 | import ora from "ora"; 6 | import { NO_SLACK_CONNECT } from "./config.js"; 7 | 8 | import { Channel, Users } from "./interfaces.js"; 9 | import { downloadUser, getName } from "./users.js"; 10 | import { getWebClient } from "./web-client.js"; 11 | 12 | export function getChannelName(channel: Channel) { 13 | return ( 14 | channel.name || channel.id || channel.purpose?.value || "Unknown channel" 15 | ); 16 | } 17 | 18 | export function isPublicChannel(channel: Channel) { 19 | return !channel.is_private && !channel.is_mpim && !channel.is_im; 20 | } 21 | 22 | export function isPrivateChannel(channel: Channel) { 23 | return channel.is_private && !channel.is_im && !channel.is_mpim; 24 | } 25 | 26 | export function isDmChannel(channel: Channel, users: Users) { 27 | return channel.is_im && channel.user && !users[channel.user]?.is_bot; 28 | } 29 | 30 | export function isBotChannel(channel: Channel, users: Users) { 31 | return channel.user && users[channel.user]?.is_bot; 32 | } 33 | 34 | function isChannels(input: any): input is ConversationsListResponse { 35 | return !!input.channels; 36 | } 37 | 38 | export async function downloadChannels( 39 | options: ConversationsListArguments, 40 | users: Users 41 | ): Promise> { 42 | const channels: Array = []; 43 | 44 | if (NO_SLACK_CONNECT) { 45 | return channels; 46 | } 47 | 48 | const spinner = ora("Downloading channels").start(); 49 | 50 | for await (const page of getWebClient().paginate( 51 | "conversations.list", 52 | options 53 | )) { 54 | if (isChannels(page)) { 55 | spinner.text = `Found ${page.channels?.length} channels (found so far: ${ 56 | channels.length + (page.channels?.length || 0) 57 | })`; 58 | 59 | const pageChannels = (page.channels || []).filter((c) => !!c.id); 60 | 61 | for (const channel of pageChannels) { 62 | if (channel.is_im) { 63 | const user = await downloadUser(channel, users); 64 | channel.name = 65 | channel.name || `${getName(user?.id, users)} (${user?.name})`; 66 | } 67 | 68 | if (channel.is_mpim) { 69 | channel.name = channel.purpose?.value; 70 | } 71 | } 72 | 73 | channels.push(...pageChannels); 74 | } 75 | } 76 | 77 | spinner.succeed(`Found ${channels.length} channels`); 78 | 79 | return channels; 80 | } 81 | -------------------------------------------------------------------------------- /src/users.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | 3 | import { getWebClient } from "./web-client.js"; 4 | import { Message, User, Users } from "./interfaces.js"; 5 | import { getAvatarFilePath } from "./config.js"; 6 | import { getUsers } from "./data-load.js"; 7 | import { downloadURL } from "./download-files.js"; 8 | import ora from "ora"; 9 | 10 | // We'll redownload users every run, but only once per user 11 | // To keep track, we'll keep the ids in this array 12 | export const usersRefetchedThisRun: Array = []; 13 | export const avatarsRefetchedThisRun: Array = []; 14 | 15 | export async function downloadUser( 16 | item: Message | any, 17 | users: Users 18 | ): Promise { 19 | if (!item.user) return null; 20 | 21 | // If we already have this user *and* downloaded them before, 22 | // return cached version 23 | if (users[item.user] && usersRefetchedThisRun.includes(item.user)) 24 | return users[item.user]; 25 | 26 | const spinner = ora(`Downloading info for user ${item.user}...`).start(); 27 | const user = (item.user === 'U00') ? {} as User : ( 28 | await getWebClient().users.info({ 29 | user: item.user, 30 | }) 31 | ).user; 32 | 33 | if (user) { 34 | usersRefetchedThisRun.push(item.user); 35 | spinner.succeed(`Downloaded info for user ${item.user} (${user.name})`); 36 | return (users[item.user] = user); 37 | } 38 | 39 | return null; 40 | } 41 | 42 | export async function downloadAvatars() { 43 | const users = await getUsers(); 44 | const userIds = Object.keys(users); 45 | const spinner = ora(`Downloading avatars (0/${userIds.length})`).start(); 46 | 47 | for (const [i, userId] of userIds.entries()) { 48 | spinner.text = `Downloading avatars (${i + 1}/${userIds.length})`; 49 | await downloadAvatarForUser(users[userId]); 50 | } 51 | 52 | spinner.stop(); 53 | } 54 | 55 | export async function downloadAvatarForUser(user?: User | null) { 56 | if (!user || !user.id || avatarsRefetchedThisRun.includes(user.id)) { 57 | return; 58 | } 59 | 60 | const { profile } = user; 61 | 62 | if (!profile || !profile.image_512) { 63 | return; 64 | } 65 | 66 | try { 67 | const filePath = getAvatarFilePath( 68 | user.id!, 69 | path.extname(profile.image_512) 70 | ); 71 | await downloadURL(profile.image_512, filePath, { 72 | authorize: false, 73 | force: true, 74 | }); 75 | avatarsRefetchedThisRun.push(user.id!); 76 | } catch (error) { 77 | console.warn(`Failed to download avatar for user ${user.id!}`, error); 78 | } 79 | } 80 | 81 | export function getName(userId: string | undefined, users: Users) { 82 | if (!userId) return "Unknown"; 83 | const user = users[userId]; 84 | if (!user) return userId; 85 | 86 | return user.profile?.display_name || user.profile?.real_name || user.name; 87 | } 88 | -------------------------------------------------------------------------------- /src/backup.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs-extra"; 2 | import inquirer from "inquirer"; 3 | import path from "path"; 4 | import trash from "trash"; 5 | import { rimraf } from "rimraf"; 6 | 7 | import { AUTOMATIC_MODE, DATA_DIR, NO_BACKUP, OUT_DIR } from "./config.js"; 8 | 9 | const { prompt } = inquirer; 10 | 11 | let backupDir = `${DATA_DIR}_backup_${Date.now()}`; 12 | 13 | export async function createBackup() { 14 | if (NO_BACKUP || !fs.existsSync(DATA_DIR)) { 15 | return; 16 | } 17 | 18 | const hasFiles = fs.readdirSync(DATA_DIR); 19 | 20 | if (hasFiles.length === 0) { 21 | return; 22 | } 23 | 24 | console.log(`Existing data directory found. Creating backup: ${backupDir}`); 25 | 26 | await fs.copy(DATA_DIR, backupDir); 27 | 28 | console.log(`Backup created.\n`); 29 | } 30 | 31 | export async function deleteBackup() { 32 | if (!fs.existsSync(backupDir)) { 33 | return; 34 | } 35 | 36 | console.log( 37 | `Cleaning up backup: If anything went wrong, you'll find it in your system's trash.` 38 | ); 39 | 40 | try { 41 | // NB: trash doesn't work on many Linux distros 42 | await trash(backupDir); 43 | return; 44 | } catch (error) { 45 | console.log('Moving backup to trash failed.'); 46 | } 47 | 48 | if (!process.env['TRASH_HARDER']) { 49 | console.log(`Set TRASH_HARDER=1 to delete files permanently.`); 50 | return; 51 | } 52 | 53 | try { 54 | await rimraf(backupDir); 55 | } catch (error) { 56 | console.log(`Deleting backup permanently failed. Aborting here.`); 57 | } 58 | } 59 | 60 | export async function deleteOlderBackups() { 61 | try { 62 | const oldBackupNames: Array = []; 63 | const oldBackupPaths: Array = []; 64 | 65 | for (const entry of fs.readdirSync(OUT_DIR)) { 66 | const isBackup = entry.startsWith("data_backup_"); 67 | if (!isBackup) continue; 68 | 69 | const dir = path.join(OUT_DIR, entry); 70 | const { isDirectory } = fs.statSync(dir); 71 | if (!isDirectory) continue; 72 | 73 | oldBackupPaths.push(dir); 74 | oldBackupNames.push(entry); 75 | } 76 | 77 | if (oldBackupPaths.length === 0) return; 78 | 79 | if (AUTOMATIC_MODE) { 80 | console.log( 81 | `Found existing older backups, but in automatic mode: Proceeding without deleting them.` 82 | ); 83 | return; 84 | } 85 | 86 | const { del } = await prompt([ 87 | { 88 | type: "confirm", 89 | default: true, 90 | name: "del", 91 | message: `We've found existing backups (${oldBackupNames.join( 92 | ", " 93 | )}). Do you want to delete them?`, 94 | }, 95 | ]); 96 | 97 | if (del) { 98 | oldBackupPaths.forEach((v) => fs.removeSync(v)); 99 | } 100 | } catch (error) { 101 | // noop 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/config.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { dirname } from "path"; 3 | import { fileURLToPath } from "url"; 4 | 5 | const __dirname = dirname(fileURLToPath(import.meta.url)); 6 | 7 | export const config = { 8 | token: process.env.SLACK_TOKEN, 9 | }; 10 | 11 | function findCliParameter(param: string) { 12 | const args = process.argv; 13 | 14 | for (const arg of args) { 15 | if (arg === param) { 16 | return true; 17 | } 18 | } 19 | 20 | return false; 21 | } 22 | 23 | function getCliParameter(param: string) { 24 | const args = process.argv; 25 | 26 | for (const [i, arg] of args.entries()) { 27 | if (arg === param) { 28 | return args[i + 1]; 29 | } 30 | } 31 | 32 | return null; 33 | } 34 | 35 | export const AUTOMATIC_MODE = findCliParameter("--automatic"); 36 | export const USE_PREVIOUS_CHANNEL_CONFIG = findCliParameter( 37 | "--use-previous-channel-config" 38 | ); 39 | export const CHANNEL_TYPES = getCliParameter("--channel-types"); 40 | export const NO_BACKUP = findCliParameter("--no-backup"); 41 | export const NO_SEARCH = findCliParameter("--no-search"); 42 | export const NO_FILE_DOWNLOAD = findCliParameter("--no-file-download"); 43 | export const NO_SLACK_CONNECT = findCliParameter("--no-slack-connect"); 44 | export const FORCE_HTML_GENERATION = findCliParameter( 45 | "--force-html-generation" 46 | ); 47 | export const EXCLUDE_CHANNELS = getCliParameter("--exclude-channels"); 48 | export const BASE_DIR = process.cwd(); 49 | export const OUT_DIR = path.join(BASE_DIR, "slack-archive"); 50 | export const TOKEN_FILE = path.join(OUT_DIR, ".token"); 51 | export const DATE_FILE = path.join(OUT_DIR, ".last-successful-run"); 52 | export const DATA_DIR = path.join(OUT_DIR, "data"); 53 | export const HTML_DIR = path.join(OUT_DIR, "html"); 54 | export const FILES_DIR = path.join(HTML_DIR, "files"); 55 | export const AVATARS_DIR = path.join(HTML_DIR, "avatars"); 56 | export const EMOJIS_DIR = path.join(HTML_DIR, "emojis"); 57 | 58 | export const INDEX_PATH = path.join(OUT_DIR, "index.html"); 59 | export const SEARCH_PATH = path.join(OUT_DIR, "search.html"); 60 | export const MESSAGES_JS_PATH = path.join(__dirname, "../static/scroll.js"); 61 | export const SEARCH_TEMPLATE_PATH = path.join( 62 | __dirname, 63 | "../static/search.html" 64 | ); 65 | export const CHANNELS_DATA_PATH = path.join(DATA_DIR, "channels.json"); 66 | export const USERS_DATA_PATH = path.join(DATA_DIR, "users.json"); 67 | export const EMOJIS_DATA_PATH = path.join(DATA_DIR, "emojis.json"); 68 | export const SLACK_ARCHIVE_DATA_PATH = path.join( 69 | DATA_DIR, 70 | "slack-archive.json" 71 | ); 72 | export const SEARCH_DATA_PATH = path.join(DATA_DIR, "search.js"); 73 | 74 | export function getChannelDataFilePath(channelId: string) { 75 | return path.join(DATA_DIR, `${channelId}.json`); 76 | } 77 | 78 | export function getChannelUploadFilePath(channelId: string, fileName: string) { 79 | return path.join(FILES_DIR, channelId, fileName); 80 | } 81 | 82 | export function getHTMLFilePath(channelId: string, index: number) { 83 | return path.join(HTML_DIR, `${channelId}-${index}.html`); 84 | } 85 | 86 | export function getAvatarFilePath(userId: string, extension: string) { 87 | return path.join(AVATARS_DIR, `${userId}${extension}`); 88 | } 89 | -------------------------------------------------------------------------------- /src/download-files.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | import fs from "fs-extra"; 3 | import esMain from "es-main"; 4 | import ora, { Ora } from "ora"; 5 | import path from "path"; 6 | 7 | import { File } from "./interfaces.js"; 8 | import { 9 | getChannelUploadFilePath, 10 | config, 11 | NO_FILE_DOWNLOAD, 12 | } from "./config.js"; 13 | import { getChannels, getMessages } from "./data-load.js"; 14 | import { downloadAvatars } from "./users.js"; 15 | 16 | export interface DownloadUrlOptions { 17 | authorize?: boolean; 18 | force?: boolean; 19 | } 20 | 21 | export async function downloadURL( 22 | url: string, 23 | filePath: string, 24 | options: DownloadUrlOptions = {} 25 | ) { 26 | const authorize = options.authorize === undefined ? true : options.authorize; 27 | 28 | if (!options.force && fs.existsSync(filePath)) { 29 | return; 30 | } 31 | 32 | const { token } = config; 33 | const headers: HeadersInit = authorize 34 | ? { 35 | Authorization: `Bearer ${token}`, 36 | } 37 | : {}; 38 | 39 | try { 40 | const response = await fetch(url, { headers }); 41 | const buffer = await response.buffer(); 42 | fs.outputFileSync(filePath, buffer); 43 | } catch (error) { 44 | console.warn(`Failed to download file ${url}`, error); 45 | } 46 | } 47 | 48 | async function downloadFile( 49 | file: File, 50 | channelId: string, 51 | i: number, 52 | total: number, 53 | spinner: Ora 54 | ) { 55 | const { url_private, id, is_external, mimetype } = file; 56 | const { thumb_1024, thumb_720, thumb_480, thumb_pdf } = file as any; 57 | 58 | const fileUrl = is_external 59 | ? thumb_1024 || thumb_720 || thumb_480 || thumb_pdf 60 | : url_private; 61 | 62 | if (!fileUrl) return; 63 | 64 | spinner.text = `Downloading ${i}/${total}: ${fileUrl}`; 65 | 66 | const extension = path.extname(fileUrl); 67 | const filePath = getChannelUploadFilePath(channelId, `${id}${extension}`); 68 | 69 | await downloadURL(fileUrl, filePath); 70 | 71 | if (mimetype === "application/pdf" && thumb_pdf) { 72 | spinner.text = `Downloading ${i}/${total}: ${thumb_pdf}`; 73 | const thumbFile = filePath.replace(extension, ".png"); 74 | await downloadURL(thumb_pdf, thumbFile); 75 | } 76 | } 77 | 78 | export async function downloadFilesForChannel(channelId: string, spinner: Ora) { 79 | if (NO_FILE_DOWNLOAD) { 80 | return; 81 | } 82 | 83 | const messages = await getMessages(channelId); 84 | const channels = await getChannels(); 85 | const channel = channels.find(({ id }) => id === channelId); 86 | const fileMessages = messages.filter( 87 | (m) => (m.files?.length || m.replies?.length || 0) > 0 88 | ); 89 | const getSpinnerText = (i: number, ri?: number) => { 90 | let reply = ""; 91 | if (ri !== undefined) { 92 | reply = ` (reply ${ri})`; 93 | } 94 | 95 | return `Downloading ${i}/${ 96 | fileMessages.length 97 | }${reply} messages with files for channel ${channel?.name || channelId}...`; 98 | }; 99 | 100 | spinner.text = getSpinnerText(0); 101 | 102 | for (const [i, fileMessage] of fileMessages.entries()) { 103 | if (!fileMessage.files && !fileMessage.replies) { 104 | continue; 105 | } 106 | 107 | if (fileMessage.files) { 108 | for (const file of fileMessage.files) { 109 | spinner.text = getSpinnerText(i); 110 | spinner.render(); 111 | await downloadFile(file, channelId, i, fileMessages.length, spinner); 112 | } 113 | } 114 | 115 | if (fileMessage.replies) { 116 | for (const [ri, reply] of fileMessage.replies.entries()) { 117 | if (reply.files) { 118 | for (const file of reply.files) { 119 | spinner.text = getSpinnerText(i, ri); 120 | spinner.render(); 121 | await downloadFile( 122 | file, 123 | channelId, 124 | i, 125 | fileMessages.length, 126 | spinner 127 | ); 128 | } 129 | } 130 | } 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/emoji.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import ora from "ora"; 3 | import fs from "fs"; 4 | import { createRequire } from "node:module"; 5 | 6 | import { EMOJIS_DIR, NO_SLACK_CONNECT } from "./config.js"; 7 | import { downloadURL } from "./download-files.js"; 8 | import { ArchiveMessage, Emojis } from "./interfaces.js"; 9 | import { getWebClient } from "./web-client.js"; 10 | 11 | const require = createRequire(import.meta.url); 12 | const emojiData = require("emoji-datasource"); 13 | 14 | let _unicodeEmoji: Record; 15 | function getUnicodeEmoji() { 16 | if (_unicodeEmoji) { 17 | return _unicodeEmoji; 18 | } 19 | 20 | _unicodeEmoji = {}; 21 | for (const emoji of emojiData) { 22 | _unicodeEmoji[emoji.short_name as string] = emoji.unified; 23 | } 24 | 25 | return _unicodeEmoji; 26 | } 27 | 28 | export function getEmojiFilePath(name: string, extension?: string) { 29 | // If we have an extension, return the correct path 30 | if (extension) { 31 | return path.join(EMOJIS_DIR, `${name}${extension}`); 32 | } 33 | 34 | // If we don't have an extension, return the first path that exists 35 | // regardless of extension 36 | const extensions = [".png", ".jpg", ".gif"]; 37 | for (const ext of extensions) { 38 | if (fs.existsSync(path.join(EMOJIS_DIR, `${name}${ext}`))) { 39 | return path.join(EMOJIS_DIR, `${name}${ext}`); 40 | } 41 | } 42 | } 43 | 44 | export function isEmojiUnicode(name: string) { 45 | const unicodeEmoji = getUnicodeEmoji(); 46 | return !!unicodeEmoji[name]; 47 | } 48 | 49 | export function getEmojiUnicode(name: string) { 50 | const unicodeEmoji = getUnicodeEmoji(); 51 | const unified = unicodeEmoji[name]; 52 | const split = unified.split("-"); 53 | 54 | return split 55 | .map((code) => { 56 | return String.fromCodePoint(parseInt(code, 16)); 57 | }) 58 | .join(""); 59 | } 60 | 61 | export async function downloadEmojiList(): Promise { 62 | if (NO_SLACK_CONNECT) { 63 | return {}; 64 | } 65 | 66 | const response = await getWebClient().emoji.list(); 67 | 68 | if (response.ok) { 69 | return response.emoji!; 70 | } else { 71 | return {}; 72 | } 73 | } 74 | 75 | export async function downloadEmoji( 76 | name: string, 77 | url: string, 78 | emojis: Emojis 79 | ): Promise { 80 | // Alias? 81 | if (url.startsWith("alias:")) { 82 | const alias = getEmojiAlias(url); 83 | 84 | if (!emojis[alias]) { 85 | console.warn( 86 | `Found emoji alias ${alias}, which does not exist in master emoji list` 87 | ); 88 | return; 89 | } else { 90 | return downloadEmoji(alias, emojis[alias], emojis); 91 | } 92 | } 93 | 94 | const extension = path.extname(url); 95 | const filePath = getEmojiFilePath(name, extension); 96 | 97 | return downloadURL(url, filePath!); 98 | } 99 | 100 | export function getEmojiAlias(name: string): string { 101 | // Ugh regex methods - this should turn "alias:hi-bob" into "hi-bob" 102 | const alias = [...name.matchAll(/alias:(.*)/g)][0][1]!; 103 | return alias!; 104 | } 105 | 106 | export async function downloadEmojis( 107 | messages: Array, 108 | emojis: Emojis 109 | ) { 110 | const regex = /:[^:\s]*(?:::[^:\s]*)*:/g; 111 | 112 | const spinner = ora( 113 | `Scanning 0/${messages.length} messages for emoji shortcodes...` 114 | ).start(); 115 | let downloaded = 0; 116 | 117 | for (const [i, message] of messages.entries()) { 118 | spinner.text = `Scanning ${i}/${messages.length} messages for emoji shortcodes...`; 119 | 120 | // Reactions 121 | if (message.reactions && message.reactions.length > 0) { 122 | for (const reaction of message.reactions) { 123 | const reactEmoji = emojis[reaction.name!]; 124 | if (reactEmoji) { 125 | downloaded++; 126 | await downloadEmoji(reaction.name!, reactEmoji, emojis); 127 | } 128 | } 129 | } 130 | } 131 | 132 | spinner.succeed( 133 | `Scanned ${messages.length} messages for emoji (and downloaded ${downloaded})` 134 | ); 135 | } 136 | -------------------------------------------------------------------------------- /src/search.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs-extra"; 2 | import ora, { Ora } from "ora"; 3 | import { getChannelName } from "./channels.js"; 4 | 5 | import { 6 | NO_SEARCH, 7 | SEARCH_DATA_PATH, 8 | SEARCH_PATH, 9 | SEARCH_TEMPLATE_PATH, 10 | } from "./config.js"; 11 | import { SearchFile, SearchMessage, SearchPageIndex } from "./interfaces"; 12 | import { 13 | getChannels, 14 | getMessages, 15 | getSearchFile, 16 | getUsers, 17 | } from "./data-load.js"; 18 | 19 | // Format: 20 | // channelId: [ timestamp0, timestamp1, timestamp2, ... ] 21 | // 22 | // channelId: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ] 23 | // pages: { 24 | // 0: [ 10, 9, 8 ] 25 | // 1: [ 7, 6, 5 ] 26 | // 2: [ 4, 3, 2 ] 27 | // 3: [ 1, 0 ] 28 | // } 29 | // INDEX_OF_PAGES: { 30 | // channelId: [8, 5, 2, 0] 31 | // } 32 | // 33 | // For channelId, a message older than timestamp 0 but younger than timestamp1 is on page 1. 34 | // In our example above, the message with timestamp 6 is older than 5 but younger than 8. 35 | const INDEX_OF_PAGES: SearchPageIndex = {}; 36 | 37 | export function recordPage(channelId?: string, timestamp?: string) { 38 | if (!channelId || !timestamp) { 39 | console.warn( 40 | `Search: Cannot record page: channelId: ${channelId} timestamp: ${timestamp}` 41 | ); 42 | return; 43 | } 44 | 45 | if (!INDEX_OF_PAGES[channelId]) { 46 | INDEX_OF_PAGES[channelId] = []; 47 | } 48 | 49 | INDEX_OF_PAGES[channelId].push(timestamp); 50 | } 51 | 52 | export async function createSearch() { 53 | if (NO_SEARCH) return; 54 | 55 | const spinner = ora(`Creating search file...`).start(); 56 | spinner.render(); 57 | 58 | await createSearchFile(spinner); 59 | await createSearchHTML(); 60 | 61 | spinner.succeed(`Search file created`); 62 | } 63 | 64 | async function createSearchFile(spinner: Ora) { 65 | const existingData = await getSearchFile(); 66 | const users = await getUsers(); 67 | const channels = await getChannels(); 68 | const result: SearchFile = { 69 | channels: {}, 70 | users: {}, 71 | messages: {}, 72 | pages: { ...existingData.pages, ...INDEX_OF_PAGES }, 73 | }; 74 | 75 | // Users 76 | for (const user in users) { 77 | result.users[user] = users[user].name || users[user].real_name || "Unknown"; 78 | } 79 | 80 | // Channels & Messages 81 | for (const [i, channel] of channels.entries()) { 82 | if (!channel.id) { 83 | console.warn( 84 | `Can't create search file for channel ${channel.name}: No id found`, 85 | channel 86 | ); 87 | continue; 88 | } 89 | 90 | const name = getChannelName(channel); 91 | 92 | spinner.text = `Creating search messages for channel ${name}`; 93 | spinner.render(); 94 | 95 | const messages = (await getMessages(channel.id, true)).map((message) => { 96 | const searchMessage: SearchMessage = { 97 | m: message.text, 98 | u: message.user, 99 | t: message.ts, 100 | }; 101 | 102 | return searchMessage; 103 | }); 104 | 105 | result.messages![channel.id] = messages; 106 | result.channels[channel.id] = name; 107 | } 108 | 109 | const jsContent = `window.search_data = ${JSON.stringify(result)};`; 110 | await fs.outputFile(SEARCH_DATA_PATH, jsContent); 111 | } 112 | 113 | async function createSearchHTML() { 114 | let template = fs.readFileSync(SEARCH_TEMPLATE_PATH, "utf8"); 115 | 116 | template = template.replace( 117 | "", 118 | getScript(`react@18.2.0/umd/react.production.min.js`) 119 | ); 120 | template = template.replace( 121 | "", 122 | getScript(`react-dom@18.2.0/umd/react-dom.production.min.js`) 123 | ); 124 | template = template.replace( 125 | ``, 126 | getScript(`babel-standalone@6.26.0/babel.min.js`) 127 | ); 128 | template = template.replace( 129 | ``, 130 | getScript("minisearch@5.0.0/dist/umd/index.min.js") 131 | ); 132 | 133 | template = template.replace(``, getSize()); 134 | 135 | fs.outputFileSync(SEARCH_PATH, template); 136 | } 137 | 138 | function getSize() { 139 | const mb = fs.statSync(SEARCH_DATA_PATH).size / 1048576; //MB 140 | return `Loading ${Math.round(mb)}MB of data`; 141 | } 142 | 143 | function getScript(script: string) { 144 | return ``; 145 | } 146 | -------------------------------------------------------------------------------- /src/messages.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ConversationsHistoryResponse, 3 | ConversationsListArguments, 4 | ConversationsListResponse, 5 | } from "@slack/web-api"; 6 | import { Channel } from "@slack/web-api/dist/response/ConversationsListResponse"; 7 | import ora from "ora"; 8 | 9 | import { ArchiveMessage, Message, Users } from "./interfaces.js"; 10 | import { getMessages } from "./data-load.js"; 11 | import { isThread } from "./threads.js"; 12 | import { downloadUser, getName } from "./users.js"; 13 | import { getWebClient } from "./web-client.js"; 14 | 15 | function isConversation(input: any): input is ConversationsHistoryResponse { 16 | return !!input.messages; 17 | } 18 | 19 | interface DownloadMessagesResult { 20 | messages: Array; 21 | new: number; 22 | } 23 | 24 | export async function downloadMessages( 25 | channel: Channel, 26 | i: number, 27 | channelCount: number 28 | ): Promise { 29 | let result: DownloadMessagesResult = { 30 | messages: [], 31 | new: 0, 32 | }; 33 | 34 | if (!channel.id) { 35 | console.warn(`Channel without id`, channel); 36 | return result; 37 | } 38 | 39 | for (const message of await getMessages(channel.id)) { 40 | result.messages.push(message); 41 | } 42 | 43 | const oldest = 44 | result.messages.length > 0 ? parseInt(result.messages[0].ts || "0", 10) : 0; 45 | const name = 46 | channel.name || channel.id || channel.purpose?.value || "Unknown channel"; 47 | 48 | const spinner = ora( 49 | `Downloading messages for channel ${i + 1}/${channelCount} (${name})...` 50 | ).start(); 51 | 52 | for await (const page of getWebClient().paginate("conversations.history", { 53 | channel: channel.id, 54 | oldest, 55 | })) { 56 | if (isConversation(page)) { 57 | const pageLength = page.messages?.length || 0; 58 | const fetched = `Fetched ${pageLength} messages`; 59 | const total = `(total so far: ${result.messages.length + pageLength}`; 60 | 61 | spinner.text = `Downloading ${ 62 | i + 1 63 | }/${channelCount} ${name}: ${fetched} ${total})`; 64 | 65 | result.new = result.new + (page.messages || []).length; 66 | 67 | result.messages.unshift(...(page.messages || [])); 68 | } 69 | } 70 | 71 | spinner.succeed( 72 | `Downloaded messages for channel ${i + 1}/${channelCount} (${name})` 73 | ); 74 | 75 | return result; 76 | } 77 | 78 | export async function downloadReplies( 79 | channel: Channel, 80 | message: ArchiveMessage 81 | ): Promise> { 82 | if (!channel.id || !message.ts) { 83 | console.warn("Could not find channel or message id", channel, message); 84 | return []; 85 | } 86 | 87 | if (!message.reply_count) { 88 | console.warn("Message has no reply count", message); 89 | return []; 90 | } 91 | 92 | // Do we already have all replies? 93 | if (message.replies && message.replies.length >= message.reply_count) { 94 | return message.replies; 95 | } 96 | 97 | const replies = message.replies || []; 98 | // Oldest is the last entry 99 | const oldest = replies.length > 0 ? replies[replies.length - 1].ts : "0"; 100 | const result = await getWebClient().conversations.replies({ 101 | channel: channel.id, 102 | ts: message.ts, 103 | oldest, 104 | }); 105 | 106 | // First message is the parent 107 | return (result.messages || []).slice(1); 108 | } 109 | 110 | export async function downloadExtras( 111 | channel: Channel, 112 | messages: Array, 113 | users: Users 114 | ) { 115 | const spinner = ora( 116 | `Downloading threads and users for ${channel.name || channel.id}...` 117 | ).start(); 118 | 119 | // Then, all messages and threads 120 | let processedThreads = 0; 121 | const totalThreads = messages.filter(isThread).length; 122 | for (const message of messages) { 123 | // Download threads 124 | if (isThread(message)) { 125 | processedThreads++; 126 | spinner.text = `Downloading threads (${processedThreads}/${totalThreads}) for ${ 127 | channel.name || channel.id 128 | }...`; 129 | message.replies = await downloadReplies(channel, message); 130 | } 131 | 132 | // Download users and avatars 133 | if (message.user) { 134 | await downloadUser(message, users); 135 | } 136 | } 137 | 138 | spinner.succeed( 139 | `Downloaded ${totalThreads} threads and users for ${ 140 | channel.name || channel.id 141 | }.` 142 | ); 143 | } 144 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Export your Slack workspace as static HTML 2 | 3 | Alright, so you want to export all your messages on Slack. You want them in a format that you 4 | can still enjoy in 20 years. This tool will help you do that. 5 | 6 | * **Completely static**: The generated files are pure HTML and will still work in 50 years. 7 | * **Everything you care about**: This tool downloads messages, files, and avatars. 8 | * **Nothing you do not care about**: Choose exactly which channels and DMs to download. 9 | * **All types of conversations**: We'll fetch public channels, private channels, DMs, and multi-person DMs. 10 | * **Incremental backups**: If you already have local data, we'll extend it - no need to download existing stuff again. 11 | * **JSON included**: All data is also stored as JSON, so you can consume it with other tools later. 12 | * **No cloud, free**: Do all of this for free, without giving anyone your information. 13 | * **Basic search**: Offers basic search functionality. 14 | 15 | Screen Shot 2021-09-09 at 6 43 55 PM 16 | 17 | ## Using it 18 | 19 | 1. Do you already have a user token for your workspace? If not, read on below on how to get a token. 20 | 2. Make sure you have [`node` and `npm`](https://nodejs.org/en/) installed, ideally something newer than Node v14. 21 | 3. Run `slack-archive`, which will interactively guide you through the options. 22 | 23 | ```sh 24 | npx slack-archive 25 | ``` 26 | 27 | ### Parameters 28 | 29 | ``` 30 | --automatic: Don't prompt and automatically fetch all messages from all channels. 31 | --use-previous-channel-config: Fetch messages from channels selected in previous run instead of prompting. 32 | --channel-types Comma-separated list of channel types to fetch messages from. 33 | (public_channel, private_channel, mpim, im) 34 | --exclude-channels Comma-separated list of channels to exclude, in automatic mode 35 | --no-backup: Don't create backups. Not recommended. 36 | --no-search: Don't create a search file, saving disk space. 37 | --no-file-download: Don't download files. 38 | --no-slack-connect: Don't connect to Slack, just generate HTML from local data. 39 | --force-html-generation: Force regeneration of HTML files. Useful after slack-archive upgrades. 40 | ``` 41 | 42 | ## Getting a token 43 | 44 | In order to download messages from private channels and direct messages, we will need a "user 45 | token". Slack uses the token to identify what permissions it'll give this app. We used to be able 46 | to just copy a token out of your Slack app, but now, we'll need to create a custom app and jump 47 | through a few hoops. 48 | 49 | This will be mostly painless, I promise. 50 | 51 | ### 1) Make a custom app 52 | 53 | Head over to https://api.slack.com/apps and `Create New App`. Select `From scratch`. 54 | Give it a name and choose the workspace you'd like to export. 55 | 56 | Then, from the `Features` menu on the left, select `OAuth & Permission`. 57 | 58 | As a redirect URL, enter something random that doesn't actually exist, or a domain you control. For instace: 59 | 60 | ``` 61 | https://notarealurl.com/ 62 | ``` 63 | 64 | (Note that redirects will take a _very_ long time if using a domain that doesn't actually exist) 65 | 66 | Then, add the following `User Token Scopes`: 67 | 68 | * channels:history 69 | * channels:read 70 | * files:read 71 | * groups:history 72 | * groups:read 73 | * im:history 74 | * im:read 75 | * mpim:history 76 | * mpim:read 77 | * remote_files:read 78 | * users:read 79 | 80 | Finally, head back to `Basic Information` and make a note of your app's `client 81 | id` and `client secret`. We'll need both later. 82 | 83 | ### 2) Authorize 84 | 85 | Make sure you have your Slack workspace `URL` (aka team name) and your app's `client id`. 86 | Then, in a browser, open this URL - replacing `{your-team-name}` and `{your-client-id}` 87 | with your values. 88 | 89 | ``` 90 | https://{your-team-name}.slack.com/oauth/authorize?client_id={your-client-id}&scope=client 91 | ``` 92 | 93 | Confirm everything until Slack sends you to the mentioned non-existent URL. Look at your 94 | browser's address bar - it should contain an URL that looks like this: 95 | 96 | ``` 97 | https://notarealurl.com/?code={code}&state= 98 | ``` 99 | 100 | Copy everything between `?code=` and `&state`. This is your `code`. We'll need it in the 101 | next step. 102 | 103 | Next, we'll exchange your code for a token. To do so, we'll also need your `client secret` 104 | from the first step when we created your app. In a browser, open this URL - replacing 105 | `{your-team-name}`, `{your-client-id}`, `{your-code}` and `{your-client-secret}` with 106 | your values. 107 | 108 | ``` 109 | https://{your-team-name}.slack.com/api/oauth.access?client_id={your-client-id}&client_secret={your-client-secret}&code={your-code} 110 | ``` 111 | 112 | Your browser should now be returning some JSON including a token. Make a note of it - that's what we'll use. Paste it in the command line, OR create a file called `.token` in the slack-archive directory (created when the command is first run) and paste it in there. 113 | -------------------------------------------------------------------------------- /static/style.css: -------------------------------------------------------------------------------- 1 | /* Reset */ 2 | 3 | /* Box sizing rules */ 4 | *, 5 | *::before, 6 | *::after { 7 | box-sizing: border-box; 8 | } 9 | 10 | /* Remove default margin */ 11 | body, 12 | h1, 13 | h2, 14 | h3, 15 | h4, 16 | p, 17 | figure, 18 | blockquote, 19 | dl, 20 | dd { 21 | margin: 0; 22 | } 23 | 24 | /* Remove list styles on ul, ol elements with a list role, which suggests default styling will be removed */ 25 | ul[role='list'], 26 | ol[role='list'] { 27 | list-style: none; 28 | } 29 | 30 | /* Set core root defaults */ 31 | html:focus-within { 32 | scroll-behavior: smooth; 33 | } 34 | 35 | /* Set core body defaults */ 36 | body { 37 | min-height: 100vh; 38 | text-rendering: optimizeSpeed; 39 | line-height: 1.5; 40 | } 41 | 42 | /* A elements that don't have a class get default styles */ 43 | a:not([class]) { 44 | text-decoration-skip-ink: auto; 45 | } 46 | 47 | /* Make images easier to work with */ 48 | img, 49 | picture { 50 | max-width: 100%; 51 | display: block; 52 | } 53 | 54 | /* Inherit fonts for inputs and buttons */ 55 | input, 56 | button, 57 | textarea, 58 | select { 59 | font: inherit; 60 | } 61 | 62 | /* Remove all animations, transitions and smooth scroll for people that prefer not to see them */ 63 | @media (prefers-reduced-motion: reduce) { 64 | html:focus-within { 65 | scroll-behavior: auto; 66 | } 67 | 68 | *, 69 | *::before, 70 | *::after { 71 | animation-duration: 0.01ms !important; 72 | animation-iteration-count: 1 !important; 73 | transition-duration: 0.01ms !important; 74 | scroll-behavior: auto !important; 75 | } 76 | } 77 | 78 | @font-face { 79 | font-family: "Lato"; 80 | src: url('fonts/Lato-Regular.ttf') format('truetype'); 81 | font-weight: normal; 82 | font-style: normal; 83 | } 84 | 85 | @font-face { 86 | font-family: "Lato"; 87 | src: url('fonts/Lato-Bold.ttf') format('truetype'); 88 | font-weight: bold; 89 | font-style: normal; 90 | } 91 | 92 | body, html { 93 | font-family: 'Lato', sans-serif; 94 | font-size: 14px; 95 | color: rgb(29, 28, 29); 96 | } 97 | 98 | a { 99 | color: rgb(18, 100, 163); 100 | } 101 | 102 | audio, video { 103 | max-width: 400px; 104 | } 105 | 106 | .messages-list { 107 | padding-bottom: 20px; 108 | } 109 | 110 | .messages-list .avatar { 111 | height: 36px; 112 | width: 36px; 113 | border-radius: 7px; 114 | margin-right: 10px; 115 | background: #c1c1c1; 116 | } 117 | 118 | .message-gutter { 119 | display: flex; 120 | margin: 10px; 121 | scroll-margin-top: 120px; 122 | } 123 | 124 | .message-gutter:target { 125 | background-color: #fafafa; 126 | border: 2px solid #39113E; 127 | padding: 10px; 128 | border-radius: 5px; 129 | } 130 | 131 | .message-gutter div:first-of-type { 132 | flex-shrink: 0; 133 | } 134 | 135 | .message-gutter > .message-gutter { 136 | /** i.e. replies in thread. Just here to be easily findable */ 137 | } 138 | 139 | .sender { 140 | font-weight: 800; 141 | margin-right: 10px; 142 | } 143 | 144 | .timestamp { 145 | font-weight: 200; 146 | font-size: 13px; 147 | color: rgb(97, 96, 97); 148 | } 149 | 150 | .header { 151 | position: sticky; 152 | background: #fff; 153 | color: #616061; 154 | top: 0; 155 | left: 0; 156 | padding: 10px; 157 | min-height: 70px; 158 | border-bottom: 1px solid #E2E2E2; 159 | box-sizing: border-box; 160 | } 161 | 162 | .header h1 { 163 | font-size: 16px; 164 | color: #1D1C1D; 165 | display: inline-block; 166 | } 167 | 168 | .header a { 169 | color: #616061; 170 | } 171 | 172 | .header a:active, .header a.current { 173 | color: #000; 174 | } 175 | 176 | .header .created { 177 | float: right; 178 | } 179 | 180 | .jumper { 181 | display: inline-block; 182 | } 183 | 184 | .jumper a { 185 | margin: 2px; 186 | } 187 | 188 | .text { 189 | overflow-wrap: break-word; 190 | } 191 | 192 | .file { 193 | max-height: 270px; 194 | margin-right: 10px; 195 | margin-top: 10px; 196 | border-radius: 4px; 197 | border: 1px solid #80808045; 198 | outline: none; 199 | } 200 | 201 | .reaction { 202 | background-color: #eaeaea; 203 | display: inline-block; 204 | border-radius: 10px; 205 | font-size: .7em; 206 | padding-left: 6px; 207 | padding-right: 6px; 208 | padding-bottom: 4px; 209 | margin-right: 5px; 210 | padding-top: 4px; 211 | } 212 | 213 | .reaction img { 214 | height: 16px; 215 | width: 16px; 216 | margin-right: 3px; 217 | vertical-align: middle; 218 | display: inline-block; 219 | } 220 | 221 | .reaction span { 222 | position: relative; 223 | top: 1px; 224 | } 225 | 226 | #index { 227 | display: flex; 228 | height: calc(100vh - 4px); 229 | } 230 | 231 | #channels { 232 | background: #39113E; 233 | width: 250px; 234 | color: #CDC3CE; 235 | padding-top: 10px; 236 | overflow: scroll; 237 | padding-bottom: 20px; 238 | } 239 | 240 | #channels ul { 241 | margin: 0; 242 | padding: 0; 243 | list-style: none; 244 | } 245 | 246 | #channels p { 247 | padding-left: 20px; 248 | } 249 | 250 | #channels .section { 251 | font-weight: 800; 252 | color: #fff; 253 | margin-top: 10px; 254 | } 255 | 256 | #channels .section:first-of-type { 257 | margin-top: 0; 258 | } 259 | 260 | #channels a { 261 | padding: 5px; 262 | display: block; 263 | color: #CDC3CE; 264 | text-decoration: none; 265 | padding-left: 20px; 266 | display: flex; 267 | max-height: 28px; 268 | white-space: pre; 269 | text-overflow: ellipsis; 270 | overflow: hidden; 271 | } 272 | 273 | #channels a .avatar { 274 | height: 20px; 275 | width: 20px; 276 | border-radius: 3px; 277 | margin-right: 10px; 278 | object-fit: contain; 279 | } 280 | 281 | #channels a:hover { 282 | background: #301034; 283 | color: #edeced; 284 | } 285 | 286 | #messages { 287 | flex-grow: 1; 288 | } 289 | 290 | #messages iframe { 291 | height: 100%; 292 | width: calc(100vw - 250px); 293 | border: none; 294 | } 295 | 296 | #search { 297 | margin: 10px; 298 | text-align: center; 299 | } 300 | 301 | #search ul { 302 | list-style: none; 303 | display: flex; 304 | flex-direction: column; 305 | align-items: center; 306 | } 307 | 308 | #search li { 309 | padding: 5px; 310 | border-bottom: 1px solid #E2E2E2; 311 | background: hsl(0deg 0% 98%); 312 | border-radius: 5px; 313 | width: 600px; 314 | text-align: left; 315 | margin-bottom: 5px; 316 | } 317 | 318 | #search a { 319 | text-decoration: none; 320 | color: unset; 321 | } -------------------------------------------------------------------------------- /static/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Message Search 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 223 | 230 | 231 | 232 | 233 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "jsx": "react", 4 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 5 | 6 | /* Projects */ 7 | // "incremental": true, /* Enable incremental compilation */ 8 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ 9 | // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ 10 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ 11 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ 12 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ 13 | 14 | /* Language and Environment */ 15 | "target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ 16 | // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ 17 | // "jsx": "preserve", /* Specify what JSX code is generated. */ 18 | // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ 19 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ 20 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ 21 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ 22 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ 23 | // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ 24 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ 25 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ 26 | 27 | /* Modules */ 28 | "module": "ES2020", /* Specify what module code is generated. */ 29 | // "rootDir": "./", /* Specify the root folder within your source files. */ 30 | "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ 31 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ 32 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ 33 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ 34 | // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ 35 | // "types": [], /* Specify type package names to be included without being referenced in a source file. */ 36 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 37 | // "resolveJsonModule": true, /* Enable importing .json files */ 38 | // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ 39 | 40 | /* JavaScript Support */ 41 | // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */ 42 | // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ 43 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ 44 | 45 | /* Emit */ 46 | // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ 47 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */ 48 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ 49 | // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ 50 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ 51 | "outDir": "./lib", /* Specify an output folder for all emitted files. */ 52 | // "removeComments": true, /* Disable emitting comments. */ 53 | // "noEmit": true, /* Disable emitting files from a compilation. */ 54 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ 55 | // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ 56 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ 57 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ 58 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 59 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ 60 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ 61 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ 62 | // "newLine": "crlf", /* Set the newline character for emitting files. */ 63 | // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ 64 | // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ 65 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ 66 | // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ 67 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ 68 | 69 | /* Interop Constraints */ 70 | // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ 71 | "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ 72 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */ 73 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ 74 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 75 | 76 | /* Type Checking */ 77 | "strict": true, /* Enable all strict type-checking options. */ 78 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ 79 | // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ 80 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ 81 | // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ 82 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ 83 | // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ 84 | // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ 85 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ 86 | // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ 87 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ 88 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ 89 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ 90 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ 91 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 92 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ 93 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ 94 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ 95 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ 96 | 97 | /* Completeness */ 98 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ 99 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 100 | }, 101 | "ts-node": { 102 | "esm": true, 103 | "transpileOnly": true 104 | }, 105 | } 106 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | import { uniqBy } from "lodash-es"; 2 | import inquirer from "inquirer"; 3 | import fs from "fs-extra"; 4 | import { User } from "@slack/web-api/dist/response/UsersInfoResponse"; 5 | import { Channel } from "@slack/web-api/dist/response/ConversationsListResponse"; 6 | import ora from "ora"; 7 | 8 | import { 9 | CHANNELS_DATA_PATH, 10 | USERS_DATA_PATH, 11 | getChannelDataFilePath, 12 | OUT_DIR, 13 | config, 14 | TOKEN_FILE, 15 | AUTOMATIC_MODE, 16 | USE_PREVIOUS_CHANNEL_CONFIG, 17 | CHANNEL_TYPES, 18 | DATE_FILE, 19 | EMOJIS_DATA_PATH, 20 | NO_SLACK_CONNECT, 21 | EXCLUDE_CHANNELS, 22 | } from "./config.js"; 23 | import { downloadExtras } from "./messages.js"; 24 | import { downloadMessages } from "./messages.js"; 25 | import { downloadFilesForChannel } from "./download-files.js"; 26 | import { 27 | createHtmlForChannels, 28 | getChannelsToCreateFilesFor, 29 | } from "./create-html.js"; 30 | import { createBackup, deleteBackup, deleteOlderBackups } from "./backup.js"; 31 | import { isValid, parseISO } from "date-fns"; 32 | import { createSearch } from "./search.js"; 33 | import { write, writeAndMerge } from "./data-write.js"; 34 | import { messagesCache, getUsers, getChannels } from "./data-load.js"; 35 | import { getSlackArchiveData, setSlackArchiveData } from "./archive-data.js"; 36 | import { downloadEmojiList, downloadEmojis } from "./emoji.js"; 37 | import { downloadAvatars } from "./users.js"; 38 | import { downloadChannels } from "./channels.js"; 39 | import { authTest } from "./web-client.js"; 40 | import { SlackArchiveChannelData } from "./interfaces.js"; 41 | 42 | const { prompt } = inquirer; 43 | 44 | async function selectMergeFiles(): Promise { 45 | const defaultResponse = true; 46 | 47 | if (!fs.existsSync(CHANNELS_DATA_PATH)) { 48 | return false; 49 | } 50 | 51 | // We didn't download any data. Merge. 52 | if (AUTOMATIC_MODE || NO_SLACK_CONNECT) { 53 | return defaultResponse; 54 | } 55 | 56 | const { merge } = await prompt([ 57 | { 58 | type: "confirm", 59 | default: defaultResponse, 60 | name: "merge", 61 | message: `We've found existing archive files. Do you want to append new data (recommended)? \n If you select "No", we'll delete the existing data.`, 62 | }, 63 | ]); 64 | 65 | if (!merge) { 66 | fs.emptyDirSync(OUT_DIR); 67 | } 68 | 69 | return merge; 70 | } 71 | 72 | async function selectChannels( 73 | channels: Array, 74 | previouslyDownloadedChannels: Record 75 | ): Promise> { 76 | if (USE_PREVIOUS_CHANNEL_CONFIG) { 77 | const selectedChannels: Array = channels.filter( 78 | (channel) => channel.id && channel.id in previouslyDownloadedChannels 79 | ); 80 | const selectedChannelNames = selectedChannels.map( 81 | (channel) => channel.name || channel.id || "Unknown" 82 | ); 83 | console.log( 84 | `Downloading channels selected previously: ${selectedChannelNames}.` 85 | ); 86 | 87 | const previousChannelIds = Object.keys(previouslyDownloadedChannels); 88 | if (previousChannelIds.length != selectedChannels.length) { 89 | console.warn( 90 | "WARNING: Did not find all previously selected channel IDs." 91 | ); 92 | console.log( 93 | `Expected to find ${previousChannelIds.length} channels, but only ${selectedChannels.length} matched.` 94 | ); 95 | // Consider Looking up the user-facing names of the missing channels in the saved data. 96 | const availableChannelIds = new Set( 97 | channels.map((channel) => channel.id || "") 98 | ); 99 | const missingChannelIds = previousChannelIds.filter( 100 | (cId) => !availableChannelIds.has(cId) 101 | ); 102 | //console.log(availableChannelIds); 103 | console.log(`Missing channel ids: ${missingChannelIds}`); 104 | } else { 105 | console.log( 106 | `Matched all ${previousChannelIds.length} previously selected channels out of ${channels.length} total channels available.` 107 | ); 108 | } 109 | 110 | return selectedChannels; 111 | } 112 | 113 | const choices = channels.map((channel) => ({ 114 | name: channel.name || channel.id || "Unknown", 115 | value: channel, 116 | })); 117 | 118 | if (AUTOMATIC_MODE || NO_SLACK_CONNECT) { 119 | if (EXCLUDE_CHANNELS) { 120 | const excludeChannels = EXCLUDE_CHANNELS.split(','); 121 | return channels.filter((channel) => !excludeChannels.includes(channel.name || '')); 122 | } 123 | return channels; 124 | } 125 | 126 | const result = await prompt([ 127 | { 128 | type: "checkbox", 129 | loop: true, 130 | name: "channels", 131 | message: "Which channels do you want to download?", 132 | choices, 133 | }, 134 | ]); 135 | 136 | return result.channels; 137 | } 138 | 139 | async function selectChannelTypes(): Promise> { 140 | const choices = [ 141 | { 142 | name: "Public Channels", 143 | value: "public_channel", 144 | }, 145 | { 146 | name: "Private Channels", 147 | value: "private_channel", 148 | }, 149 | { 150 | name: "Multi-Person Direct Message", 151 | value: "mpim", 152 | }, 153 | { 154 | name: "Direct Messages", 155 | value: "im", 156 | }, 157 | ]; 158 | 159 | if (CHANNEL_TYPES) { 160 | return CHANNEL_TYPES.split(","); 161 | } 162 | 163 | if (AUTOMATIC_MODE || USE_PREVIOUS_CHANNEL_CONFIG || NO_SLACK_CONNECT) { 164 | return ["public_channel", "private_channel", "mpim", "im"]; 165 | } 166 | 167 | const result = await prompt([ 168 | { 169 | type: "checkbox", 170 | loop: true, 171 | name: "channel-types", 172 | message: `Which channel types do you want to download?`, 173 | choices, 174 | }, 175 | ]); 176 | 177 | return result["channel-types"]; 178 | } 179 | 180 | async function getToken() { 181 | if (NO_SLACK_CONNECT) { 182 | return; 183 | } 184 | 185 | if (config.token) { 186 | console.log(`Using token ${config.token}`); 187 | return; 188 | } 189 | 190 | if (fs.existsSync(TOKEN_FILE)) { 191 | config.token = fs.readFileSync(TOKEN_FILE, "utf-8").trim(); 192 | return; 193 | } 194 | 195 | const result = await prompt([ 196 | { 197 | name: "token", 198 | type: "input", 199 | message: 200 | "Please enter your Slack token (xoxp-...). See README for more details.", 201 | }, 202 | ]); 203 | 204 | config.token = result.token; 205 | } 206 | 207 | async function writeLastSuccessfulArchive() { 208 | const now = new Date(); 209 | write(DATE_FILE, now.toISOString()); 210 | } 211 | 212 | function getLastSuccessfulRun() { 213 | if (!fs.existsSync(DATE_FILE)) { 214 | return ""; 215 | } 216 | 217 | const lastSuccessfulArchive = fs.readFileSync(DATE_FILE, "utf-8"); 218 | 219 | let date = null; 220 | 221 | try { 222 | date = parseISO(lastSuccessfulArchive); 223 | } catch (error) { 224 | return ""; 225 | } 226 | 227 | if (date && isValid(date)) { 228 | return `. Last successful run: ${date.toLocaleString()}`; 229 | } 230 | 231 | return ""; 232 | } 233 | 234 | async function getAuthTest() { 235 | if (NO_SLACK_CONNECT) { 236 | return; 237 | } 238 | 239 | const spinner = ora("Testing authentication with Slack...").start(); 240 | const result = await authTest(); 241 | 242 | if (!result.ok) { 243 | spinner.fail(`Authentication with Slack failed.`); 244 | 245 | console.log( 246 | `Authentication with Slack failed. The error was: ${result.error}` 247 | ); 248 | console.log( 249 | `The provided token was ${config.token}. Double-check the token and try again.` 250 | ); 251 | console.log( 252 | `For more information on the error code, see the error table at https://api.slack.com/methods/auth.test` 253 | ); 254 | console.log(`This tool will now exit.`); 255 | 256 | await deleteBackup(); 257 | process.exit(-1); 258 | } else { 259 | spinner.succeed(`Successfully authorized with Slack as ${result.user}\n`); 260 | } 261 | 262 | return result; 263 | } 264 | 265 | export async function main() { 266 | console.log(`Welcome to slack-archive${getLastSuccessfulRun()}`); 267 | 268 | if (AUTOMATIC_MODE) { 269 | console.log(`Running in fully automatic mode without prompts`); 270 | } 271 | 272 | if (NO_SLACK_CONNECT) { 273 | console.log(`Not connecting to Slack and skipping all Slack API calls`); 274 | } 275 | 276 | await getToken(); 277 | await createBackup(); 278 | 279 | const slackArchiveData = await getSlackArchiveData(); 280 | const users: Record = await getUsers(); 281 | const channelTypes = (await selectChannelTypes()).join(","); 282 | 283 | slackArchiveData.auth = await getAuthTest(); 284 | 285 | const channels = await downloadChannels({ types: channelTypes }, users); 286 | const selectedChannels = await selectChannels( 287 | channels, 288 | slackArchiveData.channels 289 | ); 290 | const newMessages: Record = {}; 291 | 292 | // Emoji 293 | // We don't actually download the images here, we'll 294 | // do that as needed 295 | const emojis = await downloadEmojiList(); 296 | await writeAndMerge(EMOJIS_DATA_PATH, emojis); 297 | 298 | // Do we want to merge data? 299 | await selectMergeFiles(); 300 | await writeAndMerge(CHANNELS_DATA_PATH, selectedChannels); 301 | 302 | // Download messages and extras for each channel 303 | await downloadEachChannel(); 304 | 305 | // Save data 306 | await setSlackArchiveData(slackArchiveData); 307 | 308 | // Create HTML, but only for channels with new messages 309 | // - or channels that we didn't make HTML for yet 310 | const channelsToCreateFilesFor = await getChannelsToCreateFilesFor( 311 | selectedChannels, 312 | newMessages 313 | ); 314 | await createHtmlForChannels(channelsToCreateFilesFor); 315 | 316 | // Create search file 317 | await createSearch(); 318 | 319 | // Cleanup and finalize 320 | await deleteBackup(); 321 | await deleteOlderBackups(); 322 | await writeLastSuccessfulArchive(); 323 | 324 | console.log(`All done.`); 325 | 326 | async function downloadEachChannel() { 327 | if (NO_SLACK_CONNECT) return; 328 | 329 | for (const [i, channel] of selectedChannels.entries()) { 330 | if (!channel.id) { 331 | console.warn(`Selected channel does not have an id`, channel); 332 | continue; 333 | } 334 | 335 | // Do we already have everything? 336 | slackArchiveData.channels[channel.id] = 337 | slackArchiveData.channels[channel.id] || {}; 338 | if (slackArchiveData.channels[channel.id].fullyDownloaded) { 339 | continue; 340 | } 341 | 342 | // Download messages & users 343 | let downloadData = await downloadMessages( 344 | channel, 345 | i, 346 | selectedChannels.length 347 | ); 348 | let result = downloadData.messages; 349 | newMessages[channel.id] = downloadData.new; 350 | 351 | await downloadExtras(channel, result, users); 352 | await downloadEmojis(result, emojis); 353 | await downloadAvatars(); 354 | 355 | // Sort messages 356 | const spinner = ora( 357 | `Saving message data for ${channel.name || channel.id} to disk` 358 | ).start(); 359 | spinner.render(); 360 | 361 | result = uniqBy(result, "ts"); 362 | result = result.sort((a, b) => { 363 | return parseFloat(b.ts || "0") - parseFloat(a.ts || "0"); 364 | }); 365 | 366 | await writeAndMerge(USERS_DATA_PATH, users); 367 | fs.outputFileSync( 368 | getChannelDataFilePath(channel.id), 369 | JSON.stringify(result, undefined, 2) 370 | ); 371 | 372 | // Download files. This needs to run after the messages are saved to disk 373 | // since it uses the message data to find which files to download. 374 | await downloadFilesForChannel(channel.id!, spinner); 375 | 376 | // Update the data load cache 377 | messagesCache[channel.id!] = result; 378 | 379 | // Update the data 380 | const { is_archived, is_im, is_user_deleted } = channel; 381 | if (is_archived || (is_im && is_user_deleted)) { 382 | slackArchiveData.channels[channel.id].fullyDownloaded = true; 383 | } 384 | slackArchiveData.channels[channel.id].messages = result.length; 385 | 386 | spinner.succeed(`Saved message data for ${channel.name || channel.id}`); 387 | } 388 | } 389 | } 390 | 391 | main(); 392 | -------------------------------------------------------------------------------- /src/create-html.tsx: -------------------------------------------------------------------------------- 1 | import { format } from "date-fns"; 2 | import fs from "fs-extra"; 3 | import path from "path"; 4 | import React from "react"; 5 | import ReactDOMServer from "react-dom/server.js"; 6 | import ora, { Ora } from "ora"; 7 | import { chunk, sortBy } from "lodash-es"; 8 | import { dirname } from "path"; 9 | import { fileURLToPath } from "url"; 10 | import esMain from "es-main"; 11 | import slackMarkdown from "slack-markdown"; 12 | 13 | import { getChannels, getMessages, getUsers } from "./data-load.js"; 14 | import { 15 | ArchiveMessage, 16 | Channel, 17 | ChunksInfo, 18 | Message, 19 | Reaction, 20 | SlackArchiveData, 21 | User, 22 | Users, 23 | } from "./interfaces.js"; 24 | import { 25 | getHTMLFilePath, 26 | INDEX_PATH, 27 | OUT_DIR, 28 | MESSAGES_JS_PATH, 29 | FORCE_HTML_GENERATION, 30 | } from "./config.js"; 31 | import { slackTimestampToJavaScriptTimestamp } from "./timestamp.js"; 32 | import { recordPage } from "./search.js"; 33 | import { write } from "./data-write.js"; 34 | import { getSlackArchiveData } from "./archive-data.js"; 35 | import { getEmojiFilePath, getEmojiUnicode, isEmojiUnicode } from "./emoji.js"; 36 | import { getName } from "./users.js"; 37 | import { 38 | isBotChannel, 39 | isDmChannel, 40 | isPrivateChannel, 41 | isPublicChannel, 42 | } from "./channels.js"; 43 | 44 | const _dirname = dirname(fileURLToPath(import.meta.url)); 45 | const MESSAGE_CHUNK = 1000; 46 | 47 | // This used to be a prop on the components, but passing it around 48 | // was surprisingly slow. Global variables are cool again! 49 | // Set by createHtmlForChannels(). 50 | let users: Users = {}; 51 | let slackArchiveData: SlackArchiveData = { channels: {} }; 52 | let me: User | null; 53 | 54 | // Little hack to switch between ./index.html and ./html/... 55 | let base = ""; 56 | 57 | function formatTimestamp(message: Message, dateFormat = "PPPPpppp") { 58 | const jsTs = slackTimestampToJavaScriptTimestamp(message.ts); 59 | const ts = format(jsTs, dateFormat); 60 | 61 | return ts; 62 | } 63 | 64 | interface FilesProps { 65 | message: Message; 66 | channelId: string; 67 | } 68 | const Files: React.FunctionComponent = (props) => { 69 | const { message, channelId } = props; 70 | const { files } = message; 71 | 72 | if (!files || files.length === 0) return null; 73 | 74 | const fileElements = files.map((file) => { 75 | const { thumb_1024, thumb_720, thumb_480, thumb_pdf } = file as any; 76 | const thumb = thumb_1024 || thumb_720 || thumb_480 || thumb_pdf; 77 | let src = `files/${channelId}/${file.id}.${file.filetype}`; 78 | let href = src; 79 | 80 | if (file.mimetype?.startsWith("image")) { 81 | return ( 82 | 83 | 84 | 85 | ); 86 | } 87 | 88 | if (file.mimetype?.startsWith("video")) { 89 | return