├── .gitignore ├── package.json ├── emoji_name_to_unicode.js ├── test.js ├── README.md └── slack2roam.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "slack2roam", 3 | "version": "0.1.2", 4 | "description": "Script for converting Slack export JSON into Roam import JSON", 5 | "main": "slack2roam.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git@github.com:malcolmocean/slack2roam.git" 9 | }, 10 | "author": "Malcolm Ocean (http://malcolmocean.com/)", 11 | "license": "MIT", 12 | "dependencies": { 13 | "minimist": "^1.1.0" 14 | }, 15 | "bugs": { 16 | "url": "https://github.com/malcolmocean/slack2roam/issues" 17 | }, 18 | "engines": { 19 | "node": ">=10.14.2" 20 | }, 21 | "bin": { 22 | "slack2roam": "slack2roam.js" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /emoji_name_to_unicode.js: -------------------------------------------------------------------------------- 1 | // shoutout to Aaron Parecki 2 | // https://github.com/aaronpk/Slack-IRC-Gateway/blob/541cc464e60e6146c305afd5efc521f6553f690c/emoji.js 3 | const punycode = require('punycode') 4 | const emoji_data = require('./emoji_pretty.json') 5 | const emoji_re = /\:([a-zA-Z0-9\-_\+]+)\:(?:\:([a-zA-Z0-9\-_\+]+)\:)?/g 6 | module.exports = function (text) { 7 | var new_text = text 8 | while (match = emoji_re.exec(text)) { 9 | const ed = emoji_data.find(el => el.short_name == match[1]) 10 | if (ed) { 11 | let points = ed.unified.split("-") 12 | points = points.map(p => parseInt(p, 16)) 13 | new_text = new_text.replace(match[0], punycode.ucs2.encode(points)) 14 | } 15 | } 16 | return new_text 17 | } 18 | 19 | // TODO = the emoji_pretty.json file needs to be cleaned up (it has a ton of unneeded data) 20 | // TODO = the emoji_pretty.json file needs to be updated (it's only from 2017) 21 | // TODO = grab source code from Complice that is a slightly better search above (gets alt names) 22 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | // hi and welcome to the most minimal test ever 2 | const slack2roam = require('./slack2roam') 3 | const users = [{ 4 | id: "U0ABC123", 5 | name: "malcolmocean", 6 | real_name: "Malcolm Ocean", 7 | }, { 8 | id: "U0DEF456", 9 | name: "conaw", 10 | real_name: "Conor White-Sullivan", 11 | }] 12 | const channel = {name: 'general'} 13 | const messages = [{ 14 | "type": "message", 15 | "text": "Hey it's a message in slack", 16 | "user": "U0ABC123", 17 | "ts": "1585242429.009300", 18 | }, { 19 | "type": "message", 20 | "text": "Wow, here's another it's a message in slack!", 21 | "user": "U0DEF456", 22 | "ts": "1585893429.009300", 23 | }] 24 | slack2roam.setOptions({ 25 | uPageNameKey: 'real_name', // defaults to 'name', ie username, since guaranteed unique 26 | }) 27 | slack2roam.makeUserMap(users) 28 | const result = slack2roam.slackChannelToRoamPage(channel, messages) 29 | const expected = { 30 | "title": "general", 31 | "children": [ 32 | { 33 | "string": "[[March 26th, 2020]]", 34 | "children": [ 35 | { 36 | "uid": "slack_general_1585242429_009300", 37 | "create-time": 1585242429093, 38 | "string": "[[Malcolm Ocean]] 13:07\nHey it's a message in slack" 39 | } 40 | ] 41 | }, 42 | { 43 | "string": "[[April 3rd, 2020]]", 44 | "children": [ 45 | { 46 | "uid": "slack_general_1585893429_009300", 47 | "create-time": 1585893429093, 48 | "string": "[[Conor White-Sullivan]] 01:57\nWow, here's another it's a message in slack!" 49 | } 50 | ] 51 | } 52 | ] 53 | } 54 | 55 | const assert = require('assert') 56 | try { 57 | assert.deepEqual(result, expected) 58 | console.log("result", JSON.stringify(result, 0, 2)) 59 | console.log('\x1b[32m')//, 'green') 60 | console.log("✓ there's one test and it worked") 61 | console.log('\x1b[0m') 62 | } catch (err) { 63 | console.log('\x1b[31m')//, 'Error') 64 | console.log("✗ there's one test and it just broke\n") 65 | console.log("Error", err) 66 | console.log('\x1b[33m' + "result =", JSON.stringify(result, null, 2)) 67 | console.log('\x1b[34m' + "expected =", JSON.stringify(expected, null, 2)) 68 | console.log('\x1b[0m') 69 | } 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Script for converting [Slack](https://slack.com) JSON into [Roam](https://roamresearch.com) import JSON. 2 | 3 | Works as both a node module for use in other projects, as well as a command-line interface. The cli can be used in many diverse ways. 4 | 5 | # As node module 6 | 7 | ## Install to your node project 8 | 9 | ```bash 10 | npm install --save slack2roam 11 | ``` 12 | 13 | ## Use it 14 | 15 | ```javascript 16 | const slack2roam = require('slack2roam') 17 | const users = [{ 18 | id: "U0ABC123", 19 | name: "malcolmocean", 20 | real_name: "Malcolm Ocean", 21 | }, { 22 | id: "U0DEF456", 23 | name: "conaw", 24 | real_name: "Conor White-Sullivan", 25 | }] 26 | const channel = {name: 'general'} 27 | const messages = [{ 28 | "type": "message", 29 | "text": "Hey it's a message in slack", 30 | "user": "U0ABC123", 31 | "ts": "1585242429.009300", 32 | }, { 33 | "type": "message", 34 | "text": "Wow, here's another it's a message in slack!", 35 | "user": "U0DEF456", 36 | "ts": "1585893429.009300", 37 | }] 38 | const options = {uPageNameKey: 'real_name'} // defaults to 'name', ie username 39 | slack2roam.makeUserMap(users) 40 | const roamJson = slack2roam.slackChannelToRoamPage(channel, messages, options) 41 | ``` 42 | 43 | result: 44 | ```json { 45 | "title": "general", 46 | "children": [ 47 | { 48 | "string": "[[March 26th, 2020]]", 49 | "children": [ 50 | { 51 | "uid": "slack_general_1585242429_009300", 52 | "create-time": 1585242429093, 53 | "string": "[[Malcolm Ocean]] 13:07\nHey it's a message in slack" 54 | } 55 | ] 56 | }, 57 | { 58 | "string": "[[April 3rd, 2020]]", 59 | "children": [ 60 | { 61 | "uid": "slack_general_1585893429_009300", 62 | "create-time": 1585893429093, 63 | "string": "[[Conor White-Sullivan]] 01:57\nWow, here's another it's a message in slack!" 64 | } 65 | ] 66 | } 67 | ] 68 | } 69 | ``` 70 | 71 | # As command-line script 72 | 73 | ## Install as a command-line tool 74 | 75 | If you have NodeJS installed, npm comes with it. If not, [get it here](https://nodejs.org/en/download/). 76 | 77 | ```bash 78 | npm install --global slack2roam 79 | ``` 80 | 81 | ## Use it 82 | 83 | The command-line tool expects the command to be run from a slack export folder that contains the following: 84 | 85 | - `users.json` 86 | - `channels.json` 87 | - `general/` 88 | - `2020-06-11.json` 89 | - `2020-06-12.json` 90 | - _(other dates)_ 91 | - `random/` 92 | - _(other channel folders)_ 93 | 94 | (It doesn't specifically expect `general/` and `random/`, those are just examples) 95 | 96 | Once you're in that folder, you can just run this: 97 | 98 | ```bash 99 | slack2roam -o roam_data.json 100 | ``` 101 | 102 | This will output to the file `roam_data.json`. If you omit the -o flag it will show you a preview in the console instead. 103 | 104 | Each channel listed in `channels.json` gets its own page in roam. 105 | 106 | If you want to provide options, add this flag with a JSON string 107 | 108 | --options='{"uPageNameKey":"real_name", "workspaceName":"mycompany", "timeOfDayFormat": "ampm"}' 109 | 110 | Importing in bulk is best as it will do optimal things with threading, but if you import stuff later that replies to earlier threads, then it will block reference them. This uses custom UIDs, which is the reason you might want to provide a `workspaceName` above, since slack's UIDs are probably only per-workspace and otherwise you could get a collision if you're importing from multiple workspaces. This is fairly unlikely, especially since slack2roam also puts the channel name into the UID. 111 | 112 | # Areas for improvement 113 | 114 | - custom timezone support 115 | - handle edit timestamp better if date changed 116 | - be better at reformatting text 117 | - more command-line features, like specifying only one channel 118 | - more options! 119 | 120 | # Contributing 121 | 122 | Ummm yeah hmu. I haven't done much managing of OSS projects but if you submit a pull request we can figure something out. Talk to me about it on Twitter [@Malcolm_Ocean](https://twitter.com/Malcolm_Ocean) as I don't check GitHub notifications much. 123 | 124 | See [here](https://roamresearch.com/#/app/help/page/RxZF78p60) for Roam's JSON schema. 125 | 126 | # Tip me 🤑 127 | 128 | If this is hugely valuable to you, you can tip me [here](https://paypal.me/complice), or also go check out [my meta-systematic, goal-oriented productivity app, Complice](https://complice.co/?utm_source=github&utm_medium=readme&utm_campaign=slack2roam&utm_content=msgopa). 129 | -------------------------------------------------------------------------------- /slack2roam.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | const fs = require('fs') 3 | const path = require('path') 4 | const emoji_name_to_unicode = require('./emoji_name_to_unicode') 5 | 6 | const usersById = {} 7 | const makeUserMap = function makeUserMap (users) { 8 | users.map(user => { 9 | usersById[user.id] = user 10 | }) 11 | } 12 | 13 | let options = { 14 | workspaceName: '', 15 | timeOfDayFormat: '24h', 16 | tzOffsetMinutes: 0, 17 | uPageNameKey: 'name', 18 | } 19 | function setOptions (o) { 20 | for (let key in o) { 21 | options[key] = o[key] 22 | } 23 | } 24 | 25 | function formatDateForRoam (date) { 26 | function nth (d) { 27 | if (d > 3 && d < 21) return 'th' 28 | switch (d % 10) { 29 | case 1: return "st" 30 | case 2: return "nd" 31 | case 3: return "rd" 32 | default: return "th" 33 | } 34 | } 35 | date = new Date(date) 36 | const month = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][date.getMonth()] 37 | const dom = date.getDate() 38 | return `[[${month} ${dom}${nth(dom)}, ${date.getFullYear()}]]` 39 | } 40 | 41 | function formatTimeOfDay (date) { 42 | const h = date.getHours() 43 | const m = date.getMinutes() 44 | const hh = ('0'+h).substr(-2) 45 | const mm = ('0'+m).substr(-2) 46 | if (options.timeOfDayFormat == 'ampm') { 47 | const h12 = h % 12 || '12' 48 | const ampm = h >= 12 ? 'pm' : 'am' 49 | return h12 + ':' + mm + ampm 50 | } else if (options.timeOfDayFormat == '24h') { 51 | return hh + ':' + mm 52 | } else { // TODO = smart hour? 53 | return hh + ':' + mm 54 | } 55 | } 56 | 57 | function tsToTzAdjustedDate (ts) { 58 | // this doesn't currently play nice with the thing that converts a tz to eg [[January 1st, 2020]] 59 | // if (options.tzOffsetMinutes) { // defaults to using local timezone 60 | // ts += 60*1000*options.tzOffsetMinutes 61 | // } 62 | return new Date(ts) 63 | } 64 | 65 | // TODO = this function needs lots of work 66 | // currently defaults to username for uPageNameKey, since that's guaranteed unique 67 | function format (text) { 68 | // console.log("\n\n\n\n========================================") 69 | // console.log("format input:\n"+ text) 70 | let string = text 71 | 72 | string = emoji_name_to_unicode(string) 73 | 74 | string = string.replace(/&/g, '&') 75 | string = string.replace(/>/g, '>') 76 | string = string.replace(/</g, '<') 77 | string = string.replace(/"/g, '"') 78 | string = string.replace(/“/g, '“') 79 | string = string.replace(/”/g, '”') 80 | string = string.replace(/&[lr]squo;/g, "'") 81 | string = string.replace(/–/g, '\u2013') 82 | string = string.replace(/—/g, '\u2014') 83 | string = string.replace(/…/g, '\u2026') 84 | string = string.replace(/ /g, ' ') 85 | string = string.replace(/°/g, '\u00b0') 86 | 87 | // these two probably are not adequate at all 88 | string = string.replace(/ \*/g, ' **').replace(/\* /g, '** ') 89 | string = string.replace(/ _/g, ' __').replace(/_ /g, '__ ') 90 | 91 | // <@U9JRMJ0H4> is what an at-mention looks like 92 | 93 | string = string.replace(/<@([^>|]*?)>/g, (all, uid) => usersById[uid] ? '@[['+usersById[uid][options.uPageNameKey]+']]' : all) 94 | 95 | // is what a link looks like if it has text 96 | string = string.replace(/<(.*?)\|(.*?)>/g, '[$2]($1)') 97 | 98 | // this is what a link looks like if no link text 99 | string = string.replace(/<(https?:\/\/.*?)>/g, '$1') 100 | 101 | return string 102 | } 103 | 104 | // slack timezones are in seconds with 6 decimal places 105 | // but first & second decimal places tend to be empty 106 | // yet third & fourth have something. eg 1583940932.008300 107 | // so upgrade 008300 to 083000 so it keeps that data 108 | function slackTimeToMillis (s) { 109 | s = parseFloat(s) 110 | const secs = Math.floor(s)*1000 111 | const nano = Math.floor((s%1)*1e6) 112 | let millis 113 | if (nano < 100000) { 114 | millis = Math.round(nano/100) 115 | } else { // I'm guessing this condition won't happen 116 | console.log("WHOA WHOA WHOA WHOA \nWHOA WHOA WHOA WHOA \nWHOA WHOA WHOA WHOA \n\n slackTimeToMillis s="+s) 117 | console.log("WHOA WHOA WHOA WHOA \nWHOA WHOA WHOA WHOA \nWHOA WHOA WHOA WHOA \n\n") 118 | millis = Math.ceil(nano/1000) 119 | } 120 | return secs + millis 121 | // return Math.floor(s*1000) 122 | } 123 | 124 | function slackTimeToUid (ts, channelname) { 125 | const ts_underscore = (''+ts).replace(/\./g, '_') 126 | const wn_underscore = options.workspaceName ? options.workspaceName + '_' : '' 127 | const ch_underscore = channelname ? channelname.replace(/-/g, '__') + '_' : '' 128 | return 'slack_'+wn_underscore+ch_underscore+ts_underscore 129 | } 130 | 131 | // attachments, eg expanded links to sites 132 | function attachmentToString (attachment) { 133 | let string = '' 134 | const title = attachment.title || attachment.name 135 | const name = attachment.name || attachment.text || attachment.title 136 | const link = attachment.title_link || attachment.original_url 137 | if (title) { 138 | string = `[${title}](${link})` 139 | } else { 140 | string = link 141 | } 142 | if (name !== title) { 143 | string += ': ' + name 144 | } 145 | 146 | if (/youtube.com|youtu.be/.test(link)) { 147 | string = `${name}\n{{[[youtube]]: ${link}}}` 148 | } 149 | return string 150 | } 151 | // permalink_public: "https://slack-files.com/T2U9PFM6K-F012XGKA6TZ-a90a53104d", 152 | // permalink: "https://subdomain.slack.com/files/U9JRMJ0H4/F012XGKA6TZ/filename.m4a", 153 | 154 | // files, eg slack uploads or links to google drive 155 | // or images pasted into slack 156 | function fileToString (file) { 157 | let string = '' 158 | const name = file.name || file.title 159 | const link = file.external_url || file.title_link || file.original_url || file.url_private 160 | // file.url_private only shows if user logged into slack 161 | // TODO = get roam folks to somehow support importing images/files by url...? that's complex 162 | if (link) { 163 | string = `[${name}](${link})` 164 | if (file.mimetype.startsWith('image')) { 165 | string = `${name} ![](${link})` 166 | } 167 | if (file.mimetype.startsWith('audio')) { 168 | string = `:hiccup [:span "${name}" [:br] [:audio {:controls "1"} [:source {:src "${link}", :type "${file.mimetype}"}]]]` 169 | // string = `:hiccup ["${name}" :audio {:controls "1"} [:source {:src "${file.url_private || link}", :type "${file.mimetype}"}] "audio embed"]]` 170 | } 171 | } 172 | if (!string && file.mode == 'hidden_by_limit') { 173 | string = `File hidden_by_limit [${file.id}]` 174 | } 175 | if (!string) { 176 | console.log("========================================") 177 | console.log("idk how to handle this file", file) 178 | console.log("========================================") 179 | string = `[Attachment "${name}" ...unknown url]` 180 | } 181 | return string 182 | } 183 | 184 | function messageToBlock (message, channelname) { 185 | if (!message.user && message.is_hidden_by_limit) {return} 186 | const user = usersById[message.user] 187 | const block = { 188 | 'uid': slackTimeToUid(message.ts, channelname), // because this is used as a thread-id, for some reason 189 | 'create-time': slackTimeToMillis(message.ts), 190 | } 191 | if (user) { 192 | block.string = `[[${user[options.uPageNameKey]}]] ` 193 | if (user.email) { 194 | block['create-email'] = user.email 195 | } 196 | } else { 197 | console.log(`Warning: no user found for message "${message.text}"`) 198 | } 199 | const createTime = tsToTzAdjustedDate(block['create-time']) 200 | const createTime_formatted = formatTimeOfDay(createTime) 201 | block.string += `${createTime_formatted}` 202 | if (message.edited && message.edited.ts) { 203 | block['edit-time'] = slackTimeToMillis(message.edited.ts) 204 | const editTime = tsToTzAdjustedDate(block['edit-time']) 205 | const editTime_formatted = formatTimeOfDay(editTime) 206 | // TODO: fix date if date changed 207 | block.string += ` (edited ${editTime_formatted})` 208 | } 209 | block.string += '\n' + format(message.text) 210 | 211 | if (message.attachments && message.attachments.length) { 212 | block.string += '\n\n' + message.attachments.map(a => attachmentToString(a)).join('\n\n') 213 | // block.children = block.children.concat(...message.attachments.map(a => ({string: attachmentToString(a)}))) 214 | } 215 | if (message.files && message.files.length) { 216 | block.string += '\n\n' + message.files.map(f => fileToString(f)).join('\n\n') 217 | // block.children = block.children.concat(...message.files.map(f => ({string: fileToString(f)}))) 218 | } 219 | if (message.thread_ts && message.thread_ts !== message.ts) { 220 | if (message.thread_ts == 1585103710.000700) { 221 | console.log('messageToBlock: ' + new Date(block['create-time']) + ' -> ' + block.string) 222 | } 223 | return { 224 | 'string': `In reply to ${slackTimeToUid(message.thread_ts, channelname)}`, 225 | 'create-time': block['create-time'], 226 | 'children': [block], 227 | 'replyTo': slackTimeToUid(message.thread_ts, channelname), // used by this script 228 | } 229 | // TODO = "not sure what the plan is here" 230 | // and should it be different depending on whether the original message for thread_ts is in this batch or not? 231 | } 232 | 233 | // TODO = resolve links to other messages?? 234 | return block 235 | } 236 | 237 | function mergeThreads (blocks) { 238 | const messagesById = {} 239 | const oldThreadsById = {} 240 | blocks.map(b => b.uid && (messagesById[b.uid] = b)) 241 | blocks = blocks.filter(b => { 242 | if (b.replyTo) { 243 | if (messagesById[b.replyTo]) { 244 | if (!messagesById[b.replyTo].children) { 245 | messagesById[b.replyTo].children = [] 246 | } 247 | messagesById[b.replyTo].children.push(b.children[0]) 248 | } else if (oldThreadsById[b.replyTo]) { 249 | oldThreadsById[b.replyTo].children.push(b.children[0]) 250 | } else { 251 | oldThreadsById[b.replyTo] = b 252 | return true 253 | } 254 | return false 255 | } 256 | return true 257 | }) 258 | return blocks 259 | } 260 | 261 | function splitByDate (blocks) { 262 | const dateBlocksByDate = {} 263 | const dateBlocks = [] 264 | 265 | blocks.map(block => { 266 | let date = tsToTzAdjustedDate(block['create-time']) 267 | const roamDate = formatDateForRoam(date) 268 | if (!dateBlocksByDate[roamDate]) { 269 | dateBlocksByDate[roamDate] = { 270 | string: roamDate, 271 | children: [], 272 | } 273 | dateBlocks.push(dateBlocksByDate[roamDate]) 274 | } 275 | dateBlocksByDate[roamDate].children.push(block) 276 | }) 277 | // also do I want to insert the time of day into these messages? 278 | return dateBlocks 279 | } 280 | 281 | function slackChannelToRoamPage (channel, messages) { 282 | const page = { 283 | 'title': channel.name, 284 | } 285 | page.children = messages.map(message => messageToBlock(message, channel.name)).filter(Boolean) 286 | page.children = mergeThreads(page.children) 287 | page.children = splitByDate(page.children) 288 | return page 289 | } 290 | 291 | async function importChannel (rootPath, channelName) { 292 | const dir = await fs.promises.readdir(path.join(rootPath, channelName)) 293 | let messages = [] 294 | for (let filename of dir) { 295 | const json = await readFileToJson(path.join(rootPath, channelName, filename)) 296 | messages = messages.concat(json) 297 | } 298 | if (messages.length > 1 && messages[0].ts > messages[1].ts) { 299 | messages.reverse() 300 | } 301 | return messages 302 | } 303 | 304 | async function readFileToJson (_path) { 305 | // return fs.promises.readFile(_path) 306 | return JSON.parse(await fs.promises.readFile(_path)) 307 | } 308 | 309 | async function writeJsonToFile (_path, json) { 310 | console.log("writing JSON to " + _path) 311 | return fs.promises.writeFile(_path, JSON.stringify(json, 0, 2)) 312 | // return fs.promises.writeFile(_path, JSON.stringify(json)) 313 | } 314 | 315 | async function slack2roam_cli () { 316 | const argv = require('minimist')(process.argv.slice(2)) 317 | if (argv.options) { 318 | setOptions(JSON.parse(argv.options)) 319 | console.log("options", options) 320 | } 321 | // const command = argv._[0] 322 | // const extra = argv._[1] 323 | const dirname = argv.dirname || process.cwd() 324 | const users = await readFileToJson(path.join(dirname, 'users.json')) 325 | console.log(users.length + ' users loaded from users.json') 326 | makeUserMap(users) 327 | const channels = await readFileToJson(path.join(dirname, 'channels.json')) 328 | // console.log("channels", channels) 329 | const messagesAllChannels = await Promise.all(channels.map(async channel => { 330 | const messages = await importChannel(dirname, channel.name) // Malcolm says: maybe wants to be channel.name_normalized; they were all the same in my data 331 | return {channel, messages} 332 | })) 333 | 334 | const roamPages = messagesAllChannels.map(({channel, messages}) => { 335 | return slackChannelToRoamPage(channel, messages) 336 | }) 337 | 338 | console.log("roamPages", roamPages) 339 | 340 | if (argv.o || argv.output) { 341 | // LATER: confirm overwrite if exists 342 | await writeJsonToFile(argv.o || argv.output, roamPages) 343 | } else { 344 | console.log('roamPages', JSON.stringify(roamPages, null, 2)) 345 | } 346 | } 347 | 348 | if (require.main === module) { // called directly 349 | slack2roam_cli() 350 | .catch(err => { 351 | console.log('\x1b[31m')//, 'Error') 352 | console.log(err) 353 | console.log('\x1b[0m') 354 | }) 355 | } else { 356 | exports.makeUserMap = makeUserMap 357 | exports.slackChannelToRoamPage = slackChannelToRoamPage 358 | exports.setOptions = setOptions 359 | } 360 | --------------------------------------------------------------------------------