├── .npmrc ├── .gitignore ├── documentation ├── images │ ├── add yml.png │ ├── Set workfow.png │ ├── add new file.png │ ├── login GitHub.png │ ├── Go to Settings.png │ ├── add R2G_EMAIL.png │ ├── add R2G_GRAPH.png │ ├── success set up.png │ ├── add R2G_PASSWORD.png │ ├── Get the graph name.png │ ├── email and password.png │ ├── Create New repository.png │ ├── private repository success.png │ ├── Create New private repository.png │ └── Signing up for a new GitHub account.png ├── Setup Instructions.md ├── Settings for main.yml.md ├── Common error causes.md └── Full Guide with Step-by-Step Screenshots.md ├── package.json ├── LICENSE ├── README.md ├── backup.js └── roam2github.js /.npmrc: -------------------------------------------------------------------------------- 1 | fund=false -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | backup 3 | tmp 4 | .env -------------------------------------------------------------------------------- /documentation/images/add yml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/add yml.png -------------------------------------------------------------------------------- /documentation/images/Set workfow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/Set workfow.png -------------------------------------------------------------------------------- /documentation/images/add new file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/add new file.png -------------------------------------------------------------------------------- /documentation/images/login GitHub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/login GitHub.png -------------------------------------------------------------------------------- /documentation/images/Go to Settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/Go to Settings.png -------------------------------------------------------------------------------- /documentation/images/add R2G_EMAIL.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/add R2G_EMAIL.png -------------------------------------------------------------------------------- /documentation/images/add R2G_GRAPH.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/add R2G_GRAPH.png -------------------------------------------------------------------------------- /documentation/images/success set up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/success set up.png -------------------------------------------------------------------------------- /documentation/images/add R2G_PASSWORD.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/add R2G_PASSWORD.png -------------------------------------------------------------------------------- /documentation/images/Get the graph name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/Get the graph name.png -------------------------------------------------------------------------------- /documentation/images/email and password.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/email and password.png -------------------------------------------------------------------------------- /documentation/images/Create New repository.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/Create New repository.png -------------------------------------------------------------------------------- /documentation/images/private repository success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/private repository success.png -------------------------------------------------------------------------------- /documentation/images/Create New private repository.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/Create New private repository.png -------------------------------------------------------------------------------- /documentation/images/Signing up for a new GitHub account.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/everruler12/roam2github/HEAD/documentation/images/Signing up for a new GitHub account.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "roam2github", 3 | "version": "1.0.6", 4 | "description": "Automatic Roam Research backups", 5 | "main": "roam2github.js", 6 | "dependencies": { 7 | "dotenv": "^8.2.0", 8 | "edn-formatter": "everruler12/edn-formatter", 9 | "extract-zip": "everruler12/extract-zip", 10 | "fs-extra": "^9.1.0", 11 | "puppeteer": "^5.5.0", 12 | "sanitize-filename": "^1.6.3" 13 | }, 14 | "devDependencies": {}, 15 | "scripts": { 16 | "start": "node roam2github.js", 17 | "backup": "node backup.js" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "https://github.com/everruler12/roam2github.git" 22 | }, 23 | "author": "everruler12", 24 | "license": "MIT" 25 | } -------------------------------------------------------------------------------- /documentation/Setup Instructions.md: -------------------------------------------------------------------------------- 1 | # Setup Instructions 2 | 3 | 1. Create a new, private repository 4 | 2. Go to Settings > Secrets and add the following Secret names and values: 5 | - `ROAM_EMAIL` - Your Roam account email 6 | - `ROAM_PASSWORD` - Your Roam account password (needs to be reset if using a Google login) 7 | - `ROAM_GRAPH` - The name of the graph to backup. For multiple graphs, add on separate lines (or separate by commas) 8 | 3. Go to Actions, then click "set up a workflow yourself →" 9 | 4. Delete the code in the editor, and copy/paste the code from here: [main.yml](https://raw.githubusercontent.com/everruler12/roam2github-actions/main/.github/workflows/main.yml) 10 | 5. Click `Start Commit` then `Commit new file` 11 | 12 | The backup will run every hour. You can view the logs in Actions and clicking on the jobs. 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Erik Newhard 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /documentation/Settings for main.yml.md: -------------------------------------------------------------------------------- 1 | # Settings for main.yml 2 | 3 | If you don't need to keep your graph name private, you can include it directly in the main.yml under `R2G_GRAPH` instead of Secrets. Just replace `${{ secrets.R2G_GRAPH }}`. For multiple graph backups, separate with a comma. 4 | 5 | In your main.yml, beneath the `R2G_GRAPH` env variable, you can add some of the following extra settings if needed: 6 | 7 | - Don't backup a specific file type. (Choose one or two to skip. Not all 3, or you won't have a backup, lol). Default is `true` when not set. 8 | 9 | ``` 10 | BACKUP_JSON: false 11 | BACKUP_EDN: false 12 | BACKUP_MARKDOWN: false 13 | ``` 14 | 15 | - Change timeout in the backup script (not the Action itself). Default is `600000` ms (10 minutes) when not set. 16 | 17 | ``` 18 | TIMEOUT: 300000 19 | ``` 20 | 21 | - Change the replacement character for illegal filenames in markdown. Default is `�` when not set. 22 | 23 | ``` 24 | MD_REPLACEMENT: _ 25 | ``` 26 | 27 | - Include blank markdown files. (This can clutter the backup with lots of unnecessary files.) Default is `true` (skip the blanks) when not set. 28 | 29 | ``` 30 | MD_SKIP_BLANKS: false 31 | ``` 32 | -------------------------------------------------------------------------------- /documentation/Common error causes.md: -------------------------------------------------------------------------------- 1 | # Common error causes 2 | 3 | - `R2G ERROR - Secrets error: R2G_EMAIL not found` (or `R2G_PASSWORD` or `R2G_GRAPH`) 4 | 5 | One of those secrets is blank or missing. Add it in Settings > Secrets 6 | 7 | - `R2G ERROR - Login error. Roam says: "There is no user record corresponding to this identifier. The user may have been deleted."` or `R2G ERROR - Login error. Roam says: "The email address is badly formatted."` 8 | 9 | Your `R2G_EMAIL` secret is incorrect. Try updating it. 10 | 11 | - `R2G ERROR - Login error. Roam says: "The password is invalid or the user does not have a password."` 12 | 13 | Your `R2G_PASSWORD` secret is incorrect. Try updating it. 14 | 15 | Make sure you're not using a Google account login, as this is not supported. (If you are, sign out of Roam, and on the sign-in page, click "Forgot your password" to set a password.) 16 | 17 | - Timed out with `R2G astrolabe spinning...` then `Error: The operation was canceled.` or `"TimeoutError: waiting for selector .loading-astrolabe to be hidden failed: timeout 600000ms exceeded"` Possible causes: 18 | 19 | - The most common reason is your `R2G_GRAPH` secret is incorrect. Try updating it (make sure it's only the graph name, not a URL) 20 | 21 | - Roam's servers happened to timeout. Try re-running the job later. 22 | 23 | - You don't have permission to view that graph (in case of trying to backup up someone else's graph). 24 | 25 | - You graph is too large to be loaded within the backup timeout (default set to 10 minutes). This is highly unlikely, as it shouldn't take 10 minutes to load. (If you still think this is the case, you could try increasing the timeout in main.yml and adding the `TIMEOUT` env setting as explained here: [Extra Options](https://github.com/everruler12/roam2github/blob/main/documentation/Settings%20for%20main.yml.md)) 26 | 27 | - `R2G ERROR - EDN formatting error: mismatch with original` 28 | 29 | The file integrity check to make sure the formatted version of the EDN file matches the downloaded EDN export failed. Please let me know if this were ever to happen. 30 | -------------------------------------------------------------------------------- /documentation/Full Guide with Step-by-Step Screenshots.md: -------------------------------------------------------------------------------- 1 | # Full Guide with Step-by-Step Screenshots 2 | 3 | This guide was generously created by [flyq](https://github.com/flyq) 4 | 5 | ## 1. Create a new, private repository 6 | 1. If you don't have the GitHub account, go to https://github.com/join to new a free personal account. More infomation: [Signing up for a new GitHub account](https://docs.github.com/en/github/getting-started-with-github/signing-up-for-a-new-github-account) 7 | ![](./images/Signing%20up%20for%20a%20new%20GitHub%20account.png) 8 | 2. Go to https://github.com/login, and sign in with your account: 9 | ![](./images/login%20GitHub.png) 10 | 3. Click `New repository`: 11 | ![](./images/Create%20New%20repository.png) 12 | 4. Create a new repository: 13 | ![](./images/Create%20New%20private%20repository.png) 14 | The `repository name` is up to you, and you should make it private to protect your privacy. 15 | 5. Congratulations on successfully creating a private repository: 16 | ![](./images/private%20repository%20success.png) 17 | 18 | ## 2. Set the GitHub Repository's Secret 19 | *Note that the Secret names in the images show the old names of `R2G_EMAIL`, etc. but current names are `ROAM_EMAIL`, etc. as detailed in the text below.* 20 | 1. Get the email and password of your roam research account: 21 | ![](./images/email%20and%20password.png) 22 | 2. Get the name of graph you want to backup: 23 | ![](./images/Get%20the%20graph%20name.png) 24 | 3. Go to your GitHub repository, and go to `Settings` > `Secrets`, and `New repository secret`: 25 | ![](./images/Go%20to%20Settings.png) 26 | 4. Add `ROAM_EMAIL` in Secret: 27 | ![](./images/add%20R2G_EMAIL.png) 28 | Name must be `ROAM_EMAIL`, and Value is the email of your roam research account. 29 | 5. Add `ROAM_PASSWORD` in Secret: 30 | ![](./images/add%20R2G_PASSWORD.png) 31 | The same to step 4, New repository secret to add `ROAM_PASSWORD`. Name must be `ROAM_PASSWORD`, and Value is the password of your roam research account. 32 | 6. Add `ROAM_GRAPH` in Secret: 33 | ![](./images/add%20R2G_GRAPH.png) 34 | The same to step 4, New repository secret to add `ROAM_GRAPH`. ame must be `ROAM_GRAPH`, and Value is the name of your roam research graph, which you get from step 2. For multiple graphs, add on separate lines (or separate by commas), Here I used commas. 35 | ## 3. Set the GitHub Repository's Actions 36 | 1. Go to Actions, then click "set up a workflow yourself →" 37 | ![](./images/Set%20workfow.png) 38 | 2. Delete the code in the editor, and copy/paste the code from here: [main.yml](https://raw.githubusercontent.com/everruler12/roam2github-actions/main/.github/workflows/main.yml) 39 | ![](./images/add%20yml.png) 40 | 3. Click `Start Commit` then `Commit new file` 41 | ![](./images/add%20new%20file.png) 42 | ## 4. Congratulations on successfully finish all of this 43 | So far, you have successfully completed all steps. 44 | And Waiting a few minutes(I waited for 5 minutes here), the actions-user backup your roam research graph automatically: 45 | ![](./images/success%20set%20up.png) 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Roam2Github 2 | 3 | ⚠️ 4 | **I no longer maintain this project, since Roam has Auto Backups included natively (and I've stopped using Roam as my primary second brain).** 5 | ⚠️ 6 | 7 | --- 8 | 9 | [Click here to view guide on setting up free, unlimited, automatic Roam backups](https://www.notion.so/Roam2Github-Backup-Guide-650925859a4a42cf940e3fb74f5189f9) 10 | 11 | [Click here for extra settings](https://github.com/everruler12/roam2github/blob/main/documentation/Settings%20for%20main.yml.md) 12 | 13 | --- 14 | 15 | This project was inspired by https://github.com/MatthieuBizien/roam-to-git 16 | 17 | Roam-to-git has offered me great peace of mind knowing my Roam data is safe. However, my backups regularly failed with unknown errors multiple times a week. People were emailing me with the same issues, and I couldn't help. Then it got to the point on 2021-01-28 where all my backups were failing. Roam-to-git's creator didn't seem active with addressing the issues, and I don't know enough Python fix his code. So I decided to write my own backup solution from scratch using Node— with clearer logging to make troubleshooting easier. 18 | 19 | ## Differences from roam-to-git 20 | 21 | - Uses Node (rather than Python) 22 | - Supports EDN in addition to JSON and Markdown (not formatted markdown though) 23 | - Multiple graph backups in the same repo 24 | - Better error debugging and active support from the developer (Erik Newhard @everruler12) to get your backups running smoothly and error-free 25 | 26 | ## ~~Future Plans~~ 27 | 28 | - [ ] ~~New, full guide with step-by-step screen recordings~~ 29 | - [ ] ~~Update code to run asynchronously, instead of linearly, to cut down on run time~~ 30 | - [ ] ~~Use fipp for faster EDN formatting~~ 31 | 32 | ## Changelog 33 | 34 | - EDN support (2021-01-31) 35 | - Multi graph support (2021-02-01) 36 | - Markdown support (2021-02-04) 37 | - Allow setup of public repo for running Actions and committing to private repo for backup, in order to bypass minute limit for private GitHub Actions (2021-02-18) 38 | 39 | ### EDN Backups are live! 40 | 41 | The backup has a check to make sure the formatted EDN (which only adds extra linebreaks and indentation) can be parsed back to match exactly with the original before saving it. It will exit with an error if it can't, so you can rest assured that the formatting doesn't mess with the file integrity. I also tested that the formatted EDN can be used to successfully restore graphs. 42 | 43 | 2021-01-31 It took all day to figure out how to use ClojureScript to prettify EDN. It was a daunting task, never having dealt with Clojure before, much less compiling it into JavaScript. But I did it! This is necessary because the exported EDN data from Roam is all in one line, meaning GitHub would have to save the entire file each time, instead of just the new lines. This would eat up the storage pretty quickly if run every hour, as unchanged notes would be duplicated each time. And you wouldn't be able to see line-by-line changes in the git history. 44 | 45 | ### Multi Graph Backups in Same Repo 46 | 47 | You can now backup multiple graphs without having to create a new GitHub repo for each one. Just add them to your `R2G_GRAPH` Secret in separate lines, or separated by commas. 48 | 49 | ### Markdown support added 50 | 51 | 2021-02-04 Markdown is now supported. Worked all day to get filename sanitization working. My backup script can even export markdown from the [official Roam help database](https://roamresearch.com/#/app/help) and Roam [book](https://roamresearch.com/#/app/roam-book-club) [clubs](https://roamresearch.com/#/app/roam-book-club-2) error-free! I have added several measures to prevent errors: 52 | 53 | - `/` slashes are replaced with full-width versions `／` 54 | - illegal filename characters are replaced with `�` 55 | - Page titles longer than 255 characters are automatically truncated (though they lose the .md extension) 56 | - no subdirectories 57 | - no blank files 58 | - ~~The logs will list the files that have been renamed or overwritten.~~ The logs no longer display file names, as this would be a privacy concern for the new way to run Actions publicly. 59 | 60 | Unfortunate side-effect with markdown backups: files with duplicate names are overwritten (like [[test]] and [[Test]]). (This was also present in roam-to-git) 61 | 62 | ### Separate backup save location and backup script actions 63 | 64 | It is possible now to run the script actions from a public repo, to not be limited by 2000 minutes/month, and save the backup to a private repo. Note that Secret names have changed with this update. (The guides at the top have been updated with the new Secret names and main.yml. The old version is still up, but will no longer be updated.) 65 | 66 | ## Support 67 | 68 | [Common error causes and their solutions](https://github.com/everruler12/roam2github/blob/main/documentation/Common%20error%20causes.md) 69 | -------------------------------------------------------------------------------- /backup.js: -------------------------------------------------------------------------------- 1 | // TODO output log file to backup repo with list of changed markdown filenames and overwritten files, in order to preserve privacy in public actions 2 | const path = require('path') 3 | const fs = require('fs-extra') 4 | const puppeteer = require('puppeteer') 5 | const extract = require('extract-zip') 6 | const sanitize = require('sanitize-filename') 7 | const edn_format = require('edn-formatter').edn_formatter.core.format 8 | 9 | console.time('R2G Exit after') 10 | 11 | if (fs.existsSync(path.join(__dirname, '.env'))) { // check for local .env 12 | require('dotenv').config() 13 | } 14 | 15 | const { ROAM_EMAIL, ROAM_PASSWORD, ROAM_GRAPH, BACKUP_JSON, BACKUP_EDN, BACKUP_MARKDOWN, MD_REPLACEMENT, MD_SKIP_BLANKS, TIMEOUT } = process.env 16 | // IDEA - MD_SEPARATE_DN put daily notes in separate directory. Maybe option for namespaces to be in separate folders, the default behavior. 17 | 18 | if (!ROAM_EMAIL) error('Secrets error: ROAM_EMAIL not found') 19 | if (!ROAM_PASSWORD) error('Secrets error: ROAM_PASSWORD not found') 20 | if (!ROAM_GRAPH) error('Secrets error: ROAM_GRAPH not found') 21 | 22 | const graph_names = ROAM_GRAPH.split(/,|\n/) // comma or linebreak separator 23 | .map(g => g.trim())// remove extra spaces 24 | .filter(g => g != '') // remove blank lines 25 | // can also check "Not a valid name. Names can only contain letters, numbers, dashes and underscores." message that Roam gives when creating a new graph 26 | 27 | const backup_types = [ 28 | { type: "JSON", backup: BACKUP_JSON }, 29 | { type: "EDN", backup: BACKUP_EDN }, 30 | { type: "Markdown", backup: BACKUP_MARKDOWN } 31 | ].map(f => { 32 | (f.backup === undefined || f.backup.toLowerCase() === 'true') ? f.backup = true : f.backup = false 33 | return f 34 | }) 35 | // what about specifying filetype for each graph? Maybe use settings.json in root of repo. But too complicated for non-programmers to set up. 36 | 37 | const md_replacement = MD_REPLACEMENT || '�' 38 | 39 | const md_skip_blanks = (MD_SKIP_BLANKS && MD_SKIP_BLANKS.toLowerCase()) === 'false' ? false : true 40 | 41 | const timeout = TIMEOUT || 600000 // 10min default 42 | 43 | const tmp_dir = path.join(__dirname, 'tmp') 44 | 45 | // ; 46 | // (async () => { 47 | // const repo_path = await getRepoPath() 48 | const repo_path = getRepoPath() 49 | const backup_dir = repo_path ? repo_path : path.join(__dirname, 'backup') // if no repo_path use local path 50 | // })(); 51 | 52 | 53 | function getRepoPath() { 54 | const ubuntuPath = path.join('/', 'home', 'runner', 'work') 55 | const exists = fs.pathExistsSync(ubuntuPath) 56 | 57 | if (exists) { 58 | const files = fs.readdirSync(ubuntuPath) 59 | .filter(f => !f.startsWith('_')) // filters out [ '_PipelineMapping', '_actions', '_temp', ] 60 | 61 | if (files.length === 1) { 62 | repo_name = files[0] 63 | const files2 = fs.readdirSync(path.join(ubuntuPath, repo_name)) 64 | 65 | // path.join(ubuntuPath, repo_name, 'roam2github') == __dirname 66 | const withoutR2G = files2.filter(f => f != 'roam2github') // for old main.yml 67 | 68 | if (files2.length === 1 && files2[0] == repo_name) { 69 | 70 | log('Detected GitHub Actions path') 71 | return path.join(ubuntuPath, repo_name, repo_name) // actions/checkout@v2 outputs to path /home/runner/work// 72 | 73 | } if (files2.length == 2 && withoutR2G.length == 1 && withoutR2G[0] == repo_name) { 74 | 75 | log('Detected GitHub Actions path found. (Old main.yml being used, with potential "roam2github" repo name conflict)') 76 | return path.join(ubuntuPath, repo_name, repo_name) // actions/checkout@v2 outputs to path /home/runner/work// 77 | 78 | } else { 79 | // log(files, 'detected in', path.join(ubuntuPath, repo_name), '\nNot GitHub Action') 80 | log('GitHub Actions path not found. Using local path') 81 | return false 82 | } 83 | 84 | } else { 85 | // log(files, 'detected in', ubuntuPath, '\nNot GitHub Action') 86 | log('GitHub Actions path not found. Using local path') 87 | return false 88 | } 89 | 90 | } else { 91 | // log(ubuntuPath, 'does not exist. Not GitHub Action') 92 | log('GitHub Actions path not found. Using local path') 93 | return false 94 | } 95 | } 96 | 97 | 98 | init() 99 | 100 | async function init() { 101 | try { 102 | 103 | await fs.remove(tmp_dir, { recursive: true }) 104 | 105 | log('Create browser') 106 | const browser = await puppeteer.launch({ args: ['--no-sandbox'] }) // to run in GitHub Actions 107 | // const browser = await puppeteer.launch({ headless: false }) // to test locally and see what's going on 108 | 109 | 110 | log('Login') 111 | await roam_login(browser) 112 | 113 | for (const graph_name of graph_names) { 114 | 115 | const page = await newPage(browser) 116 | 117 | log('Open graph', graph_name) 118 | await roam_open_graph(page, graph_name) 119 | 120 | for (const f of backup_types) { 121 | if (f.backup) { 122 | const download_dir = path.join(tmp_dir, graph_name, f.type.toLowerCase()) 123 | await page._client.send('Page.setDownloadBehavior', { behavior: 'allow', downloadPath: download_dir }) 124 | 125 | log('Export', f.type) 126 | await roam_export(page, f.type, download_dir) 127 | 128 | log('Extract') 129 | await extract_file(download_dir) 130 | 131 | await format_and_save(f.type, download_dir, graph_name) 132 | // TODO run download and formatting operations asynchronously. Can be done since json and edn are same as graph name. 133 | // Await for counter expecting total operations to be done graph_names.length * backup_types.filter(f=>f.backup).length 134 | // or Promises.all(arr) where arr is initiated outside For loop, and arr.push result of format_and)_save 135 | } 136 | } 137 | } 138 | 139 | log('Close browser') 140 | browser.close() 141 | 142 | // await fs.remove(tmp_dir, { recursive: true }) 143 | 144 | log('DONE!') 145 | 146 | } catch (err) { error(err) } 147 | 148 | console.timeEnd('R2G Exit after') 149 | } 150 | 151 | async function newPage(browser) { 152 | const page = await browser.newPage() 153 | 154 | page.setDefaultTimeout(timeout) 155 | // page.on('console', consoleObj => console.log(consoleObj.text())) // for console.log() to work in page.evaluate() https://stackoverflow.com/a/46245945 156 | 157 | return page 158 | } 159 | 160 | async function roam_login(browser) { 161 | return new Promise(async (resolve, reject) => { 162 | try { 163 | 164 | const page = await newPage(browser) 165 | 166 | log('- Navigating to login page') 167 | await page.goto('https://roamresearch.com/#/signin') 168 | 169 | log('- Checking for email field') 170 | await page.waitForSelector('input[name="email"]') 171 | 172 | log('- (Wait for auto-refresh)') 173 | // log('- (Wait 10 seconds for auto-refresh)') 174 | // await page.waitForTimeout(10000) // because Roam auto refreshes the sign-in page, as mentioned here https://github.com/MatthieuBizien/roam-to-git/issues/87#issuecomment-763281895 (and can be seen in non-headless browser) 175 | 176 | await page.waitForSelector('.loading-astrolabe', { timeout: 20000 }) 177 | await page.waitForSelector('.loading-astrolabe', { hidden: true }) 178 | // log('- auto-refreshed') 179 | 180 | log('- Filling email field') 181 | await page.type('input[name="email"]', ROAM_EMAIL) 182 | 183 | log('- Filling password field') 184 | await page.type('input[name="password"]', ROAM_PASSWORD) 185 | 186 | log('- Checking for "Sign In" button') 187 | await page.waitForFunction(() => [...document.querySelectorAll('button.bp3-button')].find(button => button.innerText == 'Sign In')) 188 | 189 | log('- Clicking "Sign In"') 190 | await page.evaluate(() => { [...document.querySelectorAll('button.bp3-button')].find(button => button.innerText == 'Sign In').click() }) 191 | 192 | const login_error_selector = 'div[style="font-size: 12px; color: red;"]' // error message on login page 193 | const graphs_selector = '.my-graphs' // successful login, on graphs selection page 194 | 195 | await page.waitForSelector(login_error_selector + ', ' + graphs_selector) 196 | 197 | const error_el = await page.$(login_error_selector) 198 | 199 | if (error_el) { 200 | 201 | const error_message = await page.evaluate(el => el.innerText, error_el) 202 | reject(`Login error. Roam says: "${error_message}"`) 203 | 204 | } else if (await page.$(graphs_selector)) { 205 | 206 | log('Login successful!') 207 | resolve() 208 | 209 | } else { 210 | reject('Login error: unknown') 211 | } 212 | 213 | } catch (err) { reject(err) } 214 | }) 215 | } 216 | 217 | async function roam_open_graph(page, graph_name) { 218 | return new Promise(async (resolve, reject) => { 219 | try { 220 | 221 | page.on("dialog", async (dialog) => await dialog.accept()) // Handles "Changes will not be saved" dialog when trying to navigate away from official Roam help database https://roamresearch.com/#/app/help 222 | 223 | log('- Navigating to graph') 224 | await page.goto(`https://roamresearch.com/#/app/${graph_name}?disablecss=true&disablejs=true`) 225 | 226 | // log('- Checking for astrolabe spinner') 227 | await page.waitForSelector('.loading-astrolabe') 228 | log('- astrolabe spinning...') 229 | 230 | //await page.waitForSelector('.loading-astrolabe', { hidden: true }) 231 | //log('- astrolabe spinning stopped') 232 | 233 | // try { 234 | await page.waitForSelector('.roam-app') // add short timeout here, if fails, don't exit code 1, and instead CHECK if have permission to view graph 235 | // } catch (err) { 236 | // await page.waitForSelector('.navbar') // Likely screen saying 'You do not have permission to view this database' 237 | // reject() 238 | // } 239 | 240 | log('Graph loaded!') 241 | resolve(page) 242 | 243 | } catch (err) { reject(err) } 244 | }) 245 | } 246 | 247 | async function roam_export(page, filetype, download_dir) { 248 | return new Promise(async (resolve, reject) => { 249 | try { 250 | await fs.ensureDir(download_dir) 251 | 252 | // log('- Checking for "..." button', filetype) 253 | await page.waitForSelector('.bp3-icon-more') 254 | 255 | log('- (check for "Sync Quick Capture Notes")') // to check for "Sync Quick Capture Notes with Workspace" modal 256 | await page.waitForTimeout(1000) 257 | 258 | if (await page.$('.rm-quick-capture-sync-modal')) { 259 | log('- Detected "Sync Quick Capture Notes" modal. Closing') 260 | await page.keyboard.press('Escape') 261 | await page.waitForSelector('.rm-quick-capture-sync-modal', { hidden: true }) 262 | log('- "Sync Quick Capture Notes" modal closed') 263 | } 264 | 265 | if (await page.$('.rm-modal-dialog--expired-plan')) { 266 | log('- Detected "Your subscription to Roam has expired." modal. Closing') 267 | await page.keyboard.press('Escape') 268 | await page.waitForSelector('.rm-modal-dialog--expired-plan', { hidden: true }) 269 | log('- Expired subscription modal closed') 270 | } 271 | 272 | log('- Clicking "..." button') 273 | await page.click('.bp3-icon-more') 274 | 275 | log('- Checking for "Export All" option') 276 | await page.waitForFunction(() => [...document.querySelectorAll('li .bp3-fill')].find(li => li.innerText.match('Export All'))) 277 | 278 | log('- Clicking "Export All" option') 279 | await page.evaluate(() => { [...document.querySelectorAll('li .bp3-fill')].find(li => li.innerText.match('Export All')).click() }) 280 | 281 | const chosen_format_selector = '.bp3-dialog .bp3-button-text' 282 | 283 | log('- Checking for export dialog') 284 | await page.waitForSelector(chosen_format_selector) 285 | 286 | const chosen_format = (await page.$eval(chosen_format_selector, el => el.innerText)).trim() 287 | log(`- format chosen is "${chosen_format}"`) 288 | 289 | if (filetype != chosen_format) { 290 | 291 | log('- Clicking export format') 292 | await page.click(chosen_format_selector) 293 | 294 | log('- Checking for dropdown options') 295 | await page.waitForSelector('.bp3-text-overflow-ellipsis') 296 | 297 | log('- Checking for dropdown option', filetype) 298 | await page.waitForFunction((filetype) => [...document.querySelectorAll('.bp3-text-overflow-ellipsis')].find(dropdown => dropdown.innerText.match(filetype)), filetype) 299 | 300 | log('- Clicking', filetype) 301 | await page.evaluate((filetype) => { [...document.querySelectorAll('.bp3-text-overflow-ellipsis')].find(dropdown => dropdown.innerText.match(filetype)).click() }, filetype) 302 | 303 | } else { 304 | log('-', filetype, 'already selected') 305 | } 306 | 307 | log('- Checking for "Export All" button') 308 | await page.waitForFunction(() => [...document.querySelectorAll('button.bp3-button.bp3-intent-primary')].find(button => button.innerText.match('Export All'))) 309 | 310 | log('- Clicking "Export All" button') 311 | await page.evaluate(() => { [...document.querySelectorAll('button.bp3-button.bp3-intent-primary')].find(button => button.innerText.match('Export All')).click() }) 312 | 313 | log('- Waiting for download to start') 314 | await page.waitForSelector('.bp3-spinner') 315 | 316 | await page.waitForSelector('.bp3-spinner', { hidden: true }) 317 | log('- Downloading') 318 | 319 | await waitForDownload(download_dir) 320 | 321 | resolve() 322 | 323 | } catch (err) { reject(err) } 324 | }) 325 | } 326 | 327 | function waitForDownload(download_dir) { 328 | return new Promise(async (resolve, reject) => { 329 | try { 330 | 331 | checkDownloads() 332 | 333 | async function checkDownloads() { 334 | 335 | const files = await fs.readdir(download_dir) 336 | const file = files[0] 337 | 338 | if (file && file.match(/\.zip$/)) { // checks for .zip file 339 | 340 | log(file, 'downloaded!') 341 | resolve() 342 | 343 | } else checkDownloads() 344 | } 345 | 346 | } catch (err) { reject(err) } 347 | }) 348 | } 349 | 350 | async function extract_file(download_dir) { 351 | return new Promise(async (resolve, reject) => { 352 | try { 353 | 354 | const files = await fs.readdir(download_dir) 355 | 356 | if (files.length === 0) reject('Extraction error: download_dir is empty') 357 | if (files.length > 1) reject('Extraction error: download_dir contains more than one file') 358 | 359 | const file = files[0] 360 | 361 | if (!file.match(/\.zip$/)) reject('Extraction error: .zip not found') 362 | 363 | const file_fullpath = path.join(download_dir, file) 364 | const extract_dir = path.join(download_dir, '_extraction') 365 | 366 | log('- Extracting ' + file) 367 | await extract(file_fullpath, { 368 | dir: extract_dir, 369 | 370 | onEntry(entry, zipfile) { 371 | if (entry.fileName.endsWith('/')) { 372 | // log(' - Skipping subdirectory', entry.fileName) 373 | return false 374 | } 375 | 376 | if (md_skip_blanks && entry.uncompressedSize <= 3) { // files with 3 bytes just have a one blank block (like blank daily notes) 377 | // log(' - Skipping blank file', entry.fileName, `(${entry.uncompressedSize} bytes`) 378 | return false 379 | } 380 | 381 | // log(' -', entry.fileName) 382 | entry.fileName = sanitizeFileName(entry.fileName) 383 | 384 | if (fs.pathExistsSync(path.join(extract_dir, entry.fileName))) { 385 | 386 | // log('WARNING: file collision detected. Overwriting file with (sanitized) name:', entry.fileName) 387 | // reject(`Extraction error: file collision detected with sanitized filename: ${entry.fileName}`) 388 | // TODO? renaming to... 389 | } 390 | 391 | return true 392 | } 393 | }) 394 | 395 | resolve() 396 | 397 | } catch (err) { reject(err) } 398 | }) 399 | } 400 | 401 | async function format_and_save(filetype, download_dir, graph_name) { 402 | return new Promise(async (resolve, reject) => { 403 | try { 404 | 405 | const extract_dir = path.join(download_dir, '_extraction') 406 | 407 | const files = await fs.readdir(extract_dir) 408 | 409 | if (files.length === 0) reject('Extraction error: extract_dir is empty') 410 | 411 | if (filetype == 'Markdown') { 412 | 413 | const markdown_dir = path.join(backup_dir, 'markdown', graph_name) 414 | 415 | // log('- Removing old markdown directory') 416 | await fs.remove(markdown_dir, { recursive: true }) // necessary, to update renamed pages 417 | 418 | log('- Saving Markdown') 419 | 420 | for (const file of files) { 421 | 422 | const file_fullpath = path.join(extract_dir, file) 423 | const new_file_fullpath = path.join(markdown_dir, file) 424 | 425 | await fs.move(file_fullpath, new_file_fullpath, { overwrite: true }) 426 | } 427 | 428 | } else { 429 | 430 | // for (const file of files) { 431 | const file = files[0] 432 | const file_fullpath = path.join(extract_dir, file) 433 | const fileext = file.split('.').pop() 434 | const new_file_fullpath = path.join(backup_dir, fileext, file) 435 | 436 | if (fileext == 'json') { 437 | 438 | log('- Formatting JSON') 439 | const json = await fs.readJson(file_fullpath) 440 | const new_json = JSON.stringify(json, null, 2) 441 | 442 | log('- Saving formatted JSON') 443 | await fs.outputFile(new_file_fullpath, new_json) 444 | 445 | } else if (fileext == 'edn') { 446 | 447 | log('- Formatting EDN (this can take a couple minutes for large graphs)') // This could take a couple minutes for large graphs 448 | const edn = await fs.readFile(file_fullpath, 'utf-8') 449 | 450 | const edn_prefix = '#datascript/DB ' 451 | var new_edn = edn_prefix + edn_format(edn.replace(new RegExp('^' + edn_prefix), '')) 452 | checkFormattedEDN(edn, new_edn) 453 | 454 | log('- Saving formatted EDN') 455 | await fs.outputFile(new_file_fullpath, new_edn) 456 | 457 | } else reject(`format_and_save error: Unhandled filetype: ${files}`) 458 | // } 459 | } 460 | 461 | resolve() 462 | 463 | } catch (err) { reject(err) } 464 | }) 465 | } 466 | 467 | 468 | 469 | function log(...messages) { 470 | const timestamp = new Date().toISOString().replace('T', ' ').replace('Z', '') 471 | console.log(timestamp, 'R2G', ...messages) 472 | } 473 | 474 | async function error(err) { 475 | log('ERROR -', err) 476 | console.timeEnd('R2G Exit after') 477 | // await page.screenshot({ path: path.join(download_dir, 'error.png' }) // will need to pass page as parameter... or set as parent scope 478 | process.exit(1) 479 | } 480 | 481 | function checkFormattedEDN(original, formatted) { 482 | const reverse_format = formatted 483 | .trim() // remove trailing line break 484 | .split('\n') // separate by line 485 | .map(line => line.trim()) // remove indents, and one extra space at end of second to last line 486 | .join(' ') // replace line breaks with a space 487 | 488 | if (original === reverse_format) { 489 | // log('(formatted EDN check successful)') // formatted EDN successfully reversed to match exactly with original EDN 490 | return true 491 | } else { 492 | error('EDN formatting error: mismatch with original') 493 | return false 494 | } 495 | } 496 | 497 | function sanitizeFileName(fileName) { 498 | fileName = fileName.replace(/\//g, '／') 499 | 500 | const sanitized = sanitize(fileName, { replacement: md_replacement }) 501 | 502 | if (sanitized != fileName) { 503 | 504 | // log(' Sanitized:', fileName, '\n to:', sanitized) 505 | return sanitized 506 | 507 | } else return fileName 508 | } 509 | -------------------------------------------------------------------------------- /roam2github.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const fs = require('fs-extra') 3 | const puppeteer = require('puppeteer') 4 | const extract = require('extract-zip') 5 | const sanitize = require('sanitize-filename') 6 | const edn_format = require('edn-formatter').edn_formatter.core.format 7 | 8 | console.time('R2G Exit after') 9 | 10 | if (fs.existsSync(path.join(__dirname, '.env'))) { // check for local .env 11 | require('dotenv').config() 12 | } 13 | 14 | const { R2G_EMAIL, R2G_PASSWORD, R2G_GRAPH, BACKUP_JSON, BACKUP_EDN, BACKUP_MARKDOWN, BACKUP_FLAT_MARKDOWN, BACKUP_MSGPACK, MD_REPLACEMENT, MD_SKIP_BLANKS, TIMEOUT } = process.env 15 | // IDEA - MD_SEPARATE_DN put daily notes in separate directory 16 | 17 | if (!R2G_EMAIL) error('Secrets error: R2G_EMAIL not found') 18 | if (!R2G_PASSWORD) error('Secrets error: R2G_PASSWORD not found') 19 | if (!R2G_GRAPH) error('Secrets error: R2G_GRAPH not found') 20 | 21 | const graph_names = R2G_GRAPH.split(/,|\n/) // comma or linebreak separator 22 | .map(g => g.trim())// remove extra spaces 23 | .filter(g => g != '') // remove blank lines 24 | // can also check "Not a valid name. Names can only contain letters, numbers, dashes and underscores." message that Roam gives when creating a new graph 25 | 26 | const backup_types = [ 27 | { type: "JSON", backup: BACKUP_JSON, extension: ".json" }, 28 | { type: "EDN", backup: BACKUP_EDN, extension: ".edn" }, 29 | { type: "Markdown", backup: BACKUP_MARKDOWN, extension: ".zip" }, 30 | { type: "Flat Markdown", backup: BACKUP_FLAT_MARKDOWN, extension: ".md" }, 31 | { type: "msgpack", backup: BACKUP_MSGPACK, extension: ".msgpack" } 32 | ].map(f => { 33 | f.backup = (f.backup === undefined || f.backup.toLowerCase() === 'true'); 34 | return f; 35 | }) 36 | // what about specifying filetype for each graph? Maybe use settings.json in root of repo. But too complicated for non-programmers to set up. 37 | 38 | const md_replacement = MD_REPLACEMENT || '�' 39 | 40 | const md_skip_blanks = (MD_SKIP_BLANKS && MD_SKIP_BLANKS.toLowerCase()) === 'false' ? false : true 41 | 42 | const timeout = TIMEOUT || 600000 // 10min default 43 | 44 | const tmp_dir = path.join(__dirname, 'tmp') 45 | 46 | // ; 47 | // (async () => { 48 | // const repo_path = await getRepoPath() 49 | const repo_path = getRepoPath() 50 | const backup_dir = repo_path ? repo_path : path.join(__dirname, 'backup') 51 | // })(); 52 | 53 | 54 | function getRepoPath() { 55 | const ubuntuPath = path.join('/', 'home', 'runner', 'work') 56 | const exists = fs.pathExistsSync(ubuntuPath) 57 | 58 | if (exists) { 59 | const files = fs.readdirSync(ubuntuPath) 60 | .filter(f => !f.startsWith('_')) // filter out [ '_PipelineMapping', '_actions', '_temp', ] 61 | 62 | if (files.length === 1) { 63 | repo_name = files[0] 64 | const files2 = fs.readdirSync(path.join(ubuntuPath, repo_name)) 65 | 66 | // path.join(ubuntuPath, repo_name, 'roam2github') == __dirname 67 | const withoutR2G = files2.filter(f => f != 'roam2github') // for old main.yml 68 | 69 | if (files2.length === 1 && files2[0] == repo_name) { 70 | 71 | // log(files2, 'GitHub Actions path found') 72 | log('GitHub Actions path found') 73 | return path.join(ubuntuPath, repo_name, repo_name) // actions/checkout@v2 outputs to path /home/runner/work// 74 | 75 | } if (files2.length == 2 && withoutR2G.length == 1 && withoutR2G[0] == repo_name) { 76 | 77 | // log(files2, 'GitHub Actions path found. (Old main.yml being used)') 78 | log('GitHub Actions path found. (Old main.yml being used)') 79 | return path.join(ubuntuPath, repo_name, repo_name) // actions/checkout@v2 outputs to path /home/runner/work// 80 | 81 | } else { 82 | // log(files, 'detected in', path.join(ubuntuPath, repo_name), '\nNot GitHub Action') 83 | log('GitHub Actions path not found. Using local path') 84 | return false 85 | } 86 | 87 | } else { 88 | // log(files, 'detected in', ubuntuPath, '\nNot GitHub Action') 89 | log('GitHub Actions path not found. Using local path') 90 | return false 91 | } 92 | 93 | } else { 94 | // log(ubuntuPath, 'does not exist. Not GitHub Action') 95 | log('GitHub Actions path not found. Using local path') 96 | return false 97 | } 98 | } 99 | 100 | 101 | init() 102 | 103 | 104 | 105 | async function newPage(browser) { 106 | const page = await browser.newPage() 107 | 108 | page.setDefaultTimeout(timeout) 109 | // page.on('console', consoleObj => console.log(consoleObj.text())) // for console.log() to work in page.evaluate() https://stackoverflow.com/a/46245945 110 | 111 | return page 112 | } 113 | 114 | async function init() { 115 | try { 116 | 117 | await fs.remove(tmp_dir, { recursive: true }) 118 | 119 | log('Create browser') 120 | const browser = await puppeteer.launch({ args: ['--no-sandbox'] }) // to run in GitHub Actions 121 | // const browser = await puppeteer.launch({ headless: false }) // to test locally and see what's going on 122 | 123 | 124 | log('Login') 125 | await roam_login(browser) 126 | 127 | for (const graph_name of graph_names) { 128 | 129 | const page = await newPage(browser) 130 | 131 | log('Open graph', censor(graph_name)) 132 | await roam_open_graph(page, graph_name) 133 | 134 | for (const f of backup_types) { 135 | if (f.backup) { 136 | const download_dir = path.join(tmp_dir, graph_name, f.type.toLowerCase()) 137 | await page._client.send('Page.setDownloadBehavior', { behavior: 'allow', downloadPath: download_dir }) 138 | 139 | log('Export', f.type) 140 | await roam_export(page, f.type, f.extension, download_dir) 141 | 142 | log('Extract') 143 | if (f.extension == ".zip") { 144 | await extract_file(download_dir, f.extension) 145 | } 146 | await format_and_save(f.type, download_dir, graph_name) 147 | // TODO run download and formatting operations asynchronously. Can be done since json and edn are same as graph name. 148 | // Await for counter expecting total operations to be done graph_names.length * backup_types.filter(f=>f.backup).length 149 | // or Promises.all(arr) where arr is initiated outside For loop, and arr.push result of format_and)_save 150 | } 151 | } 152 | } 153 | 154 | log('Close browser') 155 | browser.close() 156 | 157 | // await fs.remove(tmp_dir, { recursive: true }) 158 | 159 | log('DONE!') 160 | 161 | } catch (err) { error(err) } 162 | 163 | console.timeEnd('R2G Exit after') 164 | } 165 | 166 | async function roam_login(browser) { 167 | return new Promise(async (resolve, reject) => { 168 | try { 169 | 170 | const page = await newPage(browser) 171 | 172 | log('- Navigating to login page') 173 | await page.goto('https://roamresearch.com/#/signin') 174 | 175 | log('- Checking for email field') 176 | await page.waitForSelector('input[name="email"]') 177 | 178 | log('- (Wait for auto-refresh)') 179 | // log('- (Wait 10 seconds for auto-refresh)') 180 | // await page.waitForTimeout(10000) // because Roam auto refreshes the sign-in page, as mentioned here https://github.com/MatthieuBizien/roam-to-git/issues/87#issuecomment-763281895 (and can be seen in non-headless browser) 181 | 182 | await page.waitForSelector('.loading-astrolabe', { timeout: 20000 }) 183 | await page.waitForSelector('.loading-astrolabe', { hidden: true }) 184 | // log('- auto-refreshed') 185 | 186 | log('- Filling email field') 187 | await page.type('input[name="email"]', R2G_EMAIL) 188 | 189 | log('- Filling password field') 190 | await page.type('input[name="password"]', R2G_PASSWORD) 191 | 192 | log('- Checking for "Sign In" button') 193 | await page.waitForFunction(() => [...document.querySelectorAll('button.bp3-button')].find(button => button.innerText == 'Sign In')) 194 | 195 | log('- Clicking "Sign In"') 196 | await page.evaluate(() => { [...document.querySelectorAll('button.bp3-button')].find(button => button.innerText == 'Sign In').click() }) 197 | 198 | const login_error_selector = 'div[style="font-size: 12px; color: red;"]' // error message on login page 199 | const graphs_selector = '.my-graphs' // successful login, on graphs selection page 200 | 201 | await page.waitForSelector(login_error_selector + ', ' + graphs_selector) 202 | 203 | const error_el = await page.$(login_error_selector) 204 | 205 | if (error_el) { 206 | 207 | const error_message = await page.evaluate(el => el.innerText, error_el) 208 | reject(`Login error. Roam says: "${error_message}"`) 209 | 210 | } else if (await page.$(graphs_selector)) { 211 | 212 | log('Login successful!') 213 | resolve() 214 | 215 | } else { 216 | reject('Login error: unknown') 217 | } 218 | 219 | } catch (err) { reject(err) } 220 | }) 221 | } 222 | 223 | async function roam_open_graph(page, graph_name) { 224 | return new Promise(async (resolve, reject) => { 225 | try { 226 | 227 | page.on("dialog", async (dialog) => await dialog.accept()) // Handles "Changes will not be saved" dialog when trying to navigate away from official Roam help database https://roamresearch.com/#/app/help 228 | 229 | log('- Navigating to graph') 230 | await page.goto(`https://roamresearch.com/#/app/${graph_name}?disablecss=true&disablejs=true`) 231 | 232 | // log('- Checking for astrolabe spinner') 233 | await page.waitForSelector('.loading-astrolabe') 234 | log('- astrolabe spinning...') 235 | 236 | await page.waitForSelector('.loading-astrolabe', { hidden: true }) 237 | log('- astrolabe spinning stopped') 238 | 239 | // try { 240 | await page.waitForSelector('.roam-app') // add short timeout here, if fails, don't exit code 1, and instead CHECK if have permission to view graph 241 | // } catch (err) { 242 | // await page.waitForSelector('.navbar') // Likely screen saying 'You do not have permission to view this database' 243 | // reject() 244 | // } 245 | 246 | log('Graph loaded!') 247 | resolve(page) 248 | 249 | } catch (err) { reject(err) } 250 | }) 251 | } 252 | 253 | async function roam_export(page, filetype, extension, download_dir) { 254 | return new Promise(async (resolve, reject) => { 255 | try { 256 | await fs.ensureDir(download_dir) 257 | 258 | // log('- Checking for "..." button', filetype) 259 | await page.waitForSelector('.bp3-icon-more') 260 | 261 | log('- (check for "Sync Quick Capture Notes")') // to check for "Sync Quick Capture Notes with Workspace" modal 262 | await page.waitForTimeout(1000) 263 | 264 | if (await page.$('.rm-quick-capture-sync-modal')) { 265 | log('- Detected "Sync Quick Capture Notes" modal. Closing') 266 | await page.keyboard.press('Escape') 267 | await page.waitForSelector('.rm-quick-capture-sync-modal', { hidden: true }) 268 | log('- "Sync Quick Capture Notes" modal closed') 269 | } 270 | 271 | log('- Clicking "..." button') 272 | await page.click('.bp3-icon-more') 273 | 274 | log('- Checking for "Export All" option') 275 | await page.waitForFunction(() => [...document.querySelectorAll('li .bp3-fill')].find(li => li.innerText.match('Export All'))) 276 | 277 | log('- Clicking "Export All" option') 278 | await page.evaluate(() => { [...document.querySelectorAll('li .bp3-fill')].find(li => li.innerText.match('Export All')).click() }) 279 | 280 | const chosen_format_selector = '.bp3-dialog .bp3-button-text' 281 | 282 | log('- Checking for export dialog') 283 | await page.waitForSelector(chosen_format_selector) 284 | 285 | const chosen_format = (await page.$eval(chosen_format_selector, el => el.innerText)).trim() 286 | log(`- format chosen is "${chosen_format}"`) 287 | 288 | if (filetype != chosen_format) { 289 | 290 | log('- Clicking export format') 291 | await page.click(chosen_format_selector) 292 | 293 | log('- Checking for dropdown options') 294 | await page.waitForSelector('.bp3-text-overflow-ellipsis') 295 | 296 | log('- Checking for dropdown option', filetype) 297 | await page.waitForFunction((filetype) => [...document.querySelectorAll('.bp3-text-overflow-ellipsis')].find(dropdown => dropdown.innerText.match(filetype)), filetype) 298 | 299 | log('- Clicking', filetype) 300 | await page.evaluate((filetype) => { [...document.querySelectorAll('.bp3-text-overflow-ellipsis')].find(dropdown => dropdown.innerText.match(filetype)).click() }, filetype) 301 | 302 | } else { 303 | log('-', filetype, 'already selected') 304 | } 305 | 306 | log('- Checking for "Export All" button') 307 | await page.waitForFunction(() => document.querySelector('button.bp3-button.bp3-intent-primary').innerText == 'Export All') 308 | 309 | log('- Clicking "Export All" button') 310 | await page.evaluate(() => { document.querySelector('button.bp3-button.bp3-intent-primary').click() }) 311 | 312 | log('- Waiting for download to start') 313 | await page.waitForSelector('.bp3-spinner') 314 | 315 | await page.waitForSelector('.bp3-spinner', { hidden: true }) 316 | log('- Downloading') 317 | 318 | await waitForDownload(download_dir, extension) 319 | 320 | resolve() 321 | 322 | } catch (err) { reject(err) } 323 | }) 324 | } 325 | 326 | function waitForDownload(download_dir, extension) { 327 | return new Promise(async (resolve, reject) => { 328 | try { 329 | 330 | checkDownloads() 331 | 332 | async function checkDownloads() { 333 | 334 | const files = await fs.readdir(download_dir) 335 | const file = files[0] 336 | 337 | if (file && file.match(new RegExp(`\\${extension}$`))) { // checks for specified extension 338 | 339 | log(file, 'downloaded!') 340 | resolve() 341 | 342 | } else checkDownloads() 343 | } 344 | 345 | } catch (err) { reject(err) } 346 | }) 347 | } 348 | 349 | async function extract_file(download_dir, extension) { 350 | return new Promise(async (resolve, reject) => { 351 | try { 352 | 353 | const files = await fs.readdir(download_dir) 354 | 355 | if (files.length === 0) reject('Extraction error: download_dir is empty') 356 | if (files.length > 1) reject('Extraction error: download_dir contains more than one file') 357 | 358 | const file = files[0] 359 | 360 | if (!file.match(/\.zip$/)) reject('Extraction error: .zip not found') 361 | 362 | const file_fullpath = path.join(download_dir, file) 363 | const extract_dir = path.join(download_dir, '_extraction') 364 | 365 | log('- Extracting ' + file) 366 | await extract(file_fullpath, { 367 | dir: extract_dir, 368 | 369 | onEntry(entry, zipfile) { 370 | if (entry.fileName.endsWith('/')) { 371 | // log(' - Skipping subdirectory', entry.fileName) 372 | return false 373 | } 374 | 375 | if (md_skip_blanks && entry.uncompressedSize <= 3) { // files with 3 bytes just have a one blank block (like blank daily notes) 376 | // log(' - Skipping blank file', entry.fileName, `(${entry.uncompressedSize} bytes`) 377 | return false 378 | } 379 | 380 | // log(' -', entry.fileName) 381 | entry.fileName = sanitizeFileName(entry.fileName) 382 | 383 | if (fs.pathExistsSync(path.join(extract_dir, entry.fileName))) { 384 | 385 | log('WARNING: file collision detected. Overwriting file with (sanitized) name:', entry.fileName) 386 | // reject(`Extraction error: file collision detected with sanitized filename: ${entry.fileName}`) 387 | // TODO? renaming to... 388 | } 389 | 390 | return true 391 | } 392 | }) 393 | 394 | resolve() 395 | 396 | } catch (err) { reject(err) } 397 | }) 398 | } 399 | 400 | async function format_and_save(filetype, download_dir, graph_name) { 401 | return new Promise(async (resolve, reject) => { 402 | try { 403 | 404 | if (filetype == 'Markdown') { 405 | 406 | const extract_dir = path.join(download_dir, '_extraction') 407 | 408 | const files = await fs.readdir(extract_dir) 409 | 410 | if (files.length === 0) reject('Extraction error: extract_dir is empty') 411 | 412 | const markdown_dir = path.join(backup_dir, 'markdown', graph_name) 413 | 414 | // log('- Removing old markdown directory') 415 | await fs.remove(markdown_dir, { recursive: true }) // necessary, to update renamed pages 416 | 417 | log('- Saving Markdown') 418 | 419 | for (const file of files) { 420 | 421 | const file_fullpath = path.join(extract_dir, file) 422 | const new_file_fullpath = path.join(markdown_dir, file) 423 | 424 | await fs.move(file_fullpath, new_file_fullpath, { overwrite: true }) 425 | } 426 | 427 | } else { 428 | 429 | const files = await fs.readdir(download_dir) 430 | const file = files[0] 431 | const file_fullpath = path.join(download_dir, file) 432 | const fileext = file.split('.').pop() 433 | const new_file_fullpath = path.join(backup_dir, fileext, file) 434 | const new_file_fullpath_nodate = new_file_fullpath.replace(/-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}/, ''); 435 | 436 | if (fileext == 'json') { 437 | 438 | log('- Formatting JSON') 439 | const json = await fs.readJson(file_fullpath) 440 | const new_json = JSON.stringify(json, null, 2) 441 | 442 | log('- Saving formatted JSON') 443 | await fs.outputFile(new_file_fullpath_nodate, new_json) 444 | 445 | } else if (fileext == 'edn') { 446 | log('- Formatting EDN (this can take a couple minutes for large graphs)') // This could take a couple minutes for large graphs 447 | const edn = await fs.readFile(file_fullpath, 'utf-8') 448 | 449 | const edn_prefix = '#datascript/DB ' 450 | var new_edn = edn_prefix + edn_format(edn.replace(new RegExp('^' + edn_prefix), '')) 451 | checkFormattedEDN(edn, new_edn) 452 | 453 | log('- Saving formatted EDN') 454 | await fs.outputFile(new_file_fullpath_nodate, new_edn) 455 | 456 | } else reject(`format_and_save error: Unhandled filetype: ${files}`) 457 | } 458 | 459 | resolve() 460 | 461 | } catch (err) { reject(err) } 462 | }) 463 | } 464 | 465 | 466 | 467 | function log(...messages) { 468 | const timestamp = new Date().toISOString().replace('T', ' ').replace('Z', '') 469 | console.log(timestamp, 'R2G', ...messages) 470 | } 471 | 472 | async function error(err) { 473 | log('ERROR -', err) 474 | console.timeEnd('R2G Exit after') 475 | // await page.screenshot({ path: path.join(download_dir, 'error.png' }) // will need to pass page as parameter... or set as parent scope 476 | process.exit(1) 477 | } 478 | 479 | // async function getRepoPath() { 480 | // return new Promise(async (resolve, reject) => { 481 | // try { 482 | 483 | // const ubuntuPath = path.join('/', 'home', 'runner', 'work') 484 | // const exists = await fs.pathExists(ubuntuPath) 485 | 486 | // if (exists) { 487 | // const files = (await fs.readdir(ubuntuPath)) 488 | // .filter(f => !f.startsWith('_')) // filter out [ '_PipelineMapping', '_actions', '_temp', ] 489 | 490 | // if (files.length === 1) { 491 | // repo_name = files[0] 492 | // const files2 = await fs.readdir(path.join(ubuntuPath, repo_name)) 493 | 494 | // if (files2.length === 1 && files2[0] == repo_name) { 495 | 496 | // log(files2, 'GitHub Action path found') 497 | // resolve(path.join(ubuntuPath, repo_name, repo_name)) // actions/checkout@v2 outputs to path /home/runner/work// 498 | 499 | // } else { 500 | // log(files, 'detected in', path.join(ubuntuPath, repo_name), '\nNot GitHub Action') 501 | // resolve(false) 502 | // } 503 | 504 | // } else { 505 | // log(files, 'detected in', ubuntuPath, '\nNot GitHub Action') 506 | // resolve(false) 507 | // } 508 | 509 | // } else { 510 | // log(ubuntuPath, 'does not exist. Not GitHub Action') 511 | // resolve(false) 512 | // } 513 | 514 | // } catch (err) { reject(err) } 515 | // }) 516 | // } 517 | 518 | function checkFormattedEDN(original, formatted) { 519 | const reverse_format = formatted 520 | .trim() // remove trailing line break 521 | .split('\n') // separate by line 522 | .map(line => line.trim()) // remove indents, and one extra space at end of second to last line 523 | .join(' ') // replace line breaks with a space 524 | 525 | if (original === reverse_format) { 526 | // log('(formatted EDN check successful)') // formatted EDN successfully reversed to match exactly with original EDN 527 | return true 528 | } else { 529 | error('EDN formatting error: mismatch with original') 530 | return false 531 | } 532 | } 533 | 534 | // because GitHub Actions log censors the entire name as '***', but this allows to differentiate among multiple graphs while keeping it mostly private for when getting help troubleshooting 535 | function censor(graph_name) { 536 | return graph_name.split('').map((char, i) => { 537 | if (i != 0 && i != graph_name.length - 1 && char != '-' && char != '_') return '*' // don't censor first letter, last letter, hyphens, and underscores 538 | else return char 539 | }).join('') 540 | } 541 | 542 | function sanitizeFileName(fileName) { 543 | fileName = fileName.replace(/\//g, '／') 544 | 545 | const sanitized = sanitize(fileName, { replacement: md_replacement }) 546 | 547 | if (sanitized != fileName) { 548 | 549 | log(' Sanitized:', fileName, '\n to:', sanitized) 550 | return sanitized 551 | 552 | } else return fileName 553 | } 554 | --------------------------------------------------------------------------------