├── .eslintignore ├── .eslintrc.json ├── .gitconfig ├── .gitignore ├── .npmignore ├── .npmrc ├── LICENSE ├── README.md ├── __tests__ └── app.test.js ├── app.js ├── http ├── http-client.env.json ├── http-client.private.env.example └── requests.http ├── index.js ├── lib ├── QuipProcessor.js ├── QuipService.js ├── __tests__ │ ├── QuipProcessor.test.js │ ├── QuipService.test.js │ ├── __snapshots__ │ │ └── QuipProcessor.test.js.snap │ ├── folders.json │ ├── messages.json │ └── threads.json ├── cli │ ├── CliArguments.js │ ├── __tests__ │ │ └── CliArguments.test.js │ ├── help.js │ └── options.js ├── common │ ├── LoggerAdapter.js │ ├── PinoLogger.js │ ├── TestUtils.js │ ├── blobImageToURL.js │ └── utils.js └── templates │ ├── document.css │ └── document.ejs ├── package-lock.json ├── package.json ├── public ├── demo.gif ├── example-anim.gif └── example.png └── quip-export.js /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["eslint:recommended"], 3 | "parserOptions": { 4 | "ecmaVersion": 9 5 | }, 6 | "rules": { 7 | "require-atomic-updates": "warn" 8 | }, 9 | "env": { 10 | "es6": true, 11 | "amd": true, 12 | "node": true, 13 | "jest": true 14 | } 15 | } -------------------------------------------------------------------------------- /.gitconfig: -------------------------------------------------------------------------------- 1 | # Add to .git/config 2 | # 3 | # [include] 4 | # path = ../.gitconfig 5 | 6 | [user] 7 | name = Alexander Fleming 8 | email = tech@sonnenkern.com 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | 6 | # production 7 | /build 8 | 9 | #jest 10 | /coverage 11 | 12 | # misc 13 | .DS_Store 14 | .idea/ 15 | *.iml 16 | npm-debug.log 17 | http-client.private.env.json -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | __tests__ 2 | .* 3 | /http 4 | /public 5 | /quip-export-*.tgz -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry=https://registry.npmjs.org/ 2 | proxy=null 3 | https-proxy=null 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Alexander Fleming 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Quip-Export 2 | Comprehensive full automated export (backup) tool for [Quip](https://quip.com/). 3 | 4 | Quip-Export uses official [Quip Automation API](https://quip.com/dev/automation/documentation) and exports all documents and folders from an Quip-Account. 5 | 6 | Features: 7 | 8 | * Export in HTML format with original Quip styling 9 | * Export in MS Office format: .docx for documents, .xlsx for spresdsheets _(--docx)_ 10 | * Embedded CSS for HTML-export _(--embedded-styles)_ 11 | * Embedded images for HTML-export _(--embedded-images)_ 12 | * Export of comments and conversations for HTML-export _(--comments)_ 13 | * Export of specific folders only _(--folders)_ 14 | * Export of referenced files in documents 15 | * Resolving of references between folders and documents to relative paths 16 | 17 | Slides are not supported (due to poor quality of generated PDFs). Export in PDF are not supported (due to poor quality of generated PDFs). 18 | 19 | Despite Quip-Export is designed as a standalone tool for Node.js environment, it could be also used as Quip export library for any kind of JavaScript applications. In that case, the Quip-Export project is a good usage example. 20 | 21 | Quip-Export perfectly works on Windows, Mac OS and Linux/Unix in Node.js or in pure browser environment. 22 | 23 |

24 | 25 |

26 | 27 | ## Online web app and demo 28 | Full featured web app using Quip-Export npm package with demo mode is available on [www.quip-export.com](https://www.quip-export.com) 29 | 30 |

31 | 32 |

33 | 34 | ## Install and usage 35 | As mentioned before, Quip-Export tool needs Node.js (version 10.16 or higher) environment. 36 | Node.js version check: 37 | ``` 38 | node -v 39 | ``` 40 | If Node.js is not installed or the version is lower than 10.16, please follow install/update instructions on [Node.js website](https://nodejs.org/en/). 41 | 42 | ### Use without installing locally 43 | ``` 44 | npx quip-export [options] 45 | ``` 46 | Advantage: you always run the latest version. 47 | 48 | ### Install and usage as global npm package 49 | Install: 50 | ``` 51 | npm install -g quip-export 52 | ``` 53 | 54 | Usage example: 55 | ``` 56 | quip-export [options] 57 | ``` 58 | 59 | ### Install and usage as a GitHUb project 60 | Install: 61 | ``` 62 | git clone https://github.com/sonnenkern/quip-export.git 63 | ``` 64 | 65 | Install project dependencies: 66 | ``` 67 | cd quip-export 68 | npm install 69 | ``` 70 | 71 | Usage example from project folder: 72 | ``` 73 | node quip-export [options] 74 | ``` 75 | 76 | ### Install as an npm dependency in a project 77 | Install: 78 | ``` 79 | npm install quip-export 80 | ``` 81 | 82 | ## Options 83 | ``` 84 | -h, --help Usage guide. 85 | -v, --version Print version info 86 | -t, --token "string" Quip Access Token. 87 | -d, --destination "string" Destination folder for export files 88 | -z, --zip Zip export files 89 | --embedded-styles Embedded in each document stylesheet 90 | --embedded-images Embedded images 91 | --docx Exports documents in MS-Office format (*.docx , *.xlsx) 92 | --comments Includes comments (messages) for the documents 93 | --folders "string" Comma-separated folder's IDs to export 94 | --debug Extended logging 95 | ``` 96 | 97 | To generate a personal access token, visit the page: [https://quip.com/dev/token](https://quip.com/dev/token) 98 | 99 | Be aware, the options --comments, --embedded-images, --embedded-styles don't work together with export in MS-Office format (--docx) and will be ignored. 100 | 101 | The easiest way to get to know ID of Quip fodler is just to open the folder in Quip web application in browser and look at adress line. For example the adress "https://quip.com/bGG333444111" points to the folder with ID "bGG333444111". 102 | 103 | ## Usage examples 104 | Export to folder c:\temp 105 | ``` 106 | quip-export -t "JHHHK222333444LLL1=" -d c:\temp 107 | ``` 108 | Export to current folder as destination 109 | ``` 110 | quip-export -t "JHHHK222333444LLL1=" 111 | ``` 112 | Export in Zip-file with current folder as destination 113 | ``` 114 | quip-export -t "JHHHK222333444LLL1=" -z 115 | ``` 116 | 117 | ## Logging 118 | The export errors are written to file export.log in the destination folder. 119 | 120 | ## Troubleshooting 121 | Quip-Export is strongly depending on the public [Quip Automation API](https://quip.com/dev/automation/documentation). 122 | It is possible that some problems will occur if Quip Automation API is changed. Then Quip-Export adjustment is necessary. 123 | 124 | In this case or other questions, please feel free to contact [info@quip-export.com](info@quip-export.com). -------------------------------------------------------------------------------- /__tests__/app.test.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const fs = require('fs'); 3 | 4 | const {App} = require('../app'); 5 | const {documentTemplate, documentCSS} = require('../app'); 6 | 7 | const colors = require('colors'); 8 | 9 | const Utils = require('../lib/common/utils'); 10 | Utils.getVersionInfo = jest.fn(); 11 | Utils.cliBox = jest.fn(); 12 | Utils.getVersionInfo.mockResolvedValue({localVersion: '1.33'}); 13 | Utils.writeBlobFile = jest.fn(); 14 | Utils.writeTextFile = jest.fn(); 15 | 16 | jest.mock('../lib/cli/CliArguments'); 17 | const CliArguments = require('../lib/cli/CliArguments'); 18 | 19 | jest.mock('../lib/common/PinoLogger'); 20 | const PinoLogger = require('../lib/common/PinoLogger'); 21 | 22 | jest.mock('../lib/QuipService'); 23 | const QuipService = require('../lib/QuipService'); 24 | 25 | jest.mock('../lib/QuipProcessor'); 26 | const QuipProcessor = require('../lib/QuipProcessor'); 27 | 28 | jest.mock('jszip'); 29 | const JSZip = require('jszip'); 30 | const JSZipMock = { 31 | file: jest.fn(), 32 | generateAsync: jest.fn(() => new Promise((resolve) => {resolve('CONTENT')})), 33 | folder: jest.fn(() => JSZipMock), 34 | }; 35 | JSZip.mockImplementation(() => { 36 | return JSZipMock 37 | }); 38 | 39 | let app; 40 | 41 | const pinoLoggerObj = { 42 | error: jest.fn(), 43 | debug: jest.fn() 44 | }; 45 | 46 | function initApp() { 47 | app = new App(); 48 | console.log = jest.fn(); 49 | CliArguments.mockReturnValue({ 50 | destination: 'c:/temp', 51 | token: 'TOKEN', 52 | ['embedded-styles']: true, 53 | ['embedded-images']: true, 54 | ['comments']: true, 55 | ['docx']: true 56 | }); 57 | QuipService.mockImplementation(() => { 58 | return { 59 | checkUser: () => true, 60 | setLogger: () => {} 61 | } 62 | }); 63 | 64 | QuipProcessor.mockImplementation(() => { 65 | return { 66 | startExport: jest.fn(() => new Promise((resolve) => {resolve('RESOLVED')})), 67 | setLogger: jest.fn(), 68 | quipService: {stats: 'STATS'} 69 | } 70 | }); 71 | 72 | PinoLogger.mockImplementation(() => { 73 | return pinoLoggerObj; 74 | }); 75 | } 76 | 77 | test('documentTemplate and documentCSS could be read', async () => { 78 | expect(documentTemplate.length > 0).toBe(true); 79 | expect(documentCSS.length > 0).toBe(true); 80 | }); 81 | 82 | 83 | describe('constructor() tests', () => { 84 | beforeEach(() => { 85 | initApp(); 86 | }); 87 | }); 88 | 89 | describe('main() tests', () => { 90 | beforeEach(() => { 91 | initApp(); 92 | }); 93 | 94 | test('CliArguments throws exception', async () => { 95 | CliArguments.mockImplementation(() => { 96 | throw 'Message'; 97 | }); 98 | await app.main(); 99 | expect(console.log).toHaveBeenCalledWith("Message"); 100 | expect(console.log).not.toHaveBeenCalledWith(`Quip-Export v1.33`); 101 | }); 102 | 103 | test('CliArguments not throws exception', async () => { 104 | await app.main(); 105 | expect(console.log).not.toHaveBeenCalledWith("Message"); 106 | expect(console.log).toHaveBeenCalledWith(`Quip-Export v1.33`); 107 | }); 108 | 109 | test('debug mode -> set up logger with debug level', async () => { 110 | CliArguments.mockReturnValue({destination: 'c:/temp', debug: true}); 111 | await app.main(); 112 | expect(PinoLogger).toHaveBeenCalledWith(PinoLogger.LEVELS.DEBUG, 'c:/temp/export.log'); 113 | expect(app.Logger).toBe(pinoLoggerObj); 114 | }); 115 | 116 | test('normal mode -> set up logger with info level', async () => { 117 | await app.main(); 118 | expect(PinoLogger).toHaveBeenCalledWith(PinoLogger.LEVELS.INFO, 'c:/temp/export.log'); 119 | expect(app.Logger).toBe(pinoLoggerObj); 120 | }); 121 | 122 | test('localOutdate = true', async () => { 123 | Utils.getVersionInfo.mockResolvedValue({localVersion: '1.33', localOutdate: true}); 124 | await app.main(); 125 | expect(Utils.cliBox).toHaveBeenCalled(); 126 | }); 127 | 128 | test('init QuipService', async () => { 129 | await app.main(); 130 | expect(QuipService).toHaveBeenCalledWith('TOKEN'); 131 | }); 132 | 133 | test('setLogger for QuipService', async () => { 134 | const quipServiceMock = {checkUser: () => true, setLogger: jest.fn()} 135 | QuipService.mockImplementation(() => { 136 | return quipServiceMock 137 | }); 138 | await app.main(); 139 | expect(quipServiceMock.setLogger).toHaveBeenCalledWith(app.Logger); 140 | }); 141 | 142 | test('User is available', async () => { 143 | await app.main(); 144 | expect(console.log).toHaveBeenCalledWith(`Destination folder: c:/temp`); 145 | }); 146 | 147 | test('User is not available', async () => { 148 | QuipService.mockImplementation(() => { 149 | return {checkUser: () => false, setLogger: () => {}} 150 | }); 151 | await app.main(); 152 | expect(console.log).toHaveBeenCalledWith(colors.red('ERROR: Token is wrong or expired.')); 153 | expect(console.log).not.toHaveBeenCalledWith(`Destination folder: c:/temp`); 154 | }); 155 | 156 | test('activate zip', async () => { 157 | CliArguments.mockReturnValue({destination: 'c:/temp', zip: true}); 158 | await app.main(); 159 | expect(app.zip).toBe(JSZipMock); 160 | }); 161 | 162 | test('init QuipProcessor', async () => { 163 | await app.main(); 164 | expect(QuipProcessor).toHaveBeenCalledWith('TOKEN', expect.anything(), expect.anything(), expect.anything(), 165 | { 166 | documentTemplate, 167 | documentCSS: documentCSS, 168 | embeddedImages: true, 169 | comments: true, 170 | docx: true 171 | } 172 | ); 173 | expect(app.quipProcessor.setLogger).toHaveBeenCalledWith(app.Logger); 174 | }); 175 | 176 | test('add css to zip', async () => { 177 | CliArguments.mockReturnValue({ 178 | destination: 'c:/temp', 179 | token: 'TOKEN', 180 | ['embedded-styles']: false, 181 | ['embedded-images']: true, 182 | zip: true 183 | }); 184 | await app.main(); 185 | expect(app.zip.file).toHaveBeenCalledWith('document.css', documentCSS); 186 | }); 187 | 188 | test('add css in file system', async () => { 189 | CliArguments.mockReturnValue({ 190 | destination: 'c:/temp', 191 | token: 'TOKEN', 192 | ['embedded-styles']: false, 193 | ['embedded-images']: true, 194 | zip: false 195 | }); 196 | Utils.writeTextFile = jest.fn(); 197 | await app.main(); 198 | expect(Utils.writeTextFile).toHaveBeenCalledWith(path.join('c:/temp', "quip-export", 'document.css'), documentCSS); 199 | }); 200 | 201 | test('start export: zip', async () => { 202 | CliArguments.mockReturnValue({ 203 | destination: 'c:/temp', 204 | token: 'TOKEN', 205 | ['embedded-styles']: true, 206 | ['embedded-images']: true, 207 | zip: true 208 | }); 209 | fs.writeFile = jest.fn((path, content, cb) => cb()); 210 | await app.main(); 211 | expect(app.quipProcessor.startExport).toHaveBeenCalled(); 212 | expect(app.Logger.debug).toHaveBeenCalledWith('STATS'); 213 | expect(app.zip.generateAsync).toHaveBeenCalled(); 214 | expect(fs.writeFile).toHaveBeenCalled(); 215 | 216 | expect(console.log).toHaveBeenCalledWith("Zip-file has been saved: ", path.join(app.desinationFolder, 'quip-export.zip')); 217 | }); 218 | 219 | test('folders option', async () => { 220 | const folders = ['111','222']; 221 | CliArguments.mockReturnValue({ 222 | destination: 'c:/temp', 223 | token: 'TOKEN', 224 | ['embedded-styles']: false, 225 | ['embedded-images']: false, 226 | zip: false, 227 | folders 228 | }); 229 | await app.main(); 230 | expect(app.quipProcessor.startExport).toHaveBeenCalledWith(folders); 231 | }); 232 | }); 233 | 234 | describe('fileSaver() tests', () => { 235 | const blob = { 236 | arrayBuffer () { 237 | return [1,2,3,4]; 238 | } 239 | }; 240 | 241 | const fileName = 'aaa.html'; 242 | const filePath = '/some/path'; 243 | 244 | beforeEach(() => { 245 | app = new App(); 246 | app.zip = new JSZip(); 247 | app.cliArguments = { 248 | zip: true 249 | }; 250 | app.desinationFolder = "c:\temp"; 251 | }); 252 | 253 | test('BLOB zip', async () => { 254 | app.cliArguments = { 255 | zip: true 256 | }; 257 | app.fileSaver(blob, fileName, 'BLOB', filePath); 258 | expect(app.zip.folder).toHaveBeenCalledWith(filePath); 259 | expect(app.zip.file).toHaveBeenCalledWith(fileName, [1,2,3,4]); 260 | }); 261 | 262 | test('BLOB file', async () => { 263 | app.cliArguments = { 264 | zip: false 265 | }; 266 | app.fileSaver(blob, fileName, 'BLOB', filePath); 267 | expect(Utils.writeBlobFile).toHaveBeenCalledWith(path.join(app.desinationFolder, "quip-export", filePath, fileName), blob); 268 | }); 269 | 270 | test('not a BLOB zip', async () => { 271 | app.cliArguments = { 272 | zip: true 273 | }; 274 | app.fileSaver(blob, fileName, 'NOT-BLOB', filePath); 275 | expect(app.zip.folder).toHaveBeenCalledWith(filePath); 276 | expect(app.zip.file).toHaveBeenCalledWith(fileName, blob); 277 | }); 278 | 279 | test('not a BLOB file', async () => { 280 | app.cliArguments = { 281 | zip: false 282 | }; 283 | app.fileSaver(blob, fileName, 'NOT-BLOB', filePath); 284 | expect(Utils.writeTextFile).toHaveBeenCalledWith(path.join(app.desinationFolder, "quip-export", filePath, fileName), blob); 285 | }); 286 | }); 287 | 288 | describe('progressFunc() tests', () => { 289 | beforeEach(() => { 290 | app = new App(); 291 | app.spinnerIndicator = { 292 | text: '' 293 | }; 294 | app.progressIndicator = { 295 | update: jest.fn() 296 | }; 297 | }); 298 | 299 | test('progressFunc(), phase ANALYSIS', async () => { 300 | app.phase = 'ANALYSIS'; 301 | app.progressFunc({ 302 | readFolders: 100, 303 | readThreads: 200 304 | }); 305 | expect(app.spinnerIndicator.text).toBe(` %s read 100 folder(s) | 200 thread(s)`); 306 | }); 307 | 308 | test('progressFunc(), phase EXPORT', async () => { 309 | app.phase = 'EXPORT'; 310 | app.progressFunc({ 311 | threadsProcessed: 1000 312 | }); 313 | expect(app.progressIndicator.update).toHaveBeenCalledWith(1000); 314 | }); 315 | }); 316 | 317 | describe('phaseFunc() tests', () => { 318 | beforeEach(() => { 319 | app = new App(); 320 | process.stdout.write = jest.fn(); 321 | app.spinnerIndicator = { 322 | setSpinnerDelay: jest.fn(), 323 | setSpinnerString: jest.fn(), 324 | text: 'TEXT', 325 | start: jest.fn(), 326 | stop: jest.fn(), 327 | onTick: jest.fn(), 328 | }; 329 | app.progressIndicator = { 330 | start: jest.fn(), 331 | update: jest.fn(), 332 | stop: jest.fn() 333 | }; 334 | app.quipProcessor = { 335 | foldersTotal: 100, 336 | threadsTotal: 200, 337 | quipService: { 338 | apiURL: 'http://url.com' 339 | } 340 | }; 341 | }); 342 | 343 | test("phase === 'START'", async () => { 344 | app.phaseFunc('START', ''); 345 | expect(process.stdout.write).toHaveBeenCalledWith(colors.gray(`Quip API: ${app.quipProcessor.quipService.apiURL}`)); 346 | }); 347 | 348 | test("phase === 'ANALYSIS'", async () => { 349 | app.phaseFunc('ANALYSIS', ''); 350 | expect(process.stdout.write).toHaveBeenCalledWith(colors.cyan('Analysing folders...')); 351 | expect(app.spinnerIndicator.start).toBeCalled(); 352 | }); 353 | 354 | test("prevPhase === 'ANALYSIS'", async () => { 355 | app.phaseFunc('', 'ANALYSIS'); 356 | expect(app.spinnerIndicator.onTick).toHaveBeenCalledWith(` read 100 folder(s) | 200 thread(s)`); 357 | expect(app.spinnerIndicator.stop).toBeCalled(); 358 | }); 359 | 360 | test("phase === 'EXPORT'", async () => { 361 | app.phaseFunc('EXPORT', ''); 362 | expect(process.stdout.write).toHaveBeenCalledWith(colors.cyan('Exporting...')); 363 | expect(app.progressIndicator.start).toHaveBeenCalledWith(200, 0); 364 | }); 365 | 366 | test("prevPhase === 'EXPORT'", async () => { 367 | app.phaseFunc('', 'EXPORT'); 368 | expect(app.progressIndicator.stop).toHaveBeenCalled(); 369 | }); 370 | }); 371 | 372 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const Spinner = require('cli-spinner').Spinner; 3 | const colors = require('colors'); 4 | const cliProgress = require('cli-progress'); 5 | const JSZip = require('jszip'); 6 | const fs = require('fs'); 7 | const moment = require('moment'); 8 | 9 | //PinoLogger implements LoggerAdapter-Interface 10 | //It is possible to use another logger instead of PinoLogger 11 | const PinoLogger = require('./lib/common/PinoLogger'); 12 | const QuipProcessor = require('./lib/QuipProcessor'); 13 | const QuipService = require('./lib/QuipService'); 14 | const utils = require('./lib/common/utils'); 15 | const CliArguments = require('./lib/cli/CliArguments'); 16 | 17 | //EJS template for html documents 18 | const documentTemplate = utils.readTextFile(path.join(__dirname, './lib/templates/document.ejs')); 19 | //CSS style for html documents 20 | const documentCSS = utils.readTextFile(path.join(__dirname, './lib/templates/document.css')); 21 | 22 | class App { 23 | constructor() { 24 | this.Logger = {}; 25 | this.desinationFolder; 26 | this.cliArguments; 27 | this.zip; 28 | this.quipProcessor; 29 | this.spinnerIndicator = new Spinner(' %s read 0 folder(s) | 0 thread(s)'); 30 | this.progressIndicator = new cliProgress.Bar({ 31 | format: ' |{bar}| {percentage}% | {value}/{total} threads | ETA: {eta_formatted}', 32 | barCompleteChar: '\u2588', 33 | barIncompleteChar: '\u2591', 34 | hideCursor: false 35 | }); 36 | this.phase; 37 | } 38 | 39 | /* 40 | callback-function for file saving 41 | */ 42 | fileSaver(data, fileName, type, filePath) { 43 | if(type === 'BLOB') { 44 | if(this.cliArguments.zip) { 45 | this.zip.folder(filePath).file(fileName, data.arrayBuffer()); 46 | } else { 47 | utils.writeBlobFile(path.join(this.desinationFolder, "quip-export", filePath, fileName), data); 48 | } 49 | } else { 50 | if(this.cliArguments.zip) { 51 | this.zip.folder(filePath).file(fileName, data); 52 | } else { 53 | utils.writeTextFile(path.join(this.desinationFolder, "quip-export", filePath, fileName), data); 54 | } 55 | } 56 | } 57 | 58 | /* 59 | callback-function for progress indication 60 | */ 61 | progressFunc(progress) { 62 | if(this.phase === 'ANALYSIS') { 63 | this.spinnerIndicator.text = ` %s read ${progress.readFolders} folder(s) | ${progress.readThreads} thread(s)`; 64 | } 65 | else if(this.phase === 'EXPORT') { 66 | this.progressIndicator.update(progress.threadsProcessed); 67 | } 68 | } 69 | 70 | /* 71 | callback-function for export life cycle phases 72 | available phases: 73 | START - start of process 74 | STOP - end of process 75 | ANALYSIS - folder/threads structure analysis 76 | EXPORT - export 77 | */ 78 | phaseFunc(phase, prevPhase) { 79 | this.phase = phase; 80 | if(phase === 'START') { 81 | process.stdout.write(colors.gray(`Quip API: ${this.quipProcessor.quipService.apiURL}`)); 82 | process.stdout.write('\n'); 83 | } 84 | 85 | if (phase === 'ANALYSIS'){ 86 | process.stdout.write('\n'); 87 | process.stdout.write(colors.cyan('Analysing folders...')); 88 | process.stdout.write('\n'); 89 | 90 | this.spinnerIndicator.setSpinnerDelay(80); 91 | this.spinnerIndicator.setSpinnerString("|/-\\"); 92 | 93 | this.spinnerIndicator.start(); 94 | } 95 | 96 | if(prevPhase === 'ANALYSIS') { 97 | this.spinnerIndicator.onTick(` read ${this.quipProcessor.foldersTotal} folder(s) | ${this.quipProcessor.threadsTotal} thread(s)`); 98 | this.spinnerIndicator.stop(); 99 | process.stdout.write('\n'); 100 | } 101 | 102 | if(phase === 'EXPORT') { 103 | process.stdout.write('\n'); 104 | process.stdout.write(colors.cyan('Exporting...')); 105 | process.stdout.write('\n'); 106 | 107 | this.progressIndicator.start(this.quipProcessor.threadsTotal, 0); 108 | } 109 | 110 | if(prevPhase === 'EXPORT') { 111 | this.progressIndicator.stop(); 112 | process.stdout.write('\n'); 113 | } 114 | } 115 | 116 | //main entry point 117 | async main() { 118 | console.log(); 119 | const versionInfo = await utils.getVersionInfo(); 120 | 121 | //cli arguments parsing and validation 122 | try { 123 | this.cliArguments = CliArguments(); 124 | } catch (message) { 125 | console.log(message); 126 | return; 127 | } 128 | 129 | //current folder as destination, if not set 130 | this.desinationFolder = (this.cliArguments.destination || process.cwd()); 131 | 132 | if(this.cliArguments.debug) { 133 | this.Logger = new PinoLogger(PinoLogger.LEVELS.DEBUG, `${this.desinationFolder}/export.log`); 134 | } else { 135 | this.Logger = new PinoLogger(PinoLogger.LEVELS.INFO, `${this.desinationFolder}/export.log`); 136 | } 137 | 138 | console.log(`Quip-Export v${versionInfo.localVersion}`); 139 | 140 | if(versionInfo.localOutdate) { 141 | utils.cliBox(`!!!! A new version of Quip-Export (v${versionInfo.remoteVersion}) is available.`); 142 | } 143 | 144 | if(this.cliArguments['comments'] && this.cliArguments['docx']) { 145 | console.log('Docx export: comments option will be ignored.'); 146 | } 147 | 148 | //Token verification 149 | const quipService = new QuipService(this.cliArguments.token); 150 | quipService.setLogger(this.Logger); 151 | 152 | if(!await quipService.checkUser()) { 153 | console.log(colors.red('ERROR: Token is wrong or expired.')); 154 | return; 155 | } 156 | 157 | console.log(`Destination folder: ${this.desinationFolder}`); 158 | 159 | //activate zip 160 | if(this.cliArguments.zip) { 161 | this.zip = new JSZip(); 162 | } 163 | 164 | this.quipProcessor = new QuipProcessor(this.cliArguments.token, this.fileSaver.bind(this), this.progressFunc.bind(this), this.phaseFunc.bind(this), 165 | { 166 | documentTemplate, 167 | documentCSS: this.cliArguments['embedded-styles']? documentCSS : '', 168 | embeddedImages: this.cliArguments['embedded-images'], 169 | comments: this.cliArguments['comments'], 170 | docx: this.cliArguments['docx'] 171 | }); 172 | 173 | this.quipProcessor.setLogger(this.Logger); 174 | 175 | if(!this.cliArguments['embedded-styles'] && !this.cliArguments['docx']) { 176 | if(this.cliArguments.zip) { 177 | this.zip.file('document.css', documentCSS); 178 | } else { 179 | utils.writeTextFile(path.join(this.desinationFolder, "quip-export", 'document.css'), documentCSS); 180 | } 181 | } 182 | 183 | let foldersToExport = [ 184 | //'FOLDER-1' 185 | //'FOLDER-2' 186 | //'EVZAOAW2e6U', 187 | //'UPWAOAAEpFn', //Test 188 | //'bGGAOAKTL4Y' //Test/folder1 189 | //'EJCAOAdY90Y', // Design patterns 190 | //'NBaAOAhFXJJ' //React 191 | ]; 192 | 193 | if(this.cliArguments['folders']) { 194 | foldersToExport = this.cliArguments['folders']; 195 | } 196 | 197 | const startTime = new Date().getTime(); 198 | 199 | await this.quipProcessor.startExport(foldersToExport); 200 | 201 | const durationStr = moment.utc(new Date().getTime() - startTime).format("HH:mm:ss"); 202 | 203 | this.Logger.debug(this.quipProcessor.quipService.stats); 204 | this.Logger.debug(`Export duration: ${durationStr}`); 205 | 206 | console.log(`Export duration: ${durationStr}`); 207 | 208 | if(this.cliArguments.zip) { 209 | //save zip file 210 | const content = await this.zip.generateAsync({type: "nodebuffer", compression: "DEFLATE"}); 211 | await fs.writeFile(path.join(this.desinationFolder, 'quip-export.zip'), content, () => { 212 | console.log("Zip-file has been saved: ", path.join(this.desinationFolder, 'quip-export.zip')); 213 | }); 214 | } 215 | } 216 | } 217 | 218 | module.exports = {App, documentTemplate, documentCSS}; -------------------------------------------------------------------------------- /http/http-client.env.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": { 3 | "auth_token": "#### QUIP TOKEN ###", 4 | "quip_api_url": "https://platform.quip.com:443/1" 5 | }, 6 | "demo": { 7 | "auth_token": "#### QUIP TOKEN ###", 8 | "quip_api_url": "https://platform.quip.com:443/1" 9 | } 10 | } -------------------------------------------------------------------------------- /http/http-client.private.env.example: -------------------------------------------------------------------------------- 1 | { 2 | "demo": { 3 | "auth_token": "XXXXXX" 4 | }, 5 | "private": { 6 | "auth_token": "YYYYYY" 7 | } 8 | } -------------------------------------------------------------------------------- /http/requests.http: -------------------------------------------------------------------------------- 1 | ### get user 2 | GET {{quip_api_url}}/users/current 3 | Accept: application/json 4 | Authorization: Bearer {{auth_token}} 5 | 6 | ### search threads 7 | GET https://platform.quip.com/1/threads/search?query=Programming 8 | Accept: application/json 9 | Authorization: Bearer {{auth_token}} 10 | 11 | ### get recent messages 12 | GET https://platform.quip.com/1/messages/{{thread_with_messages}} 13 | Accept: application/json 14 | Authorization: Bearer {{auth_token}} 15 | 16 | ### get thread 17 | GET https://platform.quip.com/1/threads/{{thread_with_messages}} 18 | Accept: application/json 19 | Authorization: Bearer {{auth_token}} 20 | 21 | ### get blob from thread 22 | GET https://platform.quip.com/1/blob/{{thread_with_messages}}/Ak5BJnkWubPxpIm4dtlWmg 23 | Accept: application/json 24 | Authorization: Bearer {{auth_token}} 25 | 26 | ### get user 27 | GET https://platform.quip.com/1/users/{{user_id}} 28 | Accept: application/json 29 | Authorization: Bearer {{auth_token}} 30 | 31 | ### get folders 32 | GET https://platform.quip.com/1/folders/?ids=bGGAOAKTL4Y 33 | Accept: application/json 34 | Authorization: Bearer {{auth_token}} -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const QuipService = require('./lib/QuipService'); 2 | const QuipProcessor = require('./lib/QuipProcessor'); 3 | const PinoLogger = require('./lib/common/PinoLogger'); 4 | const LoggerAdapter = require('./lib/common/LoggerAdapter'); 5 | 6 | module.exports = { 7 | QuipProcessor, 8 | QuipService, 9 | PinoLogger, 10 | LoggerAdapter 11 | }; -------------------------------------------------------------------------------- /lib/QuipProcessor.js: -------------------------------------------------------------------------------- 1 | const QuipService = require('./QuipService'); 2 | const Mime = require ('mime'); 3 | const ejs = require ('ejs'); 4 | const moment = require('moment'); 5 | const sanitizeFilename = require("sanitize-filename"); 6 | const LoggerAdapter = require('./common/LoggerAdapter'); 7 | const blobImageToURL = require('./common/blobImageToURL'); 8 | 9 | 10 | class QuipProcessor { 11 | constructor (quipToken, saveCallback = ()=>{}, progressCallback = ()=>{}, phaseCallback = ()=>{}, options={}) { 12 | this.quipToken = quipToken; 13 | this.saveCallback = saveCallback; 14 | this.progressCallback = progressCallback; 15 | this.phaseCallback = phaseCallback; 16 | this.options = options; 17 | this.logger = new LoggerAdapter(); 18 | 19 | this.start = false; 20 | this.threadsProcessed = 0; 21 | this.foldersProcessed = 0; 22 | 23 | this.threadsTotal = 0; 24 | this.foldersTotal = 0; 25 | 26 | this.referencesMap = new Map(); 27 | 28 | this.phase = 'STOP'; //START, STOP, ANALYSIS, EXPORT 29 | 30 | this.quipService = new QuipService(quipToken, options.quipApiURL); 31 | 32 | //parse options 33 | if(options.documentTemplate) { 34 | this.documentTemplate = options.documentTemplate; 35 | } else { 36 | console.error("Document template is not set!"); 37 | } 38 | } 39 | 40 | setLogger(logger) { 41 | this.logger = logger; 42 | this.logger.debug("-".repeat(80)); 43 | this.quipService.setLogger(logger); 44 | } 45 | 46 | async startExport(folderIds) { 47 | this._changePhase('START'); 48 | 49 | this.start = true; 50 | this.threadsProcessed = 0; 51 | 52 | this.quipUser = await this.quipService.getCurrentUser(); 53 | if(!this.quipUser) { 54 | this.logger.error("Can't load the User"); 55 | this.stopExport(); 56 | return; 57 | } 58 | this.logger.debug("USER-URL: " + this.quipUser.url); 59 | 60 | let folderIdsToExport = [ 61 | //this.quipUser.desktop_folder_id, 62 | //this.quipUser.archive_folder_id, 63 | //this.quipUser.starred_folder_id, 64 | this.quipUser.private_folder_id, 65 | //this.quipUser.trash_folder_id 66 | ...this.quipUser.shared_folder_ids, 67 | ...this.quipUser.group_folder_ids 68 | ]; 69 | 70 | if(folderIds && folderIds.length > 0) { 71 | folderIdsToExport = folderIds; 72 | } 73 | 74 | await this._exportFolders(folderIdsToExport); 75 | 76 | this.stopExport(); 77 | } 78 | 79 | stopExport() { 80 | this.start = false; 81 | this._changePhase('STOP'); 82 | } 83 | 84 | _changePhase(phase) { 85 | this.phaseCallback(phase, this.phase); 86 | this.phase = phase; 87 | } 88 | 89 | _getMatchGroups(text, regexStr, groups) { 90 | const regexp = new RegExp(regexStr, 'gim'); 91 | const matches = new Map(); 92 | let regexpResult; 93 | while ((regexpResult = regexp.exec(text)) != null) { 94 | const match = {match: regexpResult[0]}; 95 | for(const groupIndex in groups) { 96 | match[groups[groupIndex]] = regexpResult[+groupIndex+1]; 97 | } 98 | matches.set(regexpResult[0], match); 99 | } 100 | return Array.from(matches.values()); 101 | } 102 | 103 | async _resolveReferences(html, pathDeepness) { 104 | //look up for document or folder references 105 | const matchesReference = this._getMatchGroups(html, 106 | `href="(.*quip.com/([\\w-]+))"`, 107 | ['replacement', 'referenceId']); 108 | 109 | //replace references to documents 110 | for(const reference of matchesReference) { 111 | html = await this._processReference(html, reference, pathDeepness); 112 | } 113 | 114 | return html; 115 | } 116 | 117 | async _findReferencedUser(reference) { 118 | let referencedObject = this.referencesMap.get(reference.referenceId); 119 | if(!referencedObject) { 120 | const referencedUser = await this.quipService.getUser(reference.referenceId); 121 | if(referencedUser) { 122 | referencedObject = { 123 | user: true, 124 | name: referencedUser.name 125 | }; 126 | this.referencesMap.set(reference.referenceId, referencedObject); 127 | } 128 | } 129 | 130 | if(referencedObject && referencedObject.user) { 131 | return referencedObject; 132 | } 133 | 134 | this.logger.debug("_findReferencedThread: Couldn't find referenced user with referenceId=" + reference.referenceId); 135 | } 136 | 137 | async _findReferencedObject(reference) { 138 | let referencedObject = this.referencesMap.get(reference.referenceId); 139 | 140 | if(referencedObject) { 141 | if (referencedObject.thread && !referencedObject.title) { 142 | const referencedThread = await this.quipService.getThread(reference.referenceId); 143 | if(!referencedThread) { 144 | this.logger.debug("_processReference: Couldn't load Thread with id=" + reference.referenceId); 145 | return; 146 | } 147 | referencedObject.title = referencedThread.thread.title; 148 | referencedObject.thread = true; 149 | } 150 | 151 | return referencedObject; 152 | } else { 153 | const referencedThread = await this.quipService.getThread(reference.referenceId); 154 | if(referencedThread) { 155 | referencedObject = this.referencesMap.get(referencedThread.thread.id); 156 | if(!referencedObject) { 157 | return; 158 | } 159 | referencedObject.title = referencedThread.thread.title; 160 | referencedObject.thread = true; 161 | this.referencesMap.set(reference.referenceId, referencedObject); 162 | return referencedObject; 163 | } 164 | 165 | const referencedUser = await this._findReferencedUser(reference); 166 | 167 | if(referencedUser) { 168 | return referencedUser; 169 | } 170 | 171 | this.logger.debug("_findReferencedObject: Couldn't find referenced object with referenceId=" + reference.referenceId); 172 | } 173 | } 174 | 175 | async _processReference(html, reference, pathDeepness) { 176 | let referencedObject = await this._findReferencedObject(reference); 177 | let path; 178 | 179 | if(!referencedObject) { 180 | return html; 181 | } 182 | 183 | if(!referencedObject.folder && !referencedObject.thread) { 184 | return html; 185 | } 186 | 187 | if(referencedObject.folder) { 188 | //folder 189 | path = '../'.repeat(pathDeepness) + referencedObject.path + referencedObject.title; 190 | } else { 191 | //thread 192 | path = '../'.repeat(pathDeepness) + referencedObject.path + sanitizeFilename(referencedObject.title.trim()) + '.html'; 193 | } 194 | 195 | this.logger.debug(`_processReference: replacement=${reference.replacement}, path=${path}`); 196 | return html.replace(reference.replacement, path); 197 | } 198 | 199 | _renderDocumentHtml(html, title, path) { 200 | const pathDeepness = path.split("/").length-1; 201 | 202 | const documentRenderOptions = { 203 | title: title, 204 | body: html, 205 | stylesheet_path: '', 206 | embedded_stylesheet: this.options.documentCSS 207 | }; 208 | 209 | if(!this.options.documentCSS) { 210 | documentRenderOptions.stylesheet_path = '../'.repeat(pathDeepness) + 'document.css'; 211 | } 212 | 213 | if(this.documentTemplate) { 214 | //wrap html code 215 | return ejs.render(this.documentTemplate, documentRenderOptions); 216 | } 217 | 218 | return html; 219 | } 220 | 221 | async _getThreadMessagesHtml(quipThread, path) { 222 | let html = ''; 223 | const pathDeepness = path.split("/").length-1; 224 | //get all messages for thread without annotation-messages 225 | let messages = (await this.quipService.getThreadMessages(quipThread.thread.id)); 226 | if(messages) { 227 | //messages = messages.filter(msg => !msg.annotation) 228 | messages = messages.sort((msg1, msg2) => msg1.created_usec > msg2.created_usec? 1 : -1); 229 | } 230 | 231 | if(!messages || !messages.length) { 232 | return ''; 233 | } 234 | 235 | for(const message of messages) { 236 | let text = ''; 237 | 238 | if(message.text) { 239 | text = message.text.replace(/https/gim, ' https'); 240 | //document, user and folder references 241 | const matchesReferences = this._getMatchGroups(text, 242 | `https://.*?quip.com/([\\w-]+)`, ['referenceId']); 243 | for(const reference of matchesReferences) { 244 | if(message.mention_user_ids && message.mention_user_ids.includes(reference.referenceId)) { 245 | //user 246 | const referencedUser = await this._findReferencedUser(reference); 247 | if(referencedUser) { 248 | text = text.replace(reference.match, `@${referencedUser.name}`); 249 | } else { 250 | text = text.replace(reference.match, ``); 251 | } 252 | } else { 253 | //folder or thread 254 | const referencedObject = await this._findReferencedObject(reference); 255 | const title = referencedObject? referencedObject.title : reference.referenceId; 256 | let referenceHtml = `${title}`; 257 | referenceHtml = await this._processReference(referenceHtml, { 258 | referenceId: reference.referenceId, 259 | replacement: 'RELATIVE_PATH' 260 | }, pathDeepness); 261 | text = text.replace(reference.match, referenceHtml); 262 | } 263 | } 264 | } 265 | 266 | //file and image references 267 | if(message.files) { 268 | for(const file of message.files) { 269 | const fileMatch = { 270 | replacement: 'RELATIVE_PATH', 271 | threadId: quipThread.thread.id, 272 | blobId: file.hash 273 | }; 274 | const mimeType = Mime.getType(file.name); 275 | if(mimeType && mimeType.startsWith('image/')) { 276 | //image 277 | const imageHtml = `

`; 278 | text += await this._processFile(imageHtml, fileMatch, path, this.options.embeddedImages); 279 | } else { 280 | //file 281 | let fileName = file.name? file.name : file.hash; 282 | const fileHtml = `
${fileName}
`; 283 | text += await this._processFile(fileHtml, fileMatch, path); 284 | } 285 | 286 | text += `
`; 287 | } 288 | } 289 | 290 | let dateStr = ''; 291 | 292 | if(message.updated_usec && message.created_usec) { 293 | const updatedDate = moment(message.updated_usec/1000).format('D MMM YYYY, HH:mm'); 294 | const createdDate = moment(message.created_usec/1000).format('D MMM YYYY, HH:mm'); 295 | dateStr = updatedDate === createdDate? createdDate : `${createdDate} (Updated: ${updatedDate})`; 296 | dateStr = `, ${dateStr}`; 297 | } 298 | if(text) { 299 | html += `\n
${message.author_name}${dateStr}
${text}
`; 300 | } 301 | } 302 | 303 | return `
${html}
`; 304 | } 305 | 306 | async _processDocumentThreadDocx(quipThread, path) { 307 | const docx = await this.quipService.getDocx(quipThread.thread.id); 308 | if(docx) { 309 | this.saveCallback(docx, sanitizeFilename(`${quipThread.thread.title.trim()}.docx`), 'BLOB', path); 310 | } 311 | } 312 | 313 | async _processDocumentThreadXlsx(quipThread, path) { 314 | const xlsx = await this.quipService.getXlsx(quipThread.thread.id); 315 | if(xlsx) { 316 | this.saveCallback(xlsx, sanitizeFilename(`${quipThread.thread.title.trim()}.xlsx`), 'BLOB', path); 317 | } 318 | } 319 | 320 | async _processDocumentThread(quipThread, path) { 321 | const pathDeepness = path.split("/").length-1; 322 | let threadHtml = quipThread.html; 323 | 324 | //look up for images in html 325 | let matches = this._getMatchGroups(threadHtml, 326 | "src='(/blob/([\\w-]+)/([\\w-]+))'", 327 | ['replacement', 'threadId', 'blobId']); 328 | 329 | //replace blob references for images 330 | for(const image of matches) { 331 | threadHtml = await this._processFile(threadHtml, image, path, this.options.embeddedImages); 332 | } 333 | 334 | //look up for links in html 335 | matches = this._getMatchGroups(threadHtml, 336 | 'href="(.*/blob/(.+)/(.+)\\?name=(.+))"', 337 | ['replacement', 'threadId', 'blobId', 'fileName']); 338 | 339 | //replace blob references for links 340 | for(const link of matches) { 341 | threadHtml = await this._processFile(threadHtml, link, path); 342 | } 343 | 344 | //replace references to documents 345 | threadHtml = await this._resolveReferences(threadHtml, pathDeepness); 346 | 347 | if(this.options.comments) { 348 | threadHtml += await this._getThreadMessagesHtml(quipThread, path); 349 | } 350 | 351 | const wrappedHtml = this._renderDocumentHtml(threadHtml, quipThread.thread.title, path); 352 | 353 | this.saveCallback(wrappedHtml, sanitizeFilename(`${quipThread.thread.title.trim()}.html`), 'THREAD', path); 354 | } 355 | 356 | async _processThread(quipThread, path) { 357 | this.threadsProcessed++; 358 | 359 | if(!quipThread.thread) { 360 | const quipThreadCopy = Object.assign({}, quipThread); 361 | quipThreadCopy.html = '...'; 362 | this.logger.error("quipThread.thread is not defined, thread=" + JSON.stringify(quipThreadCopy, null, 2) + ", path=" + path); 363 | return; 364 | } 365 | 366 | if(!['document', 'spreadsheet'].includes(quipThread.thread.type)) { 367 | this.logger.warn("Thread type is not supported, thread.id=" + quipThread.thread.id + 368 | ", thread.title=" + quipThread.thread.title + 369 | ", thread.type=" + quipThread.thread.type + ", path=" + path); 370 | return; 371 | } 372 | 373 | if(this.options.docx) { 374 | if(quipThread.thread.type === 'document') { 375 | await this._processDocumentThreadDocx(quipThread, path); 376 | } else { 377 | await this._processDocumentThreadXlsx(quipThread, path); 378 | } 379 | } else { 380 | await this._processDocumentThread(quipThread, path); 381 | } 382 | } 383 | 384 | async _processFile(html, file, path, asImage=false) { 385 | const blob = await this.quipService.getBlob(file.threadId, file.blobId); 386 | if(blob) { 387 | if(asImage) { 388 | const imageURL = await blobImageToURL(blob); 389 | html = html.replace(file.replacement, imageURL); 390 | } else { 391 | let fileName; 392 | if(file.fileName) { 393 | fileName = file.fileName.trim(); 394 | } else { 395 | const extension = Mime.getExtension(blob.type); 396 | if(extension) { 397 | fileName = `${file.blobId.trim()}.${Mime.getExtension(blob.type).trim()}`; 398 | } else { 399 | fileName = `${file.blobId.trim()}`; 400 | } 401 | } 402 | fileName = sanitizeFilename(fileName); 403 | 404 | html = html.replace(file.replacement, `blobs/${fileName}`); 405 | //blob.size 406 | this.saveCallback(blob, fileName, "BLOB", `${path}blobs`); 407 | } 408 | } else { 409 | this.logger.error("Can't load the file " + file.replacement + " in path = " + path); 410 | } 411 | 412 | return html; 413 | } 414 | 415 | async _processThreads(quipThreads, path) { 416 | const promises = []; 417 | for(const index in quipThreads) { 418 | promises.push(this._processThread(quipThreads[index], path)); 419 | } 420 | await Promise.all(promises); 421 | } 422 | 423 | async _processFolders(quipFolders, path) { 424 | const promises = []; 425 | for(const index in quipFolders) { 426 | promises.push(this._processFolder(quipFolders[index], `${path}${sanitizeFilename(quipFolders[index].folder.title)}/`)); 427 | } 428 | await Promise.all(promises); 429 | } 430 | 431 | async _processFolder(quipFolder, path) { 432 | const threadIds = []; 433 | const folderIds = []; 434 | 435 | for(const index in quipFolder.children) { 436 | const quipChild = quipFolder.children[index]; 437 | 438 | if(quipChild.thread_id) { //thread 439 | threadIds.push(quipChild.thread_id); 440 | } else if(quipChild.folder_id && !quipChild.restricted) { //folder 441 | folderIds.push(quipChild.folder_id); 442 | } 443 | } 444 | 445 | if(threadIds.length > 0) { 446 | const threads = await this.quipService.getThreads(threadIds); 447 | if(threads) { 448 | await this._processThreads(threads, path); 449 | } else { 450 | this.logger.error("Can't load the Child-Threads for Folder: " + path) 451 | } 452 | } 453 | 454 | if(folderIds.length > 0) { 455 | const folders = await this.quipService.getFolders(folderIds); 456 | if(folders) { 457 | await this._processFolders(folders, path); 458 | } else { 459 | this.logger.error("Can't load the Child-Folders for Folder: " + path); 460 | } 461 | } 462 | 463 | this.foldersProcessed++; 464 | this._progressReport({ 465 | threadsProcessed: this.threadsProcessed, 466 | threadsTotal: this.threadsTotal, 467 | path: path 468 | }); 469 | } 470 | 471 | async _countThreadsAndFolders(quipFolder, path) { 472 | const threadIds = []; 473 | const folderIds = []; 474 | 475 | this.referencesMap.set(quipFolder.folder.id, { 476 | path, 477 | folder: true, 478 | title: quipFolder.folder.title 479 | }); 480 | 481 | if(!quipFolder.children || quipFolder.children.length === 0) { 482 | return; 483 | } 484 | 485 | const pathForChildren = `${path}${quipFolder.folder.title}/`; 486 | 487 | for(const index in quipFolder.children) { 488 | const quipChild = quipFolder.children[index]; 489 | if(quipChild.thread_id) { //thread 490 | threadIds.push(quipChild.thread_id); 491 | this.referencesMap.set(quipChild.thread_id, { 492 | path: pathForChildren, 493 | thread: true 494 | }); 495 | } else if(quipChild.folder_id) { //folder 496 | if (quipChild.restricted) { 497 | this.logger.debug("Folder: " + pathForChildren + " has restricted child: " + quipChild.folder_id); 498 | } else { 499 | folderIds.push(quipChild.folder_id); 500 | } 501 | } 502 | } 503 | 504 | this.threadsTotal += threadIds.length; 505 | this.foldersTotal += folderIds.length; 506 | 507 | this._progressReport({ 508 | readFolders: this.foldersTotal, 509 | readThreads: this.threadsTotal 510 | }); 511 | 512 | let childFolders = []; 513 | if(folderIds.length > 0) { 514 | childFolders = await this.quipService.getFolders(folderIds); 515 | if(!childFolders) { 516 | return; 517 | } 518 | } 519 | 520 | for(const index in childFolders) { 521 | await this._countThreadsAndFolders(childFolders[index], pathForChildren); 522 | } 523 | } 524 | 525 | async _exportFolders(folderIds) { 526 | this._changePhase('ANALYSIS'); 527 | 528 | this.threadsTotal = 0; 529 | this.foldersTotal = 0; 530 | 531 | const quipFolders = await this.quipService.getFolders(folderIds); 532 | if(!quipFolders) { 533 | this._changePhase('STOP'); 534 | this.logger.error("Can't read the root folders"); 535 | return; 536 | } 537 | 538 | for(const index in quipFolders) { 539 | this.foldersTotal++; 540 | await this._countThreadsAndFolders(quipFolders[index], ""); 541 | } 542 | 543 | this._changePhase('EXPORT'); 544 | return this._processFolders(quipFolders, ""); 545 | } 546 | 547 | _progressReport(progress) { 548 | this.progressCallback(progress); 549 | } 550 | } 551 | 552 | module.exports = QuipProcessor; 553 | -------------------------------------------------------------------------------- /lib/QuipService.js: -------------------------------------------------------------------------------- 1 | const fetch = require('node-fetch'); 2 | const LoggerAdapter = require('./common/LoggerAdapter'); 3 | 4 | const TIMES_LIMIT_503 = 10; 5 | 6 | class QuipService { 7 | constructor(accessToken, apiURL='https://platform.quip.com:443/1') { 8 | this.accessToken = accessToken; 9 | this.apiURL = apiURL; 10 | this.logger = new LoggerAdapter(); 11 | this.querries503 = new Map(); 12 | this.waitingMs = 1000; 13 | this.stats = { 14 | query_count: 0, 15 | getThread_count: 0, 16 | getThreads_count: 0, 17 | getFolder_count: 0, 18 | getFolders_count: 0, 19 | getBlob_count: 0, 20 | getPdf_count: 0, 21 | getXlsx_count: 0, 22 | getDocx_count: 0, 23 | getCurrentUser_count: 0, 24 | getThreadMessages_count: 0, 25 | getUser_count: 0 26 | }; 27 | } 28 | 29 | setLogger(logger) { 30 | this.logger = logger; 31 | } 32 | 33 | async checkUser() { 34 | this.stats.getCurrentUser_count++; 35 | 36 | const res = await fetch(`${this.apiURL}/users/current`, this._getOptions('GET')); 37 | if(res.ok) return true; 38 | 39 | return false; 40 | } 41 | 42 | async getUser(userIds) { 43 | this.stats.getUser_count++; 44 | return this._apiCallJson(`/users/${userIds}`); 45 | } 46 | 47 | async getCurrentUser() { 48 | this.stats.getCurrentUser_count++; 49 | return this._apiCallJson('/users/current'); 50 | } 51 | 52 | async getFolder(folderId) { 53 | this.stats.getFolder_count++; 54 | return this._apiCallJson(`/folders/${folderId}`); 55 | } 56 | 57 | async getThread(threadId) { 58 | this.stats.getThread_count++; 59 | return this._apiCallJson(`/threads/${threadId}`); 60 | } 61 | 62 | async getThreadMessages(threadId) { 63 | this.stats.getThreadMessages_count++; 64 | return this._apiCallJson(`/messages/${threadId}`); 65 | } 66 | 67 | async getThreads(threadIds) { 68 | this.stats.getThreads_count++; 69 | return this._apiCallJson(`/threads/?ids=${threadIds}`); 70 | } 71 | 72 | async getFolders(threadIds) { 73 | this.stats.getFolders_count++; 74 | return this._apiCallJson(`/folders/?ids=${threadIds}`); 75 | } 76 | 77 | async getBlob(threadId, blobId) { 78 | //const random = (Math.random() > 0.8) ? 'random' : ''; 79 | this.stats.getBlob_count++; 80 | return this._apiCallBlob(`/blob/${threadId}/${blobId}`); 81 | } 82 | 83 | async getPdf(threadId) { 84 | this.stats.getPdf_count++; 85 | return this._apiCallBlob(`/threads/${threadId}/export/pdf`); 86 | } 87 | 88 | async getDocx(threadId) { 89 | this.stats.getDocx_count++; 90 | return this._apiCallBlob(`/threads/${threadId}/export/docx`); 91 | } 92 | 93 | async getXlsx(threadId) { 94 | this.stats.getXlsx_count++; 95 | return this._apiCallBlob(`/threads/${threadId}/export/xlsx`); 96 | } 97 | 98 | async _apiCallBlob(url, method = 'GET') { 99 | return this._apiCall(url, method, true); 100 | } 101 | 102 | async _apiCallJson(url, method = 'GET') { 103 | return this._apiCall(url, method, false); 104 | } 105 | 106 | async _apiCall(url, method, blob) { 107 | this.stats.query_count++; 108 | 109 | try { 110 | const res = await fetch(`${this.apiURL}${url}`, this._getOptions(method)); 111 | if(!res.ok) { 112 | if(res.status === 503) { 113 | const currentTime = new Date().getTime(); 114 | const rateLimitReset = +res.headers.get('x-ratelimit-reset')*1000; 115 | let waitingInMs = this.waitingMs; 116 | if(rateLimitReset > currentTime) { 117 | waitingInMs = rateLimitReset - currentTime; 118 | } 119 | this.logger.debug(`HTTP 503: for ${url}, waiting in ms: ${waitingInMs}`); 120 | if(this._check503Query(url)) { 121 | return new Promise(resolve => setTimeout(() => { 122 | resolve(this._apiCall(url, method, blob)); 123 | }, waitingInMs)); 124 | } else { 125 | this.logger.error(`Couldn't fetch ${url}, tryed to get it ${TIMES_LIMIT_503} times`); 126 | return; 127 | } 128 | } else { 129 | this.logger.debug(`Couldn't fetch ${url}, received ${res.status}`); 130 | return; 131 | } 132 | } 133 | 134 | if(blob) { 135 | return res.blob(); 136 | } else { 137 | return res.json(); 138 | } 139 | } catch (e) { 140 | this.logger.error(`Couldn't fetch ${url}, `, e); 141 | } 142 | } 143 | 144 | _check503Query(url) { 145 | let count = this.querries503.get(url); 146 | if(!count) { 147 | count = 0; 148 | } 149 | 150 | this.querries503.set(url, ++count); 151 | if(count > TIMES_LIMIT_503) { 152 | return false; 153 | } 154 | 155 | return true; 156 | } 157 | 158 | _getOptions(method) { 159 | return { 160 | method: method, 161 | headers: { 162 | 'Authorization': 'Bearer ' + this.accessToken, 163 | 'Content-Type': 'application/json' 164 | } 165 | }; 166 | } 167 | } 168 | 169 | module.exports = QuipService; -------------------------------------------------------------------------------- /lib/__tests__/QuipProcessor.test.js: -------------------------------------------------------------------------------- 1 | const TestUtils = require('../common/TestUtils'); 2 | 3 | const QuipProcessor = require('../QuipProcessor'); 4 | 5 | jest.mock('sanitize-filename'); 6 | const sanitizeFilename = require('sanitize-filename'); 7 | sanitizeFilename.mockImplementation((fileName) => `${fileName}_SANITIZED`); 8 | 9 | jest.mock('ejs'); 10 | const ejs = require('ejs'); 11 | ejs.render.mockImplementation(() => 'TEMPLATED DOCUMENT'); 12 | 13 | jest.mock('../QuipService'); 14 | const QuipService = require('../QuipService'); 15 | 16 | jest.mock('../common/LoggerAdapter'); 17 | const LoggerAdapter = require('../common/LoggerAdapter'); 18 | 19 | jest.mock('../common/blobImageToURL'); 20 | const blobImageToURL = require('../common/blobImageToURL'); 21 | blobImageToURL.mockResolvedValue('IMAGE_URL'); 22 | 23 | const constructorParams = { 24 | token: "TOKEN", 25 | saveCallback: jest.fn(), 26 | progressCallback: jest.fn(), 27 | phaseCallback: jest.fn() 28 | }; 29 | 30 | const userFolders = { 31 | private_folder_id: 'p1', 32 | shared_folder_ids: ['s1', 's2', 's3'], 33 | group_folder_ids: ['g1', 'g2'], 34 | }; 35 | 36 | const quipUser = { 37 | url: 'https://demoaccount.quip.com', 38 | ...userFolders 39 | }; 40 | 41 | const folders = require('./folders.json'); 42 | const threads = require('./threads.json'); 43 | const messages = require('./messages.json'); 44 | 45 | const defaultOptions = { 46 | documentTemplate: "Document template", 47 | quipApiURL: "URL" 48 | }; 49 | 50 | let quipProcessor; 51 | 52 | function initQuipProcessor(options = defaultOptions) { 53 | quipProcessor = new QuipProcessor(constructorParams.token, constructorParams.saveCallback, constructorParams.progressCallback, 54 | constructorParams.phaseCallback, options); 55 | } 56 | 57 | describe('constructor tests', () => { 58 | test('init paramteres', async () => { 59 | initQuipProcessor(); 60 | expect(quipProcessor.quipToken).toBe(constructorParams.token); 61 | expect(quipProcessor.saveCallback).toBe(constructorParams.saveCallback); 62 | expect(quipProcessor.progressCallback).toBe(constructorParams.progressCallback); 63 | expect(quipProcessor.phaseCallback).toBe(constructorParams.phaseCallback); 64 | expect(quipProcessor.options).toBe(defaultOptions); 65 | expect(quipProcessor.logger).toBeInstanceOf(LoggerAdapter); 66 | 67 | expect(quipProcessor.start).toBe(false); 68 | 69 | expect(quipProcessor.threadsProcessed).toBe(0); 70 | expect(quipProcessor.foldersProcessed).toBe(0); 71 | expect(quipProcessor.threadsTotal).toBe(0); 72 | expect(quipProcessor.foldersTotal).toBe(0); 73 | 74 | expect(quipProcessor.phase).toBe('STOP'); 75 | 76 | expect(quipProcessor.referencesMap.size).toBe(0); 77 | 78 | expect(QuipService).toHaveBeenCalledWith("TOKEN", defaultOptions.quipApiURL); 79 | 80 | expect(quipProcessor.quipService).toBeInstanceOf(QuipService); 81 | 82 | expect(quipProcessor.documentTemplate).toBe(defaultOptions.documentTemplate); 83 | }); 84 | 85 | test('without document template', async () => { 86 | console.error = jest.fn(); 87 | initQuipProcessor({}); 88 | expect(console.error).toHaveBeenCalledWith("Document template is not set!"); 89 | }); 90 | }); 91 | 92 | describe('methods tests', () => { 93 | 94 | describe('setLogger', () => { 95 | beforeEach(() => { 96 | initQuipProcessor(); 97 | }); 98 | 99 | test('set logger', async () => { 100 | const customLogger = new LoggerAdapter(); 101 | quipProcessor.setLogger(customLogger); 102 | expect(quipProcessor.logger).toBe(customLogger); 103 | expect(quipProcessor.quipService.setLogger).toHaveBeenCalledWith(customLogger); 104 | }); 105 | }); 106 | 107 | describe('startExport', () => { 108 | beforeEach(() => { 109 | initQuipProcessor(); 110 | quipProcessor._exportFolders = jest.fn(); 111 | quipProcessor._changePhase = jest.fn(); 112 | quipProcessor.quipService.getCurrentUser.mockResolvedValue(quipUser); 113 | }); 114 | 115 | test('changing phase', async () => { 116 | await quipProcessor.startExport(); 117 | expect(quipProcessor._changePhase).toHaveBeenNthCalledWith(1, 'START'); 118 | expect(quipProcessor._changePhase).toHaveBeenNthCalledWith(2, 'STOP'); 119 | }); 120 | 121 | test('set up internal vars', async () => { 122 | quipProcessor.threadsProcessed = 255; 123 | await quipProcessor.startExport(); 124 | expect(quipProcessor.start).toBe(false); 125 | expect(quipProcessor.threadsProcessed).toBe(0); 126 | }); 127 | 128 | test('call once QuipService.getCurrentUser()', async () => { 129 | await quipProcessor.startExport(); 130 | expect(quipProcessor.quipService.getCurrentUser).toHaveBeenCalled(); 131 | expect(quipProcessor.quipUser).toBe(quipUser); 132 | }); 133 | 134 | test('calling QuipService.getCurrentUser() is failed', async () => { 135 | quipProcessor.quipService.getCurrentUser.mockResolvedValue(null); 136 | await quipProcessor.startExport(); 137 | expect(quipProcessor.logger.error).toHaveBeenCalledWith('Can\'t load the User'); 138 | expect(quipProcessor.start).toBe(false); 139 | }); 140 | 141 | test('using folders from QuipService.getUser()', async () => { 142 | await quipProcessor.startExport(); 143 | const folderIdsToExport = [ 144 | userFolders.private_folder_id, 145 | ...userFolders.shared_folder_ids, 146 | ...userFolders.group_folder_ids 147 | ]; 148 | expect(quipProcessor._exportFolders).toHaveBeenCalledWith(folderIdsToExport); 149 | }); 150 | 151 | test('using folders from call parameter', async () => { 152 | const folderIdsToExport = [111,222,333]; 153 | TestUtils.mockResolvedWithThen(quipProcessor._exportFolders); 154 | await quipProcessor.startExport(folderIdsToExport); 155 | expect(quipProcessor._exportFolders).toHaveBeenCalledWith(folderIdsToExport); 156 | expect(quipProcessor._exportFolders.promiseCalledWithThenTimes).toBe(1); 157 | }); 158 | }); 159 | 160 | describe('stopExport', () => { 161 | beforeEach(() => { 162 | initQuipProcessor(); 163 | quipProcessor._changePhase = jest.fn(); 164 | }); 165 | 166 | test('stop export', async () => { 167 | quipProcessor.start = true; 168 | quipProcessor.stopExport(); 169 | expect(quipProcessor.start).toBe(false); 170 | expect(quipProcessor._changePhase).toHaveBeenCalledWith('STOP'); 171 | }); 172 | }); 173 | 174 | describe('_changePhase', () => { 175 | beforeEach(() => { 176 | initQuipProcessor(); 177 | }); 178 | 179 | test('change phase', async () => { 180 | quipProcessor.phase = 'NEW_PHASE'; 181 | quipProcessor._changePhase('STOP'); 182 | expect(quipProcessor.phase).toBe('STOP'); 183 | expect(quipProcessor.phaseCallback).toHaveBeenCalledWith('STOP', 'NEW_PHASE'); 184 | }); 185 | }); 186 | 187 | describe('_getMatchGroups', () => { 188 | beforeEach(() => { 189 | initQuipProcessor(); 190 | }); 191 | 192 | test('get matches for files', async () => { 193 | const text = ` 194 | file1 195 | file2 196 | file3 197 | `; 198 | const result = quipProcessor._getMatchGroups(text, 199 | 'href="(.*/blob/(.+)/(.+)\\?name=(.+))"', 200 | ['replacement', 'threadId', 'blobId', 'fileName']); 201 | expect(result.length).toBe(3); 202 | expect(result[1].match).toBe('href="/blob/thread2/id2?name=test2.pdf"'); 203 | expect(result[1].replacement).toBe('/blob/thread2/id2?name=test2.pdf'); 204 | expect(result[1].threadId).toBe('thread2'); 205 | expect(result[1].blobId).toBe('id2'); 206 | expect(result[1].fileName).toBe('test2.pdf'); 207 | }); 208 | 209 | test('get matches for images', async () => { 210 | const text = ` 211 | 212 | 213 | 214 | `; 215 | const result = quipProcessor._getMatchGroups(text, 216 | "src='(/blob/([\\w-]+)/([\\w-]+))'", 217 | ['replacement', 'threadId', 'blobId']); 218 | expect(result.length).toBe(3); 219 | expect(result[1].match).toBe(`src='/blob/thread2/blob2'`); 220 | expect(result[1].replacement).toBe('/blob/thread2/blob2'); 221 | expect(result[1].threadId).toBe('thread2'); 222 | expect(result[1].blobId).toBe('blob2'); 223 | }); 224 | 225 | test('get matches for references', async () => { 226 | const text = ` 227 | fileA
228 | fileB
229 | folderAAA
230 | folderBBB
231 | `; 232 | const result = quipProcessor._getMatchGroups(text, 233 | `href="(.*quip.com/([\\w-]+))"`, 234 | ['replacement', 'threadId']); 235 | expect(result.length).toBe(4); 236 | expect(result[1].replacement).toBe('https://demoaccount.quip.com/reference2'); 237 | expect(result[1].threadId).toBe('reference2'); 238 | expect(result[3].replacement).toBe('https://quip.com/reference4'); 239 | expect(result[3].threadId).toBe('reference4'); 240 | }); 241 | }); 242 | 243 | describe('_exportFolders', () => { 244 | beforeEach(() => { 245 | initQuipProcessor(); 246 | quipProcessor.quipService.getFolders.mockResolvedValue(folders); 247 | quipProcessor._countThreadsAndFolders = jest.fn(); 248 | quipProcessor._processFolders = jest.fn(); 249 | quipProcessor._changePhase = jest.fn(); 250 | }); 251 | 252 | test('pahse change in normal mode', async () => { 253 | await quipProcessor._exportFolders(['f1', 'f2']); 254 | expect(quipProcessor._changePhase).toHaveBeenNthCalledWith(1, 'ANALYSIS'); 255 | expect(quipProcessor._changePhase).toHaveBeenNthCalledWith(2, 'EXPORT'); 256 | }); 257 | 258 | test('quipService.getFolders() is empty', async () => { 259 | quipProcessor.quipService.getFolders.mockResolvedValue(); 260 | await quipProcessor._exportFolders(['f1', 'f2']); 261 | expect(quipProcessor._changePhase).toHaveBeenNthCalledWith(1, 'ANALYSIS'); 262 | expect(quipProcessor._changePhase).toHaveBeenNthCalledWith(2, 'STOP'); 263 | expect(quipProcessor.logger.error).toHaveBeenCalledWith('Can\'t read the root folders'); 264 | }); 265 | 266 | test('reseting internal vars', async () => { 267 | quipProcessor.threadsTotal = 10; 268 | quipProcessor.foldersTotal = 10; 269 | await quipProcessor._exportFolders(['f1', 'f2']); 270 | expect(quipProcessor.threadsTotal).toBe(0); 271 | expect(quipProcessor.foldersTotal).toBe(2); 272 | }); 273 | 274 | test('_countThreadsAndFolders should be called two times', async () => { 275 | TestUtils.mockResolvedWithThen(quipProcessor._countThreadsAndFolders); 276 | await quipProcessor._exportFolders(['f1', 'f2']); 277 | expect(quipProcessor._countThreadsAndFolders).toHaveBeenCalledTimes(2); 278 | expect(quipProcessor._countThreadsAndFolders).toHaveBeenNthCalledWith(1, folders['FOLDER-1'], ""); 279 | expect(quipProcessor._countThreadsAndFolders).toHaveBeenNthCalledWith(2, folders['FOLDER-2'], ""); 280 | expect(quipProcessor._countThreadsAndFolders.promiseCalledWithThenTimes).toBe(2); 281 | }); 282 | 283 | test('_processFolders should be called with right parameters', async () => { 284 | await quipProcessor._exportFolders(['f1', 'f2']); 285 | expect(quipProcessor._processFolders).toHaveBeenCalledWith(folders, ""); 286 | }); 287 | }); 288 | 289 | describe('_countThreadsAndFolders', () => { 290 | beforeEach(() => { 291 | initQuipProcessor(); 292 | quipProcessor.quipService.getFolders.mockResolvedValue([ 293 | { 294 | folder: {id: 'FOLDER111'}, 295 | children: [] 296 | }, 297 | { 298 | folder: {id: 'FOLDER222'}, 299 | children: [] 300 | } 301 | ]); 302 | quipProcessor._progressReport = jest.fn(); 303 | }); 304 | 305 | test('count folders and threads, increase total counts, call progress report', async () => { 306 | quipProcessor.threadsTotal = 100; 307 | quipProcessor.foldersTotal = 100; 308 | await quipProcessor._countThreadsAndFolders(folders['FOLDER-1'], "Private/"); 309 | //one folder is restricted 310 | expect(quipProcessor.foldersTotal).toBe(100 + 12); 311 | expect(quipProcessor.threadsTotal).toBe(100 + 3); 312 | expect(quipProcessor._progressReport).toHaveBeenCalledWith({ 313 | readFolders: quipProcessor.foldersTotal, 314 | readThreads: quipProcessor.threadsTotal 315 | }); 316 | }); 317 | 318 | test('add references for folder and threads', async () => { 319 | quipProcessor.quipService.getFolders.mockResolvedValue([ 320 | { 321 | folder: { 322 | title: "TITLE_" + Math.random() * 1000, 323 | id: Math.random() * 1000 324 | }, 325 | children: [] 326 | }, 327 | { 328 | folder: { 329 | title: "TITLE_" + Math.random() * 1000, 330 | id: Math.random() * 1000 331 | }, 332 | children: [] 333 | } 334 | ]); 335 | await quipProcessor._countThreadsAndFolders(folders['FOLDER-1'], "Private/"); 336 | expect(quipProcessor.referencesMap.size).toBe(6); 337 | }); 338 | 339 | test('find and count child folders', async () => { 340 | const mock___countThreadsAndFolders = jest.spyOn(quipProcessor, '_countThreadsAndFolders'); 341 | await quipProcessor._countThreadsAndFolders(folders['FOLDER-1'], "Private/"); 342 | //one folder is restricted 343 | expect(quipProcessor.quipService.getFolders.mock.calls[0][0].length).toBe(12); 344 | expect(mock___countThreadsAndFolders).toHaveBeenCalledTimes(3); 345 | }); 346 | 347 | test('getFolders returns no folders', async () => { 348 | quipProcessor.quipService.getFolders.mockResolvedValue(undefined); 349 | const mock___countThreadsAndFolders = jest.spyOn(quipProcessor, '_countThreadsAndFolders'); 350 | await quipProcessor._countThreadsAndFolders(folders['FOLDER-1'], "Private/"); 351 | expect(mock___countThreadsAndFolders).toHaveBeenCalledTimes(1); 352 | }); 353 | }); 354 | 355 | describe('_processFolders', () => { 356 | beforeEach(() => { 357 | initQuipProcessor(); 358 | quipProcessor._processFolder = jest.fn(); 359 | }); 360 | 361 | test('call _processFolder method', async () => { 362 | const mockFolders = { 363 | folder1: { 364 | folder: {title: 'folder1'} 365 | }, 366 | folder2: { 367 | folder: {title: 'folder2'} 368 | } 369 | }; 370 | await quipProcessor._processFolders(mockFolders, "/aaa/"); 371 | expect(quipProcessor._processFolder).toHaveBeenCalledTimes(2); 372 | expect(quipProcessor._processFolder).toHaveBeenCalledWith(mockFolders.folder1, `/aaa/${mockFolders.folder1.folder.title}_SANITIZED/`); 373 | expect(quipProcessor._processFolder).toHaveBeenCalledWith(mockFolders.folder2, `/aaa/${mockFolders.folder2.folder.title}_SANITIZED/`); 374 | }); 375 | }); 376 | 377 | describe('_processFolder', () => { 378 | beforeEach(() => { 379 | initQuipProcessor(); 380 | quipProcessor.quipService.getFolders.mockResolvedValue(folders); 381 | quipProcessor.quipService.getThreads.mockResolvedValue(threads); 382 | quipProcessor._processFolders = jest.fn(); 383 | quipProcessor._processThreads = jest.fn(); 384 | quipProcessor._progressReport = jest.fn(); 385 | quipProcessor.foldersProcessed = 100; 386 | }); 387 | 388 | test('call getThreads and getFolders', async () => { 389 | await quipProcessor._processFolder(folders['FOLDER-1'], "/aaa/"); 390 | //one folder is restricted 391 | expect(quipProcessor.quipService.getFolders.mock.calls[0][0].length).toBe(12); 392 | expect(quipProcessor.quipService.getThreads.mock.calls[0][0].length).toBe(3); 393 | }); 394 | 395 | test('call _processThreads and _processFolders', async () => { 396 | TestUtils.mockResolvedWithThen(quipProcessor._processFolders); 397 | TestUtils.mockResolvedWithThen(quipProcessor._processThreads); 398 | await quipProcessor._processFolder(folders['FOLDER-1'], "/aaa/"); 399 | expect(quipProcessor._processFolders).toHaveBeenCalledWith(folders, '/aaa/'); 400 | expect(quipProcessor._processThreads).toHaveBeenCalledWith(threads, '/aaa/'); 401 | expect(quipProcessor._processFolders.promiseCalledWithThenTimes).toBe(1); 402 | expect(quipProcessor._processThreads.promiseCalledWithThenTimes).toBe(1); 403 | }); 404 | 405 | test("couldn't get folders or threads", async () => { 406 | quipProcessor.quipService.getFolders.mockResolvedValue(undefined); 407 | quipProcessor.quipService.getThreads.mockResolvedValue(undefined); 408 | await quipProcessor._processFolder(folders['FOLDER-1'], "/aaa/"); 409 | expect(quipProcessor.logger.error).toHaveBeenCalledWith("Can't load the Child-Folders for Folder: /aaa/"); 410 | expect(quipProcessor.logger.error).toHaveBeenCalledWith("Can't load the Child-Threads for Folder: /aaa/"); 411 | }); 412 | 413 | test("increasing foldersProcessed variable", async () => { 414 | await quipProcessor._processFolder(folders['FOLDER-1'], "/aaa/"); 415 | expect(quipProcessor.foldersProcessed).toBe(101); 416 | }); 417 | 418 | test("call ProgressReport", async () => { 419 | quipProcessor.threadsProcessed = 10; 420 | quipProcessor.threadsTotal = 100; 421 | await quipProcessor._processFolder(folders['FOLDER-1'], "/aaa/"); 422 | expect(quipProcessor._progressReport).toHaveBeenCalledWith({ 423 | threadsProcessed: 10, 424 | threadsTotal: 100, 425 | path: '/aaa/' 426 | }); 427 | 428 | }); 429 | }); 430 | 431 | describe('_progressReport', () => { 432 | beforeEach(() => { 433 | initQuipProcessor(); 434 | }); 435 | 436 | test('report progress', async () => { 437 | const progress = {progress: 1}; 438 | quipProcessor._progressReport(progress); 439 | expect(quipProcessor.progressCallback).toHaveBeenCalledWith(progress); 440 | }); 441 | }); 442 | 443 | describe('_processThreads', () => { 444 | beforeEach(() => { 445 | initQuipProcessor(); 446 | quipProcessor._processThread = jest.fn(); 447 | }); 448 | 449 | test('call _processThread method', async () => { 450 | const mockThreads = { 451 | thread1: { 452 | thread: {title: 'thread1'} 453 | }, 454 | thread2: { 455 | thread: {title: 'thread2'} 456 | } 457 | }; 458 | 459 | TestUtils.mockResolvedWithThen(quipProcessor._processThread); 460 | 461 | await quipProcessor._processThreads(mockThreads, "/aaa/"); 462 | expect(quipProcessor._processThread).toHaveBeenCalledTimes(2); 463 | expect(quipProcessor._processThread).toHaveBeenCalledWith(mockThreads.thread1, `/aaa/`); 464 | expect(quipProcessor._processThread).toHaveBeenCalledWith(mockThreads.thread2, `/aaa/`); 465 | expect(quipProcessor._processThread.promiseCalledWithThenTimes).toBe(2); 466 | }); 467 | }); 468 | 469 | describe('_processThread', () => { 470 | let thread; 471 | 472 | beforeEach(() => { 473 | initQuipProcessor(); 474 | quipProcessor._processDocumentThread = jest.fn(); 475 | quipProcessor._processDocumentThreadDocx = jest.fn(); 476 | quipProcessor._processDocumentThreadXlsx = jest.fn(); 477 | thread = Object.assign({}, threads['THREAD-1']); 478 | quipProcessor.threadsProcessed = 0; 479 | quipProcessor.options.docx = false; 480 | }); 481 | 482 | test('thread property is not defined', async () => { 483 | thread.thread = undefined; 484 | await quipProcessor._processThread(thread, "/aaa/"); 485 | expect(quipProcessor.logger.error.mock.calls[0][0].startsWith('quipThread.thread is not defined, thread=')).toBe(true); 486 | expect(quipProcessor.threadsProcessed).toBe(1); 487 | }); 488 | 489 | test("call _processDocumentThread() for docuement-type 'document'", async () => { 490 | thread.thread.type = 'document'; 491 | await quipProcessor._processThread(thread, "/aaa/"); 492 | expect(quipProcessor._processDocumentThread).toBeCalledWith(thread, "/aaa/"); 493 | expect(quipProcessor.threadsProcessed).toBe(1); 494 | }); 495 | 496 | test("call _processDocumentThread() for docuement-type 'spreadsheet'", async () => { 497 | thread.thread.type = 'spreadsheet'; 498 | await quipProcessor._processThread(thread, "/aaa/"); 499 | expect(quipProcessor._processDocumentThread).toBeCalledWith(thread, "/aaa/"); 500 | expect(quipProcessor.threadsProcessed).toBe(1); 501 | }); 502 | 503 | test("call _processDocumentThread() for docuement-type 'document' with --docx option", async () => { 504 | thread.thread.type = 'document'; 505 | quipProcessor.options.docx = true; 506 | await quipProcessor._processThread(thread, "/aaa/"); 507 | expect(quipProcessor._processDocumentThreadDocx).toBeCalledWith(thread, "/aaa/"); 508 | }); 509 | 510 | test("call _processDocumentThread() for docuement-type 'spreadsheet' with --docx option", async () => { 511 | thread.thread.type = 'spreadsheet'; 512 | quipProcessor.options.docx = true; 513 | await quipProcessor._processThread(thread, "/aaa/"); 514 | expect(quipProcessor._processDocumentThreadXlsx).toBeCalledWith(thread, "/aaa/"); 515 | }); 516 | 517 | test("not supported document-type in thread", async () => { 518 | thread.thread.type = 'XXX'; 519 | await quipProcessor._processThread(thread, "/aaa/"); 520 | expect(quipProcessor.logger.warn.mock.calls[0][0].startsWith('Thread type is not supported, thread.id=')).toBe(true); 521 | expect(quipProcessor.threadsProcessed).toBe(1); 522 | }); 523 | 524 | test("_processDocumentThread() called as Promise with await", async () => { 525 | thread.thread.type = 'document'; 526 | TestUtils.mockResolvedWithThen(quipProcessor._processDocumentThread); 527 | await quipProcessor._processThread(thread, "/aaa/"); 528 | expect(quipProcessor._processDocumentThread.promiseCalledWithThenTimes).toBe(1); 529 | }); 530 | }); 531 | 532 | describe('_resolveReferences', () => { 533 | const referenceMatches = [ 534 | { 535 | replacement: 'https://demoaccount.quip.com/reference2', 536 | threadId: 'reference2' 537 | }, 538 | { 539 | replacement: 'https://demoaccount.quip.com/reference3', 540 | threadId: 'reference3' 541 | }]; 542 | 543 | beforeEach(() => { 544 | initQuipProcessor(); 545 | quipProcessor.quipUser = quipUser; 546 | quipProcessor._getMatchGroups = jest.fn(() => { return referenceMatches; }); 547 | quipProcessor._processReference = jest.fn().mockResolvedValue("CHANGED-HTML"); 548 | }); 549 | 550 | test('resolve references', async () => { 551 | let result = await quipProcessor._resolveReferences("HTML", 3); 552 | expect(result).toBe("CHANGED-HTML"); 553 | expect(quipProcessor._processReference).toHaveBeenCalledTimes(2); 554 | expect(quipProcessor._processReference).toHaveBeenCalledWith('HTML', referenceMatches[0], 3); 555 | expect(quipProcessor._processReference).toHaveBeenCalledWith('CHANGED-HTML', referenceMatches[1], 3); 556 | }); 557 | }); 558 | 559 | describe('_processReference', () => { 560 | const html = ` 561 | 562 | Folder 1 563 | Thread 1 564 | 565 | `; 566 | 567 | const folderReferencedObject = { 568 | folder: true, 569 | path: 'Private/Programming/', 570 | title: 'folder1' 571 | }; 572 | 573 | const threadReferencedObject = { 574 | thread: true, 575 | path: 'Private/Programming/Angular/', 576 | title: 'thread1' 577 | }; 578 | 579 | const folderReference = { 580 | replacement: 'https://quip.com/folder1', 581 | referenceId: 'folder1' 582 | }; 583 | 584 | const threadReference = { 585 | replacement: 'https://quip.com/threadXXX', 586 | referenceId: 'threadXXX' 587 | }; 588 | 589 | const htmlFolderAfter = ` 590 | 591 | Folder 1 592 | Thread 1 593 | 594 | `; 595 | 596 | const htmlThreadAfter = ` 597 | 598 | Folder 1 599 | Thread 1 600 | 601 | `; 602 | 603 | beforeEach(() => { 604 | initQuipProcessor(); 605 | quipProcessor._findReferencedObject = jest.fn(); 606 | }); 607 | 608 | test('referenced object not found', async () => { 609 | const result = await quipProcessor._processReference(html, {}, 4); 610 | expect(result).toBe(html); 611 | }); 612 | 613 | test('referenced object is not thread or folder', async () => { 614 | quipProcessor._findReferencedObject.mockResolvedValue({user: true}); 615 | const result = await quipProcessor._processReference(html, {}, 4); 616 | expect(result).toBe(html); 617 | }); 618 | 619 | test('with folder', async () => { 620 | quipProcessor._findReferencedObject.mockResolvedValue(folderReferencedObject); 621 | const result = await quipProcessor._processReference(html, folderReference, 4); 622 | expect(result).toBe(htmlFolderAfter); 623 | }); 624 | 625 | test('with thread', async () => { 626 | quipProcessor._findReferencedObject.mockResolvedValue(threadReferencedObject); 627 | const result = await quipProcessor._processReference(html, threadReference, 4); 628 | expect(result).toBe(htmlThreadAfter); 629 | }); 630 | }); 631 | 632 | describe('_processDocumentThread', () => { 633 | const thread = threads['THREAD-3']; 634 | const sanitizedName = sanitizeFilename(`${thread.thread.title.trim()}.html`); 635 | beforeEach(() => { 636 | initQuipProcessor(); 637 | quipProcessor.quipUser = quipUser; 638 | quipProcessor._processFile = jest.fn(); 639 | quipProcessor._processFile.mockImplementation((html) => `${html}_1`); 640 | quipProcessor.options.documentCSS = undefined; 641 | quipProcessor.options.messages = false; 642 | quipProcessor.documentTemplate = undefined; 643 | quipProcessor.options.embeddedImages = undefined; 644 | quipProcessor._renderDocumentHtml = jest.fn(); 645 | quipProcessor._renderDocumentHtml.mockImplementation((html) => `${html}_WRAPPED`); 646 | quipProcessor._getThreadMessagesHtml = jest.fn(); 647 | quipProcessor._getThreadMessagesHtml.mockImplementation(() => `MESSAGES_HTML`); 648 | }); 649 | 650 | test('process document thread', async () => { 651 | await quipProcessor._processDocumentThread(thread, '/aaa/'); 652 | expect(quipProcessor._processFile).toHaveBeenCalledTimes(2); 653 | expect(quipProcessor._processFile).toHaveBeenCalledWith(`${thread.html}_1`, expect.anything(), '/aaa/'); 654 | expect(quipProcessor._processFile).toHaveBeenCalledWith(thread.html, expect.anything(), '/aaa/', undefined); 655 | expect(quipProcessor._renderDocumentHtml).toHaveBeenCalledWith(`${thread.html}_1_1`, thread.thread.title, '/aaa/'); 656 | expect(quipProcessor.saveCallback).toHaveBeenCalledWith(`${thread.html}_1_1_WRAPPED`, sanitizedName, 'THREAD', '/aaa/'); 657 | }); 658 | 659 | test('process document thread with messages', async () => { 660 | quipProcessor.options.comments = true; 661 | await quipProcessor._processDocumentThread(thread, '/aaa/'); 662 | expect(quipProcessor._getThreadMessagesHtml).toHaveBeenCalledWith(thread, '/aaa/'); 663 | expect(quipProcessor._renderDocumentHtml).toHaveBeenCalledWith(`${thread.html}_1_1MESSAGES_HTML`, thread.thread.title, '/aaa/'); 664 | }); 665 | }); 666 | 667 | describe('_processFile', () => { 668 | const blob = {data: 'bla bla bla', type:'application/pdf'}; 669 | const fileInfo = { 670 | replacement: 'REPLACEMENT', 671 | threadId: 'THREAD_ID', 672 | blobId: 'BLOB_ID', 673 | fileName: ' FILE_NAME ' //for trim test 674 | }; 675 | 676 | beforeEach(() => { 677 | initQuipProcessor(); 678 | quipProcessor.quipService.getBlob.mockResolvedValue(blob); 679 | }); 680 | 681 | test('blob read', async () => { 682 | await quipProcessor._processFile('HTML', fileInfo, '/aaa/', false); 683 | expect(quipProcessor.quipService.getBlob).toHaveBeenCalledWith(fileInfo.threadId, fileInfo.blobId); 684 | }); 685 | 686 | test('blob can not be read', async () => { 687 | quipProcessor.quipService.getBlob.mockResolvedValue(undefined); 688 | await quipProcessor._processFile('HTML', fileInfo, '/aaa/', false); 689 | expect(quipProcessor.logger.error.mock.calls[0][0].startsWith('Can\'t load the file')).toBe(true); 690 | }); 691 | 692 | test('blob processing as image', async () => { 693 | const returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', true); 694 | expect(returnedValue).toBe('HTML-IMAGE_URL'); 695 | }); 696 | 697 | test('blob processing as file with fileName', async () => { 698 | const returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false); 699 | expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'FILE_NAME_SANITIZED', 'BLOB', '/aaa/blobs'); 700 | expect(returnedValue).toBe('HTML-blobs/FILE_NAME_SANITIZED'); 701 | }); 702 | 703 | test('blob processing as file without fileName', async () => { 704 | fileInfo.fileName = undefined; 705 | const returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false); 706 | expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'BLOB_ID.pdf_SANITIZED', 'BLOB', '/aaa/blobs'); 707 | expect(returnedValue).toBe('HTML-blobs/BLOB_ID.pdf_SANITIZED'); 708 | }); 709 | 710 | test('blob processing as file without fileName and without right mime type', async () => { 711 | fileInfo.fileName = undefined; 712 | //type is not defined 713 | delete blob.type; 714 | let returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false); 715 | expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'BLOB_ID_SANITIZED', 'BLOB', '/aaa/blobs'); 716 | expect(returnedValue).toBe('HTML-blobs/BLOB_ID_SANITIZED'); 717 | //type is wrong 718 | blob.type = "dddfffggg"; 719 | returnedValue = await quipProcessor._processFile('HTML-REPLACEMENT', fileInfo, '/aaa/', false); 720 | expect(quipProcessor.saveCallback).toHaveBeenCalledWith(blob, 'BLOB_ID_SANITIZED', 'BLOB', '/aaa/blobs'); 721 | expect(returnedValue).toBe('HTML-blobs/BLOB_ID_SANITIZED'); 722 | }); 723 | }); 724 | 725 | describe('_renderDocumentHtml', () => { 726 | beforeEach(() => { 727 | initQuipProcessor(); 728 | ejs.render.mockClear(); 729 | }); 730 | 731 | test('without document template', async () => { 732 | quipProcessor.documentTemplate = undefined; 733 | const result = quipProcessor._renderDocumentHtml('HTML', 'Programming', 'aaa/bbb'); 734 | expect(result).toBe('HTML'); 735 | }); 736 | 737 | test('with document template', async () => { 738 | quipProcessor.documentTemplate = 'DOCUMENT_TEMPLATE'; 739 | quipProcessor.options.documentCSS = 'CSS'; 740 | const result = quipProcessor._renderDocumentHtml('HTML', 'Programming', 'aaa/bbb'); 741 | expect(ejs.render).toHaveBeenCalledWith('DOCUMENT_TEMPLATE', { 742 | title: 'Programming', 743 | body: 'HTML', 744 | stylesheet_path: '', 745 | embedded_stylesheet: 'CSS' 746 | }); 747 | expect(result).toBe('TEMPLATED DOCUMENT'); 748 | }); 749 | 750 | test('without embedded css', async () => { 751 | quipProcessor.documentTemplate = 'DOCUMENT_TEMPLATE'; 752 | quipProcessor.options.documentCSS = undefined; 753 | const result = quipProcessor._renderDocumentHtml('HTML', 'Programming', 'aaa/bbb'); 754 | expect(ejs.render).toHaveBeenCalledWith('DOCUMENT_TEMPLATE', { 755 | title: 'Programming', 756 | body: 'HTML', 757 | stylesheet_path: '../document.css', 758 | embedded_stylesheet: undefined 759 | }); 760 | expect(result).toBe('TEMPLATED DOCUMENT'); 761 | }); 762 | }); 763 | 764 | describe('_findReferencedObject', () => { 765 | beforeEach(() => { 766 | initQuipProcessor(); 767 | quipProcessor.referencesMap.set('folder1', { 768 | path: 'Private/Programming/', 769 | folder: true, 770 | title: 'Angular' 771 | }); 772 | quipProcessor.referencesMap.set('thread1', { 773 | path: 'Private/Programming/Angular', 774 | thread: true 775 | }); 776 | quipProcessor.quipService.getThread.mockImplementation((referenceId) => { 777 | if(referenceId === 'thread1') { 778 | return Promise.resolve({thread: {id: 'thread1_obj', title: 'Thread1 Title'}}); 779 | } 780 | if(referenceId === 'threadXXX') { 781 | return Promise.resolve({thread: {id: 'thread1', title: 'ThreadXXX Title'}}); 782 | } 783 | if(referenceId === 'threadYYY') { 784 | return Promise.resolve({thread: {id: 'thread33', title: 'ThreadYYY Title'}}); 785 | } 786 | return Promise.resolve(); 787 | }); 788 | }); 789 | 790 | test('found referenced object', async () => { 791 | const result = await quipProcessor._findReferencedObject({referenceId: 'folder1'}); 792 | expect(result).toBe(quipProcessor.referencesMap.get('folder1')); 793 | }); 794 | 795 | test('referenced object is thread but title is undefined', async () => { 796 | const result = await quipProcessor._findReferencedObject({referenceId: 'thread1'}); 797 | expect(result).toEqual({title: 'Thread1 Title', ...quipProcessor.referencesMap.get('thread1')}); 798 | }); 799 | 800 | test('referenced object is thread but could not find referenced thread', async () => { 801 | quipProcessor.quipService.getThread.mockResolvedValue(); 802 | const result = await quipProcessor._findReferencedObject({referenceId: 'thread1'}); 803 | expect(result).toBe(undefined); 804 | }); 805 | 806 | test('referenced object not found, but referenced thread could be read', async () => { 807 | const result = await quipProcessor._findReferencedObject({referenceId: 'threadXXX'}); 808 | const obj1 = quipProcessor.referencesMap.get('threadXXX'); 809 | const obj2 = quipProcessor.referencesMap.get('thread1'); 810 | expect(obj1).toEqual(obj2); 811 | expect(result).toEqual(obj1); 812 | }); 813 | 814 | test('referenced object not found, but referenced thread could not be read', async () => { 815 | const result = await quipProcessor._findReferencedObject({referenceId: 'threadYYY'}); 816 | expect(result).toBe(undefined); 817 | }); 818 | 819 | test('referenced object is user', async () => { 820 | quipProcessor._findReferencedUser = jest.fn(); 821 | quipProcessor._findReferencedUser.mockResolvedValue({name: 'Alex'}); 822 | const result = await quipProcessor._findReferencedObject({referenceId: 'user1'}); 823 | expect(result).toEqual({name: 'Alex'}); 824 | }); 825 | 826 | test('referenced object is user, and user could not be read', async () => { 827 | quipProcessor._findReferencedUser = jest.fn(); 828 | quipProcessor._findReferencedUser.mockResolvedValue(); 829 | const result = await quipProcessor._findReferencedObject({referenceId: 'user1'}); 830 | expect(result).toEqual(); 831 | }); 832 | }); 833 | 834 | describe('_findReferencedUser', () => { 835 | beforeEach(() => { 836 | initQuipProcessor(); 837 | quipProcessor.referencesMap.set('user1', { 838 | name: 'Alex', 839 | user: true 840 | }); 841 | quipProcessor.quipService.getUser.mockImplementation((referenceId) => { 842 | if(referenceId === 'userXXX') { 843 | return Promise.resolve({name: 'Bruno'}); 844 | } 845 | return Promise.resolve(); 846 | }); 847 | }); 848 | 849 | test('user reference object found', async () => { 850 | const result = await quipProcessor._findReferencedUser({referenceId:'user1'}); 851 | const obj1 = quipProcessor.referencesMap.get('user1'); 852 | expect(result).toEqual(obj1); 853 | }); 854 | 855 | test('user reference object not found, but user could be read', async () => { 856 | const result = await quipProcessor._findReferencedUser({referenceId:'userXXX'}); 857 | const obj1 = quipProcessor.referencesMap.get('userXXX'); 858 | expect(result.name).toBe('Bruno'); 859 | expect(obj1.name).toBe('Bruno'); 860 | }); 861 | 862 | test('user reference object not found, but user could not be read', async () => { 863 | const result = await quipProcessor._findReferencedUser({referenceId:'userAAA'}); 864 | expect(result).toBe(undefined); 865 | }); 866 | }); 867 | 868 | describe('_getThreadMessagesHtml', () => { 869 | beforeEach(() => { 870 | initQuipProcessor(); 871 | 872 | quipProcessor.quipService.getThreadMessages.mockResolvedValue(messages); 873 | 874 | quipProcessor._findReferencedUser = jest.fn(); 875 | quipProcessor._findReferencedUser.mockImplementation((reference) => { 876 | if(reference.referenceId === 'VKJAEAUa2PB') { 877 | return Promise.resolve({name: 'Alex', user: true}); 878 | } 879 | return Promise.resolve(); 880 | }); 881 | 882 | quipProcessor._findReferencedObject = jest.fn(); 883 | quipProcessor._findReferencedObject.mockImplementation((reference) => { 884 | if(reference.referenceId === 'UPWAOAAEpFn') { 885 | return Promise.resolve({title: 'Test', folder: true}); 886 | } 887 | if(reference.referenceId === 'bgCMALIfAuaI') { 888 | return Promise.resolve({title: 'Document-1', thread: true}); 889 | } 890 | return Promise.resolve(); 891 | }); 892 | 893 | quipProcessor._processReference = jest.fn(); 894 | quipProcessor._processReference.mockImplementation((html, reference) => { 895 | if(reference.referenceId === 'UPWAOAAEpFn') { 896 | return Promise.resolve(html + 'PROCESSED-FOLDER'); 897 | } 898 | if(reference.referenceId === 'bgCMALIfAuaI') { 899 | return Promise.resolve(html + 'PROCESSED-THREAD'); 900 | } 901 | if(reference.referenceId === 'XXXXXXXXXX') { 902 | return Promise.resolve(html + 'PROCESSED-UNKNOWN-REFERENCE'); 903 | } 904 | return Promise.resolve(); 905 | }); 906 | 907 | quipProcessor._processFile = jest.fn(); 908 | quipProcessor._processFile.mockImplementation((html, fileMatch) => { 909 | if(['1c2EXOl1sx04g5_hPNmt1A', 910 | 'sVhMwHaxsgiwmfMi4au1Zg', 911 | '1c2EXOl1sx04g5_hPNmt1A222', 912 | 'sVhMwHaxsgiwmfMi4au1Zg111'].includes(fileMatch.blobId)) { 913 | return Promise.resolve(html + 'PROCESSED-FILE'); 914 | } 915 | if(fileMatch.blobId === 'Ak5BJnkWubPxpIm4dtlWmg') { 916 | return Promise.resolve(html + 'PROCESSED-IMAGE'); 917 | } 918 | 919 | return Promise.resolve(); 920 | }); 921 | }); 922 | 923 | test('thread with messages', async () => { 924 | const result = await quipProcessor._getThreadMessagesHtml({thread: {id: 'thread123'}}, 'Test/Programming/'); 925 | expect(result).toMatchSnapshot(); 926 | }); 927 | 928 | test('thread without messages', async () => { 929 | quipProcessor.quipService.getThreadMessages.mockResolvedValue([]); 930 | let result = await quipProcessor._getThreadMessagesHtml({thread: {id: 'thread123'}}, 'Test/Programming/'); 931 | expect(result).toBe(''); 932 | }); 933 | 934 | test('thread with some error', async () => { 935 | quipProcessor.quipService.getThreadMessages.mockResolvedValue(); 936 | let result = await quipProcessor._getThreadMessagesHtml({thread: {id: 'thread123'}}, 'Test/Programming/'); 937 | expect(result).toBe(''); 938 | }); 939 | }); 940 | 941 | describe('_processDocumentThreadDocx', () => { 942 | const thread = { 943 | thread: {title: 'someDoc'} 944 | }; 945 | 946 | beforeEach(() => { 947 | initQuipProcessor(); 948 | quipProcessor.quipService.getDocx.mockResolvedValue('BLOB-DATA'); 949 | }); 950 | 951 | test('process document', async () => { 952 | await quipProcessor._processDocumentThreadDocx(thread, '/aaa/'); 953 | expect(quipProcessor.saveCallback).toHaveBeenCalledWith('BLOB-DATA', 'someDoc.docx_SANITIZED', 'BLOB', '/aaa/'); 954 | }); 955 | }); 956 | 957 | describe('_processDocumentThreadXlsx', () => { 958 | const thread = { 959 | thread: {title: 'someDoc'} 960 | }; 961 | 962 | beforeEach(() => { 963 | initQuipProcessor(); 964 | quipProcessor.quipService.getXlsx.mockResolvedValue('BLOB-DATA'); 965 | }); 966 | 967 | test('process document', async () => { 968 | await quipProcessor._processDocumentThreadXlsx(thread, '/aaa/'); 969 | expect(quipProcessor.saveCallback).toHaveBeenCalledWith('BLOB-DATA', 'someDoc.xlsx_SANITIZED', 'BLOB', '/aaa/'); 970 | }); 971 | }); 972 | 973 | }); 974 | 975 | 976 | -------------------------------------------------------------------------------- /lib/__tests__/QuipService.test.js: -------------------------------------------------------------------------------- 1 | jest.mock('node-fetch'); 2 | const fetch = require('node-fetch'); 3 | const {Response} = jest.requireActual('node-fetch'); 4 | 5 | jest.mock('../common/LoggerAdapter'); 6 | const LoggerAdapter = require('../common/LoggerAdapter'); 7 | 8 | const waitingTime = Math.ceil((new Date().getTime() / 1000) + 2 ); //2 sec 9 | const QuipService = require('../QuipService'); 10 | 11 | const blob = [1, 2, 3, 4, 5, 6]; 12 | 13 | function createResponse200() { 14 | return new Response(JSON.stringify({data:123456})); 15 | } 16 | 17 | function createResponse200Blob() { 18 | const response = new Response(blob, { 19 | headers: { 20 | 'content-type': 'image/jpeg', 21 | 'content-length': blob.length, 22 | 'content-disposition': 'attachment' 23 | } 24 | }); 25 | response.blob = jest.fn(() => blob); 26 | return response; 27 | } 28 | 29 | //general error response 30 | function createResponse500() { 31 | return new Response("", {status: 500}); 32 | } 33 | 34 | function createResponse503() { 35 | return new Response(JSON.stringify({waiting:waitingTime}), { 36 | status: 503, 37 | headers: {'x-ratelimit-reset': waitingTime} 38 | }); 39 | } 40 | 41 | const quipService = new QuipService('###TOKEN###', 'http://quip.com'); 42 | 43 | beforeEach(() => { 44 | quipService.stats.query_count = 0; 45 | quipService.querries503 = new Map(); 46 | }); 47 | 48 | test('constructor tests', async () => { 49 | expect(quipService.accessToken).toBe('###TOKEN###'); 50 | expect(quipService.apiURL).toBe('http://quip.com'); 51 | expect(quipService.logger).toBeInstanceOf(LoggerAdapter); 52 | }); 53 | 54 | test('_apiCall response with 503 code', async () => { 55 | fetch.mockReturnValue(Promise.resolve(createResponse200())).mockReturnValueOnce(Promise.resolve(createResponse503())); 56 | const res = await quipService._apiCall('/someURL'); 57 | expect(res.data).toBe(123456); 58 | expect(quipService.stats.query_count).toBe(2); 59 | }); 60 | 61 | test('_apiCall response with 503 code more than 10 times', async () => { 62 | quipService.waitingMs = 50; 63 | fetch.mockReturnValue(Promise.resolve(createResponse503())); 64 | const res = await quipService._apiCall('/someURL'); 65 | expect(res).toBe(undefined); 66 | expect(quipService.logger.error).toHaveBeenCalledWith('Couldn\'t fetch /someURL, tryed to get it 10 times'); 67 | }); 68 | 69 | test('_apiCall response with 500 code', async () => { 70 | fetch.mockReturnValue(Promise.resolve(createResponse500())); 71 | await quipService._apiCall('/someURL'); 72 | expect(quipService.logger.debug).toHaveBeenCalledWith('Couldn\'t fetch /someURL, received 500'); 73 | }); 74 | 75 | test('_apiCall: fetch with exeption', async () => { 76 | const error = new Error('Some server error'); 77 | fetch.mockImplementation(() => { 78 | throw error; 79 | }); 80 | await quipService._apiCall('/someURL'); 81 | expect(quipService.logger.error).toHaveBeenCalledWith('Couldn\'t fetch /someURL, ', error); 82 | }); 83 | 84 | test('_apiCall for JSON', async () => { 85 | fetch.mockReturnValue(Promise.resolve(createResponse200())); 86 | const res = await quipService._apiCallJson('/someURL'); 87 | expect(res.data).toBe(123456); 88 | }); 89 | 90 | test('_apiCall for Blob', async () => { 91 | fetch.mockReturnValue(Promise.resolve(createResponse200Blob())); 92 | const res = await quipService._apiCallBlob('/someURL'); 93 | expect(res).toBe(blob); 94 | }); 95 | 96 | test('_apiCallJson', async () => { 97 | quipService._apiCall = jest.fn(); 98 | await quipService._apiCallJson('/someURL'); 99 | expect(quipService._apiCall).toHaveBeenCalledWith(expect.anything(), expect.anything(), false); 100 | }); 101 | 102 | test('_apiCallBlob', async () => { 103 | quipService._apiCall = jest.fn(); 104 | await quipService._apiCallBlob('/someURL'); 105 | expect(quipService._apiCall).toHaveBeenCalledWith(expect.anything(), expect.anything(), true); 106 | }); -------------------------------------------------------------------------------- /lib/__tests__/__snapshots__/QuipProcessor.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`methods tests _getThreadMessagesHtml thread with messages 1`] = ` 4 | "
5 |
Heller Stern
Message without updated and create date
6 |
Heller Stern
Message without create date
7 |
Heller Stern, 8 May 2020, 09:18
Message with folder-link TestPROCESSED-FOLDER
8 |
Heller Stern, 8 May 2020, 09:19
Message with thread link XXXXXXXXXXPROCESSED-UNKNOWN-REFERENCE
9 |
Heller Stern, 8 May 2020, 09:19
Message with thread link Document-1PROCESSED-THREAD
10 |
Heller Stern
Message without updated date
11 |
Heller Stern, 8 May 2020, 09:20
Message with annotation @Alex
12 |
Heller Stern, 8 May 2020, 09:20
Message with unknown person-link
13 |
Heller Stern, 8 May 2020, 09:20
Message with person-link @Alex
14 |
Heller Stern, 8 May 2020, 09:20


PROCESSED-IMAGE
15 |
Heller Stern, 25 May 2020, 20:21
444444
document1.html
PROCESSED-FILE

document1.pdf
PROCESSED-FILE

document_unknown_type
PROCESSED-FILE

sVhMwHaxsgiwmfMi4au1Zg111
PROCESSED-FILE
" 16 | `; 17 | -------------------------------------------------------------------------------- /lib/__tests__/folders.json: -------------------------------------------------------------------------------- 1 | { 2 | "FOLDER-1": 3 | { 4 | "folder": 5 | { 6 | "title":"Programming", 7 | "creator_id":"CREATORID", 8 | "parent_id":"PARENTID", 9 | "color":"manila", 10 | "id":"FOLDER-1", 11 | "created_usec":1438720016926712, 12 | "updated_usec":1578634632315324 13 | }, 14 | "member_ids":[], 15 | "children": 16 | [ 17 | {"folder_id":"F1", "restricted": true}, 18 | {"folder_id":"F2"}, 19 | {"folder_id":"F3"}, 20 | {"folder_id":"F4"}, 21 | {"folder_id":"F5"}, 22 | {"folder_id":"F6"}, 23 | {"folder_id":"F7"}, 24 | {"folder_id":"F8"}, 25 | {"folder_id":"F9"}, 26 | {"thread_id":"T1"}, 27 | {"thread_id":"T2"}, 28 | {"folder_id":"F10"}, 29 | {"folder_id":"F11"}, 30 | {"thread_id":"T3"}, 31 | {"folder_id":"F12"}, 32 | {"folder_id":"F13"} 33 | ] 34 | }, 35 | "FOLDER-2": 36 | { 37 | "folder": 38 | { 39 | "title":"Git", 40 | "creator_id":"CREATORID", 41 | "parent_id":"PARENTID", 42 | "color":"manila", 43 | "id":"FOLDER-2", 44 | "created_usec":1532722544470138, 45 | "updated_usec":1577953349425750 46 | }, 47 | "member_ids":[], 48 | "children": 49 | [ 50 | {"thread_id":"T1"}, 51 | {"thread_id":"T2"}, 52 | {"thread_id":"T3"} 53 | ] 54 | } 55 | } -------------------------------------------------------------------------------- /lib/__tests__/messages.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "test_description": "Person link", 4 | "author_id": "VOQAEArheMG", 5 | "visible": true, 6 | "id": "WabADAipzzM", 7 | "created_usec": 1588922400693783, 8 | "updated_usec": 1588922400706385, 9 | "text": "Message with person-link https:\/\/quip.com\/VKJAEAUa2PB", 10 | "mention_user_ids": [ 11 | "VKJAEAUa2PB" 12 | ], 13 | "author_name": "Heller Stern" 14 | }, 15 | { 16 | "test_description": "Unknown person link", 17 | "author_id": "VOQAEArheMG", 18 | "visible": true, 19 | "id": "WabADAipzzM", 20 | "created_usec": 1588922400693783, 21 | "updated_usec": 1588922400706385, 22 | "text": "Message with unknown person-link https:\/\/quip.com\/unknownUser", 23 | "mention_user_ids": [ 24 | "unknownUser" 25 | ], 26 | "author_name": "Heller Stern" 27 | }, 28 | { 29 | "test_description": "Folder link", 30 | "author_id": "VOQAEArheMG", 31 | "visible": true, 32 | "id": "WabADAleJRK", 33 | "created_usec": 1588922338752846, 34 | "updated_usec": 1588922338763732, 35 | "text": "Message with folder-link https:\/\/quip.com\/UPWAOAAEpFn", 36 | "author_name": "Heller Stern" 37 | }, 38 | { 39 | "test_description": "Thread link", 40 | "author_id": "VOQAEArheMG", 41 | "visible": true, 42 | "id": "WabADAxrKhb", 43 | "created_usec": 1588922367022584, 44 | "updated_usec": 1588922367036276, 45 | "text": "Message with thread link https:\/\/quip.com\/bgCMALIfAuaI", 46 | "author_name": "Heller Stern" 47 | }, 48 | { 49 | "test_description": "Unknown thread link", 50 | "author_id": "VOQAEArheMG", 51 | "visible": true, 52 | "id": "WabADAxrKhb", 53 | "created_usec": 1588922367022584, 54 | "updated_usec": 1588922367036276, 55 | "text": "Message with thread link https:\/\/quip.com\/XXXXXXXXXX", 56 | "author_name": "Heller Stern" 57 | }, 58 | { 59 | "test_description": "File links", 60 | "author_id": "VOQAEArheMG", 61 | "visible": true, 62 | "id": "WabADAa1qQG", 63 | "created_usec": 1590430904798664, 64 | "updated_usec": 1590430904812335, 65 | "text": "444444", 66 | "files": [ 67 | { 68 | "name": "document1.html", 69 | "hash": "1c2EXOl1sx04g5_hPNmt1A" 70 | }, 71 | { 72 | "name": "document1.pdf", 73 | "hash": "sVhMwHaxsgiwmfMi4au1Zg" 74 | }, 75 | { 76 | "name": "document_unknown_type", 77 | "hash": "1c2EXOl1sx04g5_hPNmt1A222" 78 | }, 79 | { 80 | "hash": "sVhMwHaxsgiwmfMi4au1Zg111" 81 | } 82 | ], 83 | "author_name": "Heller Stern" 84 | }, 85 | { 86 | "test_description": "Image link", 87 | "author_id": "VOQAEArheMG", 88 | "visible": true, 89 | "id": "WabADAKZuzT", 90 | "created_usec": 1588922447536790, 91 | "updated_usec": 1588922447547165, 92 | "text": "", 93 | "files": [ 94 | { 95 | "name": "2-19-WirGlaubenAnPlatooning-startseite_1000x1500.jpg", 96 | "hash": "Ak5BJnkWubPxpIm4dtlWmg" 97 | } 98 | ], 99 | "author_name": "Heller Stern" 100 | }, 101 | { 102 | "test_description": "Message with annotation -> to be ignored", 103 | "annotation": "WabADAipzzMXXX1111", 104 | "author_id": "VOQAEArheMG", 105 | "visible": true, 106 | "id": "WabADAipzzM", 107 | "created_usec": 1588922400693783, 108 | "updated_usec": 1588922400706385, 109 | "text": "Message with annotation https:\/\/quip.com\/VKJAEAUa2PB", 110 | "mention_user_ids": [ 111 | "VKJAEAUa2PB" 112 | ], 113 | "author_name": "Heller Stern" 114 | }, 115 | { 116 | "test_description": "Message without text", 117 | "author_id": "VOQAEArheMG", 118 | "visible": true, 119 | "id": "WabADAipzzM", 120 | "created_usec": 1588922400693783, 121 | "updated_usec": 1588922400706385, 122 | "author_name": "Heller Stern" 123 | }, 124 | { 125 | "test_description": "Message without create date", 126 | "author_id": "VOQAEArheMG", 127 | "visible": true, 128 | "id": "WabADAipzzM", 129 | "text": "Message without create date", 130 | "updated_usec": 1588922400706385, 131 | "author_name": "Heller Stern" 132 | }, 133 | { 134 | "test_description": "Message without updated date", 135 | "author_id": "VOQAEArheMG", 136 | "visible": true, 137 | "id": "WabADAipzzM", 138 | "text": "Message without updated date", 139 | "created_usec": 1588922400693783, 140 | "author_name": "Heller Stern" 141 | }, 142 | { 143 | "test_description": "Message without updated and create date", 144 | "author_id": "VOQAEArheMG", 145 | "visible": true, 146 | "id": "WabADAipzzM", 147 | "text": "Message without updated and create date", 148 | "author_name": "Heller Stern" 149 | } 150 | ] 151 | -------------------------------------------------------------------------------- /lib/__tests__/threads.json: -------------------------------------------------------------------------------- 1 | { 2 | "THREAD-1": 3 | { 4 | "thread": 5 | { 6 | "author_id":"AUTHORID", 7 | "thread_class":"document", 8 | "id":"THREAD-1", 9 | "created_usec":1527841242991644, 10 | "updated_usec":1572536961860469, 11 | "title":"Tools", 12 | "link":"https:\/\/quip.com\/THREAD-1", 13 | "type":"document", 14 | "document_id":"cUCABABs027" 15 | }, 16 | "user_ids":["AUTHORID"], 17 | "shared_folder_ids":[], 18 | "expanded_user_ids":["AUTHORID"], 19 | "invited_user_emails":[], 20 | "html":"\u003ch1 id='cUCACA6AGmU'\u003eTools\u003c\/h1\u003e\n\n\u003cp id='cUCACAFL0dM' class='line'\u003e\u003cb\u003eAPI-Development\u003c\/b\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAR6DOO' class='line'\u003e\u003ca href=\"https:\/\/swagger.io\"\u003ehttps:\/\/swagger.io\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAFeVo0' class='line'\u003e\u003ca href=\"https:\/\/www.getpostman.com\/apps\"\u003ehttps:\/\/www.getpostman.com\/apps\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAWnYGZ' class='line'\u003e\u200b\u003c\/p\u003e\n\n\u003cp id='cUCACAbjfTl' class='line'\u003e\u003cb\u003eSimple http-server for test (also for angular dist folder)\u003c\/b\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAUrNEl' class='line'\u003e\u003ca href=\"https:\/\/www.npmjs.com\/package\/http-server\"\u003ehttps:\/\/www.npmjs.com\/package\/http-server\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACArir7m' class='line'\u003e\u200b\u003c\/p\u003e\n\n\u003cp id='cUCACAyy66v' class='line'\u003e\u003cb\u003eSimple http-server for test\u003c\/b\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAZ4zz4' class='line'\u003e\u003ca href=\"https:\/\/github.com\/johnpapa\/lite-server\"\u003ehttps:\/\/github.com\/johnpapa\/lite-server\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAPWGLO' class='line'\u003e\u200b\u003c\/p\u003e\n\n\u003cp id='cUCACAMGosk' class='line'\u003e\u003cb\u003eSimple http-server with benefits\u003c\/b\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAA2xpq' class='line'\u003e\u003ca href=\"https:\/\/github.com\/lwsjs\/local-web-server\"\u003ehttps:\/\/github.com\/lwsjs\/local-web-server\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACA3Cq3M' class='line'\u003e\u200b\u003c\/p\u003e\n\n\u003cp id='cUCACAO5AMi' class='line'\u003e\u003cb\u003e\u041c\u0430\u0441\u043a\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435 \u0438 \u043a\u043b\u0438\u043f\u043f\u0438\u043d\u0433\u003c\/b\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAMijgE' class='line'\u003e\u003ca href=\"https:\/\/css-tricks.com\/clipping-masking-css\/\"\u003ehttps:\/\/css-tricks.com\/clipping-masking-css\/\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAEwg8e' class='line'\u003eClip-Editor:\u003c\/p\u003e\n\n\u003cp id='cUCACAQcppU' class='line'\u003e\u003ca href=\"http:\/\/bennettfeely.com\/clippy\/\"\u003ehttp:\/\/bennettfeely.com\/clippy\/\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACA793sV' class='line'\u003e\u200b\u003c\/p\u003e\n\n\u003cp id='cUCACAdJMrX' class='line'\u003eFile examles (fake files different types and sizes for development and testing)\u003c\/p\u003e\n\n\u003cp id='cUCACA0ns2A' class='line'\u003e\u003ca href=\"https:\/\/file-examples.com\/\"\u003ehttps:\/\/file-examples.com\/\u003c\/a\u003e\u003c\/p\u003e\n\n\u003cp id='cUCACAcbODf' class='line'\u003e\u200b\u003c\/p\u003e\n\n" 21 | }, 22 | "THREAD-2": 23 | { 24 | "thread": 25 | { 26 | "author_id":"AUTHORID", 27 | "thread_class":"document", 28 | "id":"THREAD-2", 29 | "created_usec":1518528470169388, 30 | "updated_usec":1531471630282022, 31 | "title":"Web Storm Commands", 32 | "link":"https:\/\/quip.com\/THREAD-2", 33 | "type":"document", 34 | "document_id":"HRYABAf1LAv" 35 | }, 36 | "user_ids":["AUTHORID"], 37 | "shared_folder_ids":[], 38 | "expanded_user_ids":["AUTHORID"], 39 | "invited_user_emails":[], 40 | "html":"\u003ch1 id='HRYACA6bnO7'\u003eWeb Storm Commands\u003c\/h1\u003e\n\n\u003cdiv data-section-style='13'\u003e\u003ctable id='HRYACAUuX92' title='Sheet1' style='width: 50.0667em'\u003e\u003cthead\u003e\u003ctr\u003e\u003cth id='HRYACA4XfiI' class='empty' style='width: 19.4em'\u003eA\u003cbr\/\u003e\u003c\/th\u003e\u003cth id='HRYACACVIBk' class='empty' style='width: 30.6667em'\u003eB\u003cbr\/\u003e\u003c\/th\u003e\u003c\/tr\u003e\u003c\/thead\u003e\u003ctbody\u003e\u003ctr id='HRYACAILqw1'\u003e\u003ctd id='s:HRYACAILqw1_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAILqw1_HRYACAJzhmH'\u003eCtrl + Q\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAILqw1_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAILqw1_HRYACAR9Mg9'\u003eopen documentation preview\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAOBzLP'\u003e\u003ctd id='s:HRYACAOBzLP_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAOBzLP_HRYACAJzhmH'\u003eType \/**[Enter]\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAOBzLP_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAOBzLP_HRYACAR9Mg9'\u003ewrite documentation\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAyxrcg'\u003e\u003ctd id='s:HRYACAyxrcg_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAyxrcg_HRYACAJzhmH'\u003eCtrl + Shift + F\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAyxrcg_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAyxrcg_HRYACAR9Mg9'\u003efind in path\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAsnZ7H'\u003e\u003ctd id='s:HRYACAsnZ7H_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAsnZ7H_HRYACAJzhmH'\u003eCtrl + Shift + N\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAsnZ7H_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAsnZ7H_HRYACAR9Mg9'\u003efind in file name\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAtOKKT'\u003e\u003ctd id='s:HRYACAtOKKT_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAtOKKT_HRYACAJzhmH'\u003eCtrl + Alt + L\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAtOKKT_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAtOKKT_HRYACAR9Mg9'\u003ereformat code\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACALCzUI'\u003e\u003ctd id='s:HRYACALCzUI_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACALCzUI_HRYACAJzhmH'\u003eAlt + mouse selection\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACALCzUI_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACALCzUI_HRYACAR9Mg9'\u003evertical selection and edit\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACArbm7h'\u003e\u003ctd id='s:HRYACArbm7h_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACArbm7h_HRYACAJzhmH'\u003eAlt + Enter\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACArbm7h_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACArbm7h_HRYACAR9Mg9'\u003egenerate code or list of available intention actions\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAAtTiL'\u003e\u003ctd id='s:HRYACAAtTiL_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAAtTiL_HRYACAJzhmH'\u003eCtrl + D\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAAtTiL_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAAtTiL_HRYACAR9Mg9'\u003eduplicate string\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAJdrMO'\u003e\u003ctd id='s:HRYACAJdrMO_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAJdrMO_HRYACAJzhmH'\u003eF11\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAJdrMO_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAJdrMO_HRYACAR9Mg9'\u003eadd bookmark\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAcE2NT'\u003e\u003ctd id='s:HRYACAcE2NT_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAcE2NT_HRYACAJzhmH'\u003eShift + F11\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAcE2NT_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAcE2NT_HRYACAR9Mg9'\u003eshow bookmarks\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAnOuAK'\u003e\u003ctd id='s:HRYACAnOuAK_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAnOuAK_HRYACAJzhmH'\u003eCtrl + F11\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAnOuAK_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAnOuAK_HRYACAR9Mg9'\u003eadd mnemonic bookmark\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003ctr id='HRYACAUVr0E'\u003e\u003ctd id='s:HRYACAUVr0E_HRYACAJzhmH' style=''\u003e\u003cspan id='s:HRYACAUVr0E_HRYACAJzhmH'\u003e\u200b\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003ctd id='s:HRYACAUVr0E_HRYACAR9Mg9' style=''\u003e\u003cspan id='s:HRYACAUVr0E_HRYACAR9Mg9'\u003e\u200b\u003c\/span\u003e\n\n\u003cbr\/\u003e\u003c\/td\u003e\u003c\/tr\u003e\u003c\/tbody\u003e\u003c\/table\u003e\u003c\/div\u003e\u003cp id='HRYACA8oDTn' class='line'\u003e\u200b\u003c\/p\u003e\n\n" 41 | }, 42 | "THREAD-3": 43 | { 44 | "thread": 45 | { 46 | "author_id":"AUTHORID", 47 | "thread_class":"document", 48 | "id":"THREAD-3", 49 | "created_usec":1579259196274691, 50 | "updated_usec":1579259285390607, 51 | "title":"Test", 52 | "link":"https:\/\/quip.com\/THREAD-3", 53 | "type":"document", 54 | "document_id":"NKSABAh19W4" 55 | }, 56 | "user_ids":["AUTHORID"], 57 | "shared_folder_ids":[], 58 | "expanded_user_ids":["AUTHORID"], 59 | "invited_user_emails":[], 60 | "html":"\u003ch1 id='NKSACApZodO'\u003eTest\u003c\/h1\u003e\n\n\u003cp id='NKSACAZ9CCP' class='line'\u003e\u003ccontrol data-remapped=\"true\" id=\"NKSACAwmUa7\"\u003e\u003ca href=\"https:\/\/quip.com\/-\/blob\/NKSAAAGTQ2j\/1mJZTO3X-1wd5Vzd4pBy3g?name=Postman.lnk\"\u003ePostman.lnk\u003c\/a\u003e\u003c\/control\u003e\u00a0\u003c\/p\u003e\n\n\u003cp id='NKSACAzwxCc' class='line'\u003e\u200b\u003c\/p\u003e\n\n\u003cdiv data-section-style='11' style='max-width:100%'\u003e\u003cimg src='\/blob\/NKSAAAGTQ2j\/CIOl_wqnrD66BGInCm1ZXw' id='NKSACAkt88h' alt=''\u003e\u003c\/img\u003e\u003c\/div\u003e\u003cp id='NKSACAomFL2' class='line'\u003e\u200b\u003c\/p\u003e\n\n" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /lib/cli/CliArguments.js: -------------------------------------------------------------------------------- 1 | const commandLineArgs = require('command-line-args'); 2 | const colors = require('colors'); 3 | const mkdirp = require('mkdirp'); 4 | 5 | const options = require('./options'); 6 | const help = require('./help'); 7 | const pkgVersion = require('../../package.json').version; 8 | 9 | module.exports = function() { 10 | try { 11 | var cliArguments = commandLineArgs(options); 12 | } catch(e) { 13 | throw colors.red(e.message); 14 | } 15 | 16 | if(cliArguments.help || Object.keys(cliArguments).length === 0) { 17 | throw help(); 18 | } 19 | 20 | if(cliArguments.version) { 21 | throw pkgVersion; 22 | } 23 | 24 | if(!cliArguments.token) { 25 | throw colors.red('ERROR: Token is not defined.'); 26 | } 27 | 28 | if(cliArguments.destination) { 29 | try { 30 | mkdirp.sync(cliArguments.destination); 31 | } catch (e) { 32 | throw colors.red('ERROR: Destination folder is wrong.'); 33 | } 34 | } 35 | 36 | if(cliArguments.folders) { 37 | if(cliArguments.folders.trim().length === 0) { 38 | throw colors.red(`ERROR: Folder's IDs are not set.` ); 39 | } 40 | const foldersIds = cliArguments.folders.split(','); 41 | cliArguments.folders = foldersIds.map((id) => id.trim()); 42 | } 43 | 44 | return cliArguments; 45 | }; 46 | -------------------------------------------------------------------------------- /lib/cli/__tests__/CliArguments.test.js: -------------------------------------------------------------------------------- 1 | const cliArguments = require('../CliArguments'); 2 | const help = require('../help'); 3 | const pkgVersion = require('../../../package.json').version; 4 | 5 | const mkdirp = require('mkdirp'); 6 | jest.mock('mkdirp'); 7 | const rightFolder = 'c:\\temp'; 8 | mkdirp.sync.mockImplementation((folder) => { 9 | if(rightFolder === folder) { 10 | return; 11 | } 12 | throw new Error(); 13 | }); 14 | 15 | test('calling with unknown parameter', () => { 16 | process.argv = []; 17 | process.argv[2] = '--wrongParameter'; 18 | expect(() => cliArguments()).toThrow('Unknown option'); 19 | }); 20 | 21 | test('calling without any parameter', () => { 22 | process.argv = []; 23 | expect(() => cliArguments()).toThrow(help()); 24 | }); 25 | 26 | test('calling with -h parameter', () => { 27 | process.argv = []; 28 | process.argv[2] = '-h'; 29 | expect(() => cliArguments()).toThrow(help()); 30 | }); 31 | 32 | test('calling with -v parameter', () => { 33 | process.argv = []; 34 | process.argv[2] = '-v'; 35 | expect(() => cliArguments()).toThrow(pkgVersion); 36 | }); 37 | 38 | test('calling without -t parameter', () => { 39 | const params = '-d c:\\temp'; 40 | process.argv = [null, null, ...params.split(' ')]; 41 | expect(() => cliArguments()).toThrow('ERROR: Token is not defined.'); 42 | }); 43 | 44 | test('calling with right destination folder', () => { 45 | const params = '-t #TOKEN -d c:\\temp'; 46 | process.argv = [null, null, ...params.split(' ')]; 47 | expect(() => cliArguments()).not.toThrow(); 48 | }); 49 | 50 | test('calling with wrong destination folder', () => { 51 | const params = '-t #TOKEN -d c:\\temp123'; 52 | process.argv = [null, null, ...params.split(' ')]; 53 | expect(() => cliArguments()).toThrow('ERROR: Destination folder is wrong.'); 54 | }); 55 | 56 | test('default parameters', () => { 57 | const params = '-t #TOKEN -d c:\\temp'; 58 | process.argv = [null, null, ...params.split(' ')]; 59 | const result = cliArguments(); 60 | expect(result['zip']).toBeFalsy(); 61 | expect(result['embedded-images']).toBeFalsy(); 62 | expect(result['embedded-styles']).toBeFalsy(); 63 | expect(result['comments']).toBeFalsy(); 64 | expect(result['docx']).toBeFalsy(); 65 | expect(result['debug']).toBeFalsy(); 66 | }); 67 | 68 | test('setting parameters', () => { 69 | const params = '-t #TOKEN -d c:\\temp -z --embedded-styles --embedded-images --comments --debug --docx'; 70 | process.argv = [null, null, ...params.split(' ')]; 71 | const result = cliArguments(); 72 | expect(result['token']).toBe('#TOKEN'); 73 | expect(result['destination']).toBe('c:\\temp'); 74 | expect(result['zip']).toBeTruthy(); 75 | expect(result['embedded-images']).toBeTruthy(); 76 | expect(result['embedded-styles']).toBeTruthy(); 77 | expect(result['comments']).toBeTruthy(); 78 | expect(result['docx']).toBeTruthy(); 79 | expect(result['debug']).toBeTruthy(); 80 | }); 81 | 82 | test('folders option: wrong parameter', () => { 83 | const params = '-t #TOKEN -d c:\\temp --folders'; 84 | process.argv = [null, null, ...params.split(' '), ' ']; 85 | expect(() => cliArguments()).toThrow('ERROR: Folder\'s IDs are not set.'); 86 | }); 87 | 88 | test('folders option: right parameter', () => { 89 | const params = '-t #TOKEN -d c:\\temp --folders dddd,11111,3444'; 90 | process.argv = [null, null, ...params.split(' ')]; 91 | const result = cliArguments(); 92 | expect(result['folders']).toEqual(['dddd','11111','3444']); 93 | }); -------------------------------------------------------------------------------- /lib/cli/help.js: -------------------------------------------------------------------------------- 1 | const commandLineUsage = require('command-line-usage'); 2 | const options = require('./options'); 3 | const pkgVersion = require('../../package.json').version; 4 | 5 | module.exports = function() { 6 | return commandLineUsage([ 7 | { 8 | header: `Quip-Export v${pkgVersion}`, 9 | content: 'Exports all folders and referenced files from an Quip-Account. ' + 10 | 'Current version exports only the files from Private and Shared Quip-folders.' 11 | }, 12 | { 13 | header: 'Usage: quip-export package installed globally (npm install -g quip-export)', 14 | content: 'quip-export [options]' 15 | }, 16 | { 17 | header: 'Usage: quip-export project cloned from Git', 18 | content: 'node quip-export [options]' 19 | }, 20 | { 21 | header: 'Typical Example', 22 | content: [ 23 | 'quip-export -t "" -d ', 24 | 'quip-export -t "" --zip' 25 | ] 26 | }, 27 | { 28 | header: 'Options', 29 | optionList: options 30 | }, 31 | { 32 | header: 'Project home', 33 | content: [ 34 | 'GitHub: {underline https://github.com/sonnenkern/quip-export}', 35 | 'Npm: {underline https://www.npmjs.com/package/quip-export}' 36 | ] 37 | } 38 | ]); 39 | }; -------------------------------------------------------------------------------- /lib/cli/options.js: -------------------------------------------------------------------------------- 1 | module.exports = 2 | [ 3 | { 4 | name: 'help', 5 | alias: 'h', 6 | type: Boolean, 7 | description: 'Display this usage guide.' 8 | }, 9 | { 10 | name: 'version', 11 | alias: 'v', 12 | type: Boolean, 13 | description: 'Print version info' 14 | }, 15 | { 16 | name: 'token', 17 | alias: 't', 18 | type: String, 19 | description: 'Quip Access Token. To generate a personal access token, visit the page: ', 20 | typeLabel: '{underline string}' 21 | }, 22 | { 23 | name: 'destination', 24 | alias: 'd', 25 | type: String, 26 | description: 'Destination folder for export files', 27 | typeLabel: '{underline string}' 28 | }, 29 | { 30 | name: 'zip', 31 | alias: 'z', 32 | type: Boolean, 33 | description: 'Zip export files' 34 | }, 35 | { 36 | name: 'embedded-styles', 37 | type: Boolean, 38 | description: 'Embedded in each document stylesheet' 39 | }, 40 | { 41 | name: 'embedded-images', 42 | type: Boolean, 43 | description: 'Embedded images' 44 | }, 45 | { 46 | name: 'docx', 47 | type: Boolean, 48 | description: 'Exports documents in *.docx and spreadsheets in *.xlsx format' 49 | }, 50 | { 51 | name: 'comments', 52 | type: Boolean, 53 | description: 'Includes comments (messages) for the documents' 54 | }, 55 | { 56 | name: 'folders', 57 | type: String, 58 | description: 'Comma-separated folder\'s IDs to export', 59 | typeLabel: '{underline string}' 60 | }, 61 | { 62 | name: 'debug', 63 | type: Boolean, 64 | description: 'Debug mode' 65 | } 66 | ]; 67 | -------------------------------------------------------------------------------- /lib/common/LoggerAdapter.js: -------------------------------------------------------------------------------- 1 | class LoggerAdapter { 2 | constructor(level=LoggerAdapter.LEVELS.INFO, destination) { 3 | this.level = level; 4 | this.destination = destination; 5 | } 6 | 7 | debug (/*message*/) {} 8 | info (/*message*/) {} 9 | warn (/*message*/) {} 10 | error (/*message*/) {} 11 | } 12 | 13 | LoggerAdapter.LEVELS = { 14 | DEBUG: 'debug', 15 | INFO: 'info', 16 | WARN: 'warn', 17 | ERROR: 'error' 18 | }; 19 | 20 | 21 | 22 | module.exports = LoggerAdapter; -------------------------------------------------------------------------------- /lib/common/PinoLogger.js: -------------------------------------------------------------------------------- 1 | const LoggerAdapter = require('./LoggerAdapter'); 2 | const pino = require('pino'); 3 | 4 | class PinoLogger extends LoggerAdapter { 5 | constructor(level=LoggerAdapter.LEVELS.INFO, destination) { 6 | super(level, destination); 7 | this.logger = pino({ 8 | level, 9 | prettyPrint: { 10 | ignore: 'pid,hostname', 11 | translateTime: 'SYS:dd-mm-yyyy HH:MM:ss.l', 12 | colorize: destination? false : true 13 | } 14 | }, destination); 15 | } 16 | 17 | debug (message) { this.logger.debug(message); } 18 | info (message) { this.logger.info (message); } 19 | warn (message) { this.logger.warn (message); } 20 | error (message) { this.logger.error(message); } 21 | } 22 | 23 | 24 | module.exports = PinoLogger; -------------------------------------------------------------------------------- /lib/common/TestUtils.js: -------------------------------------------------------------------------------- 1 | function mockResolvedWithThen(func, resolveValue) { 2 | if(!func._isMockFunction) throw "First parameter is not a mock function!"; 3 | func.promiseCalledWithThenTimes = 0; 4 | class PromiseMock extends Promise { 5 | constructor(executor) { 6 | super(executor); 7 | } 8 | 9 | then(onFulfilled, onRejected) { 10 | func.promiseCalledWithThenTimes++; 11 | return super.then(onFulfilled, onRejected); 12 | } 13 | } 14 | 15 | func.mockImplementation(() => PromiseMock.resolve(resolveValue)); 16 | } 17 | 18 | module.exports = {mockResolvedWithThen}; -------------------------------------------------------------------------------- /lib/common/blobImageToURL.js: -------------------------------------------------------------------------------- 1 | 2 | async function blobImageToURL(blob) { 3 | return new Promise( (release) => { 4 | if(typeof window !== 'undefined') { 5 | // eslint-disable-next-line no-undef 6 | const reader = new window.FileReader(); 7 | reader.readAsDataURL(blob); 8 | reader.onloadend = function () { 9 | release(reader.result); 10 | } 11 | } else { 12 | const chunks = []; 13 | blob.stream().on('data', (chunk) => { 14 | chunks.push(chunk.toString('base64')); 15 | }).on('end', () => { 16 | release(`data:${blob.type};base64,${chunks.join('')}`); 17 | }); 18 | } 19 | }); 20 | } 21 | 22 | module.exports = blobImageToURL; -------------------------------------------------------------------------------- /lib/common/utils.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const mkdirp = require('mkdirp'); 3 | const path = require('path'); 4 | const semver = require('semver'); 5 | const npmLatest = require('npm-latest'); 6 | 7 | const packageJson = require('../../package.json'); 8 | 9 | /* 10 | Synchronous read 11 | */ 12 | function readTextFile(filename) { 13 | const nFilename = path.normalize(filename); 14 | return fs.readFileSync(nFilename, 'utf-8'); 15 | } 16 | 17 | /* 18 | Synchronous write 19 | */ 20 | function writeTextFile(filename, text) { 21 | const nFilename = path.normalize(filename); 22 | mkdirp.sync(path.dirname(nFilename)); 23 | fs.writeFileSync(nFilename, text); 24 | } 25 | /* 26 | write a blob 27 | */ 28 | function writeBlobFile(filename, blob) { 29 | const nFilename = path.normalize(filename); 30 | mkdirp.sync(path.dirname(nFilename)); 31 | return new Promise((resolve, reject) => 32 | blob.stream() 33 | .on('error', error => { 34 | if (blob.stream().truncated) 35 | // delete the truncated file 36 | fs.unlinkSync(nFilename); 37 | reject(error); 38 | }) 39 | .pipe(fs.createWriteStream(nFilename)) 40 | .on('error', error => reject(error)) 41 | .on('finish', () => resolve({ path })) 42 | ); 43 | } 44 | 45 | /* 46 | get npm package version information 47 | */ 48 | async function getVersionInfo() { 49 | const versionsInfo = { 50 | localVersion: packageJson.version, 51 | remoteVersion: packageJson.version, 52 | localOutdate: false 53 | }; 54 | 55 | try { 56 | versionsInfo.remoteVersion = (await npmLatest.getLatest('quip-esxport')).version; 57 | // compare to local version 58 | versionsInfo.localOutdate = semver.gt(versionsInfo.remoteVersion, versionsInfo.localVersion); 59 | } catch(e) { 60 | // nothing to do 61 | } 62 | 63 | return versionsInfo; 64 | } 65 | 66 | //await tryCatch(() => quipService.getUser()); 67 | async function tryCatch(func, message) { 68 | return new Promise((resolve) => { 69 | func().then(res => resolve(res)).catch((e) => { 70 | if(message) { console.error(message); } 71 | console.error(e); 72 | resolve(); 73 | }); 74 | }); 75 | } 76 | 77 | function cliBox(message) { 78 | let boxedMessage = `| ${message} |`; 79 | console.log('-'.repeat(boxedMessage.length)); 80 | console.log(boxedMessage); 81 | console.log('-'.repeat(boxedMessage.length)); 82 | } 83 | 84 | module.exports = { 85 | readTextFile, 86 | writeTextFile, 87 | writeBlobFile, 88 | getVersionInfo, 89 | cliBox, 90 | tryCatch 91 | }; -------------------------------------------------------------------------------- /lib/templates/document.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Quip 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations 14 | * under the License. 15 | */ 16 | body { 17 | font-size: 15px; 18 | color: #333; 19 | background: white; 20 | padding: 60px 95px; 21 | max-width: 900px; 22 | margin: 0 auto; 23 | text-rendering: optimizeLegibility; 24 | font-feature-settings: "kern"; 25 | font-kerning: normal; 26 | -moz-font-feature-settings: "kern"; 27 | -webkit-font-feature-settings: "kern"; 28 | } 29 | 30 | /* Headings */ 31 | h1, 32 | h2, 33 | h3, 34 | th { 35 | font-family: Roboto, sans-serif; 36 | font-weight: 700; 37 | margin: 0; 38 | margin-top: 1.25em; 39 | margin-bottom: 0.75em; 40 | } 41 | 42 | h1 { 43 | font-size: 35px; 44 | line-height: 42px; 45 | } 46 | 47 | h1:first-child { 48 | margin-top: 0; 49 | } 50 | 51 | h2 { 52 | font-size: 18px; 53 | line-height: 22px; 54 | } 55 | 56 | h3 { 57 | font-size: 13px; 58 | line-height: 16px; 59 | } 60 | 61 | .capitalize-h3 h3 { 62 | text-transform: uppercase; 63 | } 64 | 65 | /* Body text */ 66 | body, 67 | p, 68 | ul, 69 | ol, 70 | td { 71 | font-family: "Crimson Text", serif; 72 | font-size: 16px; 73 | line-height: 20px; 74 | } 75 | 76 | blockquote, 77 | q { 78 | display: block; 79 | margin: 1em 0; 80 | font-style: italic; 81 | } 82 | 83 | blockquote a, 84 | q a { 85 | text-decoration: underline; 86 | } 87 | 88 | blockquote { 89 | padding-left: 10px; 90 | border-left: 4px solid #a6a6a6; 91 | } 92 | 93 | q { 94 | color: #a6a6a6; 95 | line-height: 40px; 96 | font-size: 24px; 97 | text-align: center; 98 | quotes: none; 99 | } 100 | 101 | q a { 102 | color: #a6a6a6; 103 | } 104 | 105 | code, 106 | pre { 107 | font-family: Consolas, "Liberation Mono", Menlo, "Courier Prime Web", 108 | Courier, monospace; 109 | background: #f2f2f2; 110 | } 111 | 112 | code { 113 | padding: 1px; 114 | margin: 0 -1px; 115 | border-radius: 3px; 116 | } 117 | 118 | pre { 119 | display: block; 120 | line-height: 20px; 121 | text-shadow: 0 1px white; 122 | padding: 5px 5px 5px 30px; 123 | white-space: nowrap; 124 | position: relative; 125 | margin: 1em 0; 126 | } 127 | 128 | pre:before { 129 | content: ""; 130 | position: absolute; 131 | top: 0; 132 | bottom: 0; 133 | left: 15px; 134 | border-left: solid 1px #dadada; 135 | } 136 | 137 | /* Lists */ 138 | div[data-section-style="5"], 139 | div[data-section-style="6"], 140 | div[data-section-style="7"] { 141 | margin: 12px 0; 142 | } 143 | 144 | ul { 145 | padding: 0 0 0 40px; 146 | } 147 | 148 | ul li { 149 | margin-bottom: 0.4em; 150 | } 151 | 152 | /* Bulleted list */ 153 | div[data-section-style="5"] ul { 154 | list-style-type: disc; 155 | } 156 | div[data-section-style="5"] ul ul { 157 | list-style-type: circle; 158 | } 159 | div[data-section-style="5"] ul ul ul { 160 | list-style-type: square; 161 | } 162 | div[data-section-style="5"] ul ul ul ul { 163 | list-style-type: disc; 164 | } 165 | div[data-section-style="5"] ul ul ul ul ul { 166 | list-style-type: circle; 167 | } 168 | div[data-section-style="5"] ul ul ul ul ul ul { 169 | list-style-type: square; 170 | } 171 | 172 | /* Numbered list */ 173 | div[data-section-style="6"] ul { 174 | list-style-type: decimal; 175 | } 176 | div[data-section-style="6"] ul ul { 177 | list-style-type: lower-alpha; 178 | } 179 | div[data-section-style="6"] ul ul ul { 180 | list-style-type: lower-roman; 181 | } 182 | div[data-section-style="6"] ul ul ul ul { 183 | list-style-type: decimal; 184 | } 185 | div[data-section-style="6"] ul ul ul ul ul { 186 | list-style-type: lower-alpha; 187 | } 188 | div[data-section-style="6"] ul ul ul ul ul ul { 189 | list-style-type: lower-roman; 190 | } 191 | 192 | /* Checklist */ 193 | div[data-section-style="7"] ul { 194 | list-style-type: none; 195 | } 196 | 197 | div[data-section-style="7"] ul li:before { 198 | content: "\2610"; 199 | position: absolute; 200 | display: inline; 201 | margin-right: 1.2em; 202 | margin-left: -1.2em; 203 | } 204 | 205 | div[data-section-style="7"] ul li.parent:before { 206 | content: ""; 207 | } 208 | 209 | div[data-section-style="7"] ul li.checked { 210 | text-decoration: line-through; 211 | } 212 | 213 | div[data-section-style="7"] ul li.checked:before { 214 | content: "\2611"; 215 | text-decoration: none; 216 | } 217 | 218 | /* Tables */ 219 | div[data-section-style="8"] { 220 | margin: 12px 0; 221 | } 222 | 223 | table { 224 | border-spacing: 0; 225 | border-collapse: separate; 226 | border: solid 1px #bbb; 227 | table-layout: fixed; 228 | position: relative; 229 | } 230 | 231 | table th, 232 | table td { 233 | padding: 2px 2px 0; 234 | min-width: 1.5em; 235 | word-wrap: break-word; 236 | } 237 | 238 | table th { 239 | border-bottom: 1px solid #c7cbd1; 240 | background: #f2f2f2; 241 | font-weight: bold; 242 | vertical-align: bottom; 243 | color: #3a4449; 244 | text-align: center; 245 | } 246 | 247 | table td { 248 | padding-top: 0; 249 | border-left: 1px solid #c7cbd1; 250 | border-top: 1px solid #c7cbd1; 251 | vertical-align: top; 252 | } 253 | 254 | table td.bold { 255 | font-weight: bold; 256 | } 257 | 258 | table td.italic { 259 | font-style: italic; 260 | } 261 | 262 | table td.underline { 263 | text-decoration: underline; 264 | } 265 | 266 | table td.strikethrough { 267 | text-decoration: line-through; 268 | } 269 | 270 | table td.underline.strikethrough { 271 | text-decoration: underline line-through; 272 | } 273 | 274 | table td:first-child { 275 | border-left: hidden; 276 | } 277 | 278 | table tr:first-child td { 279 | border-top: hidden; 280 | } 281 | 282 | /* Images */ 283 | div[data-section-style="11"] { 284 | margin-top: 20px; 285 | margin-bottom: 20px; 286 | margin-left: auto; 287 | margin-right: auto; 288 | } 289 | 290 | div[data-section-style="11"][data-section-float="0"] { 291 | clear: both; 292 | text-align: center; 293 | } 294 | 295 | div[data-section-style="11"][data-section-float="1"] { 296 | float: left; 297 | clear: left; 298 | margin-right: 20px; 299 | } 300 | 301 | div[data-section-style="11"][data-section-float="2"] { 302 | float: right; 303 | clear: right; 304 | margin-left: 20px; 305 | } 306 | 307 | div[data-section-style="11"] img { 308 | display: block; 309 | max-width: 100%; 310 | height: auto; 311 | margin: auto; 312 | } 313 | 314 | hr { 315 | width: 70px; 316 | margin: 20px auto; 317 | } 318 | 319 | /* Apps */ 320 | div[data-section-style="19"].placeholder { 321 | margin: 0.8em auto; 322 | padding: 4px 0; 323 | display: block; 324 | color: #3d87f5; 325 | text-align: center; 326 | border: 1px solid rgba(41, 182, 242, 0.2); 327 | border-radius: 3px; 328 | background: #e9f8fe; 329 | font-family: Roboto, sans-serif; 330 | } 331 | 332 | div[data-section-style="19"].first-party-element { 333 | margin-bottom: 10px; 334 | background-repeat: no-repeat; 335 | background-size: contain; 336 | } 337 | 338 | div[data-section-style="19"].first-party-element.kanban { 339 | background-image: url("https://quip-cdn.com/nK0hSyhsb4jrLIL2s5Ma-g"); 340 | height: 166px; 341 | } 342 | 343 | div[data-section-style="19"].first-party-element.calendar { 344 | background-image: url("https://quip-cdn.com/OYujqLny03RILxcLIiyERg"); 345 | height: 244px; 346 | } 347 | 348 | div[data-section-style="19"].first-party-element.poll { 349 | background-image: url("https://quip-cdn.com/fbIiFrcKGv__4NB7CBfxoA"); 350 | height: 116px; 351 | } 352 | 353 | div[data-section-style="19"].first-party-element.countdown { 354 | background-image: url("https://quip-cdn.com/3bPhykD2dBei9sSjCWteTQ"); 355 | height: 96px; 356 | } 357 | 358 | div[data-section-style="19"].first-party-element.process_bar { 359 | background-image: url("https://quip-cdn.com/ybQlHnHEIIBLog5rZmYs_w"); 360 | height: 36px; 361 | } 362 | 363 | div[data-section-style="19"].first-party-element.project_tracker { 364 | background-image: url("https://quip-cdn.com/OFQU087b4Mxzz1ZaHwtjXA"); 365 | height: 164px; 366 | } 367 | 368 | div[data-section-style="19"] img { 369 | margin: 0.5em; 370 | } 371 | 372 | div[data-section-style="19"] img.masked-image { 373 | margin: 0; 374 | transform-origin: top left; 375 | } 376 | 377 | div[data-section-style="19"] .image-mask { 378 | position: relative; 379 | overflow: hidden; 380 | } 381 | /* 382 | * Copyright 2019 Quip 383 | * 384 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 385 | * not use this file except in compliance with the License. You may obtain 386 | * a copy of the License at 387 | * 388 | * http://www.apache.org/licenses/LICENSE-2.0 389 | * 390 | * Unless required by applicable law or agreed to in writing, software 391 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 392 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 393 | * License for the specific language governing permissions and limitations 394 | * under the License. 395 | */ 396 | 397 | body { 398 | counter-reset: indent0 indent1 indent2 indent3 indent4 indent5 indent6 399 | indent7 indent8; 400 | } 401 | 402 | /* Numbered list */ 403 | div[data-section-style="6"] { 404 | counter-reset: indent0 indent1 indent2 indent3 indent4 indent5 indent6 405 | indent7 indent8; 406 | } 407 | div[data-section-style="6"].list-numbering-continue { 408 | counter-reset: none; 409 | } 410 | div[data-section-style="6"].list-numbering-restart-at { 411 | counter-reset: indent0 var(--indent0) indent1 indent2 indent3 indent4 412 | indent5 indent6 indent7 indent8; 413 | } 414 | div[data-section-style="6"] ul { 415 | /* indent0 is not reset since it is shared across the div elements */ 416 | list-style-type: none !important; 417 | } 418 | div[data-section-style="6"] ul ul { 419 | counter-reset: indent1; 420 | } 421 | div[data-section-style="6"] ul ul ul { 422 | counter-reset: indent2; 423 | } 424 | div[data-section-style="6"] ul ul ul ul { 425 | counter-reset: indent3; 426 | } 427 | div[data-section-style="6"] ul ul ul ul ul { 428 | counter-reset: indent4; 429 | } 430 | div[data-section-style="6"] ul ul ul ul ul ul { 431 | counter-reset: indent5; 432 | } 433 | div[data-section-style="6"] ul li { 434 | counter-increment: indent0; 435 | } 436 | div[data-section-style="6"] ul ul li { 437 | counter-increment: indent1; 438 | } 439 | div[data-section-style="6"] ul ul ul li { 440 | counter-increment: indent2; 441 | } 442 | div[data-section-style="6"] ul ul ul ul li { 443 | counter-increment: indent3; 444 | } 445 | div[data-section-style="6"] ul ul ul ul ul li { 446 | counter-increment: indent4; 447 | } 448 | div[data-section-style="6"] ul ul ul ul ul ul li { 449 | counter-increment: indent5; 450 | } 451 | div[data-section-style="6"] ul li:before { 452 | content: counter(indent0, decimal) ". "; 453 | } 454 | div[data-section-style="6"] ul ul li:before { 455 | content: counter(indent1, lower-alpha) ". "; 456 | } 457 | div[data-section-style="6"] ul ul ul li:before { 458 | content: counter(indent2, lower-roman) ". "; 459 | } 460 | div[data-section-style="6"] ul ul ul ul li:before { 461 | content: counter(indent3, decimal) ". "; 462 | } 463 | div[data-section-style="6"] ul ul ul ul ul li:before { 464 | content: counter(indent4, lower-alpha) ". "; 465 | } 466 | div[data-section-style="6"] ul ul ul ul ul ul li:before { 467 | content: counter(indent5, lower-roman) ". "; 468 | } 469 | h3 { 470 | text-transform: uppercase; 471 | } 472 | div[data-section-style="7"] ul li.parent { 473 | font-weight: bold; 474 | } -------------------------------------------------------------------------------- /lib/templates/document.ejs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <%= title %> 6 | 7 | 8 | <%if (stylesheet_path.length > 0) { %> 9 | 10 | <%}%> 11 | 12 | <%if (embedded_stylesheet.length > 0) { %> 13 | 14 | <%}%> 15 | 33 | 34 | 35 |
36 | <%- body %> 37 |
38 | 39 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "quip-export", 3 | "version": "2.3.2", 4 | "description": "Export all folders and documents from Quip", 5 | "main": "index.js", 6 | "jest": { 7 | "verbose": true 8 | }, 9 | "scripts": { 10 | "lint": "eslint \"**/*.js\"", 11 | "test": "jest" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/sonnenkern/quip-export.git" 16 | }, 17 | "keywords": [ 18 | "quip", 19 | "quip-export", 20 | "mass-export", 21 | "export", 22 | "backup", 23 | "quip-backup", 24 | "documents", 25 | "files", 26 | "blob", 27 | "blobs" 28 | ], 29 | "engines": { 30 | "node": ">=10.16.1" 31 | }, 32 | "bin": { 33 | "quip-export": "./quip-export.js" 34 | }, 35 | "author": "Alexander Fleming ", 36 | "license": "MIT", 37 | "bugs": { 38 | "url": "https://github.com/sonnenkern/quip-export/issues" 39 | }, 40 | "homepage": "https://github.com/sonnenkern/quip-export#readme", 41 | "dependencies": { 42 | "cli-progress": "^3.6.0", 43 | "cli-spinner": "^0.2.10", 44 | "colors": "^1.4.0", 45 | "command-line-args": "^5.1.1", 46 | "command-line-usage": "^6.0.2", 47 | "ejs": "^3.0.1", 48 | "jszip": "^3.2.2", 49 | "mime": "^2.4.4", 50 | "mkdirp": "^1.0.3", 51 | "moment": "^2.26.0", 52 | "node-fetch": "^2.6.0", 53 | "npm-latest": "^2.0.0", 54 | "pino": "^5.16.0", 55 | "pino-pretty": "^3.5.0", 56 | "sanitize-filename": "^1.6.3", 57 | "semver": "^7.1.2" 58 | }, 59 | "devDependencies": { 60 | "eslint": "^6.5.1", 61 | "eslint-config-aenondynamics": "^0.2.0", 62 | "jest": "^25.1.0" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /public/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonnenkern/quip-export/8b9a9b6121330881d9291851215dfc850d28ee32/public/demo.gif -------------------------------------------------------------------------------- /public/example-anim.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonnenkern/quip-export/8b9a9b6121330881d9291851215dfc850d28ee32/public/example-anim.gif -------------------------------------------------------------------------------- /public/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonnenkern/quip-export/8b9a9b6121330881d9291851215dfc850d28ee32/public/example.png -------------------------------------------------------------------------------- /quip-export.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict'; 4 | 5 | const {App} = require('./app'); 6 | (new App()).main(); --------------------------------------------------------------------------------