├── .nvmrc ├── .npmignore ├── .prettierrc.json ├── babel.config.js ├── jest.config.js ├── __test-package__ ├── test-import.cjs ├── test-require.cjs ├── test-import.mjs ├── functions.js ├── test-creds.mjs └── index.js ├── prepublishBuild.js ├── .github └── workflows │ ├── deploy.yml │ └── verify.yml ├── __tests__ ├── index.test.js ├── load.test.js ├── node_modules │ └── test │ │ └── mocks.js └── extract-transform.test.js ├── license.md ├── eslint.config.js ├── index.js ├── .gitignore ├── package.json ├── lib ├── load.js └── extract-transform.js └── readme.md /.nvmrc: -------------------------------------------------------------------------------- 1 | 20 -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | __tests__ 2 | __test-package__ 3 | .github 4 | coverage 5 | private 6 | tmp -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "none", 4 | "arrowParens": "avoid" 5 | } 6 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | [ 4 | '@babel/preset-env', { 5 | targets: { 6 | node: '18' 7 | } 8 | } 9 | ] 10 | ] 11 | }; -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | collectCoverage: true, 3 | coverageDirectory: 'coverage', 4 | verbose: true, 5 | testEnvironment: 'node', 6 | testPathIgnorePatterns: [ 7 | '/node_modules/', 8 | '/tmp/' 9 | ] 10 | }; -------------------------------------------------------------------------------- /__test-package__/test-import.cjs: -------------------------------------------------------------------------------- 1 | const { testFn } = require('./functions'); 2 | 3 | (async function test () { 4 | const { default: gdf } = await import('package'); 5 | await testFn(gdf, 0); 6 | 7 | const { googleDriveFolder } = await import('package'); 8 | await testFn(googleDriveFolder, 1); 9 | })(); -------------------------------------------------------------------------------- /__test-package__/test-require.cjs: -------------------------------------------------------------------------------- 1 | const { testFn } = require('./functions'); 2 | const { default: gdf } = require('package'); 3 | const { googleDriveFolder } = require('package'); 4 | 5 | const functions = [ 6 | gdf, 7 | googleDriveFolder 8 | ]; 9 | 10 | (async function test () { 11 | for (let i = 0; i < functions.length; i++) { 12 | await testFn(functions[i], i); 13 | } 14 | })(); 15 | -------------------------------------------------------------------------------- /__test-package__/test-import.mjs: -------------------------------------------------------------------------------- 1 | import { testFn } from './functions.js'; 2 | import gdf from 'package'; 3 | import { googleDriveFolder } from 'package'; 4 | import * as gdfSpace from 'package'; 5 | 6 | const functions = [ 7 | gdf, 8 | googleDriveFolder, 9 | gdfSpace.default, 10 | gdfSpace.googleDriveFolder 11 | ]; 12 | 13 | (async function test () { 14 | for (let i = 0; i < functions.length; i++) { 15 | await testFn(functions[i], i); 16 | } 17 | })(); -------------------------------------------------------------------------------- /prepublishBuild.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Perform prepublishOnly build actions. 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | const fs = require('node:fs/promises'); 8 | 9 | (async function makeModuleIndex () { 10 | const indexJs = './index.js'; 11 | const indexMjs = './dist/index.mjs'; 12 | const contents = await fs.readFile(indexJs, { encoding: 'utf8' }); 13 | await fs.writeFile( 14 | indexMjs, 15 | contents 16 | ); 17 | }()); 18 | -------------------------------------------------------------------------------- /__test-package__/functions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * test functions for the test suite 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | const assert = require('node:assert'); 8 | 9 | async function testFn (fn, i) { 10 | console.log(`=== testing ${fn.name}:${i} ===`); 11 | assert(typeof fn === 'function'); 12 | let threw = false; 13 | try { 14 | await fn(); 15 | } catch (err) { 16 | assert(err instanceof Error); 17 | assert(/default credentials/i.test(err)); 18 | threw = true; 19 | } 20 | assert(threw, 'default should have thrown default credentials error'); 21 | } 22 | 23 | module.exports = { 24 | testFn 25 | }; -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy 2 | on: 3 | push: 4 | branches: [ master ] 5 | 6 | jobs: 7 | deploy: 8 | runs-on: ubuntu-24.04 9 | permissions: 10 | contents: read 11 | id-token: write 12 | steps: 13 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # 6.0.1 14 | - uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # 6.1.0 15 | with: 16 | node-version: '24.x' 17 | registry-url: 'https://registry.npmjs.org' 18 | - name: Update npm 19 | run: npm install -g npm@latest # ensure npm 11.5.1 or later 20 | - run: npm ci 21 | - name: Verify 22 | run: npm run lint && npm test 23 | - name: Publish 24 | if: ${{ success() }} 25 | run: npm publish --access public -------------------------------------------------------------------------------- /__tests__/index.test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * test index functions. 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | /* eslint-env jest */ 8 | const { 9 | mockStream, mockIndex, unmockIndex 10 | } = require('test/mocks'); 11 | 12 | require('@babel/register'); 13 | 14 | describe('index', () => { 15 | let indexModule; 16 | 17 | beforeAll(() => { 18 | mockIndex(jest); 19 | indexModule = require('../index.js'); 20 | }); 21 | 22 | afterAll(() => { 23 | unmockIndex(jest); 24 | }); 25 | 26 | test('should return stream', () => { 27 | // it does no arg checking 28 | return indexModule.default({}).then(result => { 29 | expect(result).toBeDefined(); 30 | expect(result).toEqual(mockStream); 31 | }); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /.github/workflows/verify.yml: -------------------------------------------------------------------------------- 1 | name: Verify 2 | on: 3 | pull_request: 4 | branches: [ master ] 5 | 6 | jobs: 7 | verify: 8 | 9 | runs-on: ubuntu-24.04 10 | 11 | strategy: 12 | matrix: 13 | node-version: [20.x, 22.x, 24.x] 14 | # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ 15 | 16 | steps: 17 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # 6.0.1 18 | - name: Use Node.js ${{ matrix.node-version }} 19 | uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # 6.1.0 20 | with: 21 | node-version: ${{ matrix.node-version }} 22 | - run: npm ci 23 | - name: Run Lint and Test 24 | run: npm run lint && npm test 25 | - name: Coverage Upload 26 | if: ${{ success() }} 27 | uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # 2.3.6 28 | with: 29 | github-token: ${{ secrets.GITHUB_TOKEN }} 30 | path-to-lcov: ./coverage/lcov.info -------------------------------------------------------------------------------- /license.md: -------------------------------------------------------------------------------- 1 | Copyright 2021 - 2025, Alex Grant, LocalNerve, LLC 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | const js = require('@eslint/js'); 2 | const globals = require('globals'); 3 | const jest = require('eslint-plugin-jest'); 4 | 5 | module.exports = [{ 6 | ignores: [ 7 | '__tests__/lib/**', 8 | 'coverage/**', 9 | 'dist/**', 10 | 'private/**', 11 | '**/tmp/**' 12 | ] 13 | }, { 14 | files: [ 15 | 'lib/**' 16 | ], 17 | rules: { 18 | ...js.configs.recommended.rules 19 | }, 20 | languageOptions: { 21 | sourceType: 'module', 22 | globals: { 23 | ...globals.node 24 | } 25 | } 26 | }, { 27 | files: [ 28 | '__tests__/**', 29 | '__test-package__/**/*.{js,cjs}' 30 | ], 31 | ...jest.configs['flat/recommended'], 32 | rules: { 33 | ...jest.configs['flat/recommended'].rules, 34 | 'jest/no-done-callback': 'off' 35 | }, 36 | languageOptions: { 37 | sourceType: 'commonjs', 38 | globals: { 39 | ...globals.node 40 | } 41 | } 42 | }, { 43 | files: [ 44 | '__test-package__/**/*.mjs' 45 | ], 46 | ...jest.configs['flat/recommended'], 47 | languageOptions: { 48 | sourceType: 'module', 49 | globals: { 50 | ...globals.node 51 | } 52 | } 53 | }]; -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Google Drive Folder. 3 | * 4 | * Download a google drive folder and stream it out. 5 | * Convert markdown and json. passthru the rest. 6 | * Write files to local directory if specified. 7 | * 8 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 9 | * Licensed under the MIT license. 10 | */ 11 | /* eslint-env node */ 12 | import { extractTransform } from './lib/extract-transform.js'; 13 | 14 | /** 15 | * Google Drive Folder Extract, Transform, and Load. 16 | * Environment variable SVC_ACCT_CREDENTIALS is valid path to google credential file. 17 | * @env SVC_ACCT_CREDENTIALS 18 | * Resolves to Readable object stream of objects 19 | * { input, output, converted }. 20 | * 21 | * @param {String} folderId - The folderId of the drive to read from. 22 | * @param {String} userId - The userId of the owner of the drive. 23 | * @param {Object} [options] - Additional options. 24 | * @param {Array} [options.scopes] - The scopes required by the account owner, defaults to `drive.readonly`. 25 | * @param {String} [options.fileQuery] - A query to filter the selection of files. 26 | * @see https://developers.google.com/drive/api/v3/ref-search-terms 27 | * @param {Object} [options.exportMimeMap] - The mime-types to use for export conversions. 28 | * @see https://developers.google.com/drive/api/v3/ref-export-formats 29 | * @param {String} [options.outputDirectory] - The path to the output folder. If defined, writes to directory during object stream. 30 | * @param {Function} [options.transformer] - Function receives downloaded input, returns Promise resolves to output data. 31 | * @returns {Promise} ReadableStream, unless outputDirectory was supplied. 32 | */ 33 | export default async function googleDriveFolder(folderId, userId, options = {}) { 34 | return await extractTransform(folderId, userId, options); 35 | } 36 | 37 | export { googleDriveFolder }; 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Node template 3 | # Logs 4 | /logs 5 | *.log 6 | npm-debug.log* 7 | yarn-debug.log* 8 | yarn-error.log* 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | *.pid.lock 15 | 16 | # Directory for instrumented libs generated by jscoverage/JSCover 17 | lib-cov 18 | 19 | # Coverage directory used by tools like istanbul 20 | coverage 21 | 22 | # nyc test coverage 23 | .nyc_output 24 | 25 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 26 | .grunt 27 | 28 | # Bower dependency directory (https://bower.io/) 29 | bower_components 30 | 31 | # node-waf configuration 32 | .lock-wscript 33 | 34 | # Compiled binary addons (https://nodejs.org/api/addons.html) 35 | build/Release 36 | 37 | # Dependency directories 38 | /node_modules 39 | 40 | jspm_packages/ 41 | 42 | # TypeScript v1 declaration files 43 | typings/ 44 | 45 | # Optional npm cache directory 46 | .npm 47 | 48 | # Optional eslint cache 49 | .eslintcache 50 | 51 | # Optional REPL history 52 | .node_repl_history 53 | 54 | # Output of 'npm pack' 55 | *.tgz 56 | 57 | # Yarn Integrity file 58 | .yarn-integrity 59 | 60 | # dotenv environment variables file 61 | .env 62 | 63 | # parcel-bundler cache (https://parceljs.org/) 64 | .cache 65 | 66 | # next.js build output 67 | .next 68 | 69 | # nuxt.js build output 70 | .nuxt 71 | 72 | # Nuxt generate 73 | dist 74 | 75 | # vuepress build output 76 | .vuepress/dist 77 | 78 | # Serverless directories 79 | .serverless 80 | 81 | # IDE / Editor 82 | .idea 83 | 84 | # Service worker 85 | sw.* 86 | 87 | # macOS 88 | .DS_Store 89 | 90 | # test symlink 91 | __tests__/lib 92 | 93 | # test-package node_modules 94 | __test-package__/node_modules 95 | 96 | # no tmp 97 | tmp 98 | 99 | # no private 100 | private -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@localnerve/google-drive-folder", 3 | "version": "16.0.0", 4 | "description": "Stream files from a Google Drive folder", 5 | "main": "dist/index.js", 6 | "exports": { 7 | "import": "./dist/index.mjs", 8 | "require": "./dist/index.js", 9 | "default": "./dist/index.js" 10 | }, 11 | "scripts": { 12 | "lint": "eslint .", 13 | "jest": "jest", 14 | "test-package": "node ./__test-package__/index.js", 15 | "pretest": "node -e 'try{require(\"fs\").symlinkSync(\"../lib\", \"./__tests__/lib\");}catch(e){}'", 16 | "test": "jest && npm run test-package", 17 | "pretest-only": "npm run pretest", 18 | "test-only": "jest", 19 | "test-only:debug": "node --inspect-brk node_modules/.bin/jest --testTimeout=300000", 20 | "transpile": "rimraf ./dist && babel --out-dir ./dist index.js && babel --out-dir ./dist/lib ./lib", 21 | "posttranspile": "node ./prepublishBuild.js", 22 | "prepublishOnly": "npm run transpile" 23 | }, 24 | "repository": { 25 | "type": "git", 26 | "url": "git+https://github.com/localnerve/google-drive-folder.git" 27 | }, 28 | "author": "Alex Grant (https://www.localnerve.com)", 29 | "maintainers": [ 30 | { 31 | "name": "localnerve", 32 | "email": "alex@localnerve.com", 33 | "url": "https://github.com/localnerve" 34 | } 35 | ], 36 | "license": "MIT", 37 | "bugs": { 38 | "url": "https://github.com/localnerve/google-drive-folder/issues" 39 | }, 40 | "homepage": "https://github.com/localnerve/google-drive-folder#readme", 41 | "devDependencies": { 42 | "@babel/cli": "^7.28.3", 43 | "@babel/preset-env": "^7.28.5", 44 | "@babel/register": "^7.28.3", 45 | "@eslint/js": "^9.39.2", 46 | "eslint": "^9.39.2", 47 | "eslint-plugin-jest": "^29.5.0", 48 | "globals": "^16.5.0", 49 | "jest": "^30.2.0", 50 | "prettier": "^3.7.4", 51 | "micromark": "^4.0.2", 52 | "micromark-extension-directive": "^4.0.0", 53 | "rimraf": "^6.1.2", 54 | "tar": "^7.5.2", 55 | "glob": "^13.0.0" 56 | }, 57 | "dependencies": { 58 | "@googleapis/drive": "^20.0.0" 59 | }, 60 | "engines": { 61 | "node": ">=20" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /lib/load.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Write the processed drive content to a destination. 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | /* eslint-env node */ 8 | import * as fs from 'node:fs/promises'; 9 | import * as path from 'node:path'; 10 | import { Transform } from 'node:stream'; 11 | 12 | /** 13 | * ObjectTransformStream 14 | * A Transform stream that converts data. 15 | * 16 | * The tranformer function takes an input object { data, name, ext } 17 | * and returns a promise that resolves to 18 | * { 19 | * input: { data, name, ext } 20 | * output: { data, name, ext } 21 | * converted 22 | * } 23 | */ 24 | export class ObjectTransformStream extends Transform { 25 | #customError = null; 26 | 27 | constructor(opts) { 28 | const options = { ...opts, ...{ objectMode: true } }; 29 | super(options); 30 | this.transformer = options.__transformer; 31 | } 32 | 33 | _transform(input, enc, cb) { 34 | this.transformer(input).then(data => { 35 | this.push(data); 36 | cb(); 37 | }).catch(cb); 38 | } 39 | 40 | get customError () { 41 | return this.#customError; 42 | } 43 | 44 | set customError (error) { 45 | this.#customError = error; 46 | } 47 | } 48 | 49 | /** 50 | * Create the transform stream 51 | * 52 | * @param {Function} transformer - function signature `Promise function ({ data, name, ext }). 53 | * Returned promise resolves to { input: {data, name, ext}, output: {data, name, ext }, converted }`. 54 | * @returns {ObjectTransformStream} The object transform stream. 55 | */ 56 | export function createObjectStream(transformer) { 57 | return new ObjectTransformStream({ 58 | __transformer: transformer 59 | }); 60 | } 61 | 62 | /** 63 | * Write the converted data objects contents to a directory. 64 | * 65 | * @param {String} outputDir - The output directory to put the files in. 66 | * @param {Function} handleError - handle output errors. 67 | * @param {Object} data - Completed data object. 68 | * @returns {Promise} Resolves to array of results from writing files to the directory. 69 | */ 70 | export async function writeToDirectory(outputDir, handleError, data) { 71 | const outputPath = path.join(outputDir, `${data.output.name}${data.output.ext}`); 72 | return fs.writeFile(outputPath, data.output.data) 73 | .catch(e => { 74 | handleError(e, `Failed to write file '${outputPath}'`); 75 | }); 76 | } 77 | 78 | -------------------------------------------------------------------------------- /__tests__/load.test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * test load functions. 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | /* eslint-env jest */ 8 | const { mockLoad, unmockLoad, emulateError } = require('test/mocks'); 9 | const path = require('path'); 10 | require('@babel/register'); 11 | 12 | describe('load', () => { 13 | let moduleLoad; 14 | const dir = 'testDir'; 15 | const name = 'bing'; 16 | const ext = '.bang'; 17 | const data = 'muchmuchdata'; 18 | 19 | 20 | beforeAll(() => { 21 | mockLoad(jest); 22 | moduleLoad = require('../lib/load'); 23 | }); 24 | 25 | afterAll(() => { 26 | unmockLoad(jest); 27 | }); 28 | 29 | test('createObjectStream returns ObjectTransformStream', done => { 30 | const stream = moduleLoad.createObjectStream(() => {}); 31 | expect(stream).toBeDefined(); 32 | expect(stream).toBeInstanceOf(moduleLoad.ObjectTransformStream); 33 | done(); 34 | }); 35 | 36 | test('createObjectStream composes with transformer', done => { 37 | const mockData = 'mockData'; 38 | const mockTransformer = jest.fn( 39 | passedData => { 40 | expect(passedData).toEqual(mockData); 41 | return Promise.resolve({}); 42 | } 43 | ); 44 | 45 | const stream = moduleLoad.createObjectStream(mockTransformer); 46 | stream._transform(mockData, '', () => {}); 47 | 48 | expect(mockTransformer.mock.calls.length).toEqual(1); 49 | done(); 50 | }); 51 | 52 | test('writeToDirectory calls async writeFile', () => { 53 | return moduleLoad.writeToDirectory(dir, ()=> {}, { 54 | output: { name, ext, data } 55 | }).then(result => { 56 | expect(result).toBeDefined(); 57 | expect(result.path).toEqual(path.join(dir, `${name}${ext}`)); 58 | expect(result.data).toEqual(data); 59 | }); 60 | }); 61 | 62 | test('writeToDirectory fails as expected', done => { 63 | let called = false; 64 | 65 | function handleError(e, msg) { 66 | called = true; 67 | expect(e).toEqual(emulateError); 68 | expect(msg).toContain(path.join(dir, `${name}${ext}`)); 69 | done(); 70 | } 71 | 72 | moduleLoad.writeToDirectory(dir, handleError, { 73 | output: { name, ext, data: emulateError.message } 74 | }); 75 | 76 | setTimeout(() => { 77 | if (!called) { 78 | done(new Error('Did not call error handler in time')); 79 | } 80 | }, 200); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /__test-package__/test-creds.mjs: -------------------------------------------------------------------------------- 1 | import googleDriveFolder from 'package'; 2 | import fs from 'node:fs/promises'; 3 | import path from 'node:path'; 4 | import util from 'node:util'; 5 | import { micromark } from 'micromark'; 6 | import { directive, directiveHtml } from 'micromark-extension-directive'; 7 | 8 | /** 9 | * Setup the private prerequisites for the environment. 10 | * If this is not available in the test environment, just skip. 11 | * 12 | * @returns {Object} containing the folderId and userId for the google drive. 13 | */ 14 | async function setupPrivateEnv () { 15 | const svc_acct_path = path.resolve('../../localnerve-com/private/stage-localnerve-com-3bcd66ab3f20.json'); 16 | const env_file_path = path.resolve('../../localnerve-com/private/stage-env-func.json'); 17 | 18 | let exists = false; 19 | try { 20 | await fs.access(env_file_path); 21 | await fs.access(svc_acct_path); 22 | exists = true; 23 | } catch {} 24 | 25 | if (exists) { 26 | const funcEnvJson = await fs.readFile(env_file_path); 27 | const funcEnv = JSON.parse(funcEnvJson); 28 | 29 | const folderId = funcEnv.content.folder; 30 | const userId = funcEnv.content.user; 31 | 32 | process.env.SVC_ACCT_CREDENTIALS = svc_acct_path; 33 | return { 34 | folderId, 35 | userId 36 | }; 37 | } 38 | 39 | return { 40 | folderId: 'SKIP', 41 | userId: 'SKIP' 42 | }; 43 | } 44 | 45 | /** 46 | * Micromark custom 'json' container directive handler. 47 | * 48 | * @param {Micromark.Directive} d - The micromark directive structure. 49 | */ 50 | function jsonDirective (d) { 51 | if (!(d.type === 'containerDirective' && d.name === 'json')) return false; 52 | 53 | this.raw(d.content.replace(/"/g, '"')); 54 | } 55 | 56 | /** 57 | * Convert raw data to content based on extension. 58 | * This function only knows about .md and .json. 59 | * Everything else is passed through, kept in { input }, converted is false. 60 | * 61 | * @param {Object} input - The source data. 62 | * @param {String} input.data - The source content. 63 | * @param {String} input.name - The source file name. 64 | * @param {String} input.ext - The source file extension. 65 | * @returns {Promise} Resolves to an Object { input, output, converted }. 66 | */ 67 | function transformer (input) { 68 | return new Promise(resolve => { 69 | if (input.ext === '.md') { 70 | const res = micromark(input.data, { 71 | allowDangerousHtml: true, 72 | allowDangerousProtocol: true, 73 | extensions: [directive()], 74 | htmlExtensions: [directiveHtml({json: jsonDirective})] 75 | }); 76 | resolve({ 77 | input, 78 | output: { 79 | name: input.name, 80 | ext: '.html', 81 | data: String(res) 82 | }, 83 | converted: true 84 | }); 85 | } else if (input.ext === '.json') { 86 | let inputData = input.data; 87 | // Catches EFBBBF (UTF-8 BOM) because the buffer-to-string 88 | // conversion translates it to FEFF (UTF-16 BOM) 89 | if (inputData.charCodeAt(0) === 0xfeff) { 90 | inputData = inputData.slice(1); 91 | } 92 | resolve({ 93 | input, 94 | output: { 95 | name: input.name, 96 | ext: '.json', 97 | data: JSON.stringify(JSON.parse(inputData)) 98 | }, 99 | converted: true 100 | }); 101 | } else { 102 | resolve({ 103 | input, 104 | output: input, 105 | converted: false 106 | }); 107 | } 108 | }); 109 | } 110 | 111 | (async function () { 112 | try { 113 | const { folderId, userId } = await setupPrivateEnv(); 114 | if (userId === 'SKIP') { 115 | console.log('Private environment unavailable, skipping...'); 116 | } else { 117 | const outputDirectory = path.resolve('./tmp/test-output'); 118 | await fs.mkdir(outputDirectory, { recursive: true }); 119 | const stream = await googleDriveFolder(folderId, userId, { 120 | outputDirectory, 121 | exportMimeMap: { 122 | 'application/vnd.google-apps.document': 'text/plain' 123 | }, 124 | transformer 125 | }); 126 | stream.on('data', data => { 127 | console.log('data: ', `${util.inspect(data, { 128 | maxStringLength: 10 129 | })}...`); 130 | }); 131 | stream.on('end', () => { 132 | console.log('stream ended'); 133 | }); 134 | stream.on('error', e => { 135 | console.error('stream error', e); 136 | }); 137 | } 138 | } catch (e) { 139 | console.error(e); 140 | } 141 | }()); 142 | -------------------------------------------------------------------------------- /__test-package__/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Setup the installed package and run different tests loading the package in different ways. 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | const { spawn } = require('node:child_process'); 8 | const fs = require('node:fs'); 9 | const path = require('node:path'); 10 | const tar = require('tar'); 11 | const { globSync } = require('glob'); 12 | const thisDirname = __dirname; 13 | const localNodeModulesPath = path.join(thisDirname, 'node_modules'); 14 | 15 | /** 16 | * Run the package tests 17 | */ 18 | async function runTests () { 19 | console.log('--- Run tests ---'); 20 | 21 | const testGlob = path.join(thisDirname, 'test-*'); 22 | const errors = []; 23 | const testFiles = globSync(testGlob); 24 | let result; 25 | 26 | for (const testFile of testFiles) { 27 | const testFileShort = path.basename(testFile); 28 | 29 | console.log(`=== start ${testFileShort} ===`); 30 | console.warn(`GCP check will fail and emit a MetadataLookupWarning. It\'s fine. 31 | https://github.com/googleapis/google-auth-library-nodejs/blob/main/src/auth/googleauth.ts#L475 32 | https://github.com/googleapis/gcp-metadata/blob/main/src/index.ts#L398`); 33 | 34 | await new Promise((resolve, reject) => { 35 | // const testProc = spawn('node', ['--trace-warnings', testFile], { 36 | const testProc = spawn('node', [testFile], { 37 | cwd: thisDirname, 38 | stdio: 'inherit', 39 | timeout: 40000 40 | }); 41 | testProc.on('error', reject); 42 | testProc.on('close', code => { 43 | if (code !== 0) { 44 | const msg = `${testFileShort} failed, ${result.status}`; 45 | console.error(msg); 46 | errors.push(msg); 47 | console.log(`=== ${testFileShort} FAIL ===`); 48 | } else { 49 | console.log(`=== ${testFileShort} OK ===`); 50 | } 51 | resolve(); 52 | }); 53 | }); 54 | } 55 | 56 | if (errors.length > 0) { 57 | throw new Error(errors.join('\n')); 58 | } 59 | } 60 | 61 | /** 62 | * Run `npm run transpile` 63 | * @returns Promise that resolves on success, rejects with msg on error 64 | */ 65 | function transpile () { 66 | console.log('--- transpile ---'); 67 | return new Promise((resolve, reject) => { 68 | const transpile = spawn('npm', ['run', 'transpile']); 69 | transpile.on('close', transpileCode => { 70 | if (transpileCode === 0) { 71 | resolve(); 72 | } else { 73 | reject(`npm run transpile failed: ${transpileCode}`); 74 | } 75 | }); 76 | }); 77 | } 78 | 79 | /** 80 | * Run `npm pack` 81 | * @returns Promise that resolves on success, rejects with msg on error 82 | */ 83 | function pack () { 84 | console.log('--- package ---'); 85 | return new Promise((resolve, reject) => { 86 | const pack = spawn('npm', ['pack']); 87 | pack.on('close', packCode => { 88 | if (packCode === 0) { 89 | resolve(); 90 | } else { 91 | reject(`npm pack failed: ${packCode}`); 92 | } 93 | }); 94 | }); 95 | } 96 | 97 | /** 98 | * Run `npm i` 99 | * @returns Promise that resolves on success, rejects with msg on error 100 | */ 101 | function install () { 102 | console.log('--- install ---'); 103 | return new Promise((resolve, reject) => { 104 | const install = spawn('npm', ['i', '--production=true'], { 105 | cwd: `${localNodeModulesPath}/package` 106 | }); 107 | install.on('close', installCode => { 108 | if (installCode === 0) { 109 | resolve(); 110 | } else { 111 | reject(`failed npm install: ${installCode}`); 112 | } 113 | }); 114 | }); 115 | } 116 | 117 | /** 118 | * Extract the npm packed tar to local node_modules. 119 | */ 120 | function extractTarPackage () { 121 | console.log('--- extract ---'); 122 | 123 | const tarGlob = 'localnerve-google-drive-folder*'; 124 | const tarFileName = globSync(tarGlob)[0]; 125 | 126 | // clean any existing local node_modules 127 | fs.rmSync(localNodeModulesPath, { force: true, recursive: true }); 128 | fs.mkdirSync(localNodeModulesPath, { recursive: true }); 129 | 130 | // unpack it to local `node_modules` 131 | tar.x({ 132 | C: localNodeModulesPath, 133 | file: tarFileName, 134 | sync: true 135 | }); 136 | 137 | // remove old tar file 138 | fs.rmSync(tarFileName); 139 | 140 | return Promise.resolve(); 141 | } 142 | 143 | console.log('----------------------------------------------------'); 144 | console.log('--- Test package install, loading, and integrity ---'); 145 | console.log('----------------------------------------------------'); 146 | 147 | transpile() 148 | .then(pack) 149 | .then(extractTarPackage) 150 | .then(install) 151 | .then(runTests); 152 | -------------------------------------------------------------------------------- /__tests__/node_modules/test/mocks.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Mocks for the test suite. 3 | */ 4 | 5 | function mockDependencies(jest) { 6 | jest.mock('@googleapis/drive'); 7 | } 8 | 9 | function unmockDependencies(jest) { 10 | jest.unmock('@googleapis/drive'); 11 | } 12 | 13 | // createReadableStream mock return value readableStream 14 | const mockReadableStream = 'readable'; 15 | const mockWriteResult = 'written'; 16 | const mockTransformResult = 'transformed'; 17 | const mockStream = 'mockStream'; 18 | const emulateError = new Error('emulateError'); 19 | class MockTransform { 20 | push() {} 21 | end() {} 22 | }; 23 | 24 | const mockExtractTransformImport = jest.fn( 25 | () => Promise.resolve(mockStream) 26 | ); 27 | 28 | function mockIndex(jest) { 29 | mockDependencies(jest); 30 | jest.mock('./lib/extract-transform.js', () => ({ 31 | __esModule: true, 32 | default: mockExtractTransformImport, 33 | extractTransform: mockExtractTransformImport, 34 | iAmAMock: () => {} 35 | })); 36 | } 37 | 38 | function unmockIndex(jest) { 39 | jest.unmock('./lib/extract-transform.js'); 40 | unmockDependencies(jest); 41 | } 42 | 43 | function mockFs(jest) { 44 | const mockWriteFile = jest.fn((path, data) => { 45 | if (data === emulateError.message) { 46 | return Promise.reject(emulateError); 47 | } 48 | return Promise.resolve({ 49 | path, 50 | data 51 | }); 52 | }); 53 | 54 | jest.mock('node:fs/promises', () => ({ 55 | writeFile: mockWriteFile 56 | })); 57 | 58 | return mockWriteFile; 59 | } 60 | 61 | function unmockFs(jest) { 62 | jest.unmock('node:fs/promises'); 63 | } 64 | 65 | function mockLoad(jest) { 66 | mockFs(jest); 67 | jest.mock('node:stream', () => ({ 68 | Transform: MockTransform 69 | })); 70 | } 71 | 72 | function unmockLoad(jest) { 73 | jest.unmock('node:stream'); 74 | unmockFs(jest); 75 | } 76 | 77 | const mockTestChunk = 'myspecialtestchunk'; 78 | 79 | function mockOn(name, cb) { 80 | if (name === 'data' && this.on.writeError) { 81 | cb(Buffer.from(emulateError.message)); 82 | return this; 83 | } 84 | 85 | if (name ==='data' && this.on.skipError) { 86 | cb(Buffer.from(mockTestChunk)); 87 | } else if (name === 'end' && this.on.skipError) { 88 | cb(); 89 | } else if (name === 'error' && !this.on.skipError) { 90 | cb(emulateError); 91 | } 92 | return this; 93 | } 94 | 95 | function GoogleAuth(...args) { 96 | if (GoogleAuth.mock) { 97 | GoogleAuth.mock(args); 98 | } 99 | } 100 | 101 | const defaultMockFiles = [{ 102 | id: '101010', 103 | name: 'test-file.passthru' 104 | }]; 105 | 106 | const _mockFiles = []; 107 | 108 | function driveList () { 109 | if (driveList.error) { 110 | return Promise.reject(emulateError); 111 | } 112 | 113 | return Promise.resolve({ 114 | data: { 115 | get files() { 116 | let theFiles = _mockFiles.length > 0 ? _mockFiles : defaultMockFiles; 117 | if (driveList.filter) { 118 | theFiles = driveList.filter(theFiles); 119 | } 120 | return theFiles; 121 | } 122 | } 123 | }); 124 | } 125 | 126 | function driveGet () { 127 | if (driveGet.error) { 128 | return Promise.reject(emulateError); 129 | } 130 | 131 | return Promise.resolve({ 132 | data: { 133 | on: mockOn 134 | } 135 | }); 136 | } 137 | 138 | const mockGoogleapis = { 139 | auth: { 140 | GoogleAuth 141 | }, 142 | drive: () => ({ 143 | files: { 144 | export: driveGet, 145 | get: driveGet, 146 | list: driveList 147 | } 148 | }) 149 | }; 150 | 151 | function mockExtractTransform(jest) { 152 | jest.mock('@googleapis/drive', () => mockGoogleapis); 153 | /* 154 | jest.mock('remark-html'); 155 | jest.mock('remark', () => () => ({ 156 | use: () => ({ 157 | process: (inputData, cb) => { 158 | if (inputData.mockProcessError) { 159 | cb(inputData.mockProcessError); 160 | } 161 | cb(null, inputData.mockProcessResult); 162 | } 163 | }) 164 | })); 165 | */ 166 | } 167 | 168 | function unmockExtractTransform(jest) { 169 | jest.unmock('@googleapis/drive'); 170 | /* 171 | jest.unmock('remark'); 172 | jest.unmock('remark-html'); 173 | */ 174 | } 175 | 176 | function mockFiles(files, filter = null, listError = false, getError = false) { 177 | _mockFiles.length = 0; 178 | 179 | if (filter) { 180 | driveList.filter = filter; 181 | } 182 | 183 | if (listError) { 184 | driveList.error = listError; 185 | } 186 | 187 | if (getError) { 188 | driveGet.error = getError; 189 | } 190 | 191 | Array.prototype.push.apply(_mockFiles, files); 192 | } 193 | 194 | function unmockFiles() { 195 | driveList.filter = null; 196 | driveList.error = false; 197 | driveGet.error = false; 198 | _mockFiles.length = 0; 199 | } 200 | 201 | function mockAuth(fn) { 202 | GoogleAuth.mock = fn; 203 | } 204 | 205 | function unmockAuth() { 206 | GoogleAuth.mock = null; 207 | } 208 | 209 | module.exports = { 210 | mockReadableStream, 211 | mockWriteResult, 212 | mockTransformResult, 213 | mockStream, 214 | mockIndex, 215 | unmockIndex, 216 | mockLoad, 217 | unmockLoad, 218 | emulateError, 219 | mockTestChunk, 220 | mockOn, 221 | mockGoogleapis, 222 | mockFs, 223 | unmockFs, 224 | mockFiles, 225 | unmockFiles, 226 | mockAuth, 227 | unmockAuth, 228 | mockExtractTransform, 229 | unmockExtractTransform 230 | }; 231 | -------------------------------------------------------------------------------- /lib/extract-transform.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Get files from Google Drive, convert if recognized. 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | import * as path from 'node:path'; 8 | import * as util from 'node:util'; 9 | import { drive as googleDrive, auth as googleAuth } from '@googleapis/drive'; 10 | import { createObjectStream, writeToDirectory } from './load.js'; 11 | 12 | const DEFAULT_NAME = ''; 13 | const DEFAULT_EXT = ''; 14 | 15 | /** 16 | * Update the error with additional helpful context messages. 17 | * 18 | * @param {Error} error - The error to handle 19 | * @param {Array} messages - Additional messages to add. 20 | * @returns {Error} the updated Error 21 | */ 22 | function handleError (error, ...messages) { 23 | if (messages.length > 0) { 24 | messages.push(error.message); 25 | error.message = messages.join('\n'); 26 | } 27 | return error; 28 | } 29 | 30 | /** 31 | * Download a file from google drive. 32 | * 33 | * Called in quick succession, there are problems with rate limits appearing as 403. 34 | * In the second argument to export, the options is an extention of GaxiosOptions, 35 | * so this is possible (I found it was unhelpful and ultimately unneeded). 36 | * No amount of trickery can exceed the limit of the service. 37 | { 38 | retry: true, 39 | retryConfig: { 40 | retryDelay: 1000, 41 | onRetryAttempt: err => { 42 | console.log('@@@ retry attempt', err); 43 | return Promise.resolve(); 44 | }, 45 | shouldRetry: () => true, 46 | statusCodesToRetry: [[100, 199], [403, 403], [429, 429], [500, 599]] 47 | }, 48 | responseType: 'stream' 49 | } 50 | * 51 | * @param {Google.Drive} drive - Google Drive. 52 | * @param {Google.FileResource} file - A Google File Resource object. 53 | * @param {String} file.name - The file name. 54 | * @param {String} file.id - The file id. 55 | * @param {String} file.mimeType - The file mimeType 56 | * @param {String} file.fullFileExtension - The final file extension (binary only) 57 | * @param {Object} [exportMimeMap] - Selects export method and format 58 | * @returns {Promise} Resolves to an Object { name, ext, data }. 59 | */ 60 | export async function downloadFile (drive, file, exportMimeMap) { 61 | const errMsg = `Error downloading \ 62 | '${file.name}', id: '${file.id}', mimeType: '${file.mimeType}' \ 63 | '${file.fullFileExtension}'`; 64 | 65 | let method = 'get'; 66 | const parameters = { 67 | fileId: file.id 68 | }; 69 | 70 | if (exportMimeMap) { 71 | method = 'export'; 72 | parameters.mimeType = exportMimeMap[file.mimeType] || file.mimeType; 73 | } else { 74 | parameters.alt = 'media'; 75 | } 76 | 77 | return new Promise((resolve, reject) => { 78 | const buffers = []; 79 | drive.files[method](parameters, { 80 | responseType: 'stream' 81 | }).then(res => { 82 | res.data 83 | .on('data', chunk => { 84 | buffers.push(chunk); 85 | }) 86 | .on('end', () => { 87 | const pf = path.parse(String(file.name)); 88 | resolve({ 89 | name: pf.name || DEFAULT_NAME, 90 | ext: file.fullFileExtension || pf.ext || DEFAULT_EXT, 91 | data: file.fullFileExtension ? Buffer.concat(buffers) 92 | : Buffer.concat(buffers).toString('utf8'), 93 | binary: !!file.fullFileExtension, 94 | downloadMeta: { 95 | method, 96 | parameters 97 | } 98 | }); 99 | }) 100 | .on('error', err => { 101 | reject(handleError(err, errMsg)); 102 | }); 103 | }).catch(err => { 104 | reject(handleError(err, errMsg)); 105 | }); 106 | }); 107 | } 108 | 109 | /** 110 | * startObjectFlow 111 | * 112 | * Starts and drives the object flow for the library. 113 | * Downloads the files and writes to the stream for conversion. 114 | * 115 | * @param {Google.Drive} googDrive - The google drive instance to download from. 116 | * @param {Array} files - The array of files from the folder. 117 | * @param {ObjectTransformStream} objectStream - The object stream to write to. 118 | * @param {Object} exportMimeMap - The mime map to use in 'export' 119 | */ 120 | export async function startObjectFlow (googDrive, files, objectStream, exportMimeMap) { 121 | let i = 0; 122 | 123 | objectStream.on('finish', () => { 124 | if (objectStream.customError) { 125 | objectStream.destroy(objectStream.customError); 126 | } 127 | }); 128 | 129 | try { 130 | let data; 131 | for (; i < files.length; i++) { 132 | data = await downloadFile(googDrive, files[i], exportMimeMap); 133 | if (objectStream.writableEnded || objectStream.errored) { 134 | break; 135 | } 136 | objectStream.write(data); 137 | } 138 | } catch (e) { 139 | objectStream.customError = handleError( 140 | e, 141 | `Failed downloading files at file '${files[i].name}'`, 142 | util.inspect(e, { depth: 6 }) 143 | ); 144 | } finally { 145 | if (!objectStream.writableEnded) { 146 | objectStream.end(); 147 | } 148 | } 149 | } 150 | 151 | /** 152 | * Get files from Google Drive, convert to html, json or passthru, and return them. 153 | * This function relies on env SVC_ACCT_CREDENTIALS with the path to the service account credential file. 154 | * Relies on a service account that can impersonate the owner of the content files. 155 | * 156 | * Example Workspace File exportMimeMap: 157 | * 'application/vnd.google-apps.document': 'text/plain', 158 | * 'application/vnd.google-apps.presentation': 'text/plain', 159 | * 'application/vnd.google-apps.spreadsheet': 'application/pdf', 160 | * 'application/vnd.google-apps.drawing': 'image/svg+xml', 161 | * 'application/vnd.google-apps.script': 'application/vnd.google-apps.script+json' 162 | * 163 | * @env SVC_ACCT_CREDENTIALS - Path to service account cert that can impersonate the content owner. 164 | * @param {String} folderId - The folderId of the drive to read from. 165 | * @param {String} userId - The userId of the owner of the drive. 166 | * @param {Object} [options] - Additional options. 167 | * @param {String} [options.fileQuery] - A Google drive search query for file selection. 168 | * @see https://developers.google.com/drive/api/v3/ref-search-terms 169 | * @param {Array} [options.scopes] - The scopes required by the account owner, defaults to `drive.readonly`. 170 | * @param {GoogleAuth|OAuth2Client|JWT|String} [options.auth] - An alternate, pre-resolved auth. 171 | * @param {String} [options.outputDirectory] - The path to the output folder. If defined, writes to directory during object stream. 172 | * @param {Object} [options.exportMimeMap] - The mime-type conversion map to use in 'export', implies export. 173 | * @param {Function} [options.transformer] - The conversion transformer function. 174 | * @returns {Promise} array of data objects { input, output, converted } 175 | */ 176 | export async function extractTransform (folderId, userId, { 177 | fileQuery = '', 178 | scopes = [ 179 | 'https://www.googleapis.com/auth/drive.readonly' 180 | ], 181 | auth = null, 182 | outputDirectory = '', 183 | exportMimeMap = null, 184 | transformer = input => Promise.resolve({ 185 | input, 186 | output: input, 187 | converted: false 188 | }) 189 | } = {}) { 190 | let keyFile = ''; 191 | 192 | if (!auth) { 193 | keyFile = process.env.SVC_ACCT_CREDENTIALS; 194 | auth = new googleAuth.GoogleAuth({ 195 | clientOptions: { 196 | subject: userId, // The user to impersonate 197 | forceRefreshOnFailure: true // keep the token updated 198 | // eagerRefreshThresholdMillis: 900000 // 15 * 60 * 1000 199 | }, 200 | keyFile, 201 | // scopes *apparently* need to match the service account delegation as defined exactly. 202 | scopes 203 | }); 204 | } 205 | 206 | // Google drive api as service account impersonating the drive user 207 | const googDrive = googleDrive({ 208 | version: 'v3', 209 | auth 210 | }); 211 | 212 | // Get the file list from the folder 213 | let listRes; 214 | try { 215 | listRes = await googDrive.files.list({ 216 | includeItemsFromAllDrives: true, 217 | supportsAllDrives: true, 218 | q: `'${folderId}' in parents${fileQuery ? ` and ${fileQuery}` : ' and trashed = false'}`, 219 | fields: 'files(id, name, mimeType, fullFileExtension)', 220 | spaces: 'drive' 221 | }); 222 | } catch (e) { 223 | throw handleError( 224 | e, 225 | `Failed to get file list for user '${userId}', and folder '${folderId}'`, 226 | `Auth used: '${auth}'`, 227 | `Credential path from env: '${keyFile}'`, 228 | `Scopes used: '${scopes}'`, 229 | `FileQuery used: '${fileQuery}'`, 230 | util.inspect(e, { depth: 6 }) 231 | ); 232 | } 233 | 234 | const objectStream = createObjectStream(transformer); 235 | if (outputDirectory) { 236 | objectStream.on('data', writeToDirectory.bind(null, outputDirectory, (e, msg) => { 237 | objectStream.customError = handleError(e, msg); 238 | objectStream.end(); 239 | })); 240 | } 241 | 242 | // Download the files from the content folder in series to avoid 403 limits 243 | const files = []; 244 | Array.prototype.push.apply(files, listRes.data.files); 245 | startObjectFlow(googDrive, files, objectStream, exportMimeMap); 246 | 247 | return objectStream; 248 | } 249 | 250 | export { extractTransform as default } 251 | -------------------------------------------------------------------------------- /__tests__/extract-transform.test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * test extract-transform functions. 3 | * 4 | * Copyright (c) 2021 - 2025 Alex Grant (@localnerve), LocalNerve LLC 5 | * Licensed under the MIT license. 6 | */ 7 | /* eslint-env jest */ 8 | const path = require('path'); 9 | const { 10 | mockExtractTransform, unmockExtractTransform, mockOn, 11 | mockFiles, unmockFiles, mockFs, unmockFs, 12 | mockAuth, unmockAuth, 13 | emulateError, mockTestChunk, mockGoogleapis 14 | } = require('test/mocks'); 15 | require('@babel/register'); 16 | 17 | describe('extract-transform', () => { 18 | let etModule; 19 | const processError = emulateError; 20 | let mockWriteFile; 21 | 22 | beforeAll(() => { 23 | mockExtractTransform(jest); 24 | mockWriteFile = mockFs(jest); 25 | etModule = require('../lib/extract-transform'); 26 | }); 27 | 28 | afterAll(() => { 29 | unmockFs(jest); 30 | unmockExtractTransform(jest); 31 | }); 32 | 33 | beforeEach(() => { 34 | mockOn.skipError = true; 35 | mockOn.writeError = false; 36 | }); 37 | 38 | describe('downloadFile', () => { 39 | const file = { 40 | id: '101010', 41 | name: 'mockFile.mockExt', 42 | mimeType: 'some/type', 43 | fullFileExtension: undefined 44 | }; 45 | /* 46 | const exportMimeMap = { 47 | 'application/vnd.google-apps.document': 'text/plain' 48 | }; 49 | */ 50 | const drive = mockGoogleapis.drive(); 51 | 52 | test('should succeed', () => { 53 | return etModule.downloadFile(drive, file).then(result => { 54 | expect(result).toBeDefined(); 55 | expect(result.name).toEqual(path.parse(file.name).name); 56 | expect(result.ext).toEqual(path.parse(file.name).ext); 57 | expect(result.data).toEqual(mockTestChunk); 58 | }); 59 | }); 60 | 61 | test('should fail', () => { 62 | mockOn.skipError = false; 63 | return etModule.downloadFile(drive, file).then(result => { 64 | throw new Error(`should not have succeeded: ${require('util').inspect(result)}`); 65 | }, err => { 66 | expect(err).toEqual(processError); 67 | }); 68 | }); 69 | }); 70 | 71 | describe('extractTransform', () => { 72 | let counter = 0; 73 | const googDocsType = 'application/vnd.google-apps.document'; 74 | const files = [{ 75 | name: '0.passthru', 76 | id: '123123123', 77 | mimeType: 'some/type', 78 | fullFileExtension: undefined 79 | }, { 80 | name: '1.passthru', 81 | id: '456456456', 82 | mimeType: 'some/type', 83 | fullFileExtension: undefined 84 | }]; 85 | const binaryFiles = [{ 86 | name: '0.bin', 87 | id: '567567567', 88 | mimeType: 'application/octet-stream', 89 | fullFileExtension: 'bin' 90 | }, { 91 | name: '1.bin', 92 | id: '789789789', 93 | mimeType: 'application/octet-stream', 94 | fullFileExtension: 'bin' 95 | }]; 96 | const filesWithMimeTypes = [{ 97 | name: '0.doc', 98 | id: '234234234', 99 | mimeType: googDocsType 100 | }, { 101 | name: '1.bin', 102 | id: '345345345', 103 | mimeType: 'image/jpeg' 104 | }, { 105 | name: '1.doc', 106 | id: '012012012', 107 | mimeType: googDocsType 108 | }]; 109 | 110 | const docsType = googDocsType; 111 | 112 | function filterByType(type, files) { 113 | return files.filter(file => file.mimeType.includes(type)); 114 | } 115 | 116 | test('should return stream', () => { 117 | return etModule.extractTransform('101010', 'user@domain.dom') 118 | .then(result => { 119 | expect(result).toBeDefined(); 120 | expect(result).toBeInstanceOf(require('../lib/load').ObjectTransformStream); 121 | }); 122 | }); 123 | 124 | test('should throw on drive list failure', async () => { 125 | let result; 126 | function complete(e) { 127 | unmockFiles(); 128 | return e; 129 | } 130 | mockFiles(files, null, true); 131 | 132 | try { 133 | await etModule.extractTransform('iMaFiLeIdOfSoMeKiNd', 'user@domain.dom', { 134 | outputDirectory: 'tmp/to/nowhere' 135 | }); 136 | result = complete(new Error('Should have thrown')); 137 | } 138 | catch (e) { 139 | expect(e).toEqual(emulateError); // eslint-disable-line 140 | result = complete(); 141 | } 142 | 143 | if (result) { 144 | throw result; 145 | } 146 | }); 147 | 148 | test('should error on download failure', done => { 149 | mockFiles(files, null, false, true); 150 | 151 | etModule.extractTransform('iMaFiLeIdOfSoMeKiNd', 'user@domain.dom', { 152 | outputDirectory: 'tmp/to/nowhere' 153 | }).then(stream => { 154 | stream.on('data', () => { 155 | done(new Error('received unexpected data')); 156 | }); 157 | stream.on('error', err => { 158 | expect(err).toEqual(emulateError); 159 | unmockFiles(); 160 | done(); 161 | }); 162 | }); 163 | }); 164 | 165 | test('should send data, correct structure, ref input on passthru', done => { 166 | mockFiles(files); 167 | counter = 0; 168 | etModule.extractTransform('101010', 'user@domain.dom') 169 | .then(stream => { 170 | stream.on('data', obj => { 171 | expect(obj).toBeDefined(); 172 | expect(obj).toHaveProperty('input.name'); 173 | expect(obj).toHaveProperty('input.ext'); 174 | expect(obj).toHaveProperty('input.data'); 175 | expect(obj).toHaveProperty('input.binary'); 176 | expect(obj).toHaveProperty('input.downloadMeta'); 177 | expect(obj).toHaveProperty('output.name'); 178 | expect(obj).toHaveProperty('output.ext'); 179 | expect(obj).toHaveProperty('output.data'); 180 | expect(obj).toHaveProperty('converted'); 181 | expect(obj.converted).toBeFalsy(); 182 | expect(obj.input.name).toEqual(obj.output.name); 183 | expect(parseInt(obj.output.name)).toEqual(counter); // order 184 | expect(obj.output.ext).toEqual(`.${files[counter].name.split('.')[1]}`); 185 | counter++; 186 | }); 187 | stream.on('end', () => { 188 | expect(counter).toEqual(files.length); 189 | unmockFiles(); 190 | done(); 191 | }); 192 | stream.on('error', err => { 193 | unmockFiles(); 194 | done(err); 195 | }); 196 | }); 197 | }); 198 | 199 | test('handle write errors', done => { 200 | function complete() { 201 | mockWriteFile.mockClear(); 202 | unmockFiles(); 203 | done(); 204 | } 205 | mockOn.writeError = true; 206 | mockFiles(files); 207 | mockWriteFile.mockClear(); 208 | 209 | counter = 0; 210 | etModule.extractTransform('iMaFiLeIdOfSoMeKiNd', 'user@domain.dom', { 211 | outputDirectory: 'tmp/to/nowhere' 212 | }) 213 | .then(stream => { 214 | stream.on('error', e => { 215 | expect(e).toEqual(emulateError); 216 | complete(); 217 | }); 218 | }); 219 | }); 220 | 221 | test('should send data and write file when outputDirectory is specified', done => { 222 | function complete(e) { 223 | mockWriteFile.mockClear(); 224 | unmockFiles(); 225 | done(e); 226 | } 227 | mockFiles(files); 228 | mockWriteFile.mockClear(); 229 | 230 | counter = 0; 231 | etModule.extractTransform('iMaFiLeIdOfSoMeKiNd', 'user@domain.dom', { 232 | outputDirectory: 'tmp/to/nowhere' 233 | }) 234 | .then(stream => { 235 | stream.on('data', data => { 236 | expect(data.input.binary).toBeFalsy(); 237 | expect(data.output.data).toEqual('myspecialtestchunk'); 238 | counter++; 239 | }); 240 | stream.on('end', () => { 241 | expect(counter).toEqual(files.length); 242 | expect(mockWriteFile.mock.calls.length).toEqual(files.length); 243 | complete(); 244 | }); 245 | stream.on('error', e => { 246 | complete(e); 247 | }); 248 | }); 249 | }); 250 | 251 | /* eslint-disable jest/expect-expect */ 252 | test('should use auth if supplied', done => { 253 | function complete(e) { 254 | unmockAuth(); 255 | done(e); 256 | } 257 | 258 | mockAuth(() => { 259 | done(new Error('should have used supplied auth and not have called GoogleAuth')); 260 | }); 261 | 262 | etModule.extractTransform('123456789', 'user@domain.dom', { 263 | auth: () => {} 264 | }).then(() => { 265 | complete(); 266 | }).catch(complete); 267 | }); 268 | 269 | test('should use GoogleAuth if no auth supplied', done => { 270 | function complete(e) { 271 | unmockAuth(); 272 | done(e); 273 | } 274 | 275 | mockAuth(() => { 276 | complete(); 277 | }); 278 | 279 | etModule.extractTransform('123456789', 'user@domain.dom') 280 | .then(() => {}) 281 | .catch(complete); 282 | }); 283 | /* eslint-enable jest/expect-expect */ 284 | 285 | test('should send Buffer if binary content', done => { 286 | function complete(e) { 287 | unmockFiles(); 288 | done(e); 289 | } 290 | mockFiles(binaryFiles); 291 | counter = 0; 292 | etModule.extractTransform('101010', 'user@domain.dom') 293 | .then(stream => { 294 | stream.on('data', data => { 295 | expect(data.input.binary).toEqual(true); 296 | expect(data.output.data).toBeInstanceOf(Buffer); 297 | counter++; 298 | }); 299 | stream.on('end', () => { 300 | expect(counter).toEqual(binaryFiles.length); 301 | complete(); 302 | }); 303 | stream.on('error', e => { 304 | complete(e); 305 | }) 306 | }); 307 | }); 308 | 309 | test('should filter files if fileQuery is specified', done => { 310 | function complete(e) { 311 | unmockFiles(); 312 | done(e); 313 | } 314 | 315 | mockFiles(filesWithMimeTypes, filterByType.bind(null, docsType)); 316 | counter = 0; 317 | etModule.extractTransform('imASimpleFolderId', 'owner@ofFolder.dom', { 318 | fileQuery: `mimeType = "application/vnd.${docsType}"` 319 | }) 320 | .then(stream => { 321 | stream.on('data', () => { 322 | counter++; 323 | }); 324 | stream.on('end', () => { 325 | expect(counter).toEqual(2); // only 2 google-apps.document in fileWithMimeTypes 326 | complete(); 327 | }); 328 | stream.on('error', e => { 329 | complete(e); 330 | }); 331 | }); 332 | }); 333 | 334 | test('should run export if exportMimeMap', done => { 335 | function complete(e) { 336 | unmockFiles(); 337 | done(e); 338 | } 339 | 340 | const mimeType = 'text/plain'; 341 | mockFiles(filesWithMimeTypes, filterByType.bind(null, docsType)); 342 | counter = 0; 343 | etModule.extractTransform('asdfasdfasdf', 'owner@folder.com', { 344 | exportMimeMap: { 345 | [googDocsType]: mimeType 346 | } 347 | }) 348 | .then(stream => { 349 | stream.on('data', data => { 350 | expect(data.output.downloadMeta.method).toEqual('export'); 351 | expect(data.output.downloadMeta.parameters.mimeType).toEqual(mimeType); 352 | counter++; 353 | }); 354 | stream.on('end', () => { 355 | expect(counter).toEqual(2); 356 | complete(); 357 | }); 358 | stream.on('error', e => { 359 | complete(e); 360 | }); 361 | }); 362 | }); 363 | }); 364 | }); 365 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # google-drive-folder 2 | 3 | > Streams files from a google drive folder 4 | 5 | [![npm version](https://badge.fury.io/js/%40localnerve%2Fgoogle-drive-folder.svg)](https://badge.fury.io/js/%40localnerve%2Fgoogle-drive-folder) 6 | ![Verify](https://github.com/localnerve/google-drive-folder/workflows/Verify/badge.svg) 7 | [![Coverage Status](https://coveralls.io/repos/github/localnerve/google-drive-folder/badge.svg?branch=master)](https://coveralls.io/github/localnerve/google-drive-folder?branch=master) 8 | 9 | ## Contents 10 | + [Overview](#overview) 11 | + [Why](#why-this-exists) 12 | + [Authentication and Authorization](#authentication-and-authorization) 13 | + [Conversions](#conversions) 14 | + [API](#api) 15 | + [Input Types](#input) 16 | + [Output Stream](#output) 17 | + [Stream Data Format](#stream-data-format) 18 | + [Input Detail](#input-detail) 19 | + [Google Drive Parameters](#google-drive-parameters) 20 | + [Options](#options) 21 | + [Conversion Options](#conversion-options) 22 | + [Transformer Function](#transformer-function) 23 | + [Transformer Input Object](#transformer-input-object) 24 | + [Transformer Example](#transformer-example) 25 | + [Example Usage](#example-usage) 26 | + [Example Minimal Usage](#code-example-using-minimal-options) 27 | + [Example with Most Options](#code-example-using-most-options) 28 | + [MIT License](#license) 29 | 30 | ## Overview 31 | This library streams files from a google drive folder, and applies conversions (if desired). 32 | 33 | ### Why This Exists 34 | 35 | I wrote this library to codify my solution to a painful experience in which [403](https://en.wikipedia.org/wiki/HTTP_403) errors developed during larger folder downloads. This was initially puzzling (403 code not sufficient to understand the precise issue), but I solved it by serially streaming the files. The original problem must have been that I was making too many requests all at once over a very small time period, resulting in a 403 from the Google Drive service by policy. 36 | 37 | ### Authentication and Authorization 38 | 39 | This library uses the [google api node client](https://github.com/googleapis/google-api-nodejs-client), and therefore requires credentials. By default, this library relies on Environment variable SVC_ACCT_CREDENTIALS to point to a valid Google service account credential file. Further, this credential must have the permissions to impersonate the user email passed in as `userId` for the folder `folderId`. 40 | 41 | However, If you have different requirements (OAuth2, JWT, etc), you can supply your own pre-resolved auth reference by passing [google auth](https://github.com/googleapis/google-auth-library-nodejs) as an 'auth' option. For more information, all options are [detailed here](#input-detail). 42 | 43 | ### Conversions 44 | 45 | By default, no conversion occurs, data is just passed through as a Buffer (if binary) or a utf8 encoded string. 46 | To change this, supply a `transformer` function and an optional `exportMimeMap` if the source file on Google Drive is a Google Workspace file type (Docs, Spreadsheet, Presentation, Drawing, etc). These options are supplied using the [conversion options](#conversion-options). 47 | 48 | ## API 49 | 50 | The library exports a single function that returns a Promise that resolves to a Stream. 51 | Here's the signature: 52 | 53 | ``` 54 | Promise GoogleDriveFolder (folderId, userId, Options) 55 | ``` 56 | 57 | ### Input 58 | 59 | A quick look at all the input and the types. All options are optional. For a more detailed explanation of input [skip to here](#input-detail). 60 | 61 | ``` 62 | folderId: String, 63 | userId: String, 64 | [Options: Object] 65 | outputDirectory: String, 66 | scopes: Array, 67 | fileQuery: String, 68 | auth: GoogleAuth | OAuth2Client | JWT | String, 69 | exportMimeMap: Object, 70 | transformer: Function 71 | ``` 72 | 73 | ### Output 74 | 75 | The returned Promise resolves to a `Stream` in object mode to receive data objects for each downloaded file as it arrives. 76 | 77 | #### Stream Data Format 78 | 79 | The format of the data objects you will receive on `data` events: 80 | 81 | ``` 82 | { 83 | input: { 84 | data, // Buffer | String, data from Google Drive, Buffer if binary, 'utf-8' String otherwise 85 | name, // String of the file name 86 | ext, // String of the file extension 87 | binary, // Boolean, true if binary data content 88 | downloadMeta, // Object 89 | mimeType | alt // String, mimeType for export, alt for get 90 | }, 91 | // If converted is true, the converted data. Otherwise, a reference to input 92 | output: { 93 | data, // String (utf-8) or Buffer if input was binary 94 | name, // String of the file name 95 | ext // String of the file extension 96 | }, 97 | converted // Boolean, true if conversion occurred, false otherwise 98 | } 99 | ``` 100 | 101 | The `input` is data as downloaded from Google Drive. 102 | The `output` is data as converted by a [transformer](#transformer-function). 103 | If no conversion occurs (`converted === false`), output is a referece to `input`. 104 | 105 | ## Input Detail 106 | 107 | `SVC_ACCT_CREDENTIALS` environment variable must point to a valid Google Service Account credential file **UNLESS** Auth is supplied using the `auth` option. 108 | 109 | ### Google Drive Parameters 110 | 111 | * `folderId` {String} - Uniquely identifies your folder in the google drive service. Found on the web in the url. 112 | * `userId` {String} - The email address of the folder owner that SVC_ACCT_CREDENTIALS will impersonate. If you supply the `auth` option, this parameter is ignored. 113 | 114 | ### Options 115 | 116 | All options are optional. 117 | 118 | * `[Options]` {Object} - The general options object. 119 | * `[Options.scopes]` {Array} - Scopes to use for auth (if required) in a special case. Defaults to the `drive.readonly` scope. 120 | * `[Options.fileQuery]` {String} - A file query string used to filter files to download by specific characteristics. Defaults to downloading all files in the `folderId` that are NOT deleted (`trashed = false`). @see [file reference search terms](https://developers.google.com/drive/api/v3/ref-search-terms). 121 | * `[Options.auth]` {GoogleAuth | OAuth2Client | JWT | String} - Given directly to the Google Drive NodeJS Client `auth` option. Use this option to override the default behavior of the `SVC_ACCT_CREDENTIALS` environment variable path to a service account credentials. This will also cause the `userId` parameter to be ignored. 122 | * `[Options.outputDirectory]` {String} - Absolute path to the output directory. Defaults to falsy. If supplied, files are written out as data arrives. Does not touch the directory other than to write files. The directory must already exist. 123 | 124 | #### Conversion Options 125 | 126 | * `[Options.exportMimeMap]` {Object} - A name-value map of Google Workspace mime-types to a conversion mime-type to be performed by the Google Drive service prior to sending the data to the `transformer` function. If this option is supplied, the Google Drive Files 'export' method is used, and therefore the types are presumed to conform to the service capabilities outlined in the [Google Export Reference](https://developers.google.com/drive/api/v3/ref-export-formats). For detail on Google Workspace mime-types, see [Google Workspace MimeTypes](https://developers.google.com/drive/api/v3/mime-types). If this option is not supplied, the Google Drive Files 'get' method is used for download. 127 | * `[Options.transformer]` {Function} - Transforms the input after download (or optional export conversion) and before it goes out to the stream (and optional `outputDirectory`, if supplied). Defaults to pass-through. Returns a Promise that resolves to an object that conforms to the [stream data format](#stream-data-format). 128 | 129 | ##### Transformer Function 130 | 131 | A Transformer function receives input from the download and returns a Promise that resolves to the [data stream object format](#data-stream-format) 132 | 133 | ###### Transformer Input Object 134 | 135 | A supplied `transformer` function receives a single object from the download of the following format: 136 | 137 | ``` 138 | { 139 | name: String, 140 | ext: String, 141 | data: , // Buffer if binary, 'utf-8' String otherwise 142 | binary: Boolean, 143 | downloadMeta: Object 144 | method: String, // 'get' or 'export' 145 | parameters: Object // parameters used in the download (mimeType from exportMimeMap or alt='media') 146 | } 147 | ``` 148 | 149 | ###### Transformer Example 150 | This example downloads all files from the given Google Drive Folder. 151 | It presumes they are all Google Workspace GoogleDocs files that are markdown documents. 152 | Downloads them as `text/plain`, converts them to 'html', and outputs the result to the stream. 153 | 154 | ```js 155 | import googleDriveFolder from '@localnerve/google-drive-folder'; 156 | 157 | process.env.SVC_ACCT_CREDENTIALS = '/path/to/svcacctcredential.json'; 158 | 159 | const folderId = 'ThEfOlDeRiDyOuSeEiNyOuRbRoWsErOnGoOgLeDrIvE'; 160 | const userId = 'email-of-the-folder-owner@will-be-impersonated.by-svc-acct'; 161 | 162 | // Use remark for markdown to html conversion 163 | const remark = require('remark'); 164 | const remarkHtml = require('remark-html'); 165 | 166 | /** 167 | * The transformer definition. 168 | * Receives an input object from the Google Drive download, outputs a conversion object 169 | * in the form of #stream-data-format defined in this readme.md 170 | * 171 | * @param input (Object) The file input object 172 | * @param input.name {String} The name of the file downloaded 173 | * @param input.ext {String} The extension of the file download (if available) 174 | * @param input.data {String} The utf-8 encoded string data from the 'text/plain' download. 175 | * @param input.binary {Boolean} False in this case, true if content binary. 176 | * @param input.downloadMeta {Object} download meta data 177 | * @param input.downloadMeta.method {String} 'export' or 'get', 'export' in this case. 178 | * @param input.downloadMeta.parameters {Object} the download parameters 179 | * @param input.downloadMeta.parameters.mimeType {String} The mimeType used, 'text/plain' in this case. 180 | * @param input.downloadMeta.parameters.alt {String} 'meta' when 'get' method is used, undefined in this case. 181 | */ 182 | function myTransformer (input) { 183 | return new Promise((resolve, reject) => { 184 | remark() 185 | .use(remarkHtml) 186 | .process(input.data, (err, res) => { 187 | if (err) { 188 | err.message = `Failed to convert '${input.name}'` + err.message; 189 | return reject(err); 190 | } 191 | resolve({ 192 | input, 193 | output: { 194 | name: input.name, 195 | ext: '.html', 196 | data: String(res) 197 | }, 198 | converted: true 199 | }); 200 | }); 201 | }); 202 | } 203 | 204 | // let's do this: 205 | try { 206 | // blocks until object flow begins 207 | const stream = await googleDriveFolder(folderId, userId, { 208 | exportMimeMap: { 209 | 'application/vnd.google-apps.document': 'text/plain' 210 | }, 211 | transformer: myTransformer 212 | }); 213 | stream.on('data', data => { 214 | // data.input.data has markdown 215 | // data.output.data has html 216 | console.log(`Received converted data for '${data.output.name}'`, data); 217 | }); 218 | stream.on('end', () => { 219 | console.log('downloads are done, we got all the files.'); 220 | }); 221 | stream.on('error', e => { 222 | throw e; 223 | }); 224 | } 225 | catch (e) { 226 | console.error(e); // something went wrong 227 | } 228 | ``` 229 | 230 | ## Example Usage 231 | 232 | ### Code Example using minimal options 233 | 234 | ```js 235 | import googleDriveFolder from '@localnerve/google-drive-folder'; 236 | 237 | process.env.SVC_ACCT_CREDENTIALS = '/path/to/svcacctcredential.json'; 238 | 239 | const folderId = 'ThEfOlDeRiDyOuSeEiNyOuRbRoWsErOnGoOgLeDrIvE'; 240 | const userId = 'email-of-the-folder-owner@will-be-impersonated.by-svc-acct'; 241 | 242 | try { 243 | // Blocks until object flow begins (while auth and file list is downloaded) 244 | const stream = await googleDriveFolder(folderId, userId); 245 | stream.on('data', data => { 246 | console.log(`Received a data object for '${data.input.name}'`, data); 247 | }); 248 | stream.on('end', () => { 249 | console.log('downloads are done, we got all the files'); 250 | }); 251 | stream.on('error', e => { 252 | throw e; 253 | }); 254 | } catch (e) { 255 | console.error(e); // something went wrong 256 | } 257 | ``` 258 | 259 | ### Code Example using most options 260 | 261 | All the prerequisites and possible arguments as code (except for `auth` option): 262 | 263 | ```js 264 | import googleDriveFolder from '@localnerve/google-drive-folder'; 265 | import myTransformer from './myTransformer'; // your transform function 266 | 267 | process.env.SVC_ACCT_CREDENTIALS = '/path/to/svcacctcredential.json'; 268 | 269 | const folderId = 'ThEfOlDeRiDyOuSeEiNyOuRbRoWsErOnGoOgLeDrIvE'; 270 | const userId = 'email-of-the-folder-owner@will-be-impersonated.by-svc-acct'; 271 | 272 | // all optional, if outputDirectory omitted, returns ReadableStream 273 | const options = { 274 | outputDirectory: '/tmp/mydrivefolder/mustexist', 275 | scopes: [ 276 | 'special/google.auth/scope/you/might/need/other/than/drive.readonly', 277 | 'https://www.googleapis.com/auth/drive.readonly' 278 | ], 279 | fileQuery: 'name contains ".md"', // download GoogleDocs markdown files 280 | exportMimeMap: { 281 | 'application/vnd.google-apps.document': 'text/plain' 282 | }, 283 | transformer: myTransformer 284 | }; 285 | 286 | try { 287 | // Blocks until object flow begins 288 | const stream = await googleDriveFolder(folderId, userId, options); 289 | stream.on('data', data => { 290 | console.log(`Received a data object for '${data.input.name}'`, data); 291 | }); 292 | stream.on('end', () => { 293 | console.log('downloads are done, we got all the files'); 294 | }); 295 | stream.on('error', e => { 296 | throw e; 297 | }); 298 | } catch (e) { 299 | console.error(e); // something went wrong 300 | } 301 | 302 | ``` 303 | 304 | ## LICENSE 305 | 306 | * [MIT, Alex Grant, LocalNerve, LLC](license.md) 307 | --------------------------------------------------------------------------------