├── .npmignore ├── .eslintrc ├── .vscode └── settings.json ├── periodic-test ├── .gitignore ├── apify.json ├── package.json ├── mock-data.js ├── test.js └── package-lock.json ├── .gitignore ├── apify.json ├── src ├── defaults-prefills.js ├── constants.js ├── upload.js ├── modes.js ├── loaders.js ├── transformers.js ├── validate-parse-input.js ├── tabulation.js ├── raw-data-parser.js ├── utils.js └── main.js ├── .editorconfig ├── test ├── exponential-backoff.js ├── docs-example.js ├── test.js ├── customFilerFunctions.js └── mocks.js ├── package.json ├── Dockerfile ├── CHANGELOG.md ├── INPUT_SCHEMA.json ├── README.md └── LICENSE.md /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@apify" 3 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "eslint.enable": true 3 | } -------------------------------------------------------------------------------- /periodic-test/.gitignore: -------------------------------------------------------------------------------- 1 | apify_storage 2 | node_modules -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | apify_storage 2 | node_modules 3 | .DS_Store 4 | storage -------------------------------------------------------------------------------- /periodic-test/apify.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "periodic-test", 3 | "version": "0.0", 4 | "buildTag": "latest", 5 | "env": null 6 | } 7 | -------------------------------------------------------------------------------- /apify.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "google-oauth2", 3 | "template": "puppeteer_crawler", 4 | "version": "0.1", 5 | "buildTag": "latest", 6 | "env": { "npm_config_loglevel": "silent" } 7 | } 8 | -------------------------------------------------------------------------------- /src/defaults-prefills.js: -------------------------------------------------------------------------------- 1 | // prefill transform function 2 | 3 | // this code works the same as if you delete it and leave this area blank 4 | ({ spreadsheetData, datasetData }) => { 5 | return spreadsheetData.concat(datasetData); 6 | } 7 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | end_of_line = lf 10 | # editorconfig-tools is unable to ignore longs strings or urls 11 | max_line_length = null 12 | -------------------------------------------------------------------------------- /test/exponential-backoff.js: -------------------------------------------------------------------------------- 1 | const { backOff } = require('exponential-backoff'); 2 | const Apify = require('apify'); 3 | 4 | const breakedFunction = async () => { 5 | console.log('failing'); 6 | throw new Error('failed'); 7 | }; 8 | 9 | Apify.main(async () => { 10 | await backOff({ fn: breakedFunction }, { numberOfAttempts: 20 }); 11 | }); 12 | -------------------------------------------------------------------------------- /periodic-test/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "periodic-test", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "mock-data.js", 6 | "scripts": { 7 | "start": "node test.js", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "apify": "^0.11.5" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | CLIENT_ID: '118138324390-1nmbvv5q2nffuagp65i03k086qr173jg.apps.googleusercontent.com', 3 | REDIRECT_URI: 'urn:ietf:wg:oauth:2.0:oob', 4 | CLIENT_ID_2: '211810992764-n3k9d189h8h631vnvviock7ui5mu1ak1.apps.googleusercontent.com', 5 | REDIRECT_URI_SERVER: 'https://ihlijwwwytja.runs.apify.net/authorize', 6 | CLIENT_SERVER_ID_1: '118138324390-9kind5g1u2q83lhjek6ah3v68ttkp48i.apps.googleusercontent.com', 7 | }; 8 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "actor-google-spreadsheet", 3 | "version": "0.0.1", 4 | "description": "Apify actor for importing data to Google spreadsheet", 5 | "main": "src/main.js", 6 | "dependencies": { 7 | "apify": "^2.2.2", 8 | "apify-google-auth": "^0.5.1", 9 | "csvtojson": "^2.0.8", 10 | "googleapis": "^61.0.0", 11 | "md5": "^2.2.1" 12 | }, 13 | "scripts": { 14 | "start": "node src/main.js --silent", 15 | "test": "mocha", 16 | "backoff-test": "node test/exponential-backoff.js" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "" 21 | }, 22 | "author": "", 23 | "license": "ISC", 24 | "homepage": "https://www.apify.com/lukaskrivka/google-spreadsheet", 25 | "devDependencies": { 26 | "@apify/eslint-config": "^0.2.2", 27 | "eslint": "^8.10.0" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # First, specify the base Docker image. You can read more about 2 | # the available images at https://sdk.apify.com/docs/guides/docker-images 3 | # You can also use any other image from Docker Hub. 4 | FROM apify/actor-node:16 5 | 6 | # Second, copy just package.json and package-lock.json since it should be 7 | # the only file that affects "npm install" in the next step, to speed up the build 8 | COPY package*.json ./ 9 | 10 | # Install NPM packages, skip optional and development dependencies to 11 | # keep the image small. Avoid logging too much and print the dependency 12 | # tree for debugging 13 | RUN npm --quiet set progress=false \ 14 | && npm install --only=prod --no-optional \ 15 | && echo "Installed NPM packages:" \ 16 | && (npm list --only=prod --no-optional --all || true) \ 17 | && echo "Node.js version:" \ 18 | && node --version \ 19 | && echo "NPM version:" \ 20 | && npm --version 21 | 22 | # Next, copy the remaining files and directories with the source code. 23 | # Since we do this after NPM install, quick build will be really fast 24 | # for most source file changes. 25 | COPY . ./ 26 | 27 | # Optionally, specify how to launch the source code of your actor. 28 | # By default, Apify's base Docker images define the CMD instruction 29 | # that runs the Node.js source code using the command specified 30 | # in the "scripts.start" section of the package.json file. 31 | # In short, the instruction looks something like this: 32 | # 33 | # CMD npm start -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Version 2 2 | #### 2022-03-03 3 | - Migrated to new SDK version which caused crashes for one day until hotfixed. 4 | - Better error messages 5 | 6 | #### 2020-10-08 7 | - Fixed: A bug that prevented loading big amount of data from the spreadsheets (by upgrading `googleapis` version) 8 | - Fixed: `transformFunction` was adding extra columns from the dataset even if they were renamed 9 | - Fixed: Sub-array default sorting 10 | - Feature: Added `keepSheetColumnOrder` option to input to prevent re-sorting data in your sheet 11 | 12 | #### 2019-11-17 13 | - Removed support for Apify Crawler (this product has been removed) 14 | - Webhooks work out of the box without a need to change the payload template (for tasks). 15 | - Input - `datasetOrExecutionId` changed to `datasetId`. 16 | - Added sections to input (only visual change) 17 | 18 | #### 2019-12-31 19 | - Fixed: Excess rows/columns were wrongly trimmed if the range was not the first sheet. May have caused a removal of data in the first sheet. 20 | 21 | #### 2020-01-11 22 | - **Warning!**: For running this actor's code outside of `lukaskrivka/google-sheets` official version, you will need to create your own Google Dev Console project and provide your own keys to the input! This change will apply to older versions as well! 23 | - Added: Option to read from public spreadsheets without authorization (pass `publicSpreadsheet: true` to the input). 24 | 25 | #### 2020-04-24 26 | - Added `columnsOrder` field so the user can define the order of columns 27 | 28 | #### 2020-04-29 29 | - New sheets (via `range` input) are now automatically created by the actor, no need to pre-create them anymore 30 | -------------------------------------------------------------------------------- /periodic-test/mock-data.js: -------------------------------------------------------------------------------- 1 | module.exports.mock1 = [ 2 | { 3 | 'dogs/0': 'Lara', 4 | hobby: '', 5 | job: 'developer', 6 | name: 'Lukas', 7 | }, 8 | { 9 | 'dogs/0': '', 10 | hobby: 'alcohol', 11 | job: 'seller', 12 | name: 'Adam', 13 | }, 14 | ]; 15 | 16 | module.exports.mock1Dataset = [ 17 | { 18 | dogs: ['Lara'], 19 | job: 'developer', 20 | name: 'Lukas', 21 | }, 22 | { 23 | hobby: 'alcohol', 24 | job: 'seller', 25 | name: 'Adam', 26 | }, 27 | ]; 28 | 29 | module.exports.mock2 = [ 30 | { 31 | 'dogs/0': '', 32 | hobby: 'music', 33 | job: 'developer', 34 | name: 'Sena', 35 | }, 36 | { 37 | 'dogs/0': '', 38 | hobby: 'tabletop games', 39 | job: 'none', 40 | name: 'Hout', 41 | }, 42 | ]; 43 | 44 | module.exports.mock2Dataset = [ 45 | { 46 | hobby: 'music', 47 | job: 'developer', 48 | name: 'Sena', 49 | }, 50 | { 51 | hobby: 'tabletop games', 52 | job: 'none', 53 | name: 'Hout', 54 | }, 55 | ]; 56 | 57 | module.exports.backupMock = [ 58 | ['dogs/0', 'hobby', 'job', 'name'], 59 | ['', 'alcohol', 'seller', 'Adam'], 60 | ['', 'music', 'developer', 'Sena'], 61 | ['', 'tabletop games', 'none', 'Hout'], 62 | ['Lara', '', 'developer', 'Lukas'], 63 | ]; 64 | 65 | module.exports.mockTransform = ({ spreadsheetData, datasetData }) => { 66 | const all = spreadsheetData.concat(datasetData); 67 | const nonDogs = all.reduce((acc, item) => { 68 | if (!item['dogs/0']) { 69 | acc.push(item); 70 | } 71 | return acc; 72 | }, []); 73 | return nonDogs; 74 | }; 75 | -------------------------------------------------------------------------------- /test/docs-example.js: -------------------------------------------------------------------------------- 1 | const func = (newData, oldData) => { 2 | // First we put the data together into one array 3 | const allData = newData.concat(oldData); 4 | 5 | // We define an object that will hold a state about which item is the cheapest for each country 6 | const stateObject = {}; 7 | 8 | // Now let's loop over the data and update the object 9 | allData.forEach((item) => { 10 | // If the item doesn't have price field, we will throw it away 11 | if (!item.price) return; 12 | 13 | // If the state doesn't hold the country, we will add the first item there to hold the current position of cheapest item 14 | if (!stateObject[item.country]) { 15 | stateObject[item.country] = item; 16 | } else if (item.price < stateObject[item.country].price) { 17 | // If the state already holds the country, lets compare if the new item is cheaper than the old and if so, replace them 18 | stateObject[item.country] = item; 19 | } 20 | }); 21 | 22 | // Once we went through all the item, let's convert our state object back to the right array format 23 | const finalData = Object.values(stateObject); 24 | return finalData; 25 | }; 26 | 27 | const oldData = [{ 28 | sku: '234234234', 29 | country: 'US', 30 | price: 20.99, 31 | 'sizes/0': 'S', 32 | 'sizes/1': 'M', 33 | 'sizes/2': 'L', 34 | 'sizes/3': 'XL', 35 | }, 36 | { 37 | sku: '123123123', 38 | country: 'UK', 39 | price: 48.49, 40 | 'sizes/0': 'M', 41 | 'sizes/2': 'XL', 42 | }]; 43 | 44 | const newData = [{ 45 | sku: '234234234', 46 | country: 'US', 47 | price: 29.99, 48 | 'sizes/0': 'S', 49 | 'sizes/1': 'M', 50 | 'sizes/2': 'L', 51 | 'sizes/3': 'XL', 52 | }, 53 | { 54 | sku: '123123123', 55 | country: 'UK', 56 | price: 44.49, 57 | 'sizes/0': 'M', 58 | 'sizes/2': 'XL', 59 | }]; 60 | 61 | console.dir(func(oldData, newData)); 62 | -------------------------------------------------------------------------------- /src/upload.js: -------------------------------------------------------------------------------- 1 | const { countCells, trimSheetRequest, retryingRequest } = require('./utils'); 2 | 3 | module.exports = async ({ maxCells, rowsToInsert, spreadsheetId, spreadsheetRange, values, client, targetSheetId }) => { 4 | // ensuring max cells limit 5 | const cellsToInsert = countCells(rowsToInsert); 6 | console.log(`Total rows: ${rowsToInsert.length}, total columns: ${rowsToInsert[0].length} total cells: ${cellsToInsert}`); 7 | if (cellsToInsert > maxCells) { 8 | throw `You reached the max limit of ${maxCells} cells. Try inserting less rows.`; 9 | } 10 | 11 | // inserting cells 12 | console.log('Inserting new cells'); 13 | await retryingRequest('Inserting new rows', async () => client.spreadsheets.values.update({ 14 | spreadsheetId, 15 | range: spreadsheetRange, 16 | valueInputOption: 'USER_ENTERED', 17 | resource: { values: rowsToInsert }, 18 | })); 19 | console.log('Items inserted...'); 20 | 21 | // trimming cells 22 | console.log('Maybe deleting unused cells'); 23 | const height = values && values.length > rowsToInsert.length 24 | ? rowsToInsert.length 25 | : null; 26 | const maxInSheetWidth = values ? values.reduce((max, row) => Math.max(max, row.length), 0) : 0; 27 | const maxInsertWidth = rowsToInsert.reduce((max, row) => Math.max(max, row.length), 0); 28 | const width = maxInSheetWidth > maxInsertWidth 29 | ? maxInsertWidth 30 | : null; 31 | if (height || width) { 32 | if (height) console.log('Will delete unused rows'); 33 | if (width) console.log('Will delete unused columns'); 34 | await retryingRequest('Trimming excessive cells', async () => client.spreadsheets.batchUpdate({ 35 | spreadsheetId, 36 | resource: trimSheetRequest(height, width, targetSheetId), 37 | })); 38 | } else { 39 | console.log('No need to delete any rows or columns'); 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /src/modes.js: -------------------------------------------------------------------------------- 1 | const Apify = require('apify'); 2 | 3 | const { toRows, toObjects, updateRowsObjects } = require('./transformers.js'); 4 | 5 | module.exports = async ({ mode, values, newObjects, deduplicateByField, deduplicateByEquality, transformFunction, columnsOrder, keepSheetColumnOrder, backupStore }) => { 6 | if (mode === 'replace') { 7 | const replacedObjects = updateRowsObjects({ newObjects, deduplicateByField, deduplicateByEquality, transformFunction, columnsOrder, keepSheetColumnOrder }); 8 | return toRows(replacedObjects); 9 | } 10 | if (mode === 'modify' || mode === 'read') { 11 | if (!values || values.length <= 1) { 12 | throw new Error('There are either no data in the sheet or only one header row so it cannot be modified!'); 13 | } 14 | const oldObjects = toObjects(values); 15 | const replacedObjects = updateRowsObjects({ oldObjects, deduplicateByField, deduplicateByEquality, transformFunction, columnsOrder, keepSheetColumnOrder }); 16 | 17 | if (mode === 'read') { 18 | await Apify.setValue('OUTPUT', replacedObjects); 19 | console.log('Data were read, processed and saved as OUTPUT to the default key-value store'); 20 | process.exit(0); 21 | } 22 | return toRows(replacedObjects); 23 | } 24 | if (mode === 'append') { 25 | const oldObjects = toObjects(values); // [] if zero or one rows 26 | const appendedObjects = updateRowsObjects({ oldObjects, newObjects, deduplicateByField, deduplicateByEquality, transformFunction, columnsOrder, keepSheetColumnOrder }); 27 | return toRows(appendedObjects); 28 | } 29 | if (mode === 'load backup') { 30 | const store = await Apify.openKeyValueStore(backupStore); 31 | if (!store) { 32 | throw new Error('Backup store not found under id/name:', backupStore); 33 | } 34 | const rows = await store.getValue('backup'); 35 | if (!rows) { 36 | throw new Error('We did not find any record called "backup" in this store:', backupStore); 37 | } 38 | return rows; 39 | } 40 | }; 41 | -------------------------------------------------------------------------------- /src/loaders.js: -------------------------------------------------------------------------------- 1 | const Apify = require('apify'); 2 | const csvParser = require('csvtojson'); 3 | 4 | const { log } = Apify.utils; 5 | 6 | const { retryingRequest } = require('./utils.js'); 7 | 8 | module.exports.loadFromApify = async ({ mode, datasetId, limit, offset }) => { 9 | if (mode !== 'append' && mode !== 'replace') { 10 | return; 11 | } 12 | 13 | const defaultOptions = { 14 | limit, 15 | offset, 16 | clean: true, 17 | }; 18 | 19 | const datasetClient = Apify.newClient().dataset(datasetId); 20 | let datasetInfo; 21 | try { 22 | datasetInfo = await datasetClient.get(); 23 | } catch (e) { 24 | throw `Could not find dataset with ID ${datasetId}. Perhaps you provided a wrong ID? Got error: ${e}`; 25 | } 26 | 27 | // Unfortunately, simplified parameter is no longer supported by the client so we have to do raw HTTP 28 | const isLegacyPhantom = datasetInfo.actId === 'YPh5JENjSSR6vBf2E'; 29 | let csv; 30 | if (isLegacyPhantom) { 31 | log.warning(`Requesting dataset of deprecated phantom legacy actor. Please report if the format is not correct`); 32 | const limitStr = limit ? `&limit=${limit}` : ''; 33 | const offsetStr = offset ? `&offset=${offset}` : ''; 34 | const url = `https://api.apify.com/v2/datasets/${datasetId}/items?format=csv&simplified=true&clean=true${limitStr}${offsetStr}`; 35 | csv = await Apify.utils.requestAsBrowser({ url }).then((res) => res.body.toString()); 36 | } else { 37 | csv = await datasetClient.downloadItems('csv', { 38 | ...defaultOptions, 39 | }).then((res) => res.toString()); 40 | } 41 | 42 | if (!csv) { 43 | throw new Error(`We didn't find any dataset with provided datasetId: ${datasetId}`); 44 | } 45 | 46 | console.log('Data loaded from Apify storage'); 47 | 48 | // TODO: Client now provides sorted list of fields so we should be able to get rid of this whole CSV business 49 | const newObjects = await csvParser().fromString(csv); 50 | 51 | console.log('Data parsed from CSV'); 52 | 53 | if (newObjects.length === 0) { 54 | throw new Error('We loaded 0 items from the dataset or crawler execution, finishing...'); 55 | } 56 | 57 | console.log(`We loaded ${newObjects.length} items from Apify storage \n`); 58 | 59 | return newObjects; 60 | }; 61 | 62 | module.exports.loadFromSpreadsheet = async ({ client, spreadsheetId, spreadsheetRange }) => { 63 | const rowsResponse = await retryingRequest('Getting spreadsheet rows', async () => client.spreadsheets.values.get({ 64 | spreadsheetId, 65 | range: spreadsheetRange, 66 | })); 67 | if (!rowsResponse || !rowsResponse.data) { 68 | throw new Error('We couldn\'t load current data from the spreadsheet so we cannot continue!!'); 69 | } 70 | // console.dir(rowsResponse.data.values); 71 | return rowsResponse.data.values; 72 | }; 73 | -------------------------------------------------------------------------------- /src/transformers.js: -------------------------------------------------------------------------------- 1 | const md5 = require('md5'); 2 | 3 | const { sortObj } = require('./utils'); 4 | const { flattenArrayOfObjectsAndGetKeys } = require('./tabulation'); 5 | 6 | exports.toObjects = (rows) => { 7 | if (!rows || rows.length <= 1) return []; 8 | const keys = rows[0]; 9 | return rows.slice(1).map((row) => { 10 | const obj = {}; 11 | keys.forEach((key, i) => { 12 | if (typeof row[i] === 'object') { 13 | throw new Error('TRANSFORMING ERROR - Cannot convert nested objects to rows'); 14 | } 15 | obj[key] = row[i]; 16 | }); 17 | return obj; 18 | }); 19 | }; 20 | 21 | exports.toRows = (objects) => { 22 | if (!objects || objects.length === 0) return []; 23 | const header = Object.keys(objects[0]); 24 | const values = objects.map((object) => Object.values(object)); 25 | return [header, ...values]; 26 | }; 27 | 28 | const makeUniqueRows = (oldObjects, newObjects, field, equality) => { 29 | const countHash = (row) => md5(Object.values(row).join('')); 30 | const rowIntoKey = (row) => { 31 | if (field) return row[field]; 32 | if (equality) return countHash(row); 33 | throw new Error('Nor field or equality was provided to filterUniqueRows function'); 34 | }; 35 | if (!field && !equality) return oldObjects.concat(newObjects); 36 | 37 | const tempObj = {}; 38 | oldObjects.concat(newObjects).forEach((row) => { 39 | const key = rowIntoKey(row); 40 | if (!tempObj[key]) { 41 | tempObj[key] = row; 42 | } 43 | }); 44 | const filteredRows = Object.values(tempObj).filter((row) => !!row); 45 | return filteredRows; 46 | }; 47 | 48 | // export to test 49 | exports.makeUniqueRows = makeUniqueRows; 50 | 51 | // works only if all objects in one array have the same keys 52 | exports.updateRowsObjects = ({ 53 | oldObjects = [], 54 | newObjects = [], 55 | deduplicateByField, 56 | deduplicateByEquality, 57 | transformFunction, 58 | columnsOrder, 59 | keepSheetColumnOrder, 60 | }) => { 61 | if (keepSheetColumnOrder && oldObjects[0]) { 62 | // eslint-disable-next-line prefer-destructuring 63 | columnsOrder = Object.keys(oldObjects[0]); 64 | } 65 | // if no field or equality - this is simple concat 66 | const allObjects = transformFunction 67 | ? transformFunction({ datasetData: newObjects, spreadsheetData: oldObjects }) 68 | : makeUniqueRows(oldObjects, newObjects, deduplicateByField, deduplicateByEquality); 69 | 70 | const { keys, flattenedData } = flattenArrayOfObjectsAndGetKeys(allObjects); 71 | 72 | // We have to sort and fill empty after we flatten 73 | const updatedObjects = flattenedData.map((object) => { 74 | const updatedObj = object; 75 | keys.forEach((key) => { 76 | if (!updatedObj[key]) updatedObj[key] = ''; 77 | }); 78 | return sortObj(updatedObj, columnsOrder); 79 | }); 80 | return updatedObjects; 81 | }; 82 | -------------------------------------------------------------------------------- /src/validate-parse-input.js: -------------------------------------------------------------------------------- 1 | const { parseRawData } = require('./raw-data-parser.js'); 2 | const { evalFunction } = require('./utils.js'); 3 | 4 | module.exports = async (input) => { 5 | const { 6 | spreadsheetId, 7 | publicSpreadsheet = false, 8 | mode, 9 | datasetId, 10 | rawData = [], 11 | deduplicateByField, 12 | deduplicateByEquality, 13 | transformFunction, 14 | googleCredentials, 15 | columnsOrder, 16 | } = input; 17 | 18 | const parsedRawData = await parseRawData({ mode, rawData }); 19 | 20 | if (parsedRawData.length > 0 && datasetId) { 21 | throw new Error('WRONG INPUT! - Use only one of "rawData" and "datasetId"!'); 22 | } 23 | 24 | if ( 25 | ['replace', 'append'].includes(mode) 26 | && (typeof datasetId !== 'string' || datasetId.length !== 17) 27 | && parsedRawData.length === 0 28 | ) { 29 | throw new Error('WRONG INPUT! - datasetId field needs to be a string with 17 characters!'); 30 | } 31 | if (mode !== 'load backup' && (typeof spreadsheetId !== 'string' || spreadsheetId.length !== 44)) { 32 | throw new Error('WRONG INPUT! - spreadsheetId field needs to be a string with 44 characters!'); 33 | } 34 | if (deduplicateByEquality && deduplicateByField) { 35 | throw new Error('WRONG INPUT! - deduplicateByEquality and deduplicateByField cannot be used together!'); 36 | } 37 | 38 | // Cannot write to public spreadsheet 39 | if (['replace', 'append', 'modify'].includes(mode) && publicSpreadsheet) { 40 | throw new Error('WRONG INPUT - Cannot use replace, append or modify mode for public spreadsheet. For write access, use authorization!') 41 | } 42 | 43 | // Check if googleCredentials have correct format 44 | if (googleCredentials) { 45 | if (typeof googleCredentials !== 'object' || !googleCredentials.client_id || !googleCredentials.client_secret || !googleCredentials.redirect_uri) { 46 | throw new Error('If you want to pass your own Google keys, it has to be an object with those properties: client_id, client_secret, redirect_uri'); 47 | } 48 | } 49 | 50 | // Parsing stringified function 51 | let parsedTransformFunction; 52 | if (transformFunction && transformFunction.trim()) { 53 | console.log('\nPHASE - PARSING TRANSFORM FUNCTION\n'); 54 | parsedTransformFunction = await evalFunction(transformFunction); 55 | if (typeof parsedTransformFunction === 'function' && (deduplicateByEquality || deduplicateByField)) { 56 | throw new Error('WRONG INPUT! - transformFunction cannot be used together with deduplicateByEquality or deduplicateByField!'); 57 | } 58 | console.log('Transform function parsed...'); 59 | } 60 | 61 | if (columnsOrder && !Array.isArray(columnsOrder)) { 62 | throw new Error('WRONG INPUT! - columnsOrder must be an array on of string values(keys)!') 63 | } 64 | 65 | return { transformFunction: parsedTransformFunction, rawData: parsedRawData }; 66 | } 67 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | const { toObjects, toRows, updateRowsObjects, makeUniqueRows } = require('../src/utils.js'); 4 | // const { customTransform1, reconstructArray, pseudoDeepEquals, createKeys } = require('../src/transformFunctions'); 5 | // const { customObjectsNew, customObjectsOld, customObjFlat, customObjFlat2, transformedArray } = require('./mocks'); 6 | 7 | const rows = [['a', 'b', 'c'], [2, 2, 4], [3, 2, 5], [4, 2, 5]]; 8 | const objects = [{ a: 2, b: 2, c: 4 }, { a: 3, b: 2, c: 5 }, { a: 4, b: 2, c: 5 }]; 9 | const objects2 = [{ d: 4, b: 2, c: null }, { d: 3, b: 2, c: 6 }, { d: 4, b: null, c: 5 }]; 10 | const appendedBasic = [ 11 | { a: 2, b: 2, c: 4, d: '' }, 12 | { a: 3, b: 2, c: 5, d: '' }, 13 | { a: 4, b: 2, c: 5, d: '' }, 14 | { a: '', b: 2, c: '', d: 4 }, 15 | { a: '', b: 2, c: 6, d: 3 }, 16 | { a: '', b: '', c: 5, d: 4 }, 17 | ]; 18 | const uniqueToAppend = [ 19 | { a: 2, b: 2, c: 4, d: '' }, 20 | { d: 4, b: '', c: 5, a: '' }, 21 | ]; 22 | const appendedWithFilterField = [ 23 | { a: 2, b: 2, c: 4, d: '' }, 24 | // { a: 3, b: 2, c: 5, d: null }, 25 | // { a: 4, b: 2, c: 5, d: null }, 26 | { a: '', b: '', c: 5, d: 4 }, 27 | ]; 28 | const replacedWithFilterField = [ 29 | { d: 4, b: 2, c: '' }, 30 | { d: 4, b: '', c: 5 }, 31 | ]; 32 | 33 | // const keysToCompare = createKeys(customObjFlat2, customObjFlat); 34 | // const promotionKeys = keysToCompare.filter((key) => key.startsWith('promotions/')); 35 | 36 | /* 37 | describe('pseudoDeepEquals', () => { 38 | it('works', () => { 39 | assert.deepEqual(pseudoDeepEquals(customObjFlat2, customObjFlat, keysToCompare, promotionKeys), true); 40 | }); 41 | }); 42 | 43 | describe('reconstructArray', () => { 44 | it('works', () => { 45 | assert.deepEqual(reconstructArray(customObjFlat, promotionKeys), transformedArray); 46 | }); 47 | }); 48 | 49 | describe('customTransform1', () => { 50 | const customTransform = customTransform1; 51 | const finalObjects = []; 52 | it('works', () => { 53 | assert.deepEqual(customTransform(customObjectsNew, customObjectsOld), finalObjects); 54 | }); 55 | }); 56 | */ 57 | 58 | describe('toObjects', () => { 59 | it('works', () => { 60 | // console.log('to objects') 61 | // console.dir(objects) 62 | // console.dir(toObjects(rows)) 63 | assert.deepEqual(objects, toObjects(rows)); 64 | }); 65 | }); 66 | 67 | describe('toRows', () => { 68 | it('works', () => { 69 | // console.log('to rows') 70 | // console.dir(rows) 71 | // console.dir(toRows(objects)) 72 | assert.deepEqual(rows, toRows(objects)); 73 | }); 74 | }); 75 | 76 | describe('replace', () => { 77 | it('basic', () => { 78 | assert.deepEqual(objects2, updateRowsObjects({ oldObjects: [], newObjects: objects2 })); 79 | }); 80 | it('with field filter', () => { 81 | assert.deepEqual(replacedWithFilterField, updateRowsObjects({ oldObjects: [], newObjects: objects2, filterByField: 'b' })); 82 | }); 83 | }); 84 | 85 | describe('append', () => { 86 | it('basic', () => { 87 | assert.deepEqual(appendedBasic, updateRowsObjects({ oldObjects: objects, newObjects: objects2 })); 88 | }); 89 | it('with field filter', () => { 90 | assert.deepEqual(appendedWithFilterField, updateRowsObjects({ oldObjects: objects, newObjects: objects2, filterByField: 'b' })); 91 | }); 92 | }); 93 | 94 | describe('makeUniqueRows', () => { 95 | it('works', () => { 96 | assert.deepEqual(uniqueToAppend, makeUniqueRows(objects, objects2, 'b', null)); 97 | }); 98 | }); 99 | -------------------------------------------------------------------------------- /src/tabulation.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-restricted-syntax */ 2 | // These functions are not yet fully used because we are getting CSV from dataset 3 | // but it would be better to use internal tabulation everywhere 4 | 5 | // only consider integers smaller than 1 billion, array indexes are not higher anyway 6 | const REGEXP_NUM = /^[0-9]{1,9}$/; 7 | 8 | /** 9 | * This function sorts an array with string property names using the following rules: 10 | * 1) "URL" is always first 11 | * 2) Multi-component strings with numberical components are sorted using numerical comparison, rather than string 12 | * (e.g. "array/2/xxx" < "array/10") 13 | * 3) Otherwise, use normal string 14 | * Note that the implementation of this method has some unexpected consequences, 15 | * e.g. it will sort array as ["/", " /"], while Array.sort() would produce [" /", "/"]. But we can live with that. 16 | * @param propNames A string array 17 | */ 18 | exports.sortPropertyNames = function (propNames) { 19 | propNames.sort((a, b) => { 20 | // Lowering case so Capitals are not sorted before lowers 21 | a = String(a).toLowerCase(); 22 | b = String(b).toLowerCase(); 23 | 24 | // sort index numbers numerically (e.g. "array/2" < "array/10") 25 | const partsA = a.split('/'); 26 | const partsB = b.split('/'); 27 | 28 | // Check the parts that are contained in both keys 29 | const len = Math.min(partsA.length, partsB.length); 30 | for (let i = 0; i < len; i++) { 31 | const partA = partsA[i]; 32 | const partB = partsB[i]; 33 | 34 | // if parts are the same, go to next level 35 | if (partA !== partB) { 36 | // if both parts are numbers, compare their numeric values 37 | if (REGEXP_NUM.test(partA) && REGEXP_NUM.test(partB)) return parseInt(partA, 10) - parseInt(partB, 10); 38 | 39 | // at least one part is not a number, use normal string comparison 40 | return partA < partB ? -1 : 1; 41 | } 42 | } 43 | 44 | // if the common part of the string is equal in both strings then the shorter string is before longer string 45 | return partsA.length - partsB.length; 46 | }); 47 | }; 48 | 49 | const flattenObjectMut = function (srcObj, trgObj, path) { 50 | if (srcObj instanceof Date) srcObj = srcObj.toISOString(); 51 | 52 | const type = typeof srcObj; 53 | 54 | if (type === 'number' || type === 'string' || type === 'boolean' || srcObj === null) { 55 | trgObj[path] = srcObj; 56 | } else if (type === 'object') { 57 | // srcObj is an object or array 58 | // eslint-disable-next-line guard-for-in 59 | for (const key in srcObj) { 60 | let subPath; 61 | if (key === '') subPath = path; // super-properties have the same path as the parent 62 | else if (path.length > 0) subPath = `${path}/${key}`; 63 | else subPath = key; 64 | flattenObjectMut(srcObj[key], trgObj, subPath); 65 | } 66 | } 67 | }; 68 | 69 | const flattenObject = function (item) { 70 | const flattenedObject = {}; 71 | flattenObjectMut(item, flattenedObject, ''); 72 | return flattenedObject; 73 | }; 74 | 75 | exports.flattenArrayOfObjectsAndGetKeys = function (data) { 76 | const flattenedData = []; 77 | const keySet = new Set(); 78 | for (const item of data) { 79 | const flattenedObj = flattenObject(item); 80 | flattenedData.push(flattenedObj); 81 | for (const key of Object.keys(flattenedObj)) { 82 | keySet.add(key); 83 | } 84 | } 85 | return { 86 | flattenedData, 87 | keys: Array.from(keySet), 88 | }; 89 | }; 90 | -------------------------------------------------------------------------------- /src/raw-data-parser.js: -------------------------------------------------------------------------------- 1 | const Apify = require('apify'); 2 | const csvParser = require('csvtojson'); 3 | 4 | const { toObjects } = require('./transformers.js'); 5 | const { loadFromApify } = require('./loaders'); 6 | const { sortObj } = require('./utils'); 7 | 8 | module.exports.parseRawData = async ({ mode, rawData }) => { 9 | if (!Array.isArray(rawData)) { 10 | throw new Error('WRONG INPUT! - rawData has to be an array!'); 11 | } 12 | 13 | if (rawData.length === 0) { 14 | return rawData; 15 | } 16 | 17 | if (!['append', 'replace'].includes(mode)) { 18 | throw new Error('WRONG INPUT! - Can use rawData only with "replace" or "append" mode!'); 19 | } 20 | 21 | const hasOnlyObjects = rawData.reduce((acc, item) => { 22 | if (!acc) { 23 | return false; 24 | } 25 | if (Array.isArray(item) || typeof item !== 'object') { 26 | return false; 27 | } 28 | return true; 29 | }, true); 30 | 31 | const hasOnlyArrays = rawData.reduce((acc, item) => { 32 | if (!acc) { 33 | return false; 34 | } 35 | if (!Array.isArray(item)) { 36 | return false; 37 | } 38 | return true; 39 | }, true); 40 | 41 | if (!hasOnlyObjects && !hasOnlyArrays) { 42 | throw new Error('WRONG INPUT - rawData needs to be either an array of objects or array of arrays!'); 43 | } 44 | 45 | if (hasOnlyArrays) { 46 | try { 47 | return toObjects(rawData); 48 | } catch (e) { 49 | throw new Error(`WRONG INPUT - rawData array of arrays cannot contain nested objects! Error: ${e.message}`); 50 | } 51 | } 52 | if (hasOnlyObjects) { 53 | const isNested = rawData.reduce((acc, item) => { 54 | if (acc) { 55 | return true; 56 | } 57 | for (const value of Object.values(item)) { 58 | if (typeof value === 'object') { 59 | return true; 60 | } 61 | } 62 | return false; 63 | }, false); 64 | 65 | if (!isNested) { 66 | const keysObject = rawData.reduce((acc, item) => { 67 | for (const key of Object.keys(item)) { 68 | acc[key] = true; 69 | } 70 | return acc; 71 | }, {}); 72 | const keys = Object.keys(keysObject); 73 | const updatedData = rawData.map((item) => { 74 | keys.forEach((key) => { 75 | if (!item[key]) item[key] = ''; 76 | }); 77 | return sortObj(item, []); 78 | }); 79 | return updatedData; 80 | } 81 | 82 | console.log('Raw data have nested structures. We need to use Apify API to flatten them, this may take a while on large structures. If you don\'t have Apify account, this will not work'); // eslint-disable-line 83 | 84 | if (Apify.isAtHome()) { 85 | await Apify.pushData(rawData); 86 | return loadFromApify({ mode, datasetId: process.env.APIFY_DEFAULT_DATASET_ID }); 87 | } 88 | 89 | const { datasets } = Apify.client; 90 | const datasetCollectionClient = Apify.newClient().datasets; 91 | 92 | let { id, itemCount } = await datasetCollectionClient.getOrCreate('spreadsheet-temporary-container'); 93 | if (itemCount > 0) { 94 | const datasetClient = Apify.newClient().dataset(id); 95 | await datasetClient.delete(); 96 | id = await datasetCollectionClient.getOrCreate('spreadsheet-temporary-container') 97 | .then((res) => res.id); 98 | } 99 | const datasetClient = Apify.newClient().dataset(id); 100 | await datasetClient.pushItems(rawData); 101 | 102 | const csv = (await datasetClient.downloadItems('csv')).toString(); 103 | 104 | return csvParser().fromString(csv); 105 | } 106 | }; 107 | -------------------------------------------------------------------------------- /test/customFilerFunctions.js: -------------------------------------------------------------------------------- 1 | const customFilterFunctionPat = (domainMain) => (newObjectsMain, oldObjectsMain = []) => { 2 | const union = (setA, setB) => { 3 | const unioned = new Set(setA); 4 | for (const elem of setB) { 5 | unioned.add(elem); 6 | } 7 | return Array.from(unioned); 8 | }; 9 | 10 | const createKeys = (obj1 = {}, obj2 = {}) => { 11 | const unioned = union(Object.keys(obj1), Object.keys(obj2)); 12 | return unioned.filter((key) => key.startsWith('promotions/') || key === 'price' || key === 'stock' || key === 'product_availability'); 13 | }; 14 | 15 | exports.createKeys = createKeys; 16 | 17 | const isNotEmpty = (obj) => { 18 | if (!obj) return false; 19 | const nonemptyValues = Object.values(obj).filter((val) => val !== '' && val !== undefined); 20 | return nonemptyValues.length > 0; 21 | }; 22 | 23 | const reconstructArray = (rowObject, keys) => { 24 | const arr = []; 25 | keys.forEach((key) => { 26 | const [, i, field] = key.match(/promotions\/(\d+)\/(.+)/); 27 | const index = parseInt(i, 10); 28 | if (!arr[index]) arr[index] = {}; 29 | arr[index][field] = rowObject[key]; 30 | }); 31 | return arr.filter(isNotEmpty); 32 | }; 33 | 34 | exports.reconstructArray = reconstructArray; 35 | // only one level deep 36 | const deepEqualsArray = (arr1, arr2, promotionKeys) => { 37 | // by default they are equal 38 | if (arr1.length !== arr2.length) return false; 39 | for (let i = 0; i < arr1.length; i++) { 40 | for (const key of promotionKeys) { 41 | if (arr1[key] !== arr2[key]) return false; 42 | } 43 | } 44 | return true; 45 | }; 46 | const pseudoDeepEquals = (rowObject1, rowObject2, keysToCompare, promotionKeys) => { 47 | const promotionsEquals = deepEqualsArray( 48 | reconstructArray(rowObject1, promotionKeys), 49 | reconstructArray(rowObject2, promotionKeys), 50 | promotionKeys, 51 | ); 52 | const bool = keysToCompare.reduce((accBool, key) => { 53 | if (accBool) { 54 | if (promotionKeys.includes(key)) { 55 | return promotionsEquals; 56 | } 57 | return rowObject1[key] === rowObject2[key]; 58 | } 59 | return false; 60 | }, true); 61 | return bool; 62 | }; 63 | 64 | exports.pseudoDeepEquals = pseudoDeepEquals; 65 | 66 | const customTransform1 = (newObjects, oldObjects, domain) => { 67 | let id; 68 | switch (domain) { 69 | case 'shopee': id = 'product_id'; 70 | break; 71 | case 'lazada': id = 'sku'; 72 | break; 73 | default: throw new Error('Cannot match a domain for defining what is the id property. Expected "shopee" or "lazada"'); 74 | } 75 | 76 | // We put new rows into temp object and then overwrite them with null if we find a match 77 | const tempObj = {}; 78 | const keysToCompare = createKeys(newObjects[0], oldObjects[0]); 79 | const promotionKeys = keysToCompare.filter((key) => key.startsWith('promotions/')); 80 | console.log('keys to compare', keysToCompare); 81 | console.log('newObjects.length', newObjects.length); 82 | console.log('oldObjects.length', oldObjects.length); 83 | newObjects.forEach((row) => { 84 | tempObj[row[id]] = row; 85 | }); 86 | oldObjects.forEach((row) => { 87 | const maybeRow = tempObj[row[id]]; 88 | 89 | if (maybeRow && pseudoDeepEquals(maybeRow, row, keysToCompare, promotionKeys)) { 90 | tempObj[row[id]] = null; 91 | } 92 | }); 93 | const filteredRows = Object.values(tempObj).filter((row) => !!row); 94 | 95 | console.log('transformed rows length:', filteredRows.length); 96 | console.log('sliced rows:'); 97 | console.dir(filteredRows.slice(0, 5)); 98 | return filteredRows; 99 | }; 100 | 101 | return customTransform1(newObjectsMain, oldObjectsMain, domainMain); 102 | }; 103 | 104 | exports.patShopee = customFilterFunctionPat('shopee'); 105 | exports.patLazada = customFilterFunctionPat('lazada'); 106 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | const Apify = require('apify'); 2 | 3 | const { sortPropertyNames } = require('./tabulation'); 4 | 5 | const { log } = Apify.utils; 6 | 7 | const ERRORS_TO_RETRY = [ 8 | 'The service is currently unavailable', 9 | ] 10 | 11 | const getNiceErrorMessage = (type, errorMessage) => { 12 | const baseErrorMessage = `Request ${type} failed with error ${errorMessage}`; 13 | const wrongAccountText = `Perhaps you used a wrong Google account?\n` 14 | + `If you want to use a different Google account or use multiple Google accounts, please follow the guide here:\n` 15 | + `https://apify.com/lukaskrivka/google-sheets#authentication-and-authorization\n` 16 | if (errorMessage.includes('invalid_grant')) { 17 | return `${baseErrorMessage}\n${wrongAccountText}`; 18 | } else if (errorMessage.includes('The caller does not have permission')) { 19 | return `${baseErrorMessage}\n${wrongAccountText}`; 20 | } else { 21 | return baseErrorMessage; 22 | } 23 | } 24 | 25 | exports.retryingRequest = async (type, request) => { 26 | const MAX_ATTEMPTS = 6; 27 | const SLEEP_MULTIPLIER = 3; 28 | let sleepMs = 1000; 29 | 30 | for (let i = 0; i < MAX_ATTEMPTS; i++) { 31 | let response; 32 | try { 33 | response = await request(); 34 | return response; 35 | } catch (e) { 36 | const willRetry = ERRORS_TO_RETRY.some((errorMessage) => e.message.includes(errorMessage)); 37 | if (willRetry) { 38 | log.warning(`Retrying API call for ${type} to google with attempt n. ${i + 1} for error: ${e.message}`); 39 | await Apify.utils.sleep(sleepMs); 40 | sleepMs *= SLEEP_MULTIPLIER; 41 | } else { 42 | const error = getNiceErrorMessage(type, e.message); 43 | throw error; 44 | } 45 | } 46 | } 47 | }; 48 | 49 | exports.countCells = (rows) => { 50 | if (!rows) return 0; 51 | if (!rows[0]) return 0; 52 | return rows[0].length * rows.length; 53 | }; 54 | 55 | exports.trimSheetRequest = (height, width, sheetId) => { 56 | const payload = { 57 | requests: [], 58 | }; 59 | if (height) { 60 | payload.requests.push({ 61 | deleteDimension: { 62 | range: { 63 | sheetId, 64 | dimension: 'ROWS', 65 | startIndex: height, 66 | }, 67 | }, 68 | }); 69 | } 70 | if (width) { 71 | payload.requests.push({ 72 | deleteDimension: { 73 | range: { 74 | sheetId, 75 | dimension: 'COLUMNS', 76 | startIndex: width, 77 | }, 78 | }, 79 | }); 80 | } 81 | return payload; 82 | }; 83 | 84 | module.exports.createSheetRequest = (title) => { 85 | return { 86 | requests: [{ 87 | addSheet: { 88 | properties: { title }, 89 | }, 90 | }], 91 | }; 92 | }; 93 | 94 | module.exports.saveBackup = async (createBackup, values) => { 95 | if (createBackup) { 96 | if (values) { 97 | log.info('Saving backup...'); 98 | await Apify.setValue('backup', values); 99 | } else { 100 | log.warning('There are currently no rows in the spreadsheet so we will not save backup...'); 101 | } 102 | } 103 | }; 104 | 105 | module.exports.evalFunction = (transformFunction) => { 106 | let parsedTransformFunction; 107 | if (transformFunction) { 108 | try { 109 | // Safe eval stopped working with commented code 110 | parsedTransformFunction = eval(transformFunction); // eslint-disable-line 111 | } catch (e) { 112 | throw new Error('Evaluation of the tranform function failed with error. Please check if you inserted valid javascript code:', e); 113 | } 114 | // Undefined is allowed because I wanted to allow have commented code in the transform function 115 | if (typeof parsedTransformFunction !== 'function' && parsedTransformFunction !== undefined) { 116 | throw new Error('Transform function has to be a javascript function or it has to be undefined (in case the whole code is commented out)'); 117 | } 118 | return parsedTransformFunction; 119 | } 120 | }; 121 | 122 | // I know this is very inneficient way but so far didn't hit a performance bottleneck (on 3M items) 123 | module.exports.sortObj = (obj, keys) => { 124 | const newObj = {}; 125 | // First we add user-requested sorting 126 | for (const key of keys) { 127 | newObj[key] = obj[key]; 128 | } 129 | // The we sort the rest with special algorithm 130 | // They are really only sorted mutably 131 | const sortedKeys = Object.keys(obj); 132 | sortPropertyNames(sortedKeys); 133 | 134 | for (const key of sortedKeys) { 135 | if (!keys.includes(key)) { 136 | newObj[key] = obj[key]; 137 | } 138 | } 139 | return newObj; 140 | }; 141 | -------------------------------------------------------------------------------- /periodic-test/test.js: -------------------------------------------------------------------------------- 1 | const Apify = require('apify'); 2 | const assert = require('assert'); 3 | 4 | const { mock1, mock2, mock1Dataset, mock2Dataset, mockTransform, backupMock } = require('./mock-data.js'); 5 | 6 | const NAME = 'lukaskrivka/google-sheets'; 7 | const spreadsheetId = '1jCmoAhhhHKAo5Ost3DzI4D9GgJ8VgwNBOeQk6qfXqgs'; 8 | 9 | Apify.main(async () => { 10 | const datasetOne = await Apify.openDataset('SHEETS-TEST-1', { forceCloud: true }); 11 | await datasetOne.pushData(mock1Dataset); 12 | const datasetIdOne = await datasetOne.getInfo().then((res) => res.id); 13 | 14 | const datasetTwo = await Apify.openDataset('SHEETS-TEST-2', { forceCloud: true }); 15 | await datasetTwo.pushData(mock2Dataset); 16 | const datasetIdTwo = await datasetTwo.getInfo().then((res) => res.id); 17 | 18 | try { 19 | // TEST 1 20 | console.log('TEST-1'); 21 | // REPLACE 22 | console.log('calling - replace'); 23 | await Apify.call( 24 | NAME, 25 | { 26 | datasetId: datasetIdOne, 27 | spreadsheetId, 28 | mode: 'replace', 29 | }, 30 | ); 31 | console.log('done - replace'); 32 | 33 | // REPLACE - READ 34 | console.log('calling - read'); 35 | const read1 = await Apify.call( 36 | NAME, 37 | { 38 | spreadsheetId, 39 | mode: 'read', 40 | }, 41 | ).then(((res) => res.output.body)); 42 | console.log('done - read'); 43 | 44 | console.log('trying assertion'); 45 | console.dir(read1); 46 | console.dir(mock1); 47 | assert.deepEqual(read1, mock1); 48 | console.log('assertion done'); 49 | 50 | // TEST 2 51 | console.log('TEST-2'); 52 | // APPEND 53 | console.log('calling append'); 54 | await Apify.call( 55 | NAME, 56 | { 57 | datasetId: datasetIdTwo, 58 | spreadsheetId, 59 | mode: 'append', 60 | }, 61 | ); 62 | console.log('done - append'); 63 | 64 | // APPEND - READ 65 | console.log('calling - read'); 66 | const read2 = await Apify.call( 67 | NAME, 68 | { 69 | spreadsheetId, 70 | mode: 'read', 71 | }, 72 | ).then(((res) => res.output.body)); 73 | console.log('done - read'); 74 | 75 | console.log('trying assertion'); 76 | console.dir(read2); 77 | console.dir(mock1.concat(mock2)); 78 | assert.deepEqual(read2, mock1.concat(mock2)); 79 | console.log('assertion done'); 80 | 81 | // TEST 3 82 | console.log('TEST 3'); 83 | 84 | // MODIFY 85 | console.log('calling modify'); 86 | await Apify.call( 87 | NAME, 88 | { 89 | spreadsheetId, 90 | mode: 'modify', 91 | transformFunction: mockTransform.toString(), 92 | }, 93 | ); 94 | console.log('done - modify'); 95 | 96 | // MODIFY - READ 97 | console.log('calling - read'); 98 | const read3 = await Apify.call( 99 | NAME, 100 | { 101 | spreadsheetId, 102 | mode: 'read', 103 | }, 104 | ).then(((res) => res.output.body)); 105 | console.log('done - read'); 106 | 107 | console.log('trying assertion'); 108 | assert.deepEqual(read3, mock1.concat(mock2).slice(1)); 109 | console.log('assertion done'); 110 | 111 | // TEST 4 112 | console.log('TEST-4'); 113 | 114 | // APPEND 115 | console.log('calling - append'); 116 | const runInfo = await Apify.call( 117 | NAME, 118 | { 119 | datasetId: datasetIdOne, 120 | spreadsheetId, 121 | mode: 'append', 122 | createBackup: true, 123 | deduplicateByField: 'name', 124 | }, 125 | ); 126 | const { defaultKeyValueStoreId } = runInfo; 127 | console.log('done - append'); 128 | 129 | // APPEND - READ 130 | console.log('calling - read'); 131 | const read4 = await Apify.call( 132 | NAME, 133 | { 134 | spreadsheetId, 135 | mode: 'read', 136 | }, 137 | ).then(((res) => res.output.body)); 138 | console.log('done - read'); 139 | 140 | console.log('trying assertion'); 141 | assert.deepEqual(read4, mock1.concat(mock2).slice(1).concat(mock1.slice(0,1))); 142 | console.log('assertion done'); 143 | 144 | // TEST 5 145 | console.log('TEST-5'); 146 | // 147 | console.log('calling - load backup'); 148 | await Apify.call( 149 | NAME, 150 | { 151 | spreadsheetId, 152 | mode: 'load backup', 153 | backupStore: defaultKeyValueStoreId, 154 | }, 155 | ); 156 | console.log('done - load backup'); 157 | 158 | console.log('calling - read'); 159 | const read5 = await Apify.call( 160 | NAME, 161 | { 162 | spreadsheetId, 163 | mode: 'read', 164 | }, 165 | ).then(((res) => res.output.body)); 166 | console.log('done - read'); 167 | 168 | console.log('trying assertion'); 169 | assert.deepEqual(read5, mock1.concat(mock2).slice(1)); 170 | console.log('assertion done'); 171 | 172 | console.log('TEST SUCCESSFUL!!!'); 173 | } finally { 174 | await datasetOne.delete(); 175 | await datasetTwo.delete(); 176 | } 177 | }); 178 | -------------------------------------------------------------------------------- /INPUT_SCHEMA.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Spreadsheet import input", 3 | "type": "object", 4 | "schemaVersion": 1, 5 | "required": [], 6 | "properties": { 7 | "mode": { 8 | "title": "Mode", 9 | "type": "string", 10 | "description": "What should the actor do", 11 | "enum": ["append", "replace", "modify", "read", "load backup"] 12 | }, 13 | "spreadsheetId": { 14 | "title": "Spreadsheet id", 15 | "type": "string", 16 | "description": "Id of the spreadsheet from where the old data will be loaded", 17 | "editor": "textfield" 18 | }, 19 | "publicSpreadsheet": { 20 | "title": "Public spreadsheet (read-only)", 21 | "type": "boolean", 22 | "description": "If checked, you don't need to authorize. You have to publish your spreadsheet and it works only in read mode", 23 | "default": false 24 | }, 25 | "datasetId": { 26 | "title": "Dataset ID", 27 | "type": "string", 28 | "description": "Dataset or crawler execution id where the new data will be loaded from", 29 | "editor": "textfield", 30 | "sectionCaption": "Input data", 31 | "sectionDescription": "Only fill in for `append` or `replace` mode. Choose either `Dataset ID` or `Raw data`. Limit and offset is optional." 32 | }, 33 | "rawData": { 34 | "title": "Raw data", 35 | "type": "array", 36 | "description": "Raw data JSON array. Can be array of arrays for direct row import or arrays of objects.", 37 | "editor": "json" 38 | }, 39 | "limit": { 40 | "title": "Limit items", 41 | "type": "integer", 42 | "description": "Number of items to take from the dataset. The default is 250000.", 43 | "unit": "items", 44 | "minimum": 0, 45 | "default": 250000 46 | }, 47 | "offset": { 48 | "title": "Offset items", 49 | "type": "integer", 50 | "description": "Number of items to skip from the dataset. Default is 0.", 51 | "unit": "items", 52 | "minimum": 0, 53 | "default": 0 54 | }, 55 | "deduplicateByField": { 56 | "title": "Deduplicate by field", 57 | "type": "string", 58 | "description": "Items will be deduplicated by a value of this field. Cannot be used together with 'Deduplicate by equality' or 'Transform function'.", 59 | "editor": "textfield", 60 | "sectionCaption": "Deduplication and transformation", 61 | "sectionDescription": "Choose up to one way to deduplicate or transform your data." 62 | }, 63 | "deduplicateByEquality": { 64 | "title": "Deduplicate by equality", 65 | "type": "boolean", 66 | "description": "Items will be deduplicated if they are the same. Cannot be used together with 'Deduplicate by field' or 'Transform function'." 67 | }, 68 | "transformFunction": { 69 | "title": "Transform function", 70 | "type": "string", 71 | "description": "Custom function that will take new items and old items arrays as parameters and produces final array that will be imported. Cannot be used together with 'Deduplicate by equality' or 'Deduplicate by field'", 72 | "editor": "javascript", 73 | "prefill": "// Uncomment this code only if you don't use \"Deduplicate by field\" or \"Deduplicate by equality\"\n// This code behaves as if there was no transform function\n/*({ spreadsheetData, datasetData }) => {\n return spreadsheetData.concat(datasetData);\n}*/" 74 | }, 75 | "range": { 76 | "title": "Range", 77 | "type": "string", 78 | "description": "Range of the spreadsheet in A1 notation where the actor should operate. Default is the first sheet.", 79 | "editor": "textfield", 80 | "sectionCaption": "Miscellaneous" 81 | }, 82 | "columnsOrder": { 83 | "title": "Columns order", 84 | "type": "array", 85 | "description": "Array of keys. First sorts the columns by provided keys. The rest is sorted alphabetically.", 86 | "editor": "stringList" 87 | }, 88 | "keepSheetColumnOrder": { 89 | "title": "Keep column order from sheet", 90 | "type": "boolean", 91 | "description": "If true, keeps the order of columns as they are in the sheet. If there is no sheet data yet, this does nothing." 92 | }, 93 | "tokensStore": { 94 | "title": "Google OAuth tokens store", 95 | "type": "string", 96 | "description": "Key-value store where your Google OAuth tokens will be stored so you don't have to authorize every time again. By default it is google-oauth-tokens", 97 | "editor": "textfield", 98 | "default": "google-oauth-tokens" 99 | }, 100 | "createBackup":{ 101 | "title": "Create backup", 102 | "type": "boolean", 103 | "description": "Old rows from your spreadsheet will be saved to the default key-value store before importing new rows." 104 | }, 105 | "backupStore": { 106 | "title": "Backup store id", 107 | "type": "string", 108 | "description": "Id of the key-value store where the backup you want to load is located. Can pnly be used if mode is 'load-backup'", 109 | "editor": "textfield" 110 | }, 111 | "googleCredentials": { 112 | "title": "Google Developer Console credentials", 113 | "type": "object", 114 | "description": "If you want to use this actor locally or with your own version, you have to provide your own crednetials. Check actor readme for more information.", 115 | "editor": "json" 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /test/mocks.js: -------------------------------------------------------------------------------- 1 | exports.transformedArray = [ 2 | { 3 | "code": "HPOF10", 4 | "min_spend": 100000000, 5 | "discount_value": null, 6 | "discount_percent": 10 7 | }, 8 | { 9 | "code": "HPOF10", 10 | "min_spend": 100000000, 11 | "discount_value": null, 12 | "discount_percent": 10 13 | } 14 | ] 15 | 16 | exports.customObjFlat = { 17 | "title": "HP 678 Black Ink Cartridge", 18 | "regPrice": 0, 19 | "price": 390, 20 | "stock": 4224, 21 | "seller": "HP Official Store", 22 | "price_currency": "THB", 23 | "url": "https://shopee.co.th/product/29250436/422572936", 24 | "product_id": "422572936", 25 | "country": "Thailand", 26 | "updated_date": "2018-10-16T14:32:30.775Z", 27 | "product_status": "InStock", 28 | "promotions/0/code": "HPOF10", 29 | "promotions/0/min_spend": 100000000, 30 | "promotions/0/discount_value": null, 31 | "promotions/0/discount_percent": 10, 32 | "promotions/1/code": "HPOF10", 33 | "promotions/1/min_spend": 100000000, 34 | "promotions/1/discount_value": null, 35 | "promotions/1/discount_percent": 10 36 | } 37 | 38 | exports.customObjFlat2 = { 39 | "title": "HP 678 Black Ink Cartridge", 40 | "regPrice": 0, 41 | "price": 390, 42 | "stock": 4224, 43 | "seller": "HP Official Store", 44 | "price_currency": "THB", 45 | "url": "https://shopee.co.th/product/29250436/422572936", 46 | "product_id": "422572936", 47 | "country": "Thailand", 48 | "updated_date": "2018-10-16T14:32:30.775Z", 49 | "product_status": "InStock", 50 | "promotions/0/code": "HPOF10", 51 | "promotions/0/min_spend": 100000000, 52 | "promotions/0/discount_value": null, 53 | "promotions/0/discount_percent": 10, 54 | "promotions/2/code": "HPOF10", 55 | "promotions/2/min_spend": 100000000, 56 | "promotions/2/discount_value": null, 57 | "promotions/2/discount_percent": 10 58 | } 59 | 60 | exports.customObjectsNew = [ 61 | { 62 | "title": "HP 678 Black Ink Cartridge", 63 | "regPrice": 0, 64 | "price": 390, 65 | "stock": 4224, 66 | "seller": "HP Official Store", 67 | "price_currency": "THB", 68 | "url": "https://shopee.co.th/product/29250436/422572936", 69 | "product_id": "422572936", 70 | "country": "Thailand", 71 | "updated_date": "2018-10-16T14:32:30.775Z", 72 | "product_status": "InStock", 73 | "promotions/0/code": "HPOF10", 74 | "promotions/0/min_spend": 100000000, 75 | "promotions/0/discount_value": null, 76 | "promotions/0/discount_percent": 10 77 | }, 78 | { 79 | "title": "HP 678 Black Ink Cartridge", 80 | "regPrice": 0, 81 | "price": 390, 82 | "stock": 4224, 83 | "seller": "HP Official Store", 84 | "price_currency": "THB", 85 | "url": "https://shopee.co.th/product/29250436/422572936", 86 | "product_id": "422572931", 87 | "country": "Thailand", 88 | "updated_date": "2018-10-16T14:32:30.775Z", 89 | "product_status": "InStock", 90 | "promotions/0/code": "HPOF11", 91 | "promotions/0/min_spend": 100000001, 92 | "promotions/0/discount_value": null, 93 | "promotions/0/discount_percent": 10 94 | } 95 | ] 96 | 97 | exports.customObjectsOld = [ 98 | { 99 | "title": "HP 678 Black Ink Cartridge", 100 | "regPrice": 0, 101 | "price": 390, 102 | "stock": 4224, 103 | "seller": "HP Official Store", 104 | "price_currency": "THB", 105 | "url": "https://shopee.co.th/product/29250436/422572936", 106 | "product_id": "422572936", 107 | "country": "Thailand", 108 | "updated_date": "2018-10-16T14:32:30.775Z", 109 | "product_status": "InStock", 110 | "promotions/0/code": "HPOF10", 111 | "promotions/0/min_spend": 100000000, 112 | "promotions/0/discount_value": null, 113 | "promotions/0/discount_percent": 10 114 | }, 115 | { 116 | "title": "HP 678 Black Ink Cartridge", 117 | "regPrice": 0, 118 | "price": 390, 119 | "stock": 4224, 120 | "seller": "HP Official Store", 121 | "price_currency": "THB", 122 | "url": "https://shopee.co.th/product/29250436/422572936", 123 | "product_id": "422572931", 124 | "country": "Thailand", 125 | "updated_date": "2018-10-16T14:32:30.775Z", 126 | "product_status": "InStock", 127 | "promotions/0/code": "HPOF11", 128 | "promotions/0/min_spend": 100000001, 129 | "promotions/0/discount_value": null, 130 | "promotions/0/discount_percent": 10 131 | }, 132 | { 133 | "title": "HP 678 Black Ink Cartridge", 134 | "regPrice": 0, 135 | "price": 390, 136 | "stock": 4224, 137 | "seller": "HP Official Store", 138 | "price_currency": "THB", 139 | "url": "https://shopee.co.th/product/29250436/422572936", 140 | "product_id": "422572936", 141 | "country": "Thailand", 142 | "updated_date": "2018-10-16T14:32:30.775Z", 143 | "product_status": "InStock", 144 | "promotions/0/code": "HPOF10", 145 | "promotions/0/min_spend": 100000000, 146 | "promotions/0/discount_value": null, 147 | "promotions/0/discount_percent": 10 148 | }, 149 | { 150 | "title": "HP 678 Black Ink Cartridge", 151 | "regPrice": 0, 152 | "price": 390, 153 | "stock": 4224, 154 | "seller": "HP Official Store", 155 | "price_currency": "THB", 156 | "url": "https://shopee.co.th/product/29250436/422572936", 157 | "product_id": "422572931", 158 | "country": "Thailand", 159 | "updated_date": "2018-10-16T14:32:30.775Z", 160 | "product_status": "InStock", 161 | "promotions/0/code": "HPOF11", 162 | "promotions/0/min_spend": 100000001, 163 | "promotions/0/discount_value": null, 164 | "promotions/0/discount_percent": 10 165 | } 166 | ] 167 | -------------------------------------------------------------------------------- /src/main.js: -------------------------------------------------------------------------------- 1 | const Apify = require('apify'); 2 | const { google } = require('googleapis'); 3 | const { apifyGoogleAuth } = require('apify-google-auth'); 4 | 5 | const processMode = require('./modes.js'); 6 | const { loadFromApify, loadFromSpreadsheet } = require('./loaders.js'); 7 | const upload = require('./upload.js'); 8 | const { saveBackup, retryingRequest, createSheetRequest } = require('./utils.js'); 9 | const validateAndParseInput = require('./validate-parse-input.js'); 10 | const { CLIENT_ID, REDIRECT_URI, CLIENT_ID_2, REDIRECT_URI_SERVER, CLIENT_SERVER_ID_1 } = require('./constants.js'); 11 | 12 | const { log } = Apify.utils; 13 | 14 | const MAX_CELLS = 5 * 1000 * 1000; 15 | 16 | Apify.main(async () => { 17 | const input = await Apify.getValue('INPUT'); 18 | log.info('Input:', { ...input, parsedData: 'not displayed, check input tab directly...', googleCredentials: 'not diplayed, check input tab directly...' }); 19 | 20 | log.info('\nPHASE - PARSING INPUT\n'); 21 | 22 | // We automatically make a webhook to work 23 | if (input.resource && input.resource.defaultDatasetId && !input.datasetId) { 24 | input.datasetId = input.resource.defaultDatasetId; 25 | } 26 | 27 | const { 28 | spreadsheetId, 29 | publicSpreadsheet = false, 30 | mode, 31 | datasetId, 32 | deduplicateByField, 33 | deduplicateByEquality, 34 | createBackup, 35 | tokensStore, 36 | limit, 37 | offset, 38 | range, 39 | backupStore, 40 | columnsOrder = [], 41 | keepSheetColumnOrder = false, 42 | googleCredentials = { 43 | client_id: CLIENT_ID, 44 | client_secret: process.env.CLIENT_SECRET, 45 | redirect_uri: REDIRECT_URI, 46 | // Unfortunately this is needed to hack around the 100 users limitation 47 | // because Google doesn't want to verify u 48 | additionalClients: [ 49 | { 50 | client_id: CLIENT_ID_2, 51 | client_secret: process.env.CLIENT_SECRET_2, 52 | redirect_uri: REDIRECT_URI, 53 | }, 54 | // This is the only client that works for new users because it uses the server flow 55 | { 56 | client_id: CLIENT_SERVER_ID_1, 57 | client_secret: process.env.CLIENT_SECRET_SERVER_1, 58 | redirect_uri: REDIRECT_URI_SERVER, 59 | }, 60 | ], 61 | }, 62 | } = input; 63 | 64 | // We have to do this to get rid of the global env var so it cannot be stolen in the user provided transform function 65 | const apiKey = process.env.API_KEY; 66 | delete process.env.API_KEY; 67 | delete process.env.CLIENT_SECRET; 68 | delete process.env.CLIENT_SECRET_2; 69 | delete process.env.CLIENT_SECRET_SERVER_1; 70 | 71 | const { rawData, transformFunction } = await validateAndParseInput(input); 72 | log.info('Input parsed...'); 73 | 74 | let auth; 75 | if (!publicSpreadsheet) { 76 | // Authenticate 77 | log.info('\nPHASE - AUTHORIZATION\n'); 78 | const authOptions = { 79 | scope: 'spreadsheets', 80 | tokensStore, 81 | credentials: googleCredentials, 82 | }; 83 | 84 | try { 85 | auth = await apifyGoogleAuth(authOptions); 86 | } catch (e) { 87 | log.error('Authorization failed! Ensure that you are signing up with the same account where the spreadsheet is located!'); 88 | throw e; 89 | } 90 | log.info('Authorization completed...'); 91 | } else { 92 | log.info('\nPHASE - SKIPPING AUTHORIZATION (public spreadsheet)\n'); 93 | } 94 | 95 | // Load sheets metadata 96 | log.info('\nPHASE - LOADING SPREADSHEET METADATA\n'); 97 | const client = google.sheets({ version: 'v4', auth: auth || apiKey }); 98 | 99 | const spreadsheetMetadata = await retryingRequest('Getting spreadsheet metadata', async () => client.spreadsheets.get({ spreadsheetId })); 100 | const sheetsMetadata = spreadsheetMetadata.data.sheets.map((sheet) => sheet.properties); 101 | const { title: firstSheetName, sheetId: firstSheetId } = sheetsMetadata[0]; 102 | log.info(`name of the first sheet: ${firstSheetName}`); 103 | log.info(`id of the first sheet: ${firstSheetId}`); 104 | 105 | const spreadsheetRange = range || firstSheetName; 106 | 107 | // This is important for trimming excess rows/columns 108 | let targetSheetId; 109 | if (!range) { 110 | targetSheetId = firstSheetId; 111 | } else { 112 | const maybeTargetSheet = sheetsMetadata.find((sheet) => range.startsWith(sheet.title)); 113 | if (maybeTargetSheet) { 114 | targetSheetId = maybeTargetSheet.sheetId; 115 | } else { 116 | // Sheet name is before ! or the whole range if no ! 117 | const title = range.split('!')[0]; 118 | log.warning('Cannot find target sheet. Creating new one.'); 119 | const resp = await retryingRequest('Creating new sheet', async () => client.spreadsheets.batchUpdate({ 120 | spreadsheetId, 121 | resource: createSheetRequest(title), 122 | })); 123 | targetSheetId = resp.data.replies[0].addSheet.properties.sheetId; 124 | } 125 | } 126 | log.info(`Target sheet id: ${targetSheetId}`); 127 | 128 | // Log info 129 | log.info('\nPHASE - SPREADSHEET SETUP:\n'); 130 | log.info(`Mode: ${mode}`); 131 | log.info(`Spreadsheet id: ${spreadsheetId}`); 132 | log.info(`Range: ${spreadsheetRange}`); 133 | log.info(`Deduplicate by field: ${deduplicateByField || false}`); 134 | log.info(`Deduplicated by equality: ${deduplicateByEquality || false}\n`); 135 | 136 | // Load data from Apify 137 | log.info('\nPHASE - LOADING DATA FROM APIFY\n'); 138 | const newObjects = rawData.length > 0 139 | ? rawData 140 | : await loadFromApify({ mode, datasetId, limit, offset }); 141 | log.info('Data loaded from Apify...'); 142 | 143 | // Load data from spreadsheet 144 | log.info('\nPHASE - LOADING DATA FROM SPREADSHEET\n'); 145 | const values = await loadFromSpreadsheet({ client, spreadsheetId, spreadsheetRange }); 146 | log.info(`${values ? values.length : 0} rows loaded from spreadsheet`); 147 | 148 | // Processing data (different for each mode) 149 | log.info('\nPHASE - PROCESSING DATA\n'); 150 | const rowsToInsert = await processMode({ 151 | mode, 152 | values, 153 | newObjects, 154 | deduplicateByField, 155 | deduplicateByEquality, 156 | transformFunction, 157 | columnsOrder, 158 | keepSheetColumnOrder, 159 | backupStore, 160 | }); // eslint-disable-line 161 | log.info('Data processed...'); 162 | 163 | // Save backup 164 | if (createBackup) { 165 | log.info('\nPHASE - SAVING BACKUP\n'); 166 | await saveBackup(createBackup, values); 167 | log.info('Backup saved...'); 168 | } 169 | 170 | // Upload to spreadsheet 171 | log.info('\nPHASE - UPLOADING TO SPREADSHEET\n'); 172 | await upload({ spreadsheetId, spreadsheetRange, rowsToInsert, values, client, targetSheetId, maxCells: MAX_CELLS }); 173 | log.info('Data uploaded...'); 174 | 175 | log.info('\nPHASE - ACTOR FINISHED\n'); 176 | log.info('URL of the updated spreadsheet:'); 177 | log.info(`https://docs.google.com/spreadsheets/d/${spreadsheetId}`); 178 | }); 179 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Google Sheets Import & Export Data 3 | 4 | - [Why use Google Sheets Import & Export](#why-use-google-sheets-import-and-export) 5 | - [How to use Google Sheets Import & Export](#how-to-use-google-sheets-import-and-export) 6 | - [Input settings](#input-settings) 7 | - [Limits](#limits) 8 | - [Authentication and authorization](#authentication-and-authorization) 9 | - [Public spreadsheet (no authorization)](#public-spreadsheet-no-authorization) 10 | - [Local usage](#local-usage) 11 | - [Important tips](#important-tips) 12 | - [Modes](#modes) 13 | - [Raw data import](#raw-data-import) 14 | - [Raw data table format (array of arrays)](#raw-data-table-format-array-of-arrays) 15 | - [Dataset format (array of objects)](#dataset-format-array-of-objects) 16 | - [Further reading](#further-reading) 17 | - [Changelog](#changelog) 18 | 19 | ## Why use Google Sheets Import and Export? 20 | If you're looking for an easy way to import and export data from datasets across multiple Google sheets, Google Sheets Import & Export is the Apify automation tool for you. 21 | 22 | It can process data in your current spreadsheet, import new data from [Apify datasets](https://www.apify.com/docs/storage#dataset), or a raw JSON file. It can be run both on the Apify platform or locally. 23 | 24 | You can use this actor with any programming language of your choice by calling [Apify API](https://www.apify.com/docs/api/v2). 25 | 26 | ## How to use Google Sheets Import and Export 27 | We have an in-depth [tutorial](https://blog.apify.com/google-sheets-import-data/) on how to use this tool. 28 | 29 | ## Input settings 30 | You can specify the following settings: 31 | - Mode 32 | - Spreadsheet id 33 | - Public spreadsheet (read-only) 34 | - Dataset ID 35 | - Raw data 36 | - Limit items 37 | - Offset items 38 | - Deduplicate by field 39 | - Deduplicate by equality 40 | - Transform function 41 | - Range 42 | - Columns order 43 | - Keep column order from sheet 44 | - Google OAuth tokens store 45 | - Create backup 46 | - Backup store id 47 | - Google Developer Console credentials 48 | 49 | For a complete description of all settings, see [input specification](https://apify.com/lukaskrivka/google-sheets/input-schema). 50 | 51 | ## Limits 52 | If you exceed these limits, the actor run will fail, and no data will be imported. 53 | - **Maximum runs (imports) per 100 seconds: 100** 54 | 55 | ## Authentication and authorization 56 | If you are using this actor for the first time, you have to log in with the Google account where the spreadsheet is located. You then need to authorize Apify to work with your spreadsheets. Internally at Apify, we use our small npm package [apify-google-auth](https://www.npmjs.com/package/apify-google-auth). Please check this [article](https://help.apify.com/en/articles/2424053-google-integration) on how to handle the authorization process. 57 | 58 | After authorization, tokens are stored in your key-value store, and you don't need to authorize again. So, after the first usage, you can fully automate the actor. 59 | 60 | ## Public spreadsheet (no authorization) 61 | If you don't mind publishing your spreadsheet, you can use this actor without authorization for **read mode**. 62 | Simply set the input **publicSpreadsheet** to **"true"**. 63 | 64 | To protect against the abuse of Apify's unofficial Google API, a public spreadsheet without authorization will only work on the Apify platform by using a secret environment variable. 65 | 66 | If you want to run the public mode locally, you have to create your project in the Google console and pass an API_KEY environment variable to your actor process. 67 | 68 | Example: 69 | **API_KEY=AIzaSyAPijSDFsdfSSf3kvGVsdfsdfsdsdnAVbcZb5Y apify run -p** 70 | 71 | ## Local usage 72 | The official actor relies on the CLIENT_SECRET environment variable being set. This assures that official API integration is used. 73 | 74 | If you want to use this actor locally or fork the source code, you will need to create your own project in [Google Developer Console](https://console.developers.google.com/), create your own credentials, and pass them correctly to the **googleCredentials** input variable. This is explained further in the [Apify Google Auth library](https://www.npmjs.com/package/apify-google-auth). 75 | 76 | ## Important tips 77 | * The maximum number of cells in the whole spreadsheet is 2 million! If the actor ever tries to import data over this limit, it will just throw an error, finish and not import anything. In this case, use more spreadsheets. 78 | 79 | * No matter which mode you choose, the actor recalculates how the data should be positioned in the sheet, updates all the cells, and then trims the exceeding rows and columns. This ensures that the sheet always has the optimal number of rows and columns and there is enough space for the newly generated data. 80 | 81 | * The actor parsing follows the default [Google Sheets parsing](https://developers.google.com/sheets/api/reference/rest/v4/ValueInputOption). Therefore, depending on the configuration of your system, constructions such as `"1.1"` and `"1,1"` can be interpreted either as a number or a string (text). For this reason, it is recommended that you always use valid JSON numbers (e.g. `1.1`). 82 | 83 | ## Modes 84 | This actor can be run in multiple different modes. Each run must have only one specific mode. _Mode_ also affects how other options work (details are explained in the specific options). 85 | 86 | - **replace:** If there's any old data in the sheet, it is cleaned, and then new data is imported. 87 | 88 | - **append:** This mode adds new data as additional rows below the old rows already present in the sheet. Keep in mind that the columns are recalculated so some of them may move to different cells if new columns are added in the middle. 89 | 90 | - **modify:** This mode doesn't import anything. It only loads the data from your sheets and applies any of the processing you set in the options. 91 | 92 | - **read:** This mode simply loads the data from the spreadsheet, optionally can process them, and saves them as 'OUTPUT' JSON file to the default key-value store. 93 | 94 | - **load backup:** This mode simply loads any backup rows from previous runs (look at the backup option for details) and imports it to a sheet in the replace mode. 95 | 96 | ## Raw data import 97 | If you want to send data in raw JSON format, you need to pass the data to the `rawData` input parameter. You will also need to have an Apify account so we can properly store your Google authentication tokens (you can opt out at anytime). 98 | 99 | > **Important!** - Raw data cannot exceed 9MB, as this the default limit for Apify actor inputs. If you want to upload more data, you can easily split it into more runs (they're fast and cheap). 100 | 101 | #### Raw data table format (array of arrays) 102 | `rawData` should be an array of arrays where each of the arrays represents one row in the sheet. The first row should be a header row where the field names are defined. Every other row is a data row. 103 | 104 | It is important to have a proper order in each array. If the field is null for any row, the array should contain an empty string in that index. Data rows can have a smaller length than the header row but if they are longer the extra data will be trimmed off. 105 | 106 | Arrays **cannot** contain nested structures like objects or other arrays! You have to flatten them in a format where `/` is a delimiter. E.g. `personal/hobbies/0`. 107 | 108 | ``` 109 | "rawData": [ 110 | ["name", "occupation", "email", "hobbies/0", "hobbies/1"], 111 | ["John Doe", "developer", "john@google.com", "sport", "movies with Leonardo"], 112 | ["Leonardo DiCaprio", "actor", "leonardo@google.com", "being rich", "climate change activism"] 113 | ] 114 | 115 | ``` 116 | 117 | #### Dataset format (array of objects) 118 | `rawData` should be an array of objects where each object represents one row in the sheet. The keys of the objects will be transformed to a header row and the values will be inserted into the data rows. Objects don't need to have the same keys. If an object doesn't have a key that another object has, the row will have an empty cell in that field. 119 | 120 | The object **can** contain nested structures (objects and arrays) but in that case, it will call Apify API to flatten the data which can take a little more time on large uploads so try to prefer flattened data. 121 | 122 | _Nested_: 123 | 124 | ``` 125 | "rawData": [ 126 | { 127 | "name": "John Doe", 128 | "email": "john@google.com", 129 | "hobbies": ["sport", "movies with Leonardo", "dog walking"] 130 | }, 131 | { 132 | "name": "Leonardo DiCaprio", 133 | "email": "leonardo@google.com", 134 | "hobbies": ["being rich", "climate change activism"] 135 | } 136 | ] 137 | 138 | ``` 139 | 140 | _Flattened_: 141 | 142 | ``` 143 | "rawData": [ 144 | { 145 | "name": "John Doe", 146 | "email": "john@google.com", 147 | "hobbies/0": "sport", 148 | "hobbies/1": "movies with Leonardo", 149 | "hobbies/2": "dog walking" 150 | }, 151 | { 152 | "name": "Leonardo DiCaprio", 153 | "email": "leonardo@google.com", 154 | "hobbies/0": "being rich", 155 | "hobbies/1": "climate change activism" 156 | } 157 | ] 158 | 159 | ``` 160 | ## Further reading 161 | If you want a much deeper understanding of how this actor works, consult [Google Sheets for developers](https://developers.google.com/sheets/api/). 162 | 163 | 164 | ## Changelog 165 | A detailed list of changes is in the [CHANGELOG.md](https://github.com/metalwarrior665/actor-google-sheets/blob/master/CHANGELOG.md) file. 166 | 167 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Apify Technologies s.r.o. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /periodic-test/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "periodic-test", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@apify/ps-tree": { 8 | "version": "1.1.3", 9 | "resolved": "https://registry.npmjs.org/@apify/ps-tree/-/ps-tree-1.1.3.tgz", 10 | "integrity": "sha512-+hIr8EaTRd9fsOiNNzf1Fi8Tm9qs8cdPBZjuq5fXDV6SOCdi2ZyQlcQSzc8lY0hb+UhBib1WPixtCdKLL169WA==", 11 | "requires": { 12 | "event-stream": "3.3.4" 13 | } 14 | }, 15 | "@types/node": { 16 | "version": "10.12.18", 17 | "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", 18 | "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==" 19 | }, 20 | "agent-base": { 21 | "version": "4.2.1", 22 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", 23 | "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", 24 | "optional": true, 25 | "requires": { 26 | "es6-promisify": "^5.0.0" 27 | } 28 | }, 29 | "ajv": { 30 | "version": "6.7.0", 31 | "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz", 32 | "integrity": "sha512-RZXPviBTtfmtka9n9sy1N5M5b82CbxWIR6HIis4s3WQTXDJamc/0gpCWNGz6EWdWp4DOfjzJfhz/AS9zVPjjWg==", 33 | "requires": { 34 | "fast-deep-equal": "^2.0.1", 35 | "fast-json-stable-stringify": "^2.0.0", 36 | "json-schema-traverse": "^0.4.1", 37 | "uri-js": "^4.2.2" 38 | } 39 | }, 40 | "apify": { 41 | "version": "0.11.5", 42 | "resolved": "https://registry.npmjs.org/apify/-/apify-0.11.5.tgz", 43 | "integrity": "sha512-FwhE71nO2yf4zX3GAYdpc6VnchU8rM5ZmZRZvYStfDC4dsJKZgHp2qz5QyKFNukUXc9mDdojnxtDoE6YfMi2wA==", 44 | "requires": { 45 | "@apify/ps-tree": "^1.1.3", 46 | "apify-client": "^0.5.3", 47 | "apify-shared": "^0.1.11", 48 | "bluebird": "^3.5.3", 49 | "cheerio": "^1.0.0-rc.2", 50 | "content-type": "^1.0.4", 51 | "fs-extra": "^7.0.1", 52 | "jquery": "^3.3.1", 53 | "mime": "^2.3.1", 54 | "proxy-chain": "^0.2.5", 55 | "puppeteer": "^1.11.0", 56 | "request": "^2.88.0", 57 | "request-promise-native": "^1.0.5", 58 | "rimraf": "^2.6.2", 59 | "selenium-webdriver": "^3.6.0", 60 | "underscore": "^1.9.1", 61 | "ws": "^6.1.2", 62 | "xregexp": "4.2.0" 63 | } 64 | }, 65 | "apify-client": { 66 | "version": "0.5.3", 67 | "resolved": "https://registry.npmjs.org/apify-client/-/apify-client-0.5.3.tgz", 68 | "integrity": "sha512-V98owt9QJkiuu6BO63X16CKKFsgrFsMsKfVryv6ax6nhrRe540vIgOjm02oIKsv5IzBmnvwZx6hoTSOrEBXhxQ==", 69 | "requires": { 70 | "apify-shared": "^0.1.9", 71 | "content-type": "^1.0.3", 72 | "request": "^2.81.0", 73 | "request-promise-native": "^1.0.5", 74 | "type-check": "^0.3.2", 75 | "underscore": "^1.9.0" 76 | } 77 | }, 78 | "apify-shared": { 79 | "version": "0.1.12", 80 | "resolved": "https://registry.npmjs.org/apify-shared/-/apify-shared-0.1.12.tgz", 81 | "integrity": "sha512-KHlHD+tMTIZ+F/9JBqYPfUAbgPJU+93mu2N0trHg9NqMh43r/LtaY8ffaBnbpXuZc54STHyzte76cgxhoaInCg==", 82 | "requires": { 83 | "bluebird": "^3.5.1", 84 | "clone": "^2.1.1", 85 | "request": "^2.83.0", 86 | "slugg": "^1.2.1", 87 | "underscore": "^1.8.3", 88 | "url": "^0.11.0" 89 | } 90 | }, 91 | "asn1": { 92 | "version": "0.2.4", 93 | "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", 94 | "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", 95 | "requires": { 96 | "safer-buffer": "~2.1.0" 97 | } 98 | }, 99 | "assert-plus": { 100 | "version": "1.0.0", 101 | "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", 102 | "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" 103 | }, 104 | "async-limiter": { 105 | "version": "1.0.0", 106 | "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", 107 | "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" 108 | }, 109 | "asynckit": { 110 | "version": "0.4.0", 111 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 112 | "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" 113 | }, 114 | "aws-sign2": { 115 | "version": "0.7.0", 116 | "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", 117 | "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" 118 | }, 119 | "aws4": { 120 | "version": "1.8.0", 121 | "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", 122 | "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" 123 | }, 124 | "balanced-match": { 125 | "version": "1.0.0", 126 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 127 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" 128 | }, 129 | "bcrypt-pbkdf": { 130 | "version": "1.0.2", 131 | "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", 132 | "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", 133 | "requires": { 134 | "tweetnacl": "^0.14.3" 135 | } 136 | }, 137 | "bluebird": { 138 | "version": "3.5.3", 139 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", 140 | "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==" 141 | }, 142 | "boolbase": { 143 | "version": "1.0.0", 144 | "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", 145 | "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" 146 | }, 147 | "brace-expansion": { 148 | "version": "1.1.11", 149 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 150 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 151 | "requires": { 152 | "balanced-match": "^1.0.0", 153 | "concat-map": "0.0.1" 154 | } 155 | }, 156 | "buffer-from": { 157 | "version": "1.1.1", 158 | "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", 159 | "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", 160 | "optional": true 161 | }, 162 | "caseless": { 163 | "version": "0.12.0", 164 | "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", 165 | "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" 166 | }, 167 | "cheerio": { 168 | "version": "1.0.0-rc.2", 169 | "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", 170 | "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", 171 | "requires": { 172 | "css-select": "~1.2.0", 173 | "dom-serializer": "~0.1.0", 174 | "entities": "~1.1.1", 175 | "htmlparser2": "^3.9.1", 176 | "lodash": "^4.15.0", 177 | "parse5": "^3.0.1" 178 | } 179 | }, 180 | "clone": { 181 | "version": "2.1.2", 182 | "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", 183 | "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18=" 184 | }, 185 | "combined-stream": { 186 | "version": "1.0.7", 187 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", 188 | "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", 189 | "requires": { 190 | "delayed-stream": "~1.0.0" 191 | } 192 | }, 193 | "commander": { 194 | "version": "2.19.0", 195 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", 196 | "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==" 197 | }, 198 | "concat-map": { 199 | "version": "0.0.1", 200 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 201 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" 202 | }, 203 | "concat-stream": { 204 | "version": "1.6.2", 205 | "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", 206 | "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", 207 | "optional": true, 208 | "requires": { 209 | "buffer-from": "^1.0.0", 210 | "inherits": "^2.0.3", 211 | "readable-stream": "^2.2.2", 212 | "typedarray": "^0.0.6" 213 | }, 214 | "dependencies": { 215 | "readable-stream": { 216 | "version": "2.3.6", 217 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", 218 | "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", 219 | "optional": true, 220 | "requires": { 221 | "core-util-is": "~1.0.0", 222 | "inherits": "~2.0.3", 223 | "isarray": "~1.0.0", 224 | "process-nextick-args": "~2.0.0", 225 | "safe-buffer": "~5.1.1", 226 | "string_decoder": "~1.1.1", 227 | "util-deprecate": "~1.0.1" 228 | } 229 | }, 230 | "string_decoder": { 231 | "version": "1.1.1", 232 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 233 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 234 | "optional": true, 235 | "requires": { 236 | "safe-buffer": "~5.1.0" 237 | } 238 | } 239 | } 240 | }, 241 | "content-type": { 242 | "version": "1.0.4", 243 | "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", 244 | "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" 245 | }, 246 | "core-js": { 247 | "version": "2.3.0", 248 | "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.3.0.tgz", 249 | "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU=", 250 | "optional": true 251 | }, 252 | "core-util-is": { 253 | "version": "1.0.2", 254 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 255 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" 256 | }, 257 | "css-select": { 258 | "version": "1.2.0", 259 | "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", 260 | "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", 261 | "requires": { 262 | "boolbase": "~1.0.0", 263 | "css-what": "2.1", 264 | "domutils": "1.5.1", 265 | "nth-check": "~1.0.1" 266 | } 267 | }, 268 | "css-what": { 269 | "version": "2.1.2", 270 | "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.2.tgz", 271 | "integrity": "sha512-wan8dMWQ0GUeF7DGEPVjhHemVW/vy6xUYmFzRY8RYqgA0JtXC9rJmbScBjqSu6dg9q0lwPQy6ZAmJVr3PPTvqQ==" 272 | }, 273 | "dashdash": { 274 | "version": "1.14.1", 275 | "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", 276 | "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", 277 | "requires": { 278 | "assert-plus": "^1.0.0" 279 | } 280 | }, 281 | "debug": { 282 | "version": "2.6.9", 283 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", 284 | "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", 285 | "requires": { 286 | "ms": "2.0.0" 287 | } 288 | }, 289 | "delayed-stream": { 290 | "version": "1.0.0", 291 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 292 | "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" 293 | }, 294 | "dom-serializer": { 295 | "version": "0.1.0", 296 | "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", 297 | "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", 298 | "requires": { 299 | "domelementtype": "~1.1.1", 300 | "entities": "~1.1.1" 301 | }, 302 | "dependencies": { 303 | "domelementtype": { 304 | "version": "1.1.3", 305 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", 306 | "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" 307 | } 308 | } 309 | }, 310 | "domelementtype": { 311 | "version": "1.3.1", 312 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", 313 | "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" 314 | }, 315 | "domhandler": { 316 | "version": "2.4.2", 317 | "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", 318 | "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", 319 | "requires": { 320 | "domelementtype": "1" 321 | } 322 | }, 323 | "domutils": { 324 | "version": "1.5.1", 325 | "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", 326 | "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", 327 | "requires": { 328 | "dom-serializer": "0", 329 | "domelementtype": "1" 330 | } 331 | }, 332 | "duplexer": { 333 | "version": "0.1.1", 334 | "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", 335 | "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=" 336 | }, 337 | "ecc-jsbn": { 338 | "version": "0.1.2", 339 | "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", 340 | "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", 341 | "requires": { 342 | "jsbn": "~0.1.0", 343 | "safer-buffer": "^2.1.0" 344 | } 345 | }, 346 | "entities": { 347 | "version": "1.1.2", 348 | "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", 349 | "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" 350 | }, 351 | "es6-promise": { 352 | "version": "4.2.5", 353 | "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", 354 | "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==", 355 | "optional": true 356 | }, 357 | "es6-promisify": { 358 | "version": "5.0.0", 359 | "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", 360 | "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", 361 | "optional": true, 362 | "requires": { 363 | "es6-promise": "^4.0.3" 364 | } 365 | }, 366 | "event-stream": { 367 | "version": "3.3.4", 368 | "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", 369 | "integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=", 370 | "requires": { 371 | "duplexer": "~0.1.1", 372 | "from": "~0", 373 | "map-stream": "~0.1.0", 374 | "pause-stream": "0.0.11", 375 | "split": "0.3", 376 | "stream-combiner": "~0.0.4", 377 | "through": "~2.3.1" 378 | } 379 | }, 380 | "extend": { 381 | "version": "3.0.2", 382 | "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", 383 | "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" 384 | }, 385 | "extract-zip": { 386 | "version": "1.6.7", 387 | "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", 388 | "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", 389 | "optional": true, 390 | "requires": { 391 | "concat-stream": "1.6.2", 392 | "debug": "2.6.9", 393 | "mkdirp": "0.5.1", 394 | "yauzl": "2.4.1" 395 | } 396 | }, 397 | "extsprintf": { 398 | "version": "1.3.0", 399 | "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", 400 | "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" 401 | }, 402 | "fast-deep-equal": { 403 | "version": "2.0.1", 404 | "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", 405 | "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" 406 | }, 407 | "fast-json-stable-stringify": { 408 | "version": "2.0.0", 409 | "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", 410 | "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" 411 | }, 412 | "fd-slicer": { 413 | "version": "1.0.1", 414 | "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", 415 | "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", 416 | "optional": true, 417 | "requires": { 418 | "pend": "~1.2.0" 419 | } 420 | }, 421 | "forever-agent": { 422 | "version": "0.6.1", 423 | "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", 424 | "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" 425 | }, 426 | "form-data": { 427 | "version": "2.3.3", 428 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", 429 | "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", 430 | "requires": { 431 | "asynckit": "^0.4.0", 432 | "combined-stream": "^1.0.6", 433 | "mime-types": "^2.1.12" 434 | } 435 | }, 436 | "from": { 437 | "version": "0.1.7", 438 | "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", 439 | "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=" 440 | }, 441 | "fs-extra": { 442 | "version": "7.0.1", 443 | "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", 444 | "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", 445 | "requires": { 446 | "graceful-fs": "^4.1.2", 447 | "jsonfile": "^4.0.0", 448 | "universalify": "^0.1.0" 449 | } 450 | }, 451 | "fs.realpath": { 452 | "version": "1.0.0", 453 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 454 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" 455 | }, 456 | "getpass": { 457 | "version": "0.1.7", 458 | "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", 459 | "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", 460 | "requires": { 461 | "assert-plus": "^1.0.0" 462 | } 463 | }, 464 | "glob": { 465 | "version": "7.1.3", 466 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", 467 | "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", 468 | "requires": { 469 | "fs.realpath": "^1.0.0", 470 | "inflight": "^1.0.4", 471 | "inherits": "2", 472 | "minimatch": "^3.0.4", 473 | "once": "^1.3.0", 474 | "path-is-absolute": "^1.0.0" 475 | } 476 | }, 477 | "graceful-fs": { 478 | "version": "4.1.15", 479 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", 480 | "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" 481 | }, 482 | "har-schema": { 483 | "version": "2.0.0", 484 | "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", 485 | "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" 486 | }, 487 | "har-validator": { 488 | "version": "5.1.3", 489 | "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", 490 | "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", 491 | "requires": { 492 | "ajv": "^6.5.5", 493 | "har-schema": "^2.0.0" 494 | } 495 | }, 496 | "htmlparser2": { 497 | "version": "3.10.0", 498 | "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.0.tgz", 499 | "integrity": "sha512-J1nEUGv+MkXS0weHNWVKJJ+UrLfePxRWpN3C9bEi9fLxL2+ggW94DQvgYVXsaT30PGwYRIZKNZXuyMhp3Di4bQ==", 500 | "requires": { 501 | "domelementtype": "^1.3.0", 502 | "domhandler": "^2.3.0", 503 | "domutils": "^1.5.1", 504 | "entities": "^1.1.1", 505 | "inherits": "^2.0.1", 506 | "readable-stream": "^3.0.6" 507 | } 508 | }, 509 | "http-signature": { 510 | "version": "1.2.0", 511 | "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", 512 | "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", 513 | "requires": { 514 | "assert-plus": "^1.0.0", 515 | "jsprim": "^1.2.2", 516 | "sshpk": "^1.7.0" 517 | } 518 | }, 519 | "https-proxy-agent": { 520 | "version": "2.2.1", 521 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", 522 | "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", 523 | "optional": true, 524 | "requires": { 525 | "agent-base": "^4.1.0", 526 | "debug": "^3.1.0" 527 | }, 528 | "dependencies": { 529 | "debug": { 530 | "version": "3.2.6", 531 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", 532 | "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", 533 | "optional": true, 534 | "requires": { 535 | "ms": "^2.1.1" 536 | } 537 | }, 538 | "ms": { 539 | "version": "2.1.1", 540 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", 541 | "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", 542 | "optional": true 543 | } 544 | } 545 | }, 546 | "immediate": { 547 | "version": "3.0.6", 548 | "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", 549 | "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", 550 | "optional": true 551 | }, 552 | "inflight": { 553 | "version": "1.0.6", 554 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 555 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 556 | "requires": { 557 | "once": "^1.3.0", 558 | "wrappy": "1" 559 | } 560 | }, 561 | "inherits": { 562 | "version": "2.0.3", 563 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 564 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" 565 | }, 566 | "is-typedarray": { 567 | "version": "1.0.0", 568 | "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", 569 | "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" 570 | }, 571 | "isarray": { 572 | "version": "1.0.0", 573 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 574 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", 575 | "optional": true 576 | }, 577 | "isstream": { 578 | "version": "0.1.2", 579 | "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", 580 | "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" 581 | }, 582 | "jquery": { 583 | "version": "3.3.1", 584 | "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.3.1.tgz", 585 | "integrity": "sha512-Ubldcmxp5np52/ENotGxlLe6aGMvmF4R8S6tZjsP6Knsaxd/xp3Zrh50cG93lR6nPXyUFwzN3ZSOQI0wRJNdGg==" 586 | }, 587 | "jsbn": { 588 | "version": "0.1.1", 589 | "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", 590 | "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" 591 | }, 592 | "json-schema": { 593 | "version": "0.2.3", 594 | "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", 595 | "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" 596 | }, 597 | "json-schema-traverse": { 598 | "version": "0.4.1", 599 | "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", 600 | "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" 601 | }, 602 | "json-stringify-safe": { 603 | "version": "5.0.1", 604 | "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", 605 | "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" 606 | }, 607 | "jsonfile": { 608 | "version": "4.0.0", 609 | "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", 610 | "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", 611 | "requires": { 612 | "graceful-fs": "^4.1.6" 613 | } 614 | }, 615 | "jsprim": { 616 | "version": "1.4.1", 617 | "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", 618 | "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", 619 | "requires": { 620 | "assert-plus": "1.0.0", 621 | "extsprintf": "1.3.0", 622 | "json-schema": "0.2.3", 623 | "verror": "1.10.0" 624 | } 625 | }, 626 | "jszip": { 627 | "version": "3.1.5", 628 | "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.1.5.tgz", 629 | "integrity": "sha512-5W8NUaFRFRqTOL7ZDDrx5qWHJyBXy6velVudIzQUSoqAAYqzSh2Z7/m0Rf1QbmQJccegD0r+YZxBjzqoBiEeJQ==", 630 | "optional": true, 631 | "requires": { 632 | "core-js": "~2.3.0", 633 | "es6-promise": "~3.0.2", 634 | "lie": "~3.1.0", 635 | "pako": "~1.0.2", 636 | "readable-stream": "~2.0.6" 637 | }, 638 | "dependencies": { 639 | "es6-promise": { 640 | "version": "3.0.2", 641 | "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", 642 | "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=", 643 | "optional": true 644 | }, 645 | "process-nextick-args": { 646 | "version": "1.0.7", 647 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", 648 | "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=", 649 | "optional": true 650 | }, 651 | "readable-stream": { 652 | "version": "2.0.6", 653 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", 654 | "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", 655 | "optional": true, 656 | "requires": { 657 | "core-util-is": "~1.0.0", 658 | "inherits": "~2.0.1", 659 | "isarray": "~1.0.0", 660 | "process-nextick-args": "~1.0.6", 661 | "string_decoder": "~0.10.x", 662 | "util-deprecate": "~1.0.1" 663 | } 664 | }, 665 | "string_decoder": { 666 | "version": "0.10.31", 667 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", 668 | "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", 669 | "optional": true 670 | } 671 | } 672 | }, 673 | "lie": { 674 | "version": "3.1.1", 675 | "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz", 676 | "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=", 677 | "optional": true, 678 | "requires": { 679 | "immediate": "~3.0.5" 680 | } 681 | }, 682 | "lodash": { 683 | "version": "4.17.11", 684 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", 685 | "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" 686 | }, 687 | "map-stream": { 688 | "version": "0.1.0", 689 | "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", 690 | "integrity": "sha1-5WqpTEyAVaFkBKBnS3jyFffI4ZQ=" 691 | }, 692 | "mime": { 693 | "version": "2.4.0", 694 | "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz", 695 | "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==" 696 | }, 697 | "mime-db": { 698 | "version": "1.37.0", 699 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", 700 | "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" 701 | }, 702 | "mime-types": { 703 | "version": "2.1.21", 704 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", 705 | "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", 706 | "requires": { 707 | "mime-db": "~1.37.0" 708 | } 709 | }, 710 | "minimatch": { 711 | "version": "3.0.4", 712 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 713 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 714 | "requires": { 715 | "brace-expansion": "^1.1.7" 716 | } 717 | }, 718 | "minimist": { 719 | "version": "0.0.8", 720 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 721 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", 722 | "optional": true 723 | }, 724 | "mkdirp": { 725 | "version": "0.5.1", 726 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 727 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 728 | "optional": true, 729 | "requires": { 730 | "minimist": "0.0.8" 731 | } 732 | }, 733 | "ms": { 734 | "version": "2.0.0", 735 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 736 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 737 | }, 738 | "nth-check": { 739 | "version": "1.0.2", 740 | "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", 741 | "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", 742 | "requires": { 743 | "boolbase": "~1.0.0" 744 | } 745 | }, 746 | "oauth-sign": { 747 | "version": "0.9.0", 748 | "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", 749 | "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" 750 | }, 751 | "once": { 752 | "version": "1.4.0", 753 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 754 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 755 | "requires": { 756 | "wrappy": "1" 757 | } 758 | }, 759 | "os-tmpdir": { 760 | "version": "1.0.2", 761 | "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", 762 | "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", 763 | "optional": true 764 | }, 765 | "pako": { 766 | "version": "1.0.8", 767 | "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.8.tgz", 768 | "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA==", 769 | "optional": true 770 | }, 771 | "parse5": { 772 | "version": "3.0.3", 773 | "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", 774 | "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", 775 | "requires": { 776 | "@types/node": "*" 777 | } 778 | }, 779 | "path-is-absolute": { 780 | "version": "1.0.1", 781 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 782 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" 783 | }, 784 | "pause-stream": { 785 | "version": "0.0.11", 786 | "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", 787 | "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", 788 | "requires": { 789 | "through": "~2.3" 790 | } 791 | }, 792 | "pend": { 793 | "version": "1.2.0", 794 | "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", 795 | "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=", 796 | "optional": true 797 | }, 798 | "performance-now": { 799 | "version": "2.1.0", 800 | "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", 801 | "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" 802 | }, 803 | "portastic": { 804 | "version": "1.0.1", 805 | "resolved": "https://registry.npmjs.org/portastic/-/portastic-1.0.1.tgz", 806 | "integrity": "sha1-HJgF1D+uj2pAzw28d5QJGi6dDSo=", 807 | "requires": { 808 | "bluebird": "^2.9.34", 809 | "commander": "^2.8.1", 810 | "debug": "^2.2.0" 811 | }, 812 | "dependencies": { 813 | "bluebird": { 814 | "version": "2.11.0", 815 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", 816 | "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=" 817 | } 818 | } 819 | }, 820 | "prelude-ls": { 821 | "version": "1.1.2", 822 | "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", 823 | "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" 824 | }, 825 | "process-nextick-args": { 826 | "version": "2.0.0", 827 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", 828 | "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", 829 | "optional": true 830 | }, 831 | "progress": { 832 | "version": "2.0.3", 833 | "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", 834 | "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", 835 | "optional": true 836 | }, 837 | "proxy-chain": { 838 | "version": "0.2.5", 839 | "resolved": "https://registry.npmjs.org/proxy-chain/-/proxy-chain-0.2.5.tgz", 840 | "integrity": "sha512-kxkiu0asLvBtD7hdHpj1i5ed+pzMQJZPa8gsFiDf0hMRXiBj2wsRHvfRNJ5HNg1xQgP5V7rF2OYi8D8Gb7CsCg==", 841 | "requires": { 842 | "bluebird": "^3.5.1", 843 | "portastic": "^1.0.1", 844 | "underscore": "^1.9.1" 845 | } 846 | }, 847 | "proxy-from-env": { 848 | "version": "1.0.0", 849 | "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", 850 | "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=", 851 | "optional": true 852 | }, 853 | "psl": { 854 | "version": "1.1.31", 855 | "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", 856 | "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" 857 | }, 858 | "punycode": { 859 | "version": "2.1.1", 860 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", 861 | "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" 862 | }, 863 | "puppeteer": { 864 | "version": "1.11.0", 865 | "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.11.0.tgz", 866 | "integrity": "sha512-iG4iMOHixc2EpzqRV+pv7o3GgmU2dNYEMkvKwSaQO/vMZURakwSOn/EYJ6OIRFYOque1qorzIBvrytPIQB3YzQ==", 867 | "optional": true, 868 | "requires": { 869 | "debug": "^4.1.0", 870 | "extract-zip": "^1.6.6", 871 | "https-proxy-agent": "^2.2.1", 872 | "mime": "^2.0.3", 873 | "progress": "^2.0.1", 874 | "proxy-from-env": "^1.0.0", 875 | "rimraf": "^2.6.1", 876 | "ws": "^6.1.0" 877 | }, 878 | "dependencies": { 879 | "debug": { 880 | "version": "4.1.1", 881 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", 882 | "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", 883 | "optional": true, 884 | "requires": { 885 | "ms": "^2.1.1" 886 | } 887 | }, 888 | "ms": { 889 | "version": "2.1.1", 890 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", 891 | "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", 892 | "optional": true 893 | } 894 | } 895 | }, 896 | "qs": { 897 | "version": "6.5.2", 898 | "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", 899 | "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" 900 | }, 901 | "querystring": { 902 | "version": "0.2.0", 903 | "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", 904 | "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" 905 | }, 906 | "readable-stream": { 907 | "version": "3.1.1", 908 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.1.1.tgz", 909 | "integrity": "sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA==", 910 | "requires": { 911 | "inherits": "^2.0.3", 912 | "string_decoder": "^1.1.1", 913 | "util-deprecate": "^1.0.1" 914 | } 915 | }, 916 | "request": { 917 | "version": "2.88.0", 918 | "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", 919 | "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", 920 | "requires": { 921 | "aws-sign2": "~0.7.0", 922 | "aws4": "^1.8.0", 923 | "caseless": "~0.12.0", 924 | "combined-stream": "~1.0.6", 925 | "extend": "~3.0.2", 926 | "forever-agent": "~0.6.1", 927 | "form-data": "~2.3.2", 928 | "har-validator": "~5.1.0", 929 | "http-signature": "~1.2.0", 930 | "is-typedarray": "~1.0.0", 931 | "isstream": "~0.1.2", 932 | "json-stringify-safe": "~5.0.1", 933 | "mime-types": "~2.1.19", 934 | "oauth-sign": "~0.9.0", 935 | "performance-now": "^2.1.0", 936 | "qs": "~6.5.2", 937 | "safe-buffer": "^5.1.2", 938 | "tough-cookie": "~2.4.3", 939 | "tunnel-agent": "^0.6.0", 940 | "uuid": "^3.3.2" 941 | } 942 | }, 943 | "request-promise-core": { 944 | "version": "1.1.1", 945 | "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", 946 | "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=", 947 | "requires": { 948 | "lodash": "^4.13.1" 949 | } 950 | }, 951 | "request-promise-native": { 952 | "version": "1.0.5", 953 | "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.5.tgz", 954 | "integrity": "sha1-UoF3D2jgyXGeUWP9P6tIIhX0/aU=", 955 | "requires": { 956 | "request-promise-core": "1.1.1", 957 | "stealthy-require": "^1.1.0", 958 | "tough-cookie": ">=2.3.3" 959 | } 960 | }, 961 | "rimraf": { 962 | "version": "2.6.3", 963 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", 964 | "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", 965 | "requires": { 966 | "glob": "^7.1.3" 967 | } 968 | }, 969 | "safe-buffer": { 970 | "version": "5.1.2", 971 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 972 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" 973 | }, 974 | "safer-buffer": { 975 | "version": "2.1.2", 976 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", 977 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" 978 | }, 979 | "sax": { 980 | "version": "1.2.4", 981 | "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", 982 | "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", 983 | "optional": true 984 | }, 985 | "selenium-webdriver": { 986 | "version": "3.6.0", 987 | "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-3.6.0.tgz", 988 | "integrity": "sha512-WH7Aldse+2P5bbFBO4Gle/nuQOdVwpHMTL6raL3uuBj/vPG07k6uzt3aiahu352ONBr5xXh0hDlM3LhtXPOC4Q==", 989 | "optional": true, 990 | "requires": { 991 | "jszip": "^3.1.3", 992 | "rimraf": "^2.5.4", 993 | "tmp": "0.0.30", 994 | "xml2js": "^0.4.17" 995 | } 996 | }, 997 | "slugg": { 998 | "version": "1.2.1", 999 | "resolved": "https://registry.npmjs.org/slugg/-/slugg-1.2.1.tgz", 1000 | "integrity": "sha1-51KvIkGvPycURjxd4iXOpHYIdAo=" 1001 | }, 1002 | "split": { 1003 | "version": "0.3.3", 1004 | "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", 1005 | "integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=", 1006 | "requires": { 1007 | "through": "2" 1008 | } 1009 | }, 1010 | "sshpk": { 1011 | "version": "1.16.0", 1012 | "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.0.tgz", 1013 | "integrity": "sha512-Zhev35/y7hRMcID/upReIvRse+I9SVhyVre/KTJSJQWMz3C3+G+HpO7m1wK/yckEtujKZ7dS4hkVxAnmHaIGVQ==", 1014 | "requires": { 1015 | "asn1": "~0.2.3", 1016 | "assert-plus": "^1.0.0", 1017 | "bcrypt-pbkdf": "^1.0.0", 1018 | "dashdash": "^1.12.0", 1019 | "ecc-jsbn": "~0.1.1", 1020 | "getpass": "^0.1.1", 1021 | "jsbn": "~0.1.0", 1022 | "safer-buffer": "^2.0.2", 1023 | "tweetnacl": "~0.14.0" 1024 | } 1025 | }, 1026 | "stealthy-require": { 1027 | "version": "1.1.1", 1028 | "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", 1029 | "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" 1030 | }, 1031 | "stream-combiner": { 1032 | "version": "0.0.4", 1033 | "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", 1034 | "integrity": "sha1-TV5DPBhSYd3mI8o/RMWGvPXErRQ=", 1035 | "requires": { 1036 | "duplexer": "~0.1.1" 1037 | } 1038 | }, 1039 | "string_decoder": { 1040 | "version": "1.2.0", 1041 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz", 1042 | "integrity": "sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==", 1043 | "requires": { 1044 | "safe-buffer": "~5.1.0" 1045 | } 1046 | }, 1047 | "through": { 1048 | "version": "2.3.8", 1049 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 1050 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" 1051 | }, 1052 | "tmp": { 1053 | "version": "0.0.30", 1054 | "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.30.tgz", 1055 | "integrity": "sha1-ckGdSovn1s51FI/YsyTlk6cRwu0=", 1056 | "optional": true, 1057 | "requires": { 1058 | "os-tmpdir": "~1.0.1" 1059 | } 1060 | }, 1061 | "tough-cookie": { 1062 | "version": "2.4.3", 1063 | "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", 1064 | "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", 1065 | "requires": { 1066 | "psl": "^1.1.24", 1067 | "punycode": "^1.4.1" 1068 | }, 1069 | "dependencies": { 1070 | "punycode": { 1071 | "version": "1.4.1", 1072 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", 1073 | "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" 1074 | } 1075 | } 1076 | }, 1077 | "tunnel-agent": { 1078 | "version": "0.6.0", 1079 | "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", 1080 | "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", 1081 | "requires": { 1082 | "safe-buffer": "^5.0.1" 1083 | } 1084 | }, 1085 | "tweetnacl": { 1086 | "version": "0.14.5", 1087 | "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", 1088 | "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" 1089 | }, 1090 | "type-check": { 1091 | "version": "0.3.2", 1092 | "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", 1093 | "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", 1094 | "requires": { 1095 | "prelude-ls": "~1.1.2" 1096 | } 1097 | }, 1098 | "typedarray": { 1099 | "version": "0.0.6", 1100 | "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", 1101 | "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", 1102 | "optional": true 1103 | }, 1104 | "underscore": { 1105 | "version": "1.9.1", 1106 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz", 1107 | "integrity": "sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg==" 1108 | }, 1109 | "universalify": { 1110 | "version": "0.1.2", 1111 | "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", 1112 | "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" 1113 | }, 1114 | "uri-js": { 1115 | "version": "4.2.2", 1116 | "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", 1117 | "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", 1118 | "requires": { 1119 | "punycode": "^2.1.0" 1120 | } 1121 | }, 1122 | "url": { 1123 | "version": "0.11.0", 1124 | "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", 1125 | "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", 1126 | "requires": { 1127 | "punycode": "1.3.2", 1128 | "querystring": "0.2.0" 1129 | }, 1130 | "dependencies": { 1131 | "punycode": { 1132 | "version": "1.3.2", 1133 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", 1134 | "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" 1135 | } 1136 | } 1137 | }, 1138 | "util-deprecate": { 1139 | "version": "1.0.2", 1140 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 1141 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 1142 | }, 1143 | "uuid": { 1144 | "version": "3.3.2", 1145 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", 1146 | "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" 1147 | }, 1148 | "verror": { 1149 | "version": "1.10.0", 1150 | "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", 1151 | "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", 1152 | "requires": { 1153 | "assert-plus": "^1.0.0", 1154 | "core-util-is": "1.0.2", 1155 | "extsprintf": "^1.2.0" 1156 | } 1157 | }, 1158 | "wrappy": { 1159 | "version": "1.0.2", 1160 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 1161 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 1162 | }, 1163 | "ws": { 1164 | "version": "6.1.2", 1165 | "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.2.tgz", 1166 | "integrity": "sha512-rfUqzvz0WxmSXtJpPMX2EeASXabOrSMk1ruMOV3JBTBjo4ac2lDjGGsbQSyxj8Odhw5fBib8ZKEjDNvgouNKYw==", 1167 | "requires": { 1168 | "async-limiter": "~1.0.0" 1169 | } 1170 | }, 1171 | "xml2js": { 1172 | "version": "0.4.19", 1173 | "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", 1174 | "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", 1175 | "optional": true, 1176 | "requires": { 1177 | "sax": ">=0.6.0", 1178 | "xmlbuilder": "~9.0.1" 1179 | } 1180 | }, 1181 | "xmlbuilder": { 1182 | "version": "9.0.7", 1183 | "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", 1184 | "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", 1185 | "optional": true 1186 | }, 1187 | "xregexp": { 1188 | "version": "4.2.0", 1189 | "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.2.0.tgz", 1190 | "integrity": "sha512-IyMa7SVe9FyT4WbQVW3b95mTLVceHhLEezQ02+QMvmIqDnKTxk0MLWIQPSW2MXAr1zQb+9yvwYhcyQULneh3wA==" 1191 | }, 1192 | "yauzl": { 1193 | "version": "2.4.1", 1194 | "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", 1195 | "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", 1196 | "optional": true, 1197 | "requires": { 1198 | "fd-slicer": "~1.0.1" 1199 | } 1200 | } 1201 | } 1202 | } 1203 | --------------------------------------------------------------------------------