├── .nvmrc ├── .node-version ├── .npmrc ├── src ├── helpers │ ├── scrapers.js │ ├── credentials.js │ ├── settings.js │ ├── crypto.js │ ├── files.js │ └── tasks.js ├── constants.js ├── definitions.js ├── index.js ├── setup │ ├── setup-main-menu.js │ ├── setup-scrapers.js │ └── tasks │ │ ├── delete-task-handler.js │ │ ├── setup-task.js │ │ ├── create-task-handler.js │ │ └── modify-task-handler.js └── scrape │ ├── scraping-main-menu.js │ ├── scrape-base.js │ ├── scrape-individual.js │ ├── generate-reports.js │ └── scrape-task.js ├── .babelrc ├── .eslintrc ├── LICENSE ├── .gitignore ├── package.json └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | v8.2.1 2 | -------------------------------------------------------------------------------- /.node-version: -------------------------------------------------------------------------------- 1 | 8.2.1 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry = https://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /src/helpers/scrapers.js: -------------------------------------------------------------------------------- 1 | export { SCRAPERS, createScraper } from 'israeli-bank-scrapers'; 2 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | const PASSWORD_FIELD = 'password'; 2 | const DATE_TIME_FORMAT = 'DD-MM-YYYY_HH-mm-ss'; 3 | const TRANSACTION_STATUS = { 4 | PENDING: 'pending', 5 | }; 6 | 7 | export { PASSWORD_FIELD, DATE_TIME_FORMAT, TRANSACTION_STATUS }; 8 | -------------------------------------------------------------------------------- /src/definitions.js: -------------------------------------------------------------------------------- 1 | import os from 'os'; 2 | 3 | export const CONFIG_FOLDER = `${os.homedir()}/.ynab-updater`; 4 | export const SETTINGS_FILE = `${CONFIG_FOLDER}/settings.json`; 5 | export const TASKS_FOLDER = `${CONFIG_FOLDER}/tasks`; 6 | export const DOWNLOAD_FOLDER = `${os.homedir()}/Downloads/Transactions`; 7 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", { 5 | "targets": { 6 | "node": "8" 7 | }, 8 | "include": [ 9 | "transform-es2015-arrow-functions", 10 | "transform-es2015-shorthand-properties", 11 | "transform-es2015-block-scoping" 12 | ], 13 | "useBuiltIns": true 14 | } 15 | ] 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "arrow-body-style": 0, 4 | "no-shadow": 0, 5 | "no-console": 0, 6 | "no-underscore-dangle": ["error", { "allow": ["_private"] }], 7 | "class-methods-use-this": 0, 8 | "no-await-in-loop": 0 9 | }, 10 | "globals": { 11 | "document": true, 12 | "fetch": true, 13 | "Headers": true 14 | }, 15 | "extends": "airbnb-base" 16 | } 17 | -------------------------------------------------------------------------------- /src/helpers/credentials.js: -------------------------------------------------------------------------------- 1 | import { encrypt, decrypt } from './crypto'; 2 | 3 | export function encryptCredentials(credentials) { 4 | const encrypted = {}; 5 | Object.keys(credentials).forEach((field) => { 6 | encrypted[field] = encrypt(credentials[field]); 7 | }); 8 | return encrypted; 9 | } 10 | 11 | export function decryptCredentials(credentials) { 12 | const decrypted = {}; 13 | Object.keys(credentials).forEach((field) => { 14 | decrypted[field] = decrypt(credentials[field]); 15 | }); 16 | return decrypted; 17 | } 18 | -------------------------------------------------------------------------------- /src/helpers/settings.js: -------------------------------------------------------------------------------- 1 | import { SETTINGS_FILE, DOWNLOAD_FOLDER } from '../definitions'; 2 | import { writeJsonFile, readJsonFile } from './files'; 3 | 4 | export async function readSettingsFile() { 5 | let settings = await readJsonFile(SETTINGS_FILE); 6 | if (!settings) { 7 | settings = { 8 | saveLocation: DOWNLOAD_FOLDER, 9 | }; 10 | } 11 | 12 | return settings; 13 | } 14 | 15 | export async function writeSettingsFile(settings) { 16 | if (settings) { 17 | await writeJsonFile(SETTINGS_FILE, settings); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/helpers/crypto.js: -------------------------------------------------------------------------------- 1 | import crypto from 'crypto'; 2 | 3 | const ALGORITHM = 'aes-256-ctr'; 4 | const SALT = '8cs+8Y(nxDLY'; 5 | 6 | export function encrypt(text) { 7 | const cipher = crypto.createCipher(ALGORITHM, SALT); 8 | const crypted = cipher.update(text, 'utf8', 'hex'); 9 | return crypted + cipher.final('hex'); 10 | } 11 | 12 | export function decrypt(text) { 13 | const decipher = crypto.createDecipher(ALGORITHM, SALT); 14 | const decrypted = decipher.update(text, 'hex', 'utf8'); 15 | return decrypted + decipher.final('utf8'); 16 | } 17 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import yargs from 'yargs'; 2 | import colors from 'colors/safe'; 3 | import setupMainMenu from './setup/setup-main-menu'; 4 | import scrapingMainMenu from './scrape/scraping-main-menu'; 5 | 6 | // set theme 7 | colors.setTheme({ 8 | title: 'bgCyan', 9 | notify: 'magenta', 10 | }); 11 | 12 | const args = yargs.options({ 13 | mode: { 14 | alias: 'm', 15 | describe: 'mode for running', 16 | }, 17 | show: { 18 | alias: 's', 19 | describe: 'show browser while scraping', 20 | type: 'boolean', 21 | default: false, 22 | }, 23 | task: { 24 | alias: 't', 25 | describe: 'select task to run', 26 | }, 27 | }).help().argv; 28 | 29 | if (!args.mode || args.mode === 'scrape') { 30 | scrapingMainMenu(args.show, args.task); 31 | } else if (args.mode === 'setup') { 32 | setupMainMenu(); 33 | } 34 | -------------------------------------------------------------------------------- /src/setup/setup-main-menu.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | 3 | import setupTask from './tasks/setup-task'; 4 | import setupScrapers from './setup-scrapers'; 5 | 6 | export default async function () { 7 | const SETUP_SCRAPER_ACTION = 'scraper'; 8 | const SETUP_TASK_ACTION = 'task'; 9 | const { setupType } = await inquirer.prompt({ 10 | type: 'list', 11 | name: 'setupType', 12 | message: 'What would you like to setup?', 13 | choices: [ 14 | { 15 | name: 'Setup a new scraper', 16 | value: SETUP_SCRAPER_ACTION, 17 | }, 18 | { 19 | name: 'Setup a new task', 20 | value: SETUP_TASK_ACTION, 21 | }, 22 | ], 23 | }); 24 | 25 | switch (setupType) { 26 | case SETUP_SCRAPER_ACTION: 27 | await setupScrapers(); 28 | break; 29 | case SETUP_TASK_ACTION: 30 | await setupTask(); 31 | break; 32 | default: 33 | break; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Elad Shaham 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/scrape/scraping-main-menu.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import scrapeIndividual from './scrape-individual'; 3 | import scrapeTask from './scrape-task'; 4 | 5 | async function selectAction(showBrowser) { 6 | const RUN_SCRAPER_ACTION = 'scraper'; 7 | const RUN_TASK_ACTION = 'task'; 8 | 9 | const { scrapeType } = await inquirer.prompt({ 10 | type: 'list', 11 | name: 'scrapeType', 12 | message: 'What would you like to do?', 13 | choices: [ 14 | { 15 | name: 'Run an individual scraper', 16 | value: RUN_SCRAPER_ACTION, 17 | }, 18 | { 19 | name: 'Run a task', 20 | value: RUN_TASK_ACTION, 21 | }, 22 | ], 23 | }); 24 | 25 | switch (scrapeType) { 26 | case RUN_SCRAPER_ACTION: 27 | await scrapeIndividual(showBrowser); 28 | break; 29 | case RUN_TASK_ACTION: 30 | await scrapeTask(showBrowser); 31 | break; 32 | default: 33 | break; 34 | } 35 | } 36 | 37 | export default async function (showBrowser, taskName) { 38 | if (taskName) { 39 | await scrapeTask(showBrowser, taskName); 40 | } else { 41 | selectAction(showBrowser); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # NPM package related 61 | lib/ 62 | 63 | # OS related 64 | .DS_Store 65 | 66 | # IDEs 67 | .idea 68 | .vscode -------------------------------------------------------------------------------- /src/setup/setup-scrapers.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | 3 | import { PASSWORD_FIELD } from '../constants'; 4 | import { CONFIG_FOLDER } from '../definitions'; 5 | import { SCRAPERS } from '../helpers/scrapers'; 6 | import { writeJsonFile } from '../helpers/files'; 7 | import { encryptCredentials } from '../helpers/credentials'; 8 | 9 | function validateNonEmpty(field, input) { 10 | if (input) { 11 | return true; 12 | } 13 | return `${field} must be non empty`; 14 | } 15 | 16 | export default async function () { 17 | const scraperIdResult = await inquirer.prompt([{ 18 | type: 'list', 19 | name: 'scraperId', 20 | message: 'Which scraper would you like to save credentials for?', 21 | choices: Object.keys(SCRAPERS).map((id) => { 22 | return { 23 | name: SCRAPERS[id].name, 24 | value: id, 25 | }; 26 | }), 27 | }]); 28 | const { loginFields } = SCRAPERS[scraperIdResult.scraperId]; 29 | const questions = loginFields.map((field) => { 30 | return { 31 | type: field === PASSWORD_FIELD ? PASSWORD_FIELD : 'input', 32 | name: field, 33 | message: `Enter value for ${field}:`, 34 | validate: input => validateNonEmpty(field, input), 35 | }; 36 | }); 37 | const credentialsResult = await inquirer.prompt(questions); 38 | const encryptedCredentials = encryptCredentials(credentialsResult); 39 | await writeJsonFile(`${CONFIG_FOLDER}/${scraperIdResult.scraperId}.json`, encryptedCredentials); 40 | console.log(`credentials file saved for ${scraperIdResult.scraperId}`); 41 | } 42 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "israeli-ynab-updater", 3 | "version": "0.0.1", 4 | "description": "A tool for updating YNAB using israeli-bank-scrapers", 5 | "main": "lib/index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "lint": "eslint src", 9 | "start": "babel-node src", 10 | "start:debug": "npm start -- -s true", 11 | "setup": "npm start -- -m setup" 12 | }, 13 | "pre-commit": [ 14 | "lint" 15 | ], 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/eshaham/israeli-ynab-updater.git" 19 | }, 20 | "author": "Elad Shaham", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/eshaham/israeli-ynab-updater/issues" 24 | }, 25 | "homepage": "https://github.com/eshaham/israeli-ynab-updater#readme", 26 | "devDependencies": { 27 | "babel-cli": "^6.26.0", 28 | "babel-plugin-transform-es2015-arrow-functions": "^6.22.0", 29 | "babel-plugin-transform-es2015-block-scoping": "^6.26.0", 30 | "babel-plugin-transform-es2015-shorthand-properties": "^6.24.1", 31 | "babel-preset-env": "^1.7.0", 32 | "eslint": "^5.3.0", 33 | "eslint-config-airbnb-base": "^13.2.0", 34 | "eslint-plugin-import": "^2.18.2", 35 | "pre-commit": "^1.2.2" 36 | }, 37 | "dependencies": { 38 | "babel-polyfill": "^6.26.0", 39 | "colors": "^1.3.3", 40 | "inquirer": "^6.5.0", 41 | "israeli-bank-scrapers": "^0.6.9", 42 | "json2csv": "^4.5.2", 43 | "jsonfile": "^5.0.0", 44 | "moment": "^2.24.0", 45 | "yargs": "^13.3.0" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/setup/tasks/delete-task-handler.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import colors from 'colors/safe'; 3 | import { TasksManager } from '../../helpers/tasks'; 4 | 5 | const tasksManager = new TasksManager(); 6 | const goBackOption = { 7 | name: 'Go Back', 8 | value: '', 9 | }; 10 | 11 | const DeleteTaskHandler = (function createDeleteTaskHandler() { 12 | const _private = new WeakMap(); 13 | 14 | class DeleteTaskHandler { 15 | static async createAdapter() { 16 | const answers = await inquirer.prompt([ 17 | { 18 | type: 'list', 19 | name: 'taskName', 20 | message: 'Select a task to delete', 21 | choices: [ 22 | ...await tasksManager.getTasksList(), 23 | goBackOption, 24 | ], 25 | }, 26 | { 27 | type: 'confirm', 28 | name: 'confirmDelete', 29 | message: 'Are you sure?', 30 | when: answers => !!answers.taskName, 31 | default: false, 32 | }, 33 | ]); 34 | 35 | if (answers.taskName && answers.confirmDelete) { 36 | return new DeleteTaskHandler(answers.taskName); 37 | } 38 | console.log(colors.notify('Delete task cancelled')); 39 | 40 | return null; 41 | } 42 | 43 | constructor(taskName) { 44 | _private.set(this, { taskName }); 45 | } 46 | 47 | async run() { 48 | if (!_private.get(this).taskName) { 49 | throw new Error('Missing task name'); 50 | } 51 | 52 | console.log(colors.notify(`Task '${_private.get(this).taskName}' deleted`)); 53 | await tasksManager.deleteTask(_private.get(this).taskName); 54 | } 55 | } 56 | 57 | return DeleteTaskHandler; 58 | }()); 59 | 60 | export default DeleteTaskHandler; 61 | -------------------------------------------------------------------------------- /src/setup/tasks/setup-task.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import ModifyTaskHandler from './modify-task-handler'; 3 | import CreateTaskHandler from './create-task-handler'; 4 | import DeleteTaskHandler from './delete-task-handler'; 5 | 6 | 7 | async function selectAction() { 8 | const CREATE_NEW_TASK_ACTION = 'new'; 9 | const MODIFY_TASK_ACTION = 'modify'; 10 | const DELETE_TASK_ACTION = 'delete'; 11 | const QUIT_ACTION = 'quit'; 12 | 13 | const answers = await inquirer.prompt([ 14 | { 15 | type: 'list', 16 | name: 'action', 17 | message: 'What do you want to do?', 18 | choices: [ 19 | { 20 | name: 'Create a new task', 21 | value: CREATE_NEW_TASK_ACTION, 22 | }, 23 | { 24 | name: 'Modify an existing task', 25 | value: MODIFY_TASK_ACTION, 26 | }, 27 | new inquirer.Separator(), 28 | { 29 | name: 'Delete a task', 30 | value: DELETE_TASK_ACTION, 31 | }, 32 | { 33 | name: 'Quit', 34 | value: QUIT_ACTION, 35 | }, 36 | ], 37 | }, 38 | ]); 39 | 40 | switch (answers.action) { 41 | case CREATE_NEW_TASK_ACTION: { 42 | const createNewTaskAdapter = new CreateTaskHandler(); 43 | await createNewTaskAdapter.run(); 44 | await selectAction(); 45 | } 46 | break; 47 | case MODIFY_TASK_ACTION: { 48 | const adapter = await ModifyTaskHandler.createAdapter(); 49 | 50 | if (adapter) { 51 | await adapter.run(); 52 | } 53 | 54 | await selectAction(); 55 | } 56 | break; 57 | case DELETE_TASK_ACTION: { 58 | const adapter = await DeleteTaskHandler.createAdapter(); 59 | 60 | if (adapter) { 61 | await adapter.run(); 62 | } 63 | 64 | await selectAction(); 65 | } 66 | break; 67 | default: 68 | break; 69 | } 70 | } 71 | 72 | export default selectAction; 73 | -------------------------------------------------------------------------------- /src/helpers/files.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import path from 'path'; 3 | import util from 'util'; 4 | import jsonfile from 'jsonfile'; 5 | 6 | const writeFileAsync = util.promisify(fs.writeFile); 7 | const existsAsync = util.promisify(fs.exists); 8 | const makeDirAsync = util.promisify(fs.mkdir); 9 | const readdirAsync = util.promisify(fs.readdir); 10 | const deleteFileAsync = util.promisify(fs.unlink); 11 | const readJsonFileAsync = util.promisify(jsonfile.readFile); 12 | const writeJsonFileAsync = util.promisify(jsonfile.writeFile); 13 | 14 | async function verifyFolder(folderPath) { 15 | const pathTokens = folderPath.split(path.sep); 16 | let currentPath = ''; 17 | for (let i = 0; i < pathTokens.length; i += 1) { 18 | const folder = pathTokens[i]; 19 | currentPath += folder + path.sep; 20 | if (!await existsAsync(currentPath)) { 21 | await makeDirAsync(currentPath); 22 | } 23 | } 24 | } 25 | 26 | export async function getFolderFiles(folderPath, suffix) { 27 | await verifyFolder(folderPath); 28 | const files = await readdirAsync(folderPath); 29 | if (suffix) { 30 | return files.filter(filePath => (path.extname(filePath) || '').toLowerCase() === suffix.toLowerCase()); 31 | } 32 | return files; 33 | } 34 | 35 | export async function deleteFile(filePath) { 36 | return deleteFileAsync(filePath); 37 | } 38 | 39 | export async function writeFile(filePath, data, options) { 40 | const folderPath = path.dirname(filePath); 41 | await verifyFolder(folderPath); 42 | return writeFileAsync(filePath, data, options); 43 | } 44 | 45 | export async function readJsonFile(filePath, options) { 46 | const exists = await existsAsync(filePath); 47 | if (!exists) { 48 | return null; 49 | } 50 | return readJsonFileAsync(filePath, options); 51 | } 52 | 53 | export async function writeJsonFile(filePath, obj, options) { 54 | const folderPath = path.dirname(filePath); 55 | await verifyFolder(folderPath); 56 | await writeJsonFileAsync(filePath, obj, options); 57 | } 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Israeli YNAB Updater 2 | A tool for updating YNAB using for the Israeli market. This tool uses [Israeli Banks Scrapers](https://github.com/eshaham/israeli-bank-scrapers) project as the source of fetching account data. 3 | 4 | ## Getting started 5 | 6 | ### Prerequisites 7 | 8 | In order to start using this tool, you will need to have Node.js (>= 8) installed on your machine. 9 | Go [here!](https://nodejs.org/en/download/) to download and install the latest Node.js for your operating system. 10 | 11 | ### Installation 12 | Once Node.js is installed, run the following command to fetch the code: 13 | 14 | ```bash 15 | git clone https://github.com/eshaham/israeli-ynab-updater 16 | cd israeli-ynab-updater 17 | ``` 18 | 19 | If you're using `nvm` make sure to run `nvm use` inside project folder for best compatability. 20 | If you're using `nodenv`, it should automatically pick up the correct node version. 21 | 22 | Next you will need to install dependencies by running 23 | ```bash 24 | npm install 25 | ``` 26 | 27 | ### Saving credentials 28 | The credentials for each scraper are encrypted and saved in an dedicated file on your system. 29 | To save credentials for a specific scraper, run the following command and choose the scraper: 30 | 31 | ```bash 32 | npm run setup 33 | ``` 34 | 35 | When asked 'What would you like to setup?' choose 'Scrapers'. 36 | 37 | ### Scraping 38 | Once you save the credentials for relevant scrapers, run the following command to start scraping: 39 | 40 | ```bash 41 | npm start 42 | ``` 43 | 44 | The CSV file should be created under `Transactions` folder in your Downloads folder, unless you choose a different folder. 45 | 46 | You can also scrape in debug mode by running: 47 | 48 | ```bash 49 | npm run start:debug 50 | ``` 51 | 52 | If you want to run a specific `task` configured in the `setup`, you can add the `-- -t taskName` to the above commands, and the task will run immediately, without going through any menu. For example: 53 | 54 | ```bash 55 | npm run start:debug -- -t myTask 56 | ``` 57 | -------------------------------------------------------------------------------- /src/setup/tasks/create-task-handler.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import colors from 'colors/safe'; 3 | import { DOWNLOAD_FOLDER } from '../../definitions'; 4 | import ModifyTaskHandler from './modify-task-handler'; 5 | import { TasksManager } from '../../helpers/tasks'; 6 | 7 | const tasksManager = new TasksManager(); 8 | 9 | function createEmptyTaskData() { 10 | return { 11 | scrapers: [], 12 | options: { 13 | combineInstallments: false, 14 | dateDiffByMonth: 3, 15 | }, 16 | output: { 17 | saveLocation: DOWNLOAD_FOLDER, 18 | combineReport: true, 19 | includeFutureTransactions: false, 20 | includePendingTransactions: false, 21 | }, 22 | }; 23 | } 24 | 25 | export default class { 26 | async run() { 27 | const answers = await inquirer.prompt([ 28 | { 29 | type: 'input', 30 | name: 'name', 31 | message: 'What is the task name (leave blank to cancel)', 32 | validate: async (value) => { 33 | if (value) { 34 | const alreadyExists = await tasksManager.hasTask(value); 35 | 36 | if (alreadyExists) { 37 | return 'A task with this name already exists, please type a unique name'; 38 | } 39 | 40 | const invalidPattern = !tasksManager.isValidTaskName(value); 41 | 42 | if (invalidPattern) { 43 | return 'The task name must include only these characters: A-Z, 0-9, -, _'; 44 | } 45 | } 46 | 47 | return true; 48 | }, 49 | }, 50 | ]); 51 | 52 | const taskName = answers.name; 53 | 54 | if (taskName) { 55 | const taskData = createEmptyTaskData(); 56 | await tasksManager.saveTask(taskName, taskData); 57 | const modifyTaskAdapter = new ModifyTaskHandler(taskName); 58 | console.log(colors.notify(`Task '${taskName}' created`)); 59 | await modifyTaskAdapter.run(); 60 | } else { 61 | console.log(colors.notify('Task creation cancelled')); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/scrape/scrape-base.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import { SCRAPERS, createScraper } from '../helpers/scrapers'; 3 | 4 | async function prepareResults(scraperId, scraperName, scraperResult, combineInstallments) { 5 | return scraperResult.accounts.map((account) => { 6 | console.log(`${scraperName}: scraped ${account.txns.length} transactions from account ${account.accountNumber}`); 7 | 8 | const txns = account.txns.map((txn) => { 9 | return { 10 | company: scraperName, 11 | account: account.accountNumber, 12 | dateMoment: moment(txn.date), 13 | payee: txn.description, 14 | status: txn.status, 15 | amount: txn.type !== 'installments' || !combineInstallments ? txn.chargedAmount : txn.originalAmount, 16 | installment: txn.installments ? txn.installments.number : null, 17 | total: txn.installments ? txn.installments.total : null, 18 | }; 19 | }); 20 | 21 | return { 22 | scraperId, 23 | scraperName, 24 | accountNumber: account.accountNumber, 25 | txns, 26 | }; 27 | }); 28 | } 29 | 30 | export default async function (scraperId, credentials, options) { 31 | const { 32 | combineInstallments, 33 | startDate, 34 | showBrowser, 35 | } = options; 36 | 37 | const scraperOptions = { 38 | companyId: scraperId, 39 | startDate, 40 | combineInstallments, 41 | showBrowser, 42 | verbose: false, 43 | }; 44 | const scraperName = SCRAPERS[scraperId] ? SCRAPERS[scraperId].name : null; 45 | 46 | if (!scraperName) { 47 | throw new Error(`unknown scraper with id ${scraperId}`); 48 | } 49 | console.log(`scraping ${scraperName}`); 50 | 51 | const scraper = createScraper(scraperOptions); 52 | scraper.onProgress((companyId, payload) => { 53 | console.log(`${scraperName}: ${payload.type}`); 54 | }); 55 | const scraperResult = await scraper.scrape(credentials); 56 | 57 | console.log(`success: ${scraperResult.success}`); 58 | if (!scraperResult.success) { 59 | console.log(`error type: ${scraperResult.errorType}`); 60 | console.log('error:', scraperResult.errorMessage); 61 | throw new Error(scraperResult.errorMessage); 62 | } 63 | 64 | return prepareResults(scraperId, scraperName, scraperResult, combineInstallments); 65 | } 66 | -------------------------------------------------------------------------------- /src/helpers/tasks.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import colors from 'colors/safe'; 3 | import moment from 'moment'; 4 | import { 5 | writeJsonFile, 6 | readJsonFile, 7 | getFolderFiles, 8 | deleteFile, 9 | } from './files'; 10 | import { TASKS_FOLDER } from '../definitions'; 11 | import { SCRAPERS } from './scrapers'; 12 | 13 | function writeSummaryLine(key, value) { 14 | console.log(`- ${colors.bold(key)}: ${value}`); 15 | } 16 | 17 | function printTaskSummary(taskData, shouldCalculateStartDate = false) { 18 | if (taskData) { 19 | const scrapers = taskData.scrapers || []; 20 | const { 21 | dateDiffByMonth, 22 | combineInstallments, 23 | } = taskData.options; 24 | const { 25 | combineReport, 26 | saveLocation, 27 | includeFutureTransactions, 28 | includePendingTransactions, 29 | } = taskData.output; 30 | console.log(colors.underline.bold('Task Summary')); 31 | writeSummaryLine('Scrapers', scrapers.map(scraper => SCRAPERS[scraper.id].name).join(', ')); 32 | 33 | if (shouldCalculateStartDate) { 34 | const substractValue = dateDiffByMonth - 1; 35 | const startMoment = moment().subtract(substractValue, 'month').startOf('month'); 36 | writeSummaryLine('Start scraping from', startMoment.format('ll')); 37 | } else { 38 | writeSummaryLine('Scrape # of months', dateDiffByMonth); 39 | } 40 | 41 | writeSummaryLine('Combine installments', combineInstallments ? 'Yes' : 'No'); 42 | writeSummaryLine('Save to location', saveLocation); 43 | writeSummaryLine('Create single report', combineReport ? 'Yes' : 'No'); 44 | writeSummaryLine('Include future Transactions', includeFutureTransactions ? 'Yes' : 'No'); 45 | writeSummaryLine('Include pending Transactions', includePendingTransactions ? 'Yes' : 'No'); 46 | } 47 | } 48 | 49 | class TasksManager { 50 | async getTasksList() { 51 | const files = await getFolderFiles(TASKS_FOLDER, '.json'); 52 | const result = files.map(file => path.basename(file, '.json')); 53 | return result; 54 | } 55 | 56 | async hasTask(taskName) { 57 | const tasksList = await this.getTasksList(); 58 | return taskName && !!tasksList.find((taskListName) => { 59 | return taskListName.toLowerCase() === taskName.toLowerCase(); 60 | }); 61 | } 62 | 63 | isValidTaskName(taskName) { 64 | return taskName && taskName.match(/^[a-zA-Z0-9_-]+$/); 65 | } 66 | 67 | async loadTask(taskName) { 68 | if (taskName && this.hasTask(taskName)) { 69 | return readJsonFile(`${TASKS_FOLDER}/${taskName}.json`); 70 | } 71 | 72 | throw new Error(`failed to find a task named ${taskName}`); 73 | } 74 | 75 | async saveTask(taskName, taskData) { 76 | if (taskName && this.isValidTaskName(taskName)) { 77 | return writeJsonFile(`${TASKS_FOLDER}/${taskName}.json`, taskData); 78 | } 79 | 80 | throw new Error(`invalid task name provided ${taskName}`); 81 | } 82 | 83 | async deleteTask(taskName) { 84 | if (taskName && this.hasTask(taskName)) { 85 | await deleteFile(`${TASKS_FOLDER}/${taskName}.json`); 86 | } else { 87 | throw new Error(`invalid task name provided ${taskName}`); 88 | } 89 | } 90 | } 91 | 92 | export { TasksManager, printTaskSummary }; 93 | -------------------------------------------------------------------------------- /src/scrape/scrape-individual.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import inquirer from 'inquirer'; 3 | 4 | import { CONFIG_FOLDER } from '../definitions'; 5 | import { readJsonFile } from '../helpers/files'; 6 | import { decryptCredentials } from '../helpers/credentials'; 7 | import { SCRAPERS } from '../helpers/scrapers'; 8 | import { readSettingsFile, writeSettingsFile } from '../helpers/settings'; 9 | import scrape from './scrape-base'; 10 | import { generateSeparatedReports } from './generate-reports'; 11 | 12 | async function getParameters() { 13 | const settings = await readSettingsFile(); 14 | const { 15 | combineInstallments, 16 | saveLocation, 17 | includeFutureTransactions, 18 | includePendingTransactions, 19 | } = settings; 20 | 21 | const startOfMonthMoment = moment().startOf('month'); 22 | const monthOptions = []; 23 | for (let i = 0; i < 6; i += 1) { 24 | const monthMoment = startOfMonthMoment.clone().subtract(i, 'month'); 25 | monthOptions.push({ 26 | name: monthMoment.format('ll'), 27 | value: monthMoment, 28 | }); 29 | } 30 | const result = await inquirer.prompt([ 31 | { 32 | type: 'list', 33 | name: 'scraperId', 34 | message: 'Which bank would you like to scrape?', 35 | choices: Object.keys(SCRAPERS).map((id) => { 36 | return { 37 | name: SCRAPERS[id].name, 38 | value: id, 39 | }; 40 | }), 41 | }, 42 | { 43 | type: 'confirm', 44 | name: 'combineInstallments', 45 | message: 'Combine installment transactions?', 46 | default: !!combineInstallments, 47 | }, 48 | { 49 | type: 'list', 50 | name: 'startDate', 51 | message: 'What date would you like to start scraping from?', 52 | choices: monthOptions, 53 | }, 54 | { 55 | type: 'input', 56 | name: 'saveLocation', 57 | message: 'Save folder?', 58 | default: saveLocation, 59 | }, 60 | { 61 | type: 'confirm', 62 | name: 'includeFutureTransactions', 63 | message: 'Include future transactions?', 64 | default: !!includeFutureTransactions, 65 | }, 66 | { 67 | type: 'confirm', 68 | name: 'includePendingTransactions', 69 | message: 'Include pending transactions?', 70 | default: !!includePendingTransactions, 71 | }, 72 | ]); 73 | 74 | settings.combineInstallments = result.combineInstallments; 75 | settings.startDate = result.startDate; 76 | settings.saveLocation = result.saveLocation; 77 | settings.includeFutureTransactions = result.includeFutureTransactions; 78 | settings.includePendingTransactions = result.includePendingTransactions; 79 | await writeSettingsFile(settings); 80 | 81 | return result; 82 | } 83 | 84 | export default async function (showBrowser) { 85 | const { 86 | scraperId, 87 | combineInstallments, 88 | startDate, 89 | saveLocation, 90 | includeFutureTransactions, 91 | includePendingTransactions, 92 | } = await getParameters(); 93 | 94 | const encryptedCredentials = await readJsonFile(`${CONFIG_FOLDER}/${scraperId}.json`); 95 | if (encryptedCredentials) { 96 | const credentials = decryptCredentials(encryptedCredentials); 97 | const options = { 98 | startDate: startDate.toDate(), 99 | combineInstallments, 100 | showBrowser, 101 | }; 102 | 103 | try { 104 | const scrapedAccounts = await scrape(scraperId, credentials, options); 105 | await generateSeparatedReports( 106 | scrapedAccounts, 107 | saveLocation, 108 | includeFutureTransactions, 109 | includePendingTransactions, 110 | ); 111 | } catch (e) { 112 | console.error(e); 113 | } 114 | } else { 115 | console.log('Could not find credentials file'); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/scrape/generate-reports.js: -------------------------------------------------------------------------------- 1 | import { parse as objToCsv } from 'json2csv'; 2 | import colors from 'colors/safe'; 3 | import moment from 'moment'; 4 | import { DATE_TIME_FORMAT, TRANSACTION_STATUS } from '../constants'; 5 | import { writeFile } from '../helpers/files'; 6 | 7 | function getReportFields(isSingleReport) { 8 | const result = [ 9 | { 10 | label: 'Date', 11 | value: row => row.dateMoment.format('DD/MM/YYYY'), 12 | }, 13 | { 14 | label: 'Payee', 15 | value: 'payee', 16 | }, 17 | { 18 | label: 'Inflow', 19 | value: 'amount', 20 | }, 21 | { 22 | label: 'Status', 23 | value: 'status', 24 | }, 25 | { 26 | label: 'Installment', 27 | value: 'installment', 28 | }, 29 | { 30 | label: 'Total', 31 | value: 'total', 32 | }, 33 | ]; 34 | 35 | if (isSingleReport) { 36 | result.unshift( 37 | { 38 | label: 'Company', 39 | value: 'company', 40 | }, 41 | { 42 | label: 'Account', 43 | value: 'account', 44 | }, 45 | ); 46 | } 47 | 48 | return result; 49 | } 50 | 51 | function filterTransactions(transactions, includeFutureTransactions, includePendingTransactions) { 52 | let result = transactions; 53 | 54 | if (!includeFutureTransactions) { 55 | const nowMoment = moment(); 56 | result = result.filter((txn) => { 57 | const txnMoment = moment(txn.dateMoment); 58 | return txnMoment.isSameOrBefore(nowMoment, 'day'); 59 | }); 60 | } 61 | 62 | if (!includePendingTransactions) { 63 | result = result.filter(txn => txn.status !== TRANSACTION_STATUS.PENDING); 64 | } 65 | 66 | return result; 67 | } 68 | 69 | async function exportAccountData(txns, scraperName, accountNumber, saveLocation) { 70 | const fields = getReportFields(false); 71 | const csv = objToCsv(txns, { fields, withBOM: true }); 72 | await writeFile(`${saveLocation}/${scraperName} (${accountNumber}).csv`, csv); 73 | } 74 | 75 | export async function generateSeparatedReports( 76 | scrapedAccounts, 77 | saveLocation, 78 | includeFutureTransactions, 79 | includePendingTransactions, 80 | ) { 81 | let numFiles = 0; 82 | for (let i = 0; i < scrapedAccounts.length; i += 1) { 83 | const { 84 | txns: accountTxns, 85 | accountNumber, 86 | scraperName, 87 | } = scrapedAccounts[i]; 88 | 89 | const filteredTxns = filterTransactions( 90 | accountTxns, 91 | includeFutureTransactions, 92 | includePendingTransactions, 93 | ); 94 | if (filteredTxns.length) { 95 | console.log(colors.notify(`exporting ${accountTxns.length} transactions for account # ${accountNumber}`)); 96 | await exportAccountData(filteredTxns, scraperName, accountNumber, saveLocation); 97 | numFiles += 1; 98 | } else { 99 | console.log(`no transactions for account # ${accountNumber}`); 100 | } 101 | } 102 | 103 | console.log(colors.notify(`${numFiles} csv files saved under ${saveLocation}`)); 104 | } 105 | 106 | export async function generateSingleReport( 107 | scrapedAccounts, 108 | saveLocation, 109 | includeFutureTransactions, 110 | includePendingTransactions, 111 | ) { 112 | const fileTransactions = scrapedAccounts.reduce((acc, account) => { 113 | const filteredTransactions = filterTransactions( 114 | account.txns, 115 | includeFutureTransactions, 116 | includePendingTransactions, 117 | ); 118 | acc.push(...filteredTransactions); 119 | return acc; 120 | }, []); 121 | const filePath = `${saveLocation}/${moment().format(DATE_TIME_FORMAT)}.csv`; 122 | const fileFields = getReportFields(true); 123 | const fileContent = objToCsv(fileTransactions, { fields: fileFields, withBOM: true }); 124 | await writeFile(filePath, fileContent); 125 | console.log(colors.notify(`created file ${filePath}`)); 126 | } 127 | -------------------------------------------------------------------------------- /src/scrape/scrape-task.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import inquirer from 'inquirer'; 3 | import colors from 'colors/safe'; 4 | import { DATE_TIME_FORMAT } from '../constants'; 5 | import { decryptCredentials } from '../helpers/credentials'; 6 | import scrape from './scrape-base'; 7 | import { TasksManager, printTaskSummary } from '../helpers/tasks'; 8 | import { generateSingleReport, generateSeparatedReports } from './generate-reports'; 9 | 10 | const tasksManager = new TasksManager(); 11 | 12 | async function getParameters() { 13 | const availableTasks = await tasksManager.getTasksList(); 14 | 15 | if (availableTasks && availableTasks.length) { 16 | const answers = await inquirer.prompt([ 17 | { 18 | type: 'list', 19 | name: 'taskName', 20 | message: 'Select a task to execute', 21 | validate: (answer) => { 22 | if (!answer) { 23 | return 'Task name must be provided'; 24 | } 25 | return true; 26 | }, 27 | choices: [...availableTasks], 28 | }, 29 | ]); 30 | 31 | return { taskName: answers.taskName }; 32 | } 33 | 34 | console.log(colors.notify('No tasks created, please run command \'npm run setup\' to create a task')); 35 | return { taskName: null }; 36 | } 37 | 38 | async function runSelectedTask(showBrowser, taskName) { 39 | if (taskName) { 40 | console.log(colors.title(`Running task '${taskName}'`)); 41 | const taskData = await tasksManager.loadTask(taskName); 42 | const scrapersOfTask = taskData.scrapers || []; 43 | 44 | if (scrapersOfTask.length === 0) { 45 | console.log(colors.notify('Task has no scrapers defined.\nplease run command \'npm run setup\' and update task scrapers')); 46 | return; 47 | } 48 | 49 | printTaskSummary(taskData, true); 50 | 51 | const { 52 | dateDiffByMonth, 53 | combineInstallments, 54 | } = taskData.options; 55 | const { 56 | combineReport, 57 | saveLocation: saveLocationRootPath, 58 | includeFutureTransactions, 59 | includePendingTransactions, 60 | } = taskData.output; 61 | const substractValue = dateDiffByMonth - 1; 62 | const startMoment = moment().subtract(substractValue, 'month').startOf('month'); 63 | const reportAccounts = []; 64 | 65 | console.log(colors.title('Run task scrapers')); 66 | 67 | for (let i = 0; i < scrapersOfTask.length; i += 1) { 68 | const scraperOfTask = scrapersOfTask[i]; 69 | const credentials = decryptCredentials(scraperOfTask.credentials); 70 | 71 | const options = { 72 | companyId: scraperOfTask.id, 73 | startDate: startMoment, 74 | combineInstallments, 75 | showBrowser, 76 | verbose: false, 77 | }; 78 | 79 | try { 80 | const scrapedAccounts = await scrape(scraperOfTask.id, credentials, options); 81 | reportAccounts.push(...scrapedAccounts); 82 | } catch (e) { 83 | console.error(e); 84 | throw e; 85 | } 86 | } 87 | 88 | if (combineReport) { 89 | const saveLocation = `${saveLocationRootPath}/tasks/${taskName}`; 90 | await generateSingleReport( 91 | reportAccounts, 92 | saveLocation, 93 | includeFutureTransactions, 94 | includePendingTransactions, 95 | ); 96 | } else { 97 | const currentExecutionFolder = moment().format(DATE_TIME_FORMAT); 98 | const saveLocation = `${saveLocationRootPath}/tasks/${taskName}/${currentExecutionFolder}`; 99 | await generateSeparatedReports( 100 | reportAccounts, 101 | saveLocation, 102 | includeFutureTransactions, 103 | includePendingTransactions, 104 | ); 105 | } 106 | } 107 | } 108 | 109 | export default async function (showBrowser, taskName) { 110 | if (taskName) { 111 | await runSelectedTask(showBrowser, taskName); 112 | } else { 113 | const { taskName: selectedTaskName } = await getParameters(); 114 | await runSelectedTask(showBrowser, selectedTaskName); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/setup/tasks/modify-task-handler.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import colors from 'colors/safe'; 3 | import { PASSWORD_FIELD } from '../../constants'; 4 | import { SCRAPERS } from '../../helpers/scrapers'; 5 | import { encryptCredentials } from '../../helpers/credentials'; 6 | import { TasksManager, printTaskSummary } from '../../helpers/tasks'; 7 | 8 | const tasksManager = new TasksManager(); 9 | 10 | const goBackOption = { 11 | name: 'Go Back', 12 | value: '', 13 | }; 14 | 15 | function getListOfScrapers(existingTaskScrapers) { 16 | return Object.keys(SCRAPERS).map((scraperId) => { 17 | const result = { value: scraperId, name: SCRAPERS[scraperId].name }; 18 | const hasCredentials = existingTaskScrapers 19 | .find(scraper => scraper.id === scraperId); 20 | result.name = `${hasCredentials ? 'Edit' : 'Add'} ${result.name}`; 21 | 22 | return result; 23 | }); 24 | } 25 | 26 | function validateNonEmpty(field, input) { 27 | if (input) { 28 | return true; 29 | } 30 | return `${field} must be non empty`; 31 | } 32 | 33 | const ModifyTaskHandler = (function createModifyTaskHandler() { 34 | const _private = new WeakMap(); 35 | 36 | class ModifyTaskHandler { 37 | static async createAdapter() { 38 | const answers = await inquirer.prompt([ 39 | { 40 | type: 'list', 41 | name: 'taskName', 42 | message: 'Select a task to modify', 43 | choices: [...await tasksManager.getTasksList(), goBackOption], 44 | }, 45 | ]); 46 | 47 | if (answers.taskName) { 48 | return new ModifyTaskHandler(answers.taskName); 49 | } 50 | 51 | return null; 52 | } 53 | 54 | constructor(taskName) { 55 | _private.set(this, { taskName }); 56 | } 57 | 58 | /* 59 | * @private 60 | */ 61 | async manageOptions() { 62 | const { 63 | combineInstallments, 64 | dateDiffByMonth, 65 | } = _private.get(this).taskData.options; 66 | 67 | const { 68 | saveLocation, 69 | combineReport, 70 | includeFutureTransactions, 71 | includePendingTransactions, 72 | } = _private.get(this).taskData.output; 73 | 74 | const answers = await inquirer.prompt([ 75 | { 76 | type: 'confirm', 77 | name: 'combineInstallments', 78 | message: 'Combine installment transactions?', 79 | default: combineInstallments, 80 | }, 81 | { 82 | type: 'input', 83 | name: 'dateDiffByMonth', 84 | message: 'How many months do you want to scrape (1-12)?', 85 | default: dateDiffByMonth, 86 | filter: (value) => { 87 | if (Number.isFinite(value) && !Number.isNaN(value)) { 88 | return value; 89 | } 90 | 91 | if (typeof value === 'string' && value.match(/^[0-9]+$/)) { 92 | return value * 1; 93 | } 94 | 95 | return null; 96 | }, 97 | validate: (value) => { 98 | const pass = value !== null && value >= 1 && value <= 12; 99 | 100 | if (pass) { 101 | return true; 102 | } 103 | 104 | return 'Please enter a value between 1 and 12'; 105 | }, 106 | }, 107 | { 108 | type: 'input', 109 | name: 'saveLocation', 110 | message: 'Save folder?', 111 | default: saveLocation, 112 | }, 113 | { 114 | type: 'confirm', 115 | name: 'combineReport', 116 | message: 'Combine all accounts into a single report?', 117 | default: !!combineReport, 118 | }, 119 | { 120 | type: 'confirm', 121 | name: 'includeFutureTransactions', 122 | message: 'Include future transactions?', 123 | default: !!includeFutureTransactions, 124 | }, 125 | { 126 | type: 'confirm', 127 | name: 'includePendingTransactions', 128 | message: 'Include pending transactions?', 129 | default: !!includePendingTransactions, 130 | }, 131 | ]); 132 | 133 | const { taskData } = _private.get(this); 134 | taskData.options.combineInstallments = answers.combineInstallments; 135 | taskData.options.dateDiffByMonth = answers.dateDiffByMonth; 136 | taskData.output.saveLocation = answers.saveLocation; 137 | taskData.output.combineReport = answers.combineReport; 138 | taskData.output.includeFutureTransactions = answers.includeFutureTransactions; 139 | taskData.output.includePendingTransactions = answers.includePendingTransactions; 140 | console.log(colors.notify('Changes saved')); 141 | await this.saveTask(); 142 | } 143 | 144 | /* 145 | * @private 146 | */ 147 | async manageScrapers() { 148 | const MODIFY_ACTION = 'modify'; 149 | const DELETE_ACTION = 'delete'; 150 | 151 | const answers = await inquirer.prompt([ 152 | { 153 | type: 'list', 154 | name: 'action', 155 | message: 'What do you want to do?', 156 | choices: [ 157 | { 158 | name: 'Add / Edit a scraper', 159 | value: MODIFY_ACTION, 160 | }, 161 | { 162 | name: 'Delete a scraper', 163 | value: DELETE_ACTION, 164 | }, 165 | goBackOption, 166 | ], 167 | }, 168 | { 169 | type: 'list', 170 | name: 'scraperId', 171 | message: 'Select a scraper', 172 | when: (answers) => { 173 | if (answers.action === DELETE_ACTION) { 174 | const hasScrapers = _private.get(this).taskData.scrapers.length !== 0; 175 | 176 | if (!hasScrapers) { 177 | console.log(colors.notify('task has no scrapers defined')); 178 | } 179 | 180 | return hasScrapers; 181 | } 182 | 183 | const isRelevantQuestion = answers.action === MODIFY_ACTION; 184 | return isRelevantQuestion; 185 | }, 186 | choices: (answers) => { 187 | if (answers.action === MODIFY_ACTION) { 188 | return [...getListOfScrapers(_private.get(this).taskData.scrapers), goBackOption]; 189 | } 190 | 191 | const taskScrapers = _private.get(this).taskData.scrapers.map(scraper => ( 192 | { 193 | value: scraper.id, 194 | name: SCRAPERS[scraper.id].name, 195 | })); 196 | 197 | return [...taskScrapers, goBackOption]; 198 | }, 199 | }, 200 | { 201 | type: 'confirm', 202 | name: 'confirmDelete', 203 | when: answers => answers.action === DELETE_ACTION && answers.scraperId, 204 | message: 'Are you sure?', 205 | default: false, 206 | }, 207 | ]); 208 | 209 | const { scraperId, action, confirmDelete } = answers; 210 | 211 | if (scraperId) { 212 | if (action === DELETE_ACTION) { 213 | if (confirmDelete) { 214 | console.log(colors.notify(`Scraper ${scraperId} deleted`)); 215 | _private.get(this).taskData.scrapers = _private.get(this) 216 | .taskData.scrapers.filter(item => item.id !== scraperId); 217 | await this.saveTask(); 218 | } else { 219 | console.log(colors.notify('Delete scraper cancelled')); 220 | } 221 | } else if (action === MODIFY_ACTION) { 222 | const { loginFields } = SCRAPERS[scraperId]; 223 | const questions = loginFields.map((field) => { 224 | return { 225 | type: field === PASSWORD_FIELD ? PASSWORD_FIELD : 'input', 226 | name: field, 227 | message: `Enter value for ${field}:`, 228 | validate: input => validateNonEmpty(field, input), 229 | }; 230 | }); 231 | const credentialsResult = await inquirer.prompt(questions); 232 | const encryptedCredentials = encryptCredentials(credentialsResult); 233 | 234 | const scraperData = _private.get(this).taskData.scrapers 235 | .find(scraper => scraper.id === scraperId); 236 | if (!scraperData) { 237 | _private.get(this).taskData.scrapers 238 | .push({ id: scraperId, credentials: encryptedCredentials }); 239 | console.log(colors.notify(`'${scraperId}' scrapper added`)); 240 | } else { 241 | scraperData.credentials = encryptedCredentials; 242 | console.log(colors.notify(`'${scraperId}' scrapper updated`)); 243 | } 244 | await this.saveTask(); 245 | } 246 | } 247 | } 248 | 249 | /* 250 | * @private 251 | */ 252 | async saveTask() { 253 | await tasksManager.saveTask(_private.get(this).taskName, _private.get(this).taskData); 254 | } 255 | 256 | async run() { 257 | const VIEW_SUMMARY_ACTION = 'summary'; 258 | const UPDATE_SCRAPERS_LIST_ACTION = 'scrapers-list'; 259 | const UPDATE_OPTIONS_ACTION = 'scraping-options'; 260 | 261 | let firstTimeEntering = false; 262 | 263 | if (!_private.get(this).taskName) { 264 | throw new Error('missing task name'); 265 | } 266 | 267 | if (!_private.get(this).taskData) { 268 | firstTimeEntering = true; 269 | _private.get(this).taskData = await tasksManager.loadTask(_private.get(this).taskName); 270 | } 271 | 272 | if (_private.get(this).taskData) { 273 | if (firstTimeEntering) { 274 | console.log(colors.title(`Editing task '${_private.get(this).taskName}'`)); 275 | } 276 | 277 | const answers = await inquirer.prompt([ 278 | { 279 | type: 'list', 280 | name: 'action', 281 | message: 'What do you want to do?', 282 | choices: [ 283 | { 284 | name: 'View task summary', 285 | value: VIEW_SUMMARY_ACTION, 286 | }, 287 | { 288 | name: 'Update scrapers list', 289 | value: UPDATE_SCRAPERS_LIST_ACTION, 290 | }, 291 | { 292 | name: 'Update scraping options', 293 | value: UPDATE_OPTIONS_ACTION, 294 | }, 295 | goBackOption, 296 | ], 297 | }, 298 | ]); 299 | 300 | switch (answers.action) { 301 | case VIEW_SUMMARY_ACTION: 302 | console.log(''); // print empty line 303 | printTaskSummary(_private.get(this).taskData, false); 304 | console.log(''); // print empty line 305 | await this.run(); 306 | break; 307 | case UPDATE_SCRAPERS_LIST_ACTION: 308 | await this.manageScrapers(); 309 | await this.run(); 310 | break; 311 | case UPDATE_OPTIONS_ACTION: 312 | await this.manageOptions(); 313 | await this.run(); 314 | break; 315 | default: 316 | break; 317 | } 318 | } 319 | } 320 | } 321 | 322 | return ModifyTaskHandler; 323 | }()); 324 | 325 | export default ModifyTaskHandler; 326 | --------------------------------------------------------------------------------