├── .nvmrc ├── .node-version ├── .npmrc ├── src ├── helpers │ ├── scrapers.js │ ├── credentials.js │ ├── settings.js │ ├── crypto.js │ ├── files.js │ └── tasks.js ├── constants.js ├── definitions.js ├── index.js ├── setup │ ├── setup-main-menu.js │ ├── setup-scrapers.js │ └── tasks │ │ ├── delete-task-handler.js │ │ ├── setup-task.js │ │ ├── create-task-handler.js │ │ └── modify-task-handler.js └── scrape │ ├── scraping-main-menu.js │ ├── scrape-base.js │ ├── generate-reports.js │ ├── scrape-individual.js │ └── scrape-task.js ├── .babelrc ├── .eslintrc ├── LICENSE ├── .gitignore ├── package.json └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | 14.4.0 2 | -------------------------------------------------------------------------------- /.node-version: -------------------------------------------------------------------------------- 1 | 10.22.0 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry = https://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /src/helpers/scrapers.js: -------------------------------------------------------------------------------- 1 | export { SCRAPERS, createScraper } from 'israeli-bank-scrapers'; 2 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | const PASSWORD_FIELD = 'password'; 2 | const DATE_TIME_FORMAT = 'DD-MM-YYYY_HH-mm-ss'; 3 | const TRANSACTION_STATUS = { 4 | PENDING: 'pending', 5 | }; 6 | 7 | export { PASSWORD_FIELD, DATE_TIME_FORMAT, TRANSACTION_STATUS }; 8 | -------------------------------------------------------------------------------- /src/definitions.js: -------------------------------------------------------------------------------- 1 | import os from 'os'; 2 | 3 | export const CONFIG_FOLDER = `${os.homedir()}/.ynab-updater`; 4 | export const SETTINGS_FILE = `${CONFIG_FOLDER}/settings.json`; 5 | export const TASKS_FOLDER = `${CONFIG_FOLDER}/tasks`; 6 | export const DOWNLOAD_FOLDER = `${os.homedir()}/Downloads/Transactions`; 7 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", { 5 | "targets": { 6 | "node": "8" 7 | }, 8 | "include": [ 9 | "transform-es2015-arrow-functions", 10 | "transform-es2015-shorthand-properties", 11 | "transform-es2015-block-scoping" 12 | ], 13 | "useBuiltIns": true 14 | } 15 | ] 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "arrow-body-style": 0, 4 | "class-methods-use-this": 0, 5 | "comma-dangle": 0, 6 | "implicit-arrow-linebreak": 0, 7 | "no-shadow": 0, 8 | "no-console": 0, 9 | "no-underscore-dangle": ["error", { "allow": ["_private"] }], 10 | "no-await-in-loop": 0, 11 | "operator-linebreak": 0, 12 | "wrap-iife": 0 13 | }, 14 | "globals": { 15 | "document": true, 16 | "fetch": true, 17 | "Headers": true 18 | }, 19 | "extends": "airbnb-base" 20 | } 21 | -------------------------------------------------------------------------------- /src/helpers/credentials.js: -------------------------------------------------------------------------------- 1 | import { encrypt, decrypt } from './crypto'; 2 | 3 | export function encryptCredentials(credentials) { 4 | const encrypted = {}; 5 | Object.keys(credentials).forEach((field) => { 6 | encrypted[field] = encrypt(credentials[field]); 7 | }); 8 | return encrypted; 9 | } 10 | 11 | export function decryptCredentials(credentials) { 12 | const decrypted = {}; 13 | Object.keys(credentials).forEach((field) => { 14 | decrypted[field] = decrypt(credentials[field]); 15 | }); 16 | return decrypted; 17 | } 18 | -------------------------------------------------------------------------------- /src/helpers/settings.js: -------------------------------------------------------------------------------- 1 | import { SETTINGS_FILE, DOWNLOAD_FOLDER } from '../definitions'; 2 | import { writeJsonFile, readJsonFile } from './files'; 3 | 4 | export async function readSettingsFile() { 5 | let settings = await readJsonFile(SETTINGS_FILE); 6 | if (!settings) { 7 | settings = { 8 | saveLocation: DOWNLOAD_FOLDER, 9 | }; 10 | } 11 | 12 | return settings; 13 | } 14 | 15 | export async function writeSettingsFile(settings) { 16 | if (settings) { 17 | await writeJsonFile(SETTINGS_FILE, settings); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/helpers/crypto.js: -------------------------------------------------------------------------------- 1 | import crypto from 'crypto'; 2 | 3 | const ALGORITHM = 'aes-256-ctr'; 4 | const SALT = '8cs+8Y(nxDLY'; 5 | 6 | export function encrypt(text) { 7 | const cipher = crypto.createCipher(ALGORITHM, SALT); 8 | const crypted = cipher.update(text, 'utf8', 'hex'); 9 | return crypted + cipher.final('hex'); 10 | } 11 | 12 | export function decrypt(text) { 13 | const decipher = crypto.createDecipher(ALGORITHM, SALT); 14 | const decrypted = decipher.update(text, 'hex', 'utf8'); 15 | return decrypted + decipher.final('utf8'); 16 | } 17 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import yargs from 'yargs'; 2 | import colors from 'colors/safe'; 3 | import setupMainMenu from './setup/setup-main-menu'; 4 | import scrapingMainMenu from './scrape/scraping-main-menu'; 5 | 6 | // set theme 7 | colors.setTheme({ 8 | title: 'bgCyan', 9 | notify: 'magenta', 10 | }); 11 | 12 | const args = yargs.options({ 13 | mode: { 14 | alias: 'm', 15 | describe: 'mode for running', 16 | }, 17 | show: { 18 | alias: 's', 19 | describe: 'show browser while scraping', 20 | type: 'boolean', 21 | default: false, 22 | }, 23 | }).help().argv; 24 | 25 | if (!args.mode || args.mode === 'scrape') { 26 | scrapingMainMenu(args.show); 27 | } else if (args.mode === 'setup') { 28 | setupMainMenu(); 29 | } 30 | -------------------------------------------------------------------------------- /src/setup/setup-main-menu.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | 3 | import setupTask from './tasks/setup-task'; 4 | import setupScrapers from './setup-scrapers'; 5 | 6 | export default async function setupMainMenu() { 7 | const SETUP_SCRAPER_ACTION = 'scraper'; 8 | const SETUP_TASK_ACTION = 'task'; 9 | const { setupType } = await inquirer.prompt({ 10 | type: 'list', 11 | name: 'setupType', 12 | message: 'What would you like to setup?', 13 | choices: [ 14 | { 15 | name: 'Setup a new scraper', 16 | value: SETUP_SCRAPER_ACTION, 17 | }, 18 | { 19 | name: 'Setup a new task', 20 | value: SETUP_TASK_ACTION, 21 | }, 22 | ], 23 | }); 24 | 25 | switch (setupType) { 26 | case SETUP_SCRAPER_ACTION: 27 | await setupScrapers(); 28 | break; 29 | case SETUP_TASK_ACTION: 30 | await setupTask(); 31 | break; 32 | default: 33 | break; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/scrape/scraping-main-menu.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import scrapeIndividual from './scrape-individual'; 3 | import scrapeTask from './scrape-task'; 4 | 5 | export default async function scrapingMainMenu(showBrowser) { 6 | const RUN_SCRAPER_ACTION = 'scraper'; 7 | const RUN_TASK_ACTION = 'task'; 8 | 9 | const { scrapeType } = await inquirer.prompt({ 10 | type: 'list', 11 | name: 'scrapeType', 12 | message: 'What would you like to do?', 13 | choices: [ 14 | { 15 | name: 'Run an individual scraper', 16 | value: RUN_SCRAPER_ACTION, 17 | }, 18 | { 19 | name: 'Run a task', 20 | value: RUN_TASK_ACTION, 21 | }, 22 | ], 23 | }); 24 | 25 | switch (scrapeType) { 26 | case RUN_SCRAPER_ACTION: 27 | await scrapeIndividual(showBrowser); 28 | break; 29 | case RUN_TASK_ACTION: 30 | await scrapeTask(showBrowser); 31 | break; 32 | default: 33 | break; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Elad Shaham 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # NPM package related 61 | lib/ 62 | 63 | # OS related 64 | .DS_Store 65 | 66 | # IDEs 67 | .idea -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "israeli-ynab-updater", 3 | "version": "0.0.1", 4 | "description": "A tool for updating YNAB using israeli-bank-scrapers", 5 | "main": "lib/index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "lint": "eslint src", 9 | "start": "babel-node --inspect src", 10 | "start:debug": "npm start -- -s true", 11 | "setup": "npm start -- -m setup" 12 | }, 13 | "pre-commit": [ 14 | "lint" 15 | ], 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/eshaham/israeli-ynab-updater.git" 19 | }, 20 | "author": "Elad Shaham", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/eshaham/israeli-ynab-updater/issues" 24 | }, 25 | "homepage": "https://github.com/eshaham/israeli-ynab-updater#readme", 26 | "devDependencies": { 27 | "babel-cli": "^6.26.0", 28 | "babel-plugin-transform-es2015-arrow-functions": "^6.22.0", 29 | "babel-plugin-transform-es2015-block-scoping": "^6.26.0", 30 | "babel-plugin-transform-es2015-shorthand-properties": "^6.24.1", 31 | "babel-preset-env": "^1.7.0", 32 | "eslint": "^7.14.0", 33 | "eslint-config-airbnb-base": "^14.2.1", 34 | "eslint-plugin-import": "^2.22.1", 35 | "pre-commit": "^1.2.2" 36 | }, 37 | "dependencies": { 38 | "babel-polyfill": "^6.26.0", 39 | "colors": "^1.4.0", 40 | "inquirer": "^7.3.3", 41 | "israeli-bank-scrapers": "^3.6.0", 42 | "json2csv": "^5.0.5", 43 | "jsonfile": "^6.1.0", 44 | "moment": "^2.29.1", 45 | "yargs": "^16.1.1" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/setup/setup-scrapers.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | 3 | import { PASSWORD_FIELD } from '../constants'; 4 | import { CONFIG_FOLDER } from '../definitions'; 5 | import { SCRAPERS } from '../helpers/scrapers'; 6 | import { writeJsonFile } from '../helpers/files'; 7 | import { encryptCredentials } from '../helpers/credentials'; 8 | 9 | function validateNonEmpty(field, input) { 10 | if (input) { 11 | return true; 12 | } 13 | return `${field} must be non empty`; 14 | } 15 | 16 | export default async function setupScrapers() { 17 | const scraperIdResult = await inquirer.prompt([ 18 | { 19 | type: 'list', 20 | name: 'scraperId', 21 | message: 'Which scraper would you like to save credentials for?', 22 | choices: Object.keys(SCRAPERS).map((id) => { 23 | return { 24 | name: SCRAPERS[id].name, 25 | value: id, 26 | }; 27 | }), 28 | }, 29 | ]); 30 | const { loginFields } = SCRAPERS[scraperIdResult.scraperId]; 31 | const questions = loginFields 32 | .filter((field) => !field.startsWith('otp')) 33 | .map((field) => { 34 | return { 35 | type: field === PASSWORD_FIELD ? PASSWORD_FIELD : 'input', 36 | name: field, 37 | message: `Enter value for ${field}:`, 38 | validate: (input) => validateNonEmpty(field, input), 39 | }; 40 | }); 41 | const credentialsResult = await inquirer.prompt(questions); 42 | const encryptedCredentials = encryptCredentials(credentialsResult); 43 | await writeJsonFile( 44 | `${CONFIG_FOLDER}/${scraperIdResult.scraperId}.json`, 45 | encryptedCredentials 46 | ); 47 | console.log(`credentials file saved for ${scraperIdResult.scraperId}`); 48 | } 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Israeli YNAB Updater 2 | A tool for updating YNAB using for the Israeli market. This tool uses [Israeli Banks Scrapers](https://github.com/eshaham/israeli-bank-scrapers) project as the source of fetching account data. 3 | 4 | ## Getting started 5 | 6 | ### Prerequisites 7 | 8 | In order to start using this tool, you will need to have Node.js (>= 10) installed on your machine. 9 | Go [here!](https://nodejs.org/en/download/) to download and install the latest Node.js for your operating system. 10 | 11 | ### Installation 12 | Once Node.js is installed, run the following command to fetch the code: 13 | 14 | ```bash 15 | git clone https://github.com/eshaham/israeli-ynab-updater 16 | cd israeli-ynab-updater 17 | ``` 18 | 19 | If you're using `nvm` make sure to run `nvm use` inside project folder for best compatability. 20 | If you're using `nodenv`, it should automatically pick up the correct node version. 21 | 22 | Next you will need to install dependencies by running 23 | ```bash 24 | npm install 25 | ``` 26 | 27 | ### Saving credentials 28 | The credentials for each scraper are encrypted and saved in an dedicated file on your system. 29 | To save credentials for a specific scraper, run the following command and choose the scraper: 30 | 31 | ```bash 32 | npm run setup 33 | ``` 34 | 35 | When asked 'What would you like to setup?' choose 'Scrapers'. 36 | 37 | ### Scraping 38 | Once you save the credentials for relevant scrapers, run the following command to start scraping: 39 | 40 | ```bash 41 | npm start 42 | ``` 43 | 44 | The CSV file should be created under `Transactions` folder in your Downloads folder, unless you choose a different folder. 45 | 46 | You can also scrape in debug mode by running: 47 | 48 | ```bash 49 | npm run start:debug 50 | ``` 51 | -------------------------------------------------------------------------------- /src/setup/tasks/delete-task-handler.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import colors from 'colors/safe'; 3 | import { TasksManager } from '../../helpers/tasks'; 4 | 5 | const tasksManager = new TasksManager(); 6 | const goBackOption = { 7 | name: 'Go Back', 8 | value: '', 9 | }; 10 | 11 | const DeleteTaskHandler = (function createDeleteTaskHandler() { 12 | const _private = new WeakMap(); 13 | 14 | class DeleteTaskHandler { 15 | static async createAdapter() { 16 | const answers = await inquirer.prompt([ 17 | { 18 | type: 'list', 19 | name: 'taskName', 20 | message: 'Select a task to delete', 21 | choices: [ 22 | ...await tasksManager.getTasksList(), 23 | goBackOption, 24 | ], 25 | }, 26 | { 27 | type: 'confirm', 28 | name: 'confirmDelete', 29 | message: 'Are you sure?', 30 | when: (answers) => !!answers.taskName, 31 | default: false, 32 | }, 33 | ]); 34 | 35 | if (answers.taskName && answers.confirmDelete) { 36 | return new DeleteTaskHandler(answers.taskName); 37 | } 38 | console.log(colors.notify('Delete task cancelled')); 39 | 40 | return null; 41 | } 42 | 43 | constructor(taskName) { 44 | _private.set(this, { taskName }); 45 | } 46 | 47 | async run() { 48 | if (!_private.get(this).taskName) { 49 | throw new Error('Missing task name'); 50 | } 51 | 52 | console.log(colors.notify(`Task '${_private.get(this).taskName}' deleted`)); 53 | await tasksManager.deleteTask(_private.get(this).taskName); 54 | } 55 | } 56 | 57 | return DeleteTaskHandler; 58 | }()); 59 | 60 | export default DeleteTaskHandler; 61 | -------------------------------------------------------------------------------- /src/helpers/files.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import path from 'path'; 3 | import util from 'util'; 4 | import jsonfile from 'jsonfile'; 5 | 6 | const writeFileAsync = util.promisify(fs.writeFile); 7 | const existsAsync = util.promisify(fs.exists); 8 | const makeDirAsync = util.promisify(fs.mkdir); 9 | const readdirAsync = util.promisify(fs.readdir); 10 | const deleteFileAsync = util.promisify(fs.unlink); 11 | const readJsonFileAsync = util.promisify(jsonfile.readFile); 12 | const writeJsonFileAsync = util.promisify(jsonfile.writeFile); 13 | 14 | async function verifyFolder(folderPath) { 15 | const pathTokens = folderPath.split(path.sep); 16 | let currentPath = ''; 17 | for (let i = 0; i < pathTokens.length; i += 1) { 18 | const folder = pathTokens[i]; 19 | currentPath += folder + path.sep; 20 | if (!await existsAsync(currentPath)) { 21 | await makeDirAsync(currentPath); 22 | } 23 | } 24 | } 25 | 26 | export async function getFolderFiles(folderPath, suffix) { 27 | await verifyFolder(folderPath); 28 | const files = await readdirAsync(folderPath); 29 | if (suffix) { 30 | return files.filter((filePath) => (path.extname(filePath) || '').toLowerCase() === suffix.toLowerCase()); 31 | } 32 | return files; 33 | } 34 | 35 | export async function deleteFile(filePath) { 36 | return deleteFileAsync(filePath); 37 | } 38 | 39 | export async function writeFile(filePath, data, options) { 40 | const folderPath = path.dirname(filePath); 41 | await verifyFolder(folderPath); 42 | return writeFileAsync(filePath, data, options); 43 | } 44 | 45 | export async function readJsonFile(filePath, options) { 46 | const exists = await existsAsync(filePath); 47 | if (!exists) { 48 | return null; 49 | } 50 | return readJsonFileAsync(filePath, options); 51 | } 52 | 53 | export async function writeJsonFile(filePath, obj, options) { 54 | const folderPath = path.dirname(filePath); 55 | await verifyFolder(folderPath); 56 | await writeJsonFileAsync(filePath, obj, options); 57 | } 58 | -------------------------------------------------------------------------------- /src/setup/tasks/setup-task.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import ModifyTaskHandler from './modify-task-handler'; 3 | import CreateTaskHandler from './create-task-handler'; 4 | import DeleteTaskHandler from './delete-task-handler'; 5 | 6 | async function selectAction() { 7 | const CREATE_NEW_TASK_ACTION = 'new'; 8 | const MODIFY_TASK_ACTION = 'modify'; 9 | const DELETE_TASK_ACTION = 'delete'; 10 | const QUIT_ACTION = 'quit'; 11 | 12 | const answers = await inquirer.prompt([ 13 | { 14 | type: 'list', 15 | name: 'action', 16 | message: 'What do you want to do?', 17 | choices: [ 18 | { 19 | name: 'Create a new task', 20 | value: CREATE_NEW_TASK_ACTION, 21 | }, 22 | { 23 | name: 'Modify an existing task', 24 | value: MODIFY_TASK_ACTION, 25 | }, 26 | new inquirer.Separator(), 27 | { 28 | name: 'Delete a task', 29 | value: DELETE_TASK_ACTION, 30 | }, 31 | { 32 | name: 'Quit', 33 | value: QUIT_ACTION, 34 | }, 35 | ], 36 | }, 37 | ]); 38 | 39 | switch (answers.action) { 40 | case CREATE_NEW_TASK_ACTION: 41 | { 42 | const createNewTaskAdapter = new CreateTaskHandler(); 43 | await createNewTaskAdapter.run(); 44 | await selectAction(); 45 | } 46 | break; 47 | case MODIFY_TASK_ACTION: 48 | { 49 | const adapter = await ModifyTaskHandler.createAdapter(); 50 | 51 | if (adapter) { 52 | await adapter.run(); 53 | } 54 | 55 | await selectAction(); 56 | } 57 | break; 58 | case DELETE_TASK_ACTION: 59 | { 60 | const adapter = await DeleteTaskHandler.createAdapter(); 61 | 62 | if (adapter) { 63 | await adapter.run(); 64 | } 65 | 66 | await selectAction(); 67 | } 68 | break; 69 | default: 70 | break; 71 | } 72 | } 73 | 74 | export default selectAction; 75 | -------------------------------------------------------------------------------- /src/setup/tasks/create-task-handler.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import colors from 'colors/safe'; 3 | import { DOWNLOAD_FOLDER } from '../../definitions'; 4 | import ModifyTaskHandler from './modify-task-handler'; 5 | import { TasksManager } from '../../helpers/tasks'; 6 | 7 | const tasksManager = new TasksManager(); 8 | 9 | function createEmptyTaskData() { 10 | return { 11 | scrapers: [], 12 | options: { 13 | combineInstallments: false, 14 | dateDiffByMonth: 3, 15 | }, 16 | output: { 17 | saveLocation: DOWNLOAD_FOLDER, 18 | combineReport: true, 19 | includeFutureTransactions: false, 20 | includePendingTransactions: false, 21 | }, 22 | }; 23 | } 24 | 25 | export default class { 26 | async run() { 27 | const answers = await inquirer.prompt([ 28 | { 29 | type: 'input', 30 | name: 'name', 31 | message: 'What is the task name (leave blank to cancel)', 32 | validate: async (value) => { 33 | if (value) { 34 | const alreadyExists = await tasksManager.hasTask(value); 35 | 36 | if (alreadyExists) { 37 | return 'A task with this name already exists, please type a unique name'; 38 | } 39 | 40 | const invalidPattern = !tasksManager.isValidTaskName(value); 41 | 42 | if (invalidPattern) { 43 | return 'The task name must include only these characters: A-Z, 0-9, -, _'; 44 | } 45 | } 46 | 47 | return true; 48 | }, 49 | }, 50 | ]); 51 | 52 | const taskName = answers.name; 53 | 54 | if (taskName) { 55 | const taskData = createEmptyTaskData(); 56 | await tasksManager.saveTask(taskName, taskData); 57 | const modifyTaskAdapter = new ModifyTaskHandler(taskName); 58 | console.log(colors.notify(`Task '${taskName}' created`)); 59 | await modifyTaskAdapter.run(); 60 | } else { 61 | console.log(colors.notify('Task creation cancelled')); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/scrape/scrape-base.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import { SCRAPERS, createScraper } from '../helpers/scrapers'; 3 | 4 | async function prepareResults( 5 | scraperId, 6 | scraperName, 7 | scraperResult, 8 | combineInstallments 9 | ) { 10 | return scraperResult.accounts.map((account) => { 11 | console.log( 12 | `${scraperName}: scraped ${account.txns.length} transactions from account ${account.accountNumber}`, 13 | ); 14 | 15 | const txns = account.txns.map((txn) => { 16 | return { 17 | company: scraperName, 18 | account: account.accountNumber, 19 | dateMoment: moment(txn.date), 20 | payee: txn.description, 21 | status: txn.status, 22 | amount: 23 | txn.type !== 'installments' || !combineInstallments 24 | ? txn.chargedAmount 25 | : txn.originalAmount, 26 | installment: txn.installments ? txn.installments.number : null, 27 | total: txn.installments ? txn.installments.total : null, 28 | }; 29 | }); 30 | 31 | return { 32 | scraperId, 33 | scraperName, 34 | accountNumber: account.accountNumber, 35 | txns, 36 | }; 37 | }); 38 | } 39 | 40 | export default async function scrape(scraperId, credentials, options) { 41 | const { combineInstallments, startDate, showBrowser } = options; 42 | 43 | const scraperOptions = { 44 | companyId: scraperId, 45 | startDate, 46 | combineInstallments, 47 | showBrowser, 48 | verbose: false, 49 | }; 50 | const scraperName = SCRAPERS[scraperId] ? SCRAPERS[scraperId].name : null; 51 | 52 | if (!scraperName) { 53 | throw new Error(`unknown scraper with id ${scraperId}`); 54 | } 55 | console.log(`scraping ${scraperName}`); 56 | 57 | const scraper = createScraper(scraperOptions); 58 | scraper.onProgress((companyId, payload) => { 59 | console.log(`${scraperName}: ${payload.type}`); 60 | }); 61 | const scraperResult = await scraper.scrape(credentials); 62 | 63 | console.log(`success: ${scraperResult.success}`); 64 | if (!scraperResult.success) { 65 | console.log(`error type: ${scraperResult.errorType}`); 66 | console.log('error:', scraperResult.errorMessage); 67 | throw new Error(scraperResult.errorMessage); 68 | } 69 | 70 | return prepareResults( 71 | scraperId, 72 | scraperName, 73 | scraperResult, 74 | combineInstallments 75 | ); 76 | } 77 | -------------------------------------------------------------------------------- /src/helpers/tasks.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import colors from 'colors/safe'; 3 | import moment from 'moment'; 4 | import { 5 | writeJsonFile, 6 | readJsonFile, 7 | getFolderFiles, 8 | deleteFile, 9 | } from './files'; 10 | import { TASKS_FOLDER } from '../definitions'; 11 | import { SCRAPERS } from './scrapers'; 12 | 13 | function writeSummaryLine(key, value) { 14 | console.log(`- ${colors.bold(key)}: ${value}`); 15 | } 16 | 17 | function printTaskSummary(taskData, shouldCalculateStartDate = false) { 18 | if (taskData) { 19 | const scrapers = taskData.scrapers || []; 20 | const { 21 | dateDiffByMonth, 22 | combineInstallments, 23 | } = taskData.options; 24 | const { 25 | combineReport, 26 | saveLocation, 27 | includeFutureTransactions, 28 | includePendingTransactions, 29 | } = taskData.output; 30 | console.log(colors.underline.bold('Task Summary')); 31 | writeSummaryLine('Scrapers', scrapers.map((scraper) => SCRAPERS[scraper.id].name).join(', ')); 32 | 33 | if (shouldCalculateStartDate) { 34 | const substractValue = dateDiffByMonth - 1; 35 | const startMoment = moment().subtract(substractValue, 'month').startOf('month'); 36 | writeSummaryLine('Start scraping from', startMoment.format('ll')); 37 | } else { 38 | writeSummaryLine('Scrape # of months', dateDiffByMonth); 39 | } 40 | 41 | writeSummaryLine('Combine installments', combineInstallments ? 'Yes' : 'No'); 42 | writeSummaryLine('Save to location', saveLocation); 43 | writeSummaryLine('Create single report', combineReport ? 'Yes' : 'No'); 44 | writeSummaryLine('Include future Transactions', includeFutureTransactions ? 'Yes' : 'No'); 45 | writeSummaryLine('Include pending Transactions', includePendingTransactions ? 'Yes' : 'No'); 46 | } 47 | } 48 | 49 | class TasksManager { 50 | async getTasksList() { 51 | const files = await getFolderFiles(TASKS_FOLDER, '.json'); 52 | const result = files.map((file) => path.basename(file, '.json')); 53 | return result; 54 | } 55 | 56 | async hasTask(taskName) { 57 | const tasksList = await this.getTasksList(); 58 | return taskName && !!tasksList.find((taskListName) => { 59 | return taskListName.toLowerCase() === taskName.toLowerCase(); 60 | }); 61 | } 62 | 63 | isValidTaskName(taskName) { 64 | return taskName && taskName.match(/^[a-zA-Z0-9_-]+$/); 65 | } 66 | 67 | async loadTask(taskName) { 68 | if (taskName && this.hasTask(taskName)) { 69 | return readJsonFile(`${TASKS_FOLDER}/${taskName}.json`); 70 | } 71 | 72 | throw new Error(`failed to find a task named ${taskName}`); 73 | } 74 | 75 | async saveTask(taskName, taskData) { 76 | if (taskName && this.isValidTaskName(taskName)) { 77 | return writeJsonFile(`${TASKS_FOLDER}/${taskName}.json`, taskData); 78 | } 79 | 80 | throw new Error(`invalid task name provided ${taskName}`); 81 | } 82 | 83 | async deleteTask(taskName) { 84 | if (taskName && this.hasTask(taskName)) { 85 | await deleteFile(`${TASKS_FOLDER}/${taskName}.json`); 86 | } else { 87 | throw new Error(`invalid task name provided ${taskName}`); 88 | } 89 | } 90 | } 91 | 92 | export { TasksManager, printTaskSummary }; 93 | -------------------------------------------------------------------------------- /src/scrape/generate-reports.js: -------------------------------------------------------------------------------- 1 | import { parse as objToCsv } from 'json2csv'; 2 | import colors from 'colors/safe'; 3 | import moment from 'moment'; 4 | import { DATE_TIME_FORMAT, TRANSACTION_STATUS } from '../constants'; 5 | import { writeFile } from '../helpers/files'; 6 | 7 | function getReportFields(isSingleReport) { 8 | const result = [ 9 | { 10 | label: 'Date', 11 | value: (row) => row.dateMoment.format('DD/MM/YYYY'), 12 | }, 13 | { 14 | label: 'Payee', 15 | value: 'payee', 16 | }, 17 | { 18 | label: 'Inflow', 19 | value: 'amount', 20 | }, 21 | { 22 | label: 'Status', 23 | value: 'status', 24 | }, 25 | { 26 | label: 'Installment', 27 | value: 'installment', 28 | }, 29 | { 30 | label: 'Total', 31 | value: 'total', 32 | }, 33 | ]; 34 | 35 | if (isSingleReport) { 36 | result.unshift( 37 | { 38 | label: 'Company', 39 | value: 'company', 40 | }, 41 | { 42 | label: 'Account', 43 | value: 'account', 44 | }, 45 | ); 46 | } 47 | 48 | return result; 49 | } 50 | 51 | function filterTransactions(transactions, includeFutureTransactions, includePendingTransactions) { 52 | let result = transactions; 53 | 54 | if (!includeFutureTransactions) { 55 | const nowMoment = moment(); 56 | result = result.filter((txn) => { 57 | const txnMoment = moment(txn.dateMoment); 58 | return txnMoment.isSameOrBefore(nowMoment, 'day'); 59 | }); 60 | } 61 | 62 | if (!includePendingTransactions) { 63 | result = result.filter((txn) => txn.status !== TRANSACTION_STATUS.PENDING); 64 | } 65 | 66 | return result; 67 | } 68 | 69 | async function exportAccountData(txns, scraperName, accountNumber, saveLocation) { 70 | const fields = getReportFields(false); 71 | const csv = objToCsv(txns, { fields, withBOM: true }); 72 | await writeFile(`${saveLocation}/${scraperName} (${accountNumber}).csv`, csv); 73 | } 74 | 75 | export async function generateSeparatedReports( 76 | scrapedAccounts, 77 | saveLocation, 78 | includeFutureTransactions, 79 | includePendingTransactions, 80 | ) { 81 | let numFiles = 0; 82 | for (let i = 0; i < scrapedAccounts.length; i += 1) { 83 | const { 84 | txns: accountTxns, 85 | accountNumber, 86 | scraperName, 87 | } = scrapedAccounts[i]; 88 | 89 | const filteredTxns = filterTransactions( 90 | accountTxns, 91 | includeFutureTransactions, 92 | includePendingTransactions, 93 | ); 94 | if (filteredTxns.length) { 95 | console.log(colors.notify(`exporting ${accountTxns.length} transactions for account # ${accountNumber}`)); 96 | await exportAccountData(filteredTxns, scraperName, accountNumber, saveLocation); 97 | numFiles += 1; 98 | } else { 99 | console.log(`no transactions for account # ${accountNumber}`); 100 | } 101 | } 102 | 103 | console.log(colors.notify(`${numFiles} csv files saved under ${saveLocation}`)); 104 | } 105 | 106 | export async function generateSingleReport( 107 | scrapedAccounts, 108 | saveLocation, 109 | includeFutureTransactions, 110 | includePendingTransactions, 111 | ) { 112 | const fileTransactions = scrapedAccounts.reduce((acc, account) => { 113 | const filteredTransactions = filterTransactions( 114 | account.txns, 115 | includeFutureTransactions, 116 | includePendingTransactions, 117 | ); 118 | acc.push(...filteredTransactions); 119 | return acc; 120 | }, []); 121 | const filePath = `${saveLocation}/${moment().format(DATE_TIME_FORMAT)}.csv`; 122 | const fileFields = getReportFields(true); 123 | const fileContent = objToCsv(fileTransactions, { fields: fileFields, withBOM: true }); 124 | await writeFile(filePath, fileContent); 125 | console.log(colors.notify(`created file ${filePath}`)); 126 | } 127 | -------------------------------------------------------------------------------- /src/scrape/scrape-individual.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import inquirer from 'inquirer'; 3 | 4 | import { CONFIG_FOLDER } from '../definitions'; 5 | import { readJsonFile } from '../helpers/files'; 6 | import { decryptCredentials } from '../helpers/credentials'; 7 | import { SCRAPERS } from '../helpers/scrapers'; 8 | import { readSettingsFile, writeSettingsFile } from '../helpers/settings'; 9 | import scrape from './scrape-base'; 10 | import { generateSeparatedReports } from './generate-reports'; 11 | 12 | async function getParameters() { 13 | const settings = await readSettingsFile(); 14 | const { 15 | combineInstallments, 16 | saveLocation, 17 | includeFutureTransactions, 18 | includePendingTransactions, 19 | } = settings; 20 | 21 | const startOfMonthMoment = moment().startOf('month'); 22 | const monthOptions = []; 23 | for (let i = 0; i < 6; i += 1) { 24 | const monthMoment = startOfMonthMoment.clone().subtract(i, 'month'); 25 | monthOptions.push({ 26 | name: monthMoment.format('ll'), 27 | value: monthMoment, 28 | }); 29 | } 30 | const result = await inquirer.prompt([ 31 | { 32 | type: 'list', 33 | name: 'scraperId', 34 | message: 'Which bank would you like to scrape?', 35 | choices: Object.keys(SCRAPERS).map((id) => { 36 | return { 37 | name: SCRAPERS[id].name, 38 | value: id, 39 | }; 40 | }), 41 | }, 42 | { 43 | type: 'confirm', 44 | name: 'combineInstallments', 45 | message: 'Combine installment transactions?', 46 | default: !!combineInstallments, 47 | }, 48 | { 49 | type: 'list', 50 | name: 'startDate', 51 | message: 'What date would you like to start scraping from?', 52 | choices: monthOptions, 53 | }, 54 | { 55 | type: 'input', 56 | name: 'saveLocation', 57 | message: 'Save folder?', 58 | default: saveLocation, 59 | }, 60 | { 61 | type: 'confirm', 62 | name: 'includeFutureTransactions', 63 | message: 'Include future transactions?', 64 | default: !!includeFutureTransactions, 65 | }, 66 | { 67 | type: 'confirm', 68 | name: 'includePendingTransactions', 69 | message: 'Include pending transactions?', 70 | default: !!includePendingTransactions, 71 | }, 72 | ]); 73 | 74 | settings.combineInstallments = result.combineInstallments; 75 | settings.startDate = result.startDate; 76 | settings.saveLocation = result.saveLocation; 77 | settings.includeFutureTransactions = result.includeFutureTransactions; 78 | settings.includePendingTransactions = result.includePendingTransactions; 79 | await writeSettingsFile(settings); 80 | 81 | return result; 82 | } 83 | 84 | export default async function scrapeIndividual(showBrowser) { 85 | const { 86 | scraperId, 87 | combineInstallments, 88 | startDate, 89 | saveLocation, 90 | includeFutureTransactions, 91 | includePendingTransactions, 92 | } = await getParameters(); 93 | 94 | const encryptedCredentials = await readJsonFile( 95 | `${CONFIG_FOLDER}/${scraperId}.json` 96 | ); 97 | if (encryptedCredentials) { 98 | const credentials = decryptCredentials(encryptedCredentials); 99 | const credentialsWithOtp = Object.assign(credentials, { 100 | otpCodeRetriever: async () => { 101 | const { otpCode } = await inquirer.prompt([ 102 | { 103 | type: 'input', 104 | name: 'otpCode', 105 | message: 'Enter OTP code:', 106 | }, 107 | ]); 108 | return otpCode; 109 | }, 110 | }); 111 | const options = { 112 | startDate: startDate.toDate(), 113 | combineInstallments, 114 | showBrowser, 115 | }; 116 | 117 | try { 118 | const scrapedAccounts = await scrape( 119 | scraperId, 120 | credentialsWithOtp, 121 | options 122 | ); 123 | await generateSeparatedReports( 124 | scrapedAccounts, 125 | saveLocation, 126 | includeFutureTransactions, 127 | includePendingTransactions 128 | ); 129 | } catch (e) { 130 | console.error(e); 131 | } 132 | } else { 133 | console.log('Could not find credentials file'); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/scrape/scrape-task.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import inquirer from 'inquirer'; 3 | import colors from 'colors/safe'; 4 | import { DATE_TIME_FORMAT } from '../constants'; 5 | import { decryptCredentials } from '../helpers/credentials'; 6 | import scrape from './scrape-base'; 7 | import { TasksManager, printTaskSummary } from '../helpers/tasks'; 8 | import { 9 | generateSingleReport, 10 | generateSeparatedReports, 11 | } from './generate-reports'; 12 | 13 | const tasksManager = new TasksManager(); 14 | 15 | async function getParameters() { 16 | const availableTasks = await tasksManager.getTasksList(); 17 | 18 | if (availableTasks && availableTasks.length) { 19 | const answers = await inquirer.prompt([ 20 | { 21 | type: 'list', 22 | name: 'taskName', 23 | message: 'Select a task to execute', 24 | validate: (answer) => { 25 | if (!answer) { 26 | return 'Task name must be provided'; 27 | } 28 | return true; 29 | }, 30 | choices: [...availableTasks], 31 | }, 32 | ]); 33 | 34 | return { taskName: answers.taskName }; 35 | } 36 | 37 | console.log( 38 | colors.notify( 39 | "No tasks created, please run command 'npm run setup' to create a task" 40 | ) 41 | ); 42 | return { taskName: null }; 43 | } 44 | 45 | export default async function scrapeTask(showBrowser) { 46 | const { taskName } = await getParameters(); 47 | 48 | if (taskName) { 49 | console.log(colors.title(`Running task '${taskName}'`)); 50 | const taskData = await tasksManager.loadTask(taskName); 51 | const scrapersOfTask = taskData.scrapers || []; 52 | 53 | if (scrapersOfTask.length === 0) { 54 | console.log( 55 | colors.notify( 56 | "Task has no scrapers defined.\nplease run command 'npm run setup' and update task scrapers" 57 | ) 58 | ); 59 | return; 60 | } 61 | 62 | printTaskSummary(taskData, true); 63 | 64 | const { dateDiffByMonth, combineInstallments } = taskData.options; 65 | const { 66 | combineReport, 67 | saveLocation: saveLocationRootPath, 68 | includeFutureTransactions, 69 | includePendingTransactions, 70 | } = taskData.output; 71 | const substractValue = dateDiffByMonth - 1; 72 | const startMoment = moment() 73 | .subtract(substractValue, 'month') 74 | .startOf('month'); 75 | const reportAccounts = []; 76 | 77 | console.log(colors.title('Run task scrapers')); 78 | 79 | for (let i = 0; i < scrapersOfTask.length; i += 1) { 80 | const scraperOfTask = scrapersOfTask[i]; 81 | const credentials = decryptCredentials(scraperOfTask.credentials); 82 | const credentialsWithOtp = Object.assign(credentials, { 83 | otpCodeRetriever: async () => { 84 | const { otpCode } = await inquirer.prompt([ 85 | { 86 | type: 'input', 87 | name: 'otpCode', 88 | message: 'Enter OTP code:', 89 | }, 90 | ]); 91 | return otpCode; 92 | }, 93 | }); 94 | 95 | const options = { 96 | companyId: scraperOfTask.id, 97 | startDate: startMoment, 98 | combineInstallments, 99 | showBrowser, 100 | verbose: false, 101 | }; 102 | 103 | try { 104 | const scrapedAccounts = await scrape( 105 | scraperOfTask.id, 106 | credentialsWithOtp, 107 | options 108 | ); 109 | reportAccounts.push(...scrapedAccounts); 110 | } catch (e) { 111 | console.error(e); 112 | throw e; 113 | } 114 | } 115 | 116 | if (combineReport) { 117 | const saveLocation = `${saveLocationRootPath}/tasks/${taskName}`; 118 | await generateSingleReport( 119 | reportAccounts, 120 | saveLocation, 121 | includeFutureTransactions, 122 | includePendingTransactions 123 | ); 124 | } else { 125 | const currentExecutionFolder = moment().format(DATE_TIME_FORMAT); 126 | const saveLocation = `${saveLocationRootPath}/tasks/${taskName}/${currentExecutionFolder}`; 127 | await generateSeparatedReports( 128 | reportAccounts, 129 | saveLocation, 130 | includeFutureTransactions, 131 | includePendingTransactions 132 | ); 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/setup/tasks/modify-task-handler.js: -------------------------------------------------------------------------------- 1 | import inquirer from 'inquirer'; 2 | import colors from 'colors/safe'; 3 | import { PASSWORD_FIELD } from '../../constants'; 4 | import { SCRAPERS } from '../../helpers/scrapers'; 5 | import { encryptCredentials } from '../../helpers/credentials'; 6 | import { TasksManager, printTaskSummary } from '../../helpers/tasks'; 7 | 8 | const tasksManager = new TasksManager(); 9 | 10 | const goBackOption = { 11 | name: 'Go Back', 12 | value: '', 13 | }; 14 | 15 | function getListOfScrapers(existingTaskScrapers) { 16 | return Object.keys(SCRAPERS).map((scraperId) => { 17 | const result = { value: scraperId, name: SCRAPERS[scraperId].name }; 18 | const hasCredentials = existingTaskScrapers.find( 19 | (scraper) => scraper.id === scraperId 20 | ); 21 | result.name = `${hasCredentials ? 'Edit' : 'Add'} ${result.name}`; 22 | 23 | return result; 24 | }); 25 | } 26 | 27 | function validateNonEmpty(field, input) { 28 | if (input) { 29 | return true; 30 | } 31 | return `${field} must be non empty`; 32 | } 33 | 34 | const ModifyTaskHandler = (function createModifyTaskHandler() { 35 | const _private = new WeakMap(); 36 | 37 | class ModifyTaskHandler { 38 | static async createAdapter() { 39 | const answers = await inquirer.prompt([ 40 | { 41 | type: 'list', 42 | name: 'taskName', 43 | message: 'Select a task to modify', 44 | choices: [...(await tasksManager.getTasksList()), goBackOption], 45 | }, 46 | ]); 47 | 48 | if (answers.taskName) { 49 | return new ModifyTaskHandler(answers.taskName); 50 | } 51 | 52 | return null; 53 | } 54 | 55 | constructor(taskName) { 56 | _private.set(this, { taskName }); 57 | } 58 | 59 | /* 60 | * @private 61 | */ 62 | async manageOptions() { 63 | const { combineInstallments, dateDiffByMonth } = 64 | _private.get(this).taskData.options; 65 | 66 | const { 67 | saveLocation, 68 | combineReport, 69 | includeFutureTransactions, 70 | includePendingTransactions, 71 | } = _private.get(this).taskData.output; 72 | 73 | const answers = await inquirer.prompt([ 74 | { 75 | type: 'confirm', 76 | name: 'combineInstallments', 77 | message: 'Combine installment transactions?', 78 | default: combineInstallments, 79 | }, 80 | { 81 | type: 'input', 82 | name: 'dateDiffByMonth', 83 | message: 'How many months do you want to scrape (1-12)?', 84 | default: dateDiffByMonth, 85 | filter: (value) => { 86 | if (Number.isFinite(value) && !Number.isNaN(value)) { 87 | return value; 88 | } 89 | 90 | if (typeof value === 'string' && value.match(/^[0-9]+$/)) { 91 | return value * 1; 92 | } 93 | 94 | return null; 95 | }, 96 | validate: (value) => { 97 | const pass = value !== null && value >= 1 && value <= 12; 98 | 99 | if (pass) { 100 | return true; 101 | } 102 | 103 | return 'Please enter a value between 1 and 12'; 104 | }, 105 | }, 106 | { 107 | type: 'input', 108 | name: 'saveLocation', 109 | message: 'Save folder?', 110 | default: saveLocation, 111 | }, 112 | { 113 | type: 'confirm', 114 | name: 'combineReport', 115 | message: 'Combine all accounts into a single report?', 116 | default: !!combineReport, 117 | }, 118 | { 119 | type: 'confirm', 120 | name: 'includeFutureTransactions', 121 | message: 'Include future transactions?', 122 | default: !!includeFutureTransactions, 123 | }, 124 | { 125 | type: 'confirm', 126 | name: 'includePendingTransactions', 127 | message: 'Include pending transactions?', 128 | default: !!includePendingTransactions, 129 | }, 130 | ]); 131 | 132 | const { taskData } = _private.get(this); 133 | taskData.options.combineInstallments = answers.combineInstallments; 134 | taskData.options.dateDiffByMonth = answers.dateDiffByMonth; 135 | taskData.output.saveLocation = answers.saveLocation; 136 | taskData.output.combineReport = answers.combineReport; 137 | taskData.output.includeFutureTransactions = 138 | answers.includeFutureTransactions; 139 | taskData.output.includePendingTransactions = 140 | answers.includePendingTransactions; 141 | console.log(colors.notify('Changes saved')); 142 | await this.saveTask(); 143 | } 144 | 145 | /* 146 | * @private 147 | */ 148 | async manageScrapers() { 149 | const MODIFY_ACTION = 'modify'; 150 | const DELETE_ACTION = 'delete'; 151 | 152 | const answers = await inquirer.prompt([ 153 | { 154 | type: 'list', 155 | name: 'action', 156 | message: 'What do you want to do?', 157 | choices: [ 158 | { 159 | name: 'Add / Edit a scraper', 160 | value: MODIFY_ACTION, 161 | }, 162 | { 163 | name: 'Delete a scraper', 164 | value: DELETE_ACTION, 165 | }, 166 | goBackOption, 167 | ], 168 | }, 169 | { 170 | type: 'list', 171 | name: 'scraperId', 172 | message: 'Select a scraper', 173 | when: (answers) => { 174 | if (answers.action === DELETE_ACTION) { 175 | const hasScrapers = 176 | _private.get(this).taskData.scrapers.length !== 0; 177 | 178 | if (!hasScrapers) { 179 | console.log(colors.notify('task has no scrapers defined')); 180 | } 181 | 182 | return hasScrapers; 183 | } 184 | 185 | const isRelevantQuestion = answers.action === MODIFY_ACTION; 186 | return isRelevantQuestion; 187 | }, 188 | choices: (answers) => { 189 | if (answers.action === MODIFY_ACTION) { 190 | return [ 191 | ...getListOfScrapers(_private.get(this).taskData.scrapers), 192 | goBackOption, 193 | ]; 194 | } 195 | 196 | const taskScrapers = _private 197 | .get(this) 198 | .taskData.scrapers.map((scraper) => ({ 199 | value: scraper.id, 200 | name: SCRAPERS[scraper.id].name, 201 | })); 202 | 203 | return [...taskScrapers, goBackOption]; 204 | }, 205 | }, 206 | { 207 | type: 'confirm', 208 | name: 'confirmDelete', 209 | when: (answers) => 210 | answers.action === DELETE_ACTION && answers.scraperId, 211 | message: 'Are you sure?', 212 | default: false, 213 | }, 214 | ]); 215 | 216 | const { scraperId, action, confirmDelete } = answers; 217 | 218 | if (scraperId) { 219 | if (action === DELETE_ACTION) { 220 | if (confirmDelete) { 221 | console.log(colors.notify(`Scraper ${scraperId} deleted`)); 222 | _private.get(this).taskData.scrapers = _private 223 | .get(this) 224 | .taskData.scrapers.filter((item) => item.id !== scraperId); 225 | await this.saveTask(); 226 | } else { 227 | console.log(colors.notify('Delete scraper cancelled')); 228 | } 229 | } else if (action === MODIFY_ACTION) { 230 | const { loginFields } = SCRAPERS[scraperId]; 231 | const questions = loginFields 232 | .filter((field) => !field.startsWith('otp')) 233 | .map((field) => { 234 | return { 235 | type: field === PASSWORD_FIELD ? PASSWORD_FIELD : 'input', 236 | name: field, 237 | message: `Enter value for ${field}:`, 238 | validate: (input) => validateNonEmpty(field, input), 239 | }; 240 | }); 241 | const credentialsResult = await inquirer.prompt(questions); 242 | const encryptedCredentials = encryptCredentials(credentialsResult); 243 | 244 | const scraperData = _private 245 | .get(this) 246 | .taskData.scrapers.find((scraper) => scraper.id === scraperId); 247 | if (!scraperData) { 248 | _private.get(this).taskData.scrapers.push({ 249 | id: scraperId, 250 | credentials: encryptedCredentials, 251 | }); 252 | console.log(colors.notify(`'${scraperId}' scrapper added`)); 253 | } else { 254 | scraperData.credentials = encryptedCredentials; 255 | console.log(colors.notify(`'${scraperId}' scrapper updated`)); 256 | } 257 | await this.saveTask(); 258 | } 259 | } 260 | } 261 | 262 | /* 263 | * @private 264 | */ 265 | async saveTask() { 266 | await tasksManager.saveTask( 267 | _private.get(this).taskName, 268 | _private.get(this).taskData 269 | ); 270 | } 271 | 272 | async run() { 273 | const VIEW_SUMMARY_ACTION = 'summary'; 274 | const UPDATE_SCRAPERS_LIST_ACTION = 'scrapers-list'; 275 | const UPDATE_OPTIONS_ACTION = 'scraping-options'; 276 | 277 | let firstTimeEntering = false; 278 | 279 | if (!_private.get(this).taskName) { 280 | throw new Error('missing task name'); 281 | } 282 | 283 | if (!_private.get(this).taskData) { 284 | firstTimeEntering = true; 285 | _private.get(this).taskData = await tasksManager.loadTask( 286 | _private.get(this).taskName 287 | ); 288 | } 289 | 290 | if (_private.get(this).taskData) { 291 | if (firstTimeEntering) { 292 | console.log( 293 | colors.title(`Editing task '${_private.get(this).taskName}'`) 294 | ); 295 | } 296 | 297 | const answers = await inquirer.prompt([ 298 | { 299 | type: 'list', 300 | name: 'action', 301 | message: 'What do you want to do?', 302 | choices: [ 303 | { 304 | name: 'View task summary', 305 | value: VIEW_SUMMARY_ACTION, 306 | }, 307 | { 308 | name: 'Update scrapers list', 309 | value: UPDATE_SCRAPERS_LIST_ACTION, 310 | }, 311 | { 312 | name: 'Update scraping options', 313 | value: UPDATE_OPTIONS_ACTION, 314 | }, 315 | goBackOption, 316 | ], 317 | }, 318 | ]); 319 | 320 | switch (answers.action) { 321 | case VIEW_SUMMARY_ACTION: 322 | console.log(''); // print empty line 323 | printTaskSummary(_private.get(this).taskData, false); 324 | console.log(''); // print empty line 325 | await this.run(); 326 | break; 327 | case UPDATE_SCRAPERS_LIST_ACTION: 328 | await this.manageScrapers(); 329 | await this.run(); 330 | break; 331 | case UPDATE_OPTIONS_ACTION: 332 | await this.manageOptions(); 333 | await this.run(); 334 | break; 335 | default: 336 | break; 337 | } 338 | } 339 | } 340 | } 341 | 342 | return ModifyTaskHandler; 343 | })(); 344 | 345 | export default ModifyTaskHandler; 346 | --------------------------------------------------------------------------------