├── .nvmrc ├── .node-version ├── tests ├── .gitignore └── readme.md ├── .npmrc ├── src ├── tests │ ├── .gitignore │ ├── .eslintrc.js │ ├── jest-setup.ts │ ├── .tests-config.tpl.js │ └── tests-utils.ts ├── helpers │ ├── debug.ts │ ├── storage.ts │ ├── dates.ts │ ├── navigation.ts │ ├── waiting.ts │ ├── transactions.ts │ ├── fetch.ts │ └── elements-interactions.ts ├── puppeteer-config.json ├── scrapers │ ├── amex.ts │ ├── isracard.ts │ ├── massad.ts │ ├── beinleumi.ts │ ├── factory.test.ts │ ├── errors.ts │ ├── max.test.ts │ ├── leumi.test.ts │ ├── union-bank.test.ts │ ├── beinleumi.test.ts │ ├── hapoalim.test.ts │ ├── amex.test.ts │ ├── otsar-hahayal.test.ts │ ├── yahav.test.ts │ ├── discount.test.ts │ ├── beyahad-bishvilha.test.ts │ ├── isracard.test.ts │ ├── visa-cal.test.ts │ ├── base-scraper-with-browser.test.ts │ ├── one-zero.test.ts │ ├── mizrahi.test.ts │ ├── factory.ts │ ├── base-scraper.ts │ ├── interface.ts │ ├── discount.ts │ ├── beyahad-bishvilha.ts │ ├── otsar-hahayal.ts │ ├── mizrahi.ts │ ├── leumi.ts │ ├── hapoalim.ts │ ├── yahav.ts │ ├── base-scraper-with-browser.ts │ ├── one-zero.ts │ ├── union-bank.ts │ ├── one-zero-queries.ts │ └── max.ts ├── constants.ts ├── index.ts ├── transactions.ts └── definitions.ts ├── logo.png ├── .github ├── workflows │ ├── lint.yml │ ├── nodeCI.yml │ └── release.yml └── stale.yml ├── utils ├── core-utils.js ├── jscodeshift │ ├── index.js │ └── puppeteer-imports.js ├── pre-publish.js ├── prepare-israeli-bank-scrapers-core.js └── update-puppeteer-config.js ├── .babelrc.js ├── jest.config.js ├── tsconfig.json ├── LICENSE ├── .gitignore ├── .eslintrc.js ├── package.json └── CONTRIBUTING.md /.nvmrc: -------------------------------------------------------------------------------- 1 | v14.14.0 2 | -------------------------------------------------------------------------------- /.node-version: -------------------------------------------------------------------------------- 1 | 14.14.0 2 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | **/* 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry = https://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /src/tests/.gitignore: -------------------------------------------------------------------------------- 1 | .*.* 2 | .* 3 | !.tests-config.tpl.js 4 | snapshots/** 5 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baruchiro/israeli-bank-scrapers/master/logo.png -------------------------------------------------------------------------------- /src/helpers/debug.ts: -------------------------------------------------------------------------------- 1 | import debug from 'debug'; 2 | 3 | export function getDebug(name: string) { 4 | return debug(`israeli-bank-scrapers:${name}`); 5 | } 6 | -------------------------------------------------------------------------------- /src/puppeteer-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "__comment": "This file is auto generated by script '../utils/update-puppeteer-config.js'. Don't change it manually.", 3 | "chromiumRevision": "843427" 4 | } -------------------------------------------------------------------------------- /src/tests/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "rules": { 3 | "import/no-extraneous-dependencies": 0, 4 | "import/no-dynamic-require": 0, 5 | "global-require": 0, 6 | "no-console": 0 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/tests/jest-setup.ts: -------------------------------------------------------------------------------- 1 | import * as SourceMap from 'source-map-support'; 2 | import { getTestsConfig } from './tests-utils'; 3 | 4 | SourceMap.install(); 5 | // Try to get test configuration object, no need to do anything beside that 6 | getTestsConfig(); 7 | -------------------------------------------------------------------------------- /tests/readme.md: -------------------------------------------------------------------------------- 1 | > Don't remove file .gitignore otherwise people might commit their credentials 2 | 3 | 1. This folder is obslete. It was moved to src/tests. 4 | 2. If you already have file `.tests-config.js`, move it to `src/tests` and change its extension to `ts`. 5 | 6 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lint: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | 13 | - name: Run npm install 14 | run: npm install 15 | 16 | - name: Run lint 17 | run: npm run lint 18 | 19 | -------------------------------------------------------------------------------- /src/helpers/storage.ts: -------------------------------------------------------------------------------- 1 | import { Page } from 'puppeteer'; 2 | 3 | export async function getFromSessionStorage(page: Page, key: string): Promise { 4 | const strData = await page.evaluate((k: string) => { 5 | return sessionStorage.getItem(k); 6 | }, key); 7 | 8 | if (!strData) return null; 9 | 10 | return JSON.parse(strData) as T; 11 | } 12 | -------------------------------------------------------------------------------- /utils/core-utils.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = function checkIfCoreVariation() { 4 | const packagePath = path.join(__dirname, '..', 'package.json'); 5 | // eslint-disable-next-line import/no-dynamic-require,global-require 6 | const packageJson = require(packagePath); 7 | 8 | return (packageJson.name === 'israeli-bank-scrapers-core'); 9 | } 10 | -------------------------------------------------------------------------------- /src/scrapers/amex.ts: -------------------------------------------------------------------------------- 1 | import IsracardAmexBaseScraper from './base-isracard-amex'; 2 | import { ScraperOptions } from './interface'; 3 | 4 | const BASE_URL = 'https://he.americanexpress.co.il'; 5 | const COMPANY_CODE = '77'; 6 | 7 | class AmexScraper extends IsracardAmexBaseScraper { 8 | constructor(options: ScraperOptions) { 9 | super(options, BASE_URL, COMPANY_CODE); 10 | } 11 | } 12 | 13 | export default AmexScraper; 14 | -------------------------------------------------------------------------------- /src/scrapers/isracard.ts: -------------------------------------------------------------------------------- 1 | import IsracardAmexBaseScraper from './base-isracard-amex'; 2 | import { ScraperOptions } from './interface'; 3 | 4 | const BASE_URL = 'https://digital.isracard.co.il'; 5 | const COMPANY_CODE = '11'; 6 | 7 | class IsracardScraper extends IsracardAmexBaseScraper { 8 | constructor(options: ScraperOptions) { 9 | super(options, BASE_URL, COMPANY_CODE); 10 | } 11 | } 12 | 13 | export default IsracardScraper; 14 | -------------------------------------------------------------------------------- /.babelrc.js: -------------------------------------------------------------------------------- 1 | const presets = [ 2 | [ 3 | "@babel/preset-env", 4 | { 5 | targets: { 6 | node: "8", 7 | }, 8 | useBuiltIns: "usage", 9 | corejs: "3", 10 | }, 11 | ], 12 | "@babel/preset-typescript", 13 | ]; 14 | 15 | module.exports = { 16 | presets, 17 | ignore: process.env.BABEL_ENV === "test" ? [] : ["**/*.test.(js,ts)", "tests/**/*", "src/tests/**/*"], 18 | plugins: ["@babel/plugin-proposal-class-properties"], 19 | }; 20 | -------------------------------------------------------------------------------- /src/scrapers/massad.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiGroupBaseScraper from './base-beinleumi-group'; 2 | 3 | 4 | class MassadScraper extends BeinleumiGroupBaseScraper { 5 | BASE_URL = 'https://online.bankmassad.co.il'; 6 | 7 | LOGIN_URL = `${this.BASE_URL}/MatafLoginService/MatafLoginServlet?bankId=MASADPRTAL&site=Private&KODSAFA=HE`; 8 | 9 | TRANSACTIONS_URL = `${this.BASE_URL}/wps/myportal/FibiMenu/Online/OnAccountMngment/OnBalanceTrans/PrivateAccountFlow`; 10 | } 11 | 12 | export default MassadScraper; 13 | -------------------------------------------------------------------------------- /src/scrapers/beinleumi.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiGroupBaseScraper from './base-beinleumi-group'; 2 | 3 | 4 | class BeinleumiScraper extends BeinleumiGroupBaseScraper { 5 | BASE_URL = 'https://online.fibi.co.il'; 6 | 7 | LOGIN_URL = `${this.BASE_URL}/MatafLoginService/MatafLoginServlet?bankId=FIBIPORTAL&site=Private&KODSAFA=HE`; 8 | 9 | TRANSACTIONS_URL = `${this.BASE_URL}/wps/myportal/FibiMenu/Online/OnAccountMngment/OnBalanceTrans/PrivateAccountFlow`; 10 | } 11 | 12 | export default BeinleumiScraper; 13 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | // For a detailed explanation regarding each configuration property, visit: 2 | // https://jestjs.io/docs/en/configuration.html 3 | 4 | module.exports = { 5 | preset: 'ts-jest/presets/js-with-babel', 6 | clearMocks: true, 7 | coverageDirectory: 'coverage', 8 | rootDir: './src', 9 | transform: { 10 | '^.+\\.ts$': 'ts-jest' 11 | }, 12 | setupFiles: [ 13 | './tests/jest-setup.ts', 14 | ], 15 | testEnvironment: 'node', 16 | globals: { 17 | 'ts-jest': { 18 | babelConfig: true, 19 | } 20 | }, 21 | }; 22 | -------------------------------------------------------------------------------- /src/scrapers/factory.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/unbound-method */ 2 | import { CompanyTypes } from '../definitions'; 3 | import createScraper from './factory'; 4 | 5 | describe('Factory', () => { 6 | test('should return a scraper instance', () => { 7 | const scraper = createScraper({ 8 | companyId: CompanyTypes.hapoalim, 9 | startDate: new Date(), 10 | }); 11 | expect(scraper).toBeDefined(); 12 | 13 | expect(scraper.scrape).toBeInstanceOf(Function); 14 | expect(scraper.onProgress).toBeInstanceOf(Function); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /utils/jscodeshift/index.js: -------------------------------------------------------------------------------- 1 | const { resolve } = require('path'); 2 | const { spawnSync } = require('child_process'); 3 | 4 | module.exports = async function() { 5 | await spawnSync('jscodeshift', 6 | [ 7 | '-t', 8 | resolve(__dirname, './puppeteer-imports.js'), 9 | resolve(__dirname, '../../src'), 10 | '--extensions', 11 | 'ts', 12 | '--parser', 13 | 'ts' 14 | ], { 15 | cwd: resolve(__dirname, '../../node_modules/.bin'), 16 | stdio: 'inherit' 17 | }) 18 | 19 | } -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | 2 | export const SHEKEL_CURRENCY_SYMBOL = '₪'; 3 | export const SHEKEL_CURRENCY_KEYWORD = 'ש"ח'; 4 | export const ALT_SHEKEL_CURRENCY = 'NIS'; 5 | export const SHEKEL_CURRENCY = 'ILS'; 6 | 7 | export const DOLLAR_CURRENCY_SYMBOL = '$'; 8 | export const DOLLAR_CURRENCY = 'USD'; 9 | 10 | export const EURO_CURRENCY_SYMBOL = '€'; 11 | export const EURO_CURRENCY = 'EUR'; 12 | 13 | export const ISO_DATE_FORMAT = 'YYYY-MM-DD[T]HH:mm:ss.SSS[Z]'; 14 | 15 | export const ISO_DATE_REGEX = /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T([0-1][0-9]|2[0-3])(:[0-5][0-9]){2}\.[0-9]{3}Z$/; 16 | -------------------------------------------------------------------------------- /utils/jscodeshift/puppeteer-imports.js: -------------------------------------------------------------------------------- 1 | import { dirname } from 'path'; 2 | 3 | const transform = (file, api) => { 4 | const j = api.jscodeshift; 5 | 6 | const root = j(file.source); 7 | root 8 | .find(j.ImportDeclaration) 9 | .find(j.Literal) 10 | .replaceWith(nodePath => { 11 | const { node } = nodePath; 12 | 13 | if (!node.value || node.value !== 'puppeteer') { 14 | return node; 15 | } 16 | 17 | node.value = 'puppeteer-core'; 18 | 19 | return node; 20 | }); 21 | 22 | return root.toSource(); 23 | }; 24 | 25 | export default transform; 26 | -------------------------------------------------------------------------------- /src/helpers/dates.ts: -------------------------------------------------------------------------------- 1 | import moment, { Moment } from 'moment'; 2 | 3 | export default function getAllMonthMoments(startMoment: Moment | string, futureMonths?: number) { 4 | let monthMoment = moment(startMoment).startOf('month'); 5 | 6 | const allMonths: Moment[] = []; 7 | let lastMonth = moment().startOf('month'); 8 | if (futureMonths && futureMonths > 0) { 9 | lastMonth = lastMonth.add(futureMonths, 'month'); 10 | } 11 | while (monthMoment.isSameOrBefore(lastMonth)) { 12 | allMonths.push(monthMoment); 13 | monthMoment = moment(monthMoment).add(1, 'month'); 14 | } 15 | 16 | return allMonths; 17 | } 18 | -------------------------------------------------------------------------------- /.github/workflows/nodeCI.yml: -------------------------------------------------------------------------------- 1 | name: Node CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | if: github.ref != 'refs/heads/master' 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | node-version: [14.x] 12 | os: [ubuntu-latest, windows-latest, macOS-latest] 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Use Node.js ${{ matrix.node-version }} 16 | uses: actions/setup-node@v1 17 | with: 18 | node-version: ${{ matrix.node-version }} 19 | - name: npm install and test 20 | run: | 21 | npm ci 22 | npm run test:ci 23 | -------------------------------------------------------------------------------- /utils/pre-publish.js: -------------------------------------------------------------------------------- 1 | const fsExtra = require('fs-extra'); 2 | const path = require('path'); 3 | const argv = require('minimist')(process.argv.slice(2), { string: 'version' }); 4 | 5 | const version = argv.version; 6 | 7 | if (!version) { 8 | console.error(`missing argument 'version'`); 9 | process.exit(1); 10 | return; 11 | } 12 | 13 | const packageJSONPath = path.resolve(__dirname, '../package.json'); 14 | 15 | const packageJSON = fsExtra.readJSONSync(packageJSONPath); 16 | 17 | 18 | packageJSON.version = version; 19 | packageJSON.private = false; 20 | 21 | fsExtra.writeJSONSync(packageJSONPath, packageJSON, { spaces: 2 }) 22 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import puppeteerConfig from './puppeteer-config.json'; 2 | 3 | export { default as createScraper } from './scrapers/factory'; 4 | export { SCRAPERS, CompanyTypes } from './definitions'; 5 | 6 | // Note: the typo ScaperScrapingResult & ScraperLoginResult (sic) are exported here for backward compatibility 7 | export { 8 | ScraperOptions, 9 | ScraperScrapingResult as ScaperScrapingResult, 10 | ScraperScrapingResult, 11 | ScraperLoginResult as ScaperLoginResult, 12 | ScraperLoginResult, 13 | Scraper, 14 | ScraperCredentials, 15 | } from './scrapers/interface'; 16 | 17 | export { default as OneZeroScraper } from './scrapers/one-zero'; 18 | 19 | export function getPuppeteerConfig() { 20 | return { ...puppeteerConfig }; 21 | } 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "esnext", 4 | "module": "commonjs", 5 | "allowJs": false, 6 | "skipLibCheck": true, 7 | "declaration": true, 8 | "outDir": "lib", 9 | "rootDir": "./src", 10 | "strict": true, 11 | "noImplicitAny": true, 12 | "resolveJsonModule": true, 13 | "noImplicitThis": true, 14 | "noUnusedLocals": true, 15 | "noUnusedParameters": true, 16 | "noImplicitReturns": true, 17 | "noFallthroughCasesInSwitch": true, 18 | "moduleResolution": "node", 19 | "allowSyntheticDefaultImports": true, 20 | "esModuleInterop": true, 21 | "forceConsistentCasingInFileNames": true 22 | }, 23 | "include": ["src/**/*"], 24 | "exclude": ["node_modules"] 25 | } 26 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | - dependencies 10 | # Label to use when marking an issue as stale 11 | staleLabel: wontfix 12 | # Comment to post when marking an issue as stale. Set to `false` to disable 13 | markComment: > 14 | This issue has been automatically marked as stale because it has not had 15 | recent activity. It will be closed if no further activity occurs. Thank you 16 | for your contributions. 17 | # Comment to post when closing a stale issue. Set to `false` to disable 18 | closeComment: false 19 | -------------------------------------------------------------------------------- /src/scrapers/errors.ts: -------------------------------------------------------------------------------- 1 | export enum ScraperErrorTypes { 2 | TwoFactorRetrieverMissing = 'TWO_FACTOR_RETRIEVER_MISSING', 3 | InvalidPassword = 'INVALID_PASSWORD', 4 | ChangePassword = 'CHANGE_PASSWORD', 5 | Timeout = 'TIMEOUT', 6 | AccountBlocked = 'ACCOUNT_BLOCKED', 7 | Generic = 'GENERIC', 8 | General = 'GENERAL_ERROR' 9 | } 10 | 11 | export type ErrorResult = { 12 | success: false; 13 | errorType: ScraperErrorTypes; 14 | errorMessage: string; 15 | }; 16 | 17 | function createErrorResult(errorType: ScraperErrorTypes, errorMessage: string): ErrorResult { 18 | return { 19 | success: false, 20 | errorType, 21 | errorMessage, 22 | }; 23 | } 24 | 25 | export function createTimeoutError(errorMessage: string): ErrorResult { 26 | return createErrorResult(ScraperErrorTypes.Timeout, errorMessage); 27 | } 28 | 29 | export function createGenericError(errorMessage: string): ErrorResult { 30 | return createErrorResult(ScraperErrorTypes.Generic, errorMessage); 31 | } 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Elad Shaham 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /utils/prepare-israeli-bank-scrapers-core.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const checkIfCoreVariation = require('./core-utils'); 4 | const transformImports = require('./jscodeshift'); 5 | 6 | function updatePackageJson() { 7 | const packagePath = path.join(__dirname, '..', 'package.json'); 8 | // eslint-disable-next-line import/no-dynamic-require,global-require 9 | const json = require(packagePath); 10 | 11 | json.dependencies['puppeteer-core'] = json.dependencies.puppeteer; 12 | delete json.dependencies.puppeteer; 13 | 14 | json.devDependencies['@types/puppeteer-core'] = '5.4.0'; 15 | delete json.devDependencies['@types/puppeteer']; 16 | json.name = 'israeli-bank-scrapers-core'; 17 | fs.writeFileSync(packagePath, JSON.stringify(json, null, ' ')); 18 | 19 | console.log('change package.json name to \'israeli-bank-scrapers-core\' and use \'puppeteer-core\''); 20 | } 21 | 22 | (async function () { 23 | if (checkIfCoreVariation()) { 24 | console.log('library is already in core variation'); 25 | process.exit(1); 26 | return; 27 | } 28 | 29 | updatePackageJson(); 30 | await transformImports(); 31 | }()); 32 | -------------------------------------------------------------------------------- /src/transactions.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface TransactionsAccount { 3 | accountNumber: string; 4 | balance?: number; 5 | txns: Transaction[]; 6 | } 7 | 8 | export enum TransactionTypes { 9 | Normal = 'normal', 10 | Installments = 'installments' 11 | } 12 | 13 | export enum TransactionStatuses { 14 | Completed = 'completed', 15 | Pending = 'pending' 16 | } 17 | 18 | export interface TransactionInstallments { 19 | /** 20 | * the current installment number 21 | */ 22 | number: number; 23 | 24 | /** 25 | * the total number of installments 26 | */ 27 | total: number; 28 | } 29 | 30 | export interface Transaction { 31 | type: TransactionTypes; 32 | /** 33 | * sometimes called Asmachta 34 | */ 35 | identifier?: string | number; 36 | /** 37 | * ISO date string 38 | */ 39 | date: string; 40 | /** 41 | * ISO date string 42 | */ 43 | processedDate: string; 44 | originalAmount: number; 45 | originalCurrency: string; 46 | chargedAmount: number; 47 | chargedCurrency?: string; 48 | description: string; 49 | memo?: string; 50 | status: TransactionStatuses; 51 | installments?: TransactionInstallments; 52 | category?: string; 53 | } 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # NPM package related 61 | lib/ 62 | 63 | # IDEs 64 | .idea 65 | .vscode 66 | .DS_Store 67 | -------------------------------------------------------------------------------- /utils/update-puppeteer-config.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const checkIfCoreVariation = require('./core-utils'); 4 | 5 | function getPuppeteerChromiumVersion() { 6 | const puppeteerLibrary = checkIfCoreVariation() ? 'puppeteer-core' : 'puppeteer'; 7 | const puppeteerPath = path.dirname(require.resolve(puppeteerLibrary)); 8 | const revisionFilePath = path.join(puppeteerPath, 'lib/cjs/puppeteer/revisions.js'); 9 | // eslint-disable-next-line import/no-dynamic-require,global-require 10 | const revisionRaw = fs.readFileSync(revisionFilePath, 'utf-8'); 11 | const [, revisionNumber] = revisionRaw.match(/chromium: ['"`](.+?)['"`][,]/); 12 | return revisionNumber; 13 | } 14 | 15 | (function updatePuppeteerConfiguration() { 16 | console.log('extract puppeteer chromium version from module \'puppeteer|pupetter-core\''); 17 | 18 | const chromiumRevision = getPuppeteerChromiumVersion(); 19 | const configPath = path.join(__dirname, '../src/puppeteer-config.json'); 20 | // eslint-disable-next-line global-require,import/no-dynamic-require 21 | const configJson = require(configPath); 22 | configJson.chromiumRevision = chromiumRevision; 23 | 24 | fs.writeFileSync(configPath, JSON.stringify(configJson, null, ' ')); 25 | 26 | console.log(`update 'src/puppeteer-config.json' file with puppeteer chroumium revision '${chromiumRevision}'`); 27 | }()); 28 | -------------------------------------------------------------------------------- /src/helpers/navigation.ts: -------------------------------------------------------------------------------- 1 | import { Frame, NavigationOptions, Page } from 'puppeteer'; 2 | import { waitUntil } from './waiting'; 3 | 4 | export async function waitForNavigation(pageOrFrame: Page | Frame, options?: NavigationOptions) { 5 | await pageOrFrame.waitForNavigation(options); 6 | } 7 | 8 | export async function waitForNavigationAndDomLoad(page: Page) { 9 | await waitForNavigation(page, { waitUntil: 'domcontentloaded' }); 10 | } 11 | 12 | export function getCurrentUrl(pageOrFrame: Page | Frame, clientSide = false) { 13 | if (clientSide) { 14 | return pageOrFrame.evaluate(() => window.location.href); 15 | } 16 | 17 | return pageOrFrame.url(); 18 | } 19 | 20 | export async function waitForRedirect(pageOrFrame: Page | Frame, timeout = 20000, 21 | clientSide = false, ignoreList: string[] = []) { 22 | const initial = await getCurrentUrl(pageOrFrame, clientSide); 23 | 24 | await waitUntil(async () => { 25 | const current = await getCurrentUrl(pageOrFrame, clientSide); 26 | return current !== initial && !ignoreList.includes(current); 27 | }, `waiting for redirect from ${initial}`, timeout, 1000); 28 | } 29 | 30 | export async function waitForUrl(pageOrFrame: Page | Frame, url: string | RegExp, timeout = 20000, clientSide = false) { 31 | await waitUntil(async () => { 32 | const current = await getCurrentUrl(pageOrFrame, clientSide); 33 | return url instanceof RegExp ? url.test(current) : url === current; 34 | }, `waiting for url to be ${url}`, timeout, 1000); 35 | } 36 | -------------------------------------------------------------------------------- /src/helpers/waiting.ts: -------------------------------------------------------------------------------- 1 | 2 | export class TimeoutError extends Error { 3 | 4 | } 5 | 6 | export const SECOND = 1000; 7 | 8 | function timeoutPromise(ms: number, promise: Promise, description: string): Promise { 9 | const timeout = new Promise((_, reject) => { 10 | const id = setTimeout(() => { 11 | clearTimeout(id); 12 | const error = new TimeoutError(description); 13 | reject(error); 14 | }, ms); 15 | }); 16 | 17 | return Promise.race([ 18 | promise, 19 | // casting to avoid type error- safe since this promise will always reject 20 | timeout as Promise, 21 | ]); 22 | } 23 | 24 | /** 25 | * Wait until a promise resolves with a truthy value or reject after a timeout 26 | */ 27 | export function waitUntil(asyncTest: () => Promise, description = '', timeout = 10000, interval = 100) { 28 | const promise = new Promise((resolve, reject) => { 29 | function wait() { 30 | asyncTest().then((value) => { 31 | if (value) { 32 | resolve(value); 33 | } else { 34 | setTimeout(wait, interval); 35 | } 36 | }).catch(() => { 37 | reject(); 38 | }); 39 | } 40 | wait(); 41 | }); 42 | return timeoutPromise(timeout, promise, description); 43 | } 44 | 45 | export function raceTimeout(ms: number, promise: Promise) { 46 | return timeoutPromise(ms, promise, 'timeout').catch((err) => { 47 | if (!(err instanceof TimeoutError)) throw err; 48 | }); 49 | } 50 | 51 | export function runSerial(actions: (() => Promise)[]): Promise { 52 | return actions.reduce((m, a) => m.then(async (x) => [...x, await a()]), Promise.resolve(new Array())); 53 | } 54 | -------------------------------------------------------------------------------- /src/tests/.tests-config.tpl.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const startDate = new Date(); 4 | startDate.setMonth(startDate.getMonth() - 1); 5 | 6 | module.exports = { 7 | options: { // options object that is passed to the scrapers. see more in readme.md file 8 | startDate, 9 | combineInstallments: false, 10 | showBrowser: true, 11 | verbose: false, 12 | args: [], 13 | storeFailureScreenShotPath: false // path.resolve(__dirname, 'snapshots/failure.jpg') 14 | }, 15 | credentials: { // commented companies will be skipped automatically, uncomment those you wish to test 16 | // hapoalim: { userCode: '', password: '' }, 17 | // leumi: { username: '', password: '' }, 18 | // hapoalimBeOnline: { userCode: '', password: '' }, 19 | // discount: { id: '', password: '', num: '' }, 20 | // otsarHahayal: { username: '', password: '' }, 21 | // max: { username: '', password: '' }, 22 | // visaCal: { username: '', password: '' }, 23 | // isracard: { id: '', password: '', card6Digits: '' }, 24 | // amex: { id: '', card6Digits: '', password: ''}, 25 | // mizrahi: { username: '', password: ''}, 26 | // union: {username:'',password:''} 27 | // beinleumi: { username: '', password: ''}, 28 | // yahav: {username: '', nationalID: '', password: ''} 29 | // beyahadBishvilha: { id: '', password: ''}, 30 | // oneZero: { email: '', password: '', otpCode: '', otpToken: null } 31 | }, 32 | companyAPI: { // enable companyAPI to execute tests against the real companies api 33 | enabled: true, 34 | excelFilesDist: '', // optional - provide exists directory path to save scraper results (csv format) 35 | invalidPassword: false, // enable to execute tests that execute with invalid credentials 36 | }, 37 | }; 38 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | "rules": { 4 | "import/prefer-default-export": 0, 5 | "no-nested-ternary": 0, 6 | "class-methods-use-this": 0, 7 | "arrow-body-style": 0, 8 | "no-shadow": 0, 9 | "no-await-in-loop": 0, 10 | "no-restricted-syntax": [ 11 | "error", 12 | "ForInStatement", 13 | "LabeledStatement", 14 | "WithStatement" 15 | ], 16 | "operator-linebreak": ["error", "after"], 17 | "max-len": ["error", 120, 2, { 18 | "ignoreUrls": true, 19 | "ignoreComments": true, 20 | "ignoreRegExpLiterals": true, 21 | "ignoreStrings": true, 22 | "ignoreTemplateLiterals": true, 23 | "ignorePattern": "^(async )?function " 24 | }], 25 | "linebreak-style": process.platform === "win32"? 0: 2, 26 | "@typescript-eslint/explicit-function-return-type": 0, 27 | "@typescript-eslint/no-explicit-any": 0, 28 | "@typescript-eslint/ban-ts-ignore": 0, 29 | "@typescript-eslint/no-non-null-assertion": 0, 30 | "@typescript-eslint/member-delimiter-style": [ "error", { 31 | multiline: { 32 | delimiter: 'semi', 33 | requireLast: true, 34 | }, 35 | singleline: { 36 | delimiter: 'comma', 37 | requireLast: false, 38 | }, 39 | }] 40 | }, 41 | "globals": { 42 | "document": true, 43 | "window": true, 44 | "fetch": true, 45 | "Headers": true 46 | }, 47 | "env": { 48 | "jest": true 49 | }, 50 | parserOptions: { 51 | project: './tsconfig.json', 52 | ecmaVersion: 2018, // Allows for the parsing of modern ECMAScript features 53 | sourceType: 'module', // Allows for the use of imports 54 | }, 55 | extends: ['airbnb-typescript/base', 56 | "plugin:@typescript-eslint/eslint-recommended", 57 | "plugin:@typescript-eslint/recommended", 58 | "plugin:@typescript-eslint/recommended-requiring-type-checking"] 59 | } 60 | -------------------------------------------------------------------------------- /src/helpers/transactions.ts: -------------------------------------------------------------------------------- 1 | import moment, { Moment } from 'moment'; 2 | import _ from 'lodash'; 3 | import { 4 | Transaction, TransactionTypes, 5 | } from '../transactions'; 6 | 7 | function isNormalTransaction(txn: any): boolean { 8 | return txn && txn.type === TransactionTypes.Normal; 9 | } 10 | 11 | function isInstallmentTransaction(txn: any): boolean { 12 | return txn && txn.type === TransactionTypes.Installments; 13 | } 14 | 15 | function isNonInitialInstallmentTransaction(txn: Transaction): boolean { 16 | return isInstallmentTransaction(txn) && !!txn.installments && txn.installments.number > 1; 17 | } 18 | 19 | function isInitialInstallmentTransaction(txn: Transaction): boolean { 20 | return isInstallmentTransaction(txn) && !!txn.installments && txn.installments.number === 1; 21 | } 22 | 23 | export function fixInstallments(txns: Transaction[]): Transaction[] { 24 | return txns.map((txn: Transaction) => { 25 | const clonedTxn = { ...txn }; 26 | 27 | if (isInstallmentTransaction(clonedTxn) && isNonInitialInstallmentTransaction(clonedTxn) && 28 | clonedTxn.installments) { 29 | const dateMoment = moment(clonedTxn.date); 30 | const actualDateMoment = dateMoment.add(clonedTxn.installments.number - 1, 'month'); 31 | clonedTxn.date = actualDateMoment.toISOString(); 32 | } 33 | return clonedTxn; 34 | }); 35 | } 36 | 37 | export function sortTransactionsByDate(txns: Transaction[]) { 38 | return _.sortBy(txns, ['date']); 39 | } 40 | 41 | export function filterOldTransactions(txns: Transaction[], 42 | startMoment: Moment, combineInstallments: boolean) { 43 | return txns.filter((txn) => { 44 | const combineNeededAndInitialOrNormal = 45 | combineInstallments && (isNormalTransaction(txn) || isInitialInstallmentTransaction(txn)); 46 | return (!combineInstallments && startMoment.isSameOrBefore(txn.date)) || 47 | (combineNeededAndInitialOrNormal && startMoment.isSameOrBefore(txn.date)); 48 | }); 49 | } 50 | -------------------------------------------------------------------------------- /src/scrapers/max.test.ts: -------------------------------------------------------------------------------- 1 | import MaxScraper from './max'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'max'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Max scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.max).toBeDefined(); 18 | expect(SCRAPERS.max.loginFields).toContain('username'); 19 | expect(SCRAPERS.max.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new MaxScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new MaxScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.max); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/leumi.test.ts: -------------------------------------------------------------------------------- 1 | import LeumiScraper from './leumi'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'leumi'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Leumi legacy scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.leumi).toBeDefined(); 18 | expect(SCRAPERS.leumi.loginFields).toContain('username'); 19 | expect(SCRAPERS.leumi.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new LeumiScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new LeumiScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.leumi); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/union-bank.test.ts: -------------------------------------------------------------------------------- 1 | import UnionBankScraper from './union-bank'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'union'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Union', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.union).toBeDefined(); 18 | expect(SCRAPERS.union.loginFields).toContain('username'); 19 | expect(SCRAPERS.union.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new UnionBankScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new UnionBankScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.union); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/beinleumi.test.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiScraper from './beinleumi'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'beinleumi'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Beinleumi', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.beinleumi).toBeDefined(); 18 | expect(SCRAPERS.beinleumi.loginFields).toContain('username'); 19 | expect(SCRAPERS.beinleumi.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new BeinleumiScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new BeinleumiScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.beinleumi); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/hapoalim.test.ts: -------------------------------------------------------------------------------- 1 | import HapoalimScraper from './hapoalim'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'hapoalim'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Hapoalim legacy scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.hapoalim).toBeDefined(); 18 | expect(SCRAPERS.hapoalim.loginFields).toContain('userCode'); 19 | expect(SCRAPERS.hapoalim.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new HapoalimScraper(options); 29 | 30 | const result = await scraper.scrape({ userCode: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new HapoalimScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.hapoalim); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/amex.test.ts: -------------------------------------------------------------------------------- 1 | import AMEXScraper from './amex'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'amex'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('AMEX legacy scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.amex).toBeDefined(); 18 | expect(SCRAPERS.amex.loginFields).toContain('id'); 19 | expect(SCRAPERS.amex.loginFields).toContain('card6Digits'); 20 | expect(SCRAPERS.amex.loginFields).toContain('password'); 21 | }); 22 | 23 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new AMEXScraper(options); 30 | 31 | const result = await scraper.scrape({ id: 'e10s12', card6Digits: '123456', password: '3f3ss3d' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new AMEXScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.amex); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/otsar-hahayal.test.ts: -------------------------------------------------------------------------------- 1 | import OtsarHahayalScraper from './otsar-hahayal'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'otsarHahayal'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('OtsarHahayal legacy scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.otsarHahayal).toBeDefined(); 18 | expect(SCRAPERS.otsarHahayal.loginFields).toContain('username'); 19 | expect(SCRAPERS.otsarHahayal.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new OtsarHahayalScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new OtsarHahayalScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.otsarHahayal); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/yahav.test.ts: -------------------------------------------------------------------------------- 1 | import YahavScraper from './yahav'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'yahav'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Yahav scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.yahav).toBeDefined(); 18 | expect(SCRAPERS.yahav.loginFields).toContain('username'); 19 | expect(SCRAPERS.yahav.loginFields).toContain('password'); 20 | expect(SCRAPERS.yahav.loginFields).toContain('nationalID'); 21 | }); 22 | 23 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new YahavScraper(options); 30 | 31 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d', nationalID: '12345679' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new YahavScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.yahav); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/discount.test.ts: -------------------------------------------------------------------------------- 1 | import DiscountScraper from './discount'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'discount'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Discount legacy scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.discount).toBeDefined(); 18 | expect(SCRAPERS.discount.loginFields).toContain('id'); 19 | expect(SCRAPERS.discount.loginFields).toContain('password'); 20 | expect(SCRAPERS.discount.loginFields).toContain('num'); 21 | }); 22 | 23 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new DiscountScraper(options); 30 | 31 | const result = await scraper.scrape({ id: 'e10s12', password: '3f3ss3d', num: '1234' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new DiscountScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.discount); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/beyahad-bishvilha.test.ts: -------------------------------------------------------------------------------- 1 | import BeyahadBishvilhaScraper from './beyahad-bishvilha'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'beyahadBishvilha'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Beyahad Bishvilha scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.beyahadBishvilha).toBeDefined(); 18 | expect(SCRAPERS.beyahadBishvilha.loginFields).toContain('id'); 19 | expect(SCRAPERS.beyahadBishvilha.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new BeyahadBishvilhaScraper(options); 29 | 30 | const result = await scraper.scrape({ id: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new BeyahadBishvilhaScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.beyahadBishvilha); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/isracard.test.ts: -------------------------------------------------------------------------------- 1 | import IsracardScraper from './isracard'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'isracard'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('Isracard legacy scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.isracard).toBeDefined(); 18 | expect(SCRAPERS.isracard.loginFields).toContain('id'); 19 | expect(SCRAPERS.isracard.loginFields).toContain('card6Digits'); 20 | expect(SCRAPERS.isracard.loginFields).toContain('password'); 21 | }); 22 | 23 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new IsracardScraper(options); 30 | 31 | const result = await scraper.scrape({ id: 'e10s12', password: '3f3ss3d', card6Digits: '123456' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new IsracardScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.isracard); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/visa-cal.test.ts: -------------------------------------------------------------------------------- 1 | import VisaCalScraper from './visa-cal'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { LoginResults } from './base-scraper-with-browser'; 7 | 8 | const COMPANY_ID = 'visaCal'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('VisaCal legacy scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.visaCal).toBeDefined(); 18 | expect(SCRAPERS.visaCal.loginFields).toContain('username'); 19 | expect(SCRAPERS.visaCal.loginFields).toContain('password'); 20 | }); 21 | 22 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new VisaCalScraper(options); 29 | 30 | const result = await scraper.scrape({ username: '971sddksmsl', password: '3f3ssdkSD3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new VisaCalScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.visaCal); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | // uncomment to test multiple accounts 50 | // expect(result?.accounts?.length).toEqual(2) 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/base-scraper-with-browser.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | extendAsyncTimeout, getTestsConfig, 3 | } from '../tests/tests-utils'; 4 | import { BaseScraperWithBrowser } from './base-scraper-with-browser'; 5 | 6 | const testsConfig = getTestsConfig(); 7 | 8 | function isNoSandbox(browser: any) { 9 | // eslint-disable-next-line no-underscore-dangle 10 | const args = browser._process.spawnargs; 11 | return args.includes('--no-sandbox'); 12 | } 13 | 14 | describe('Base scraper with browser', () => { 15 | beforeAll(() => { 16 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 17 | }); 18 | 19 | xtest('should pass custom args to scraper if provided', async () => { 20 | const options = { 21 | ...testsConfig.options, 22 | companyId: 'test', 23 | showBrowser: false, 24 | args: [], 25 | }; 26 | 27 | // avoid false-positive result by confirming that --no-sandbox is not a default flag provided by puppeteer 28 | let baseScraperWithBrowser = new BaseScraperWithBrowser(options); 29 | try { 30 | await baseScraperWithBrowser.initialize(); 31 | // @ts-ignore 32 | expect(baseScraperWithBrowser.browser).toBeDefined(); 33 | // @ts-ignore 34 | expect(isNoSandbox(baseScraperWithBrowser.browser)).toBe(false); 35 | await baseScraperWithBrowser.terminate(true); 36 | } catch (e) { 37 | await baseScraperWithBrowser.terminate(false); 38 | throw e; 39 | } 40 | 41 | // set --no-sandbox flag and expect it to be passed by puppeteer.lunch to the new created browser instance 42 | options.args = [ 43 | '--no-sandbox', 44 | '--disable-gpu', 45 | '--window-size=1920x1080', 46 | ]; 47 | baseScraperWithBrowser = new BaseScraperWithBrowser(options); 48 | try { 49 | await baseScraperWithBrowser.initialize(); 50 | // @ts-ignore 51 | expect(baseScraperWithBrowser.browser).toBeDefined(); 52 | // @ts-ignore 53 | expect(isNoSandbox(baseScraperWithBrowser.browser)).toBe(true); 54 | await baseScraperWithBrowser.terminate(true); 55 | } catch (e) { 56 | await baseScraperWithBrowser.terminate(false); 57 | throw e; 58 | } 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - master 6 | - next 7 | jobs: 8 | release: 9 | name: Release 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v1 14 | - name: Setup Node.js 15 | uses: actions/setup-node@v1 16 | with: 17 | node-version: 14 18 | registry-url: https://registry.npmjs.org/ 19 | - name: prepare default version 20 | run: npm run prepare:default 21 | - name: Release to Github 22 | uses: cycjimmy/semantic-release-action@v2 23 | id: semantic # Need an `id` for output variables 24 | with: 25 | semantic_version: 16.0.2 26 | branches: | 27 | [ 28 | '+([0-9])?(.{+([0-9]),x}).x', 29 | 'master', 30 | 'next', 31 | 'next-major', 32 | { 33 | name: 'hotfix', 34 | prerelease: true 35 | }, 36 | { 37 | name: 'beta', 38 | prerelease: true 39 | }, 40 | { 41 | name: 'alpha', 42 | prerelease: true 43 | } 44 | ] 45 | env: 46 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 47 | - name: Prepare release of default to NPM 48 | if: steps.semantic.outputs.new_release_published == 'true' 49 | run: | 50 | node utils/pre-publish.js --version ${{ steps.semantic.outputs.new_release_version }} 51 | - name: Publish to NPM 52 | if: steps.semantic.outputs.new_release_published == 'true' 53 | run: npm publish --access public --ignore-scripts 54 | env: 55 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 56 | - name: Prepare release of core to NPM 57 | if: steps.semantic.outputs.new_release_published == 'true' 58 | run: | 59 | npm run prepare:core 60 | node utils/pre-publish.js --version ${{ steps.semantic.outputs.new_release_version }} 61 | - name: Publish to NPM 62 | if: steps.semantic.outputs.new_release_published == 'true' 63 | run: npm publish --access public --ignore-scripts 64 | env: 65 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 66 | -------------------------------------------------------------------------------- /src/scrapers/one-zero.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 3 | } from '../tests/tests-utils'; 4 | import { SCRAPERS } from '../definitions'; 5 | import { LoginResults } from './base-scraper-with-browser'; 6 | import OneZeroScraper from './one-zero'; 7 | 8 | const COMPANY_ID = 'oneZero'; // TODO this property should be hard-coded in the provider 9 | const testsConfig = getTestsConfig(); 10 | 11 | describe('OneZero scraper', () => { 12 | beforeAll(() => { 13 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 14 | }); 15 | 16 | test('should expose login fields in scrapers constant', () => { 17 | expect(SCRAPERS.oneZero).toBeDefined(); 18 | expect(SCRAPERS.oneZero.loginFields).toContain('email'); 19 | expect(SCRAPERS.oneZero.loginFields).toContain('password'); 20 | expect(SCRAPERS.oneZero.loginFields).toContain('otpCodeRetriever'); 21 | expect(SCRAPERS.oneZero.loginFields).toContain('phoneNumber'); 22 | expect(SCRAPERS.oneZero.loginFields).toContain('otpLongTermToken'); 23 | }); 24 | 25 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password"', async () => { 26 | const options = { 27 | ...testsConfig.options, 28 | companyId: COMPANY_ID, 29 | }; 30 | 31 | const scraper = new OneZeroScraper(options); 32 | 33 | const result = await scraper.scrape({ email: 'e10s12@gmail.com', password: '3f3ss3d', otpLongTermToken: '11111' }); 34 | 35 | expect(result).toBeDefined(); 36 | expect(result.success).toBeFalsy(); 37 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 38 | }); 39 | 40 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 41 | const options = { 42 | ...testsConfig.options, 43 | companyId: COMPANY_ID, 44 | }; 45 | 46 | const scraper = new OneZeroScraper(options); 47 | const result = await scraper.scrape(testsConfig.credentials.oneZero); 48 | expect(result).toBeDefined(); 49 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 50 | expect(error).toBe(''); 51 | expect(result.success).toBeTruthy(); 52 | 53 | exportTransactions(COMPANY_ID, result.accounts || []); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /src/scrapers/mizrahi.test.ts: -------------------------------------------------------------------------------- 1 | import MizrahiScraper from './mizrahi'; 2 | import { 3 | maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions, 4 | } from '../tests/tests-utils'; 5 | import { SCRAPERS } from '../definitions'; 6 | import { ISO_DATE_REGEX } from '../constants'; 7 | import { LoginResults } from './base-scraper-with-browser'; 8 | import { TransactionsAccount } from '../transactions'; 9 | 10 | const COMPANY_ID = 'mizrahi'; // TODO this property should be hard-coded in the provider 11 | const testsConfig = getTestsConfig(); 12 | 13 | describe('Mizrahi scraper', () => { 14 | beforeAll(() => { 15 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 16 | }); 17 | 18 | test('should expose login fields in scrapers constant', () => { 19 | expect(SCRAPERS.mizrahi).toBeDefined(); 20 | expect(SCRAPERS.mizrahi.loginFields).toContain('username'); 21 | expect(SCRAPERS.mizrahi.loginFields).toContain('password'); 22 | }); 23 | 24 | maybeTestCompanyAPI(COMPANY_ID, (config) => config.companyAPI.invalidPassword)('should fail on invalid user/password', async () => { 25 | const options = { 26 | ...testsConfig.options, 27 | companyId: COMPANY_ID, 28 | }; 29 | 30 | const scraper = new MizrahiScraper(options); 31 | 32 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 33 | 34 | expect(result).toBeDefined(); 35 | expect(result.success).toBeFalsy(); 36 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 37 | }); 38 | 39 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions', async () => { 40 | const options = { 41 | ...testsConfig.options, 42 | companyId: COMPANY_ID, 43 | }; 44 | 45 | const scraper = new MizrahiScraper(options); 46 | const result = await scraper.scrape(testsConfig.credentials.mizrahi); 47 | expect(result).toBeDefined(); 48 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 49 | expect(error).toBe(''); 50 | expect(result.success).toBeTruthy(); 51 | expect(result.accounts).toBeDefined(); 52 | expect((result.accounts as any).length).toBeGreaterThan(0); 53 | const account: TransactionsAccount = (result as any).accounts[0]; 54 | expect(account.accountNumber).not.toBe(''); 55 | expect(account.txns[0].date).toMatch(ISO_DATE_REGEX); 56 | 57 | exportTransactions(COMPANY_ID, result.accounts || []); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /src/scrapers/factory.ts: -------------------------------------------------------------------------------- 1 | import HapoalimScraper from './hapoalim'; 2 | import OtsarHahayalScraper from './otsar-hahayal'; 3 | import LeumiScraper from './leumi'; 4 | import DiscountScraper from './discount'; 5 | import MaxScraper from './max'; 6 | import VisaCalScraper from './visa-cal'; 7 | import IsracardScraper from './isracard'; 8 | import AmexScraper from './amex'; 9 | import MizrahiScraper from './mizrahi'; 10 | import UnionBankScraper from './union-bank'; 11 | import BeinleumiScraper from './beinleumi'; 12 | import MassadScraper from './massad'; 13 | import YahavScraper from './yahav'; 14 | import { Scraper, ScraperCredentials, ScraperOptions } from './interface'; 15 | import { CompanyTypes } from '../definitions'; 16 | import BeyahadBishvilhaScraper from './beyahad-bishvilha'; 17 | import OneZeroScraper from './one-zero'; 18 | 19 | export default function createScraper(options: ScraperOptions): Scraper { 20 | switch (options.companyId) { 21 | case CompanyTypes.hapoalim: 22 | return new HapoalimScraper(options); 23 | case CompanyTypes.hapoalimBeOnline: 24 | // eslint-disable-next-line no-console 25 | console.warn("hapoalimBeOnline is deprecated, use 'hapoalim' instead"); 26 | return new HapoalimScraper(options); 27 | case CompanyTypes.leumi: 28 | return new LeumiScraper(options); 29 | case CompanyTypes.beyahadBishvilha: 30 | return new BeyahadBishvilhaScraper(options); 31 | case CompanyTypes.mizrahi: 32 | return new MizrahiScraper(options); 33 | case CompanyTypes.discount: 34 | return new DiscountScraper(options); 35 | case CompanyTypes.otsarHahayal: 36 | return new OtsarHahayalScraper(options); 37 | case CompanyTypes.visaCal: 38 | return new VisaCalScraper(options); 39 | case CompanyTypes.leumiCard: 40 | // eslint-disable-next-line no-console 41 | console.warn("leumiCard is deprecated, use 'max' instead"); 42 | return new MaxScraper(options); 43 | case CompanyTypes.max: 44 | return new MaxScraper(options); 45 | case CompanyTypes.isracard: 46 | return new IsracardScraper(options); 47 | case CompanyTypes.amex: 48 | return new AmexScraper(options); 49 | case CompanyTypes.union: 50 | return new UnionBankScraper(options); 51 | case CompanyTypes.beinleumi: 52 | return new BeinleumiScraper(options); 53 | case CompanyTypes.massad: 54 | return new MassadScraper(options); 55 | case CompanyTypes.yahav: 56 | return new YahavScraper(options); 57 | case CompanyTypes.oneZero: 58 | return new OneZeroScraper(options); 59 | default: 60 | throw new Error(`unknown company id ${options.companyId}`); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/tests/tests-utils.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import path from 'path'; 3 | import moment from 'moment'; 4 | import * as json2csv from 'json2csv'; 5 | import { TransactionsAccount } from '../transactions'; 6 | 7 | let testsConfig: Record; 8 | let configurationLoaded = false; 9 | 10 | const MISSING_ERROR_MESSAGE = 'Missing test environment configuration. To troubleshoot this issue open CONTRIBUTING.md file and read the "F.A.Q regarding the tests" section.'; 11 | 12 | export function getTestsConfig() { 13 | if (configurationLoaded) { 14 | if (!testsConfig) { 15 | throw new Error(MISSING_ERROR_MESSAGE); 16 | } 17 | 18 | return testsConfig; 19 | } 20 | 21 | configurationLoaded = true; 22 | 23 | try { 24 | const environmentConfig = process.env.TESTS_CONFIG; 25 | if (environmentConfig) { 26 | testsConfig = JSON.parse(environmentConfig); 27 | return testsConfig; 28 | } 29 | } catch (e) { 30 | throw new Error(`failed to parse environment variable 'TESTS_CONFIG' with error '${e.message}'`); 31 | } 32 | 33 | try { 34 | const configPath = path.join(__dirname, '.tests-config.js'); 35 | testsConfig = require(configPath); 36 | return testsConfig; 37 | } catch (e) { 38 | console.error(e); 39 | throw new Error(MISSING_ERROR_MESSAGE); 40 | } 41 | } 42 | 43 | export function maybeTestCompanyAPI(scraperId: string, filter?: (config: any) => boolean) { 44 | if (!configurationLoaded) { 45 | getTestsConfig(); 46 | } 47 | return testsConfig && testsConfig.companyAPI.enabled && 48 | testsConfig.credentials[scraperId] && 49 | (!filter || filter(testsConfig)) ? test : test.skip; 50 | } 51 | 52 | export function extendAsyncTimeout(timeout = 120000) { 53 | jest.setTimeout(timeout); 54 | } 55 | 56 | export function exportTransactions(fileName: string, accounts: TransactionsAccount[]) { 57 | const config = getTestsConfig(); 58 | 59 | if (!config.companyAPI.enabled || 60 | !config.companyAPI.excelFilesDist || 61 | !fs.existsSync(config.companyAPI.excelFilesDist)) { 62 | return; 63 | } 64 | 65 | let data: any = []; 66 | 67 | for (let i = 0; i < accounts.length; i += 1) { 68 | const account = accounts[i]; 69 | 70 | data = [ 71 | ...data, 72 | ...account.txns.map((txn) => { 73 | return { 74 | account: account.accountNumber, 75 | balance: `account balance: ${account.balance}`, 76 | ...txn, 77 | date: moment(txn.date).format('DD/MM/YYYY'), 78 | processedDate: moment(txn.processedDate).format('DD/MM/YYYY'), 79 | }; 80 | })]; 81 | } 82 | 83 | if (data.length === 0) { 84 | data = [ 85 | { 86 | comment: 'no transaction found for requested time frame', 87 | }, 88 | ]; 89 | } 90 | 91 | const csv = json2csv.parse(data, { withBOM: true }); 92 | const filePath = `${path.join(config.companyAPI.excelFilesDist, fileName)}.csv`; 93 | fs.writeFileSync(filePath, csv); 94 | } 95 | -------------------------------------------------------------------------------- /src/helpers/fetch.ts: -------------------------------------------------------------------------------- 1 | import nodeFetch from 'node-fetch'; 2 | import { Page } from 'puppeteer'; 3 | 4 | const JSON_CONTENT_TYPE = 'application/json'; 5 | 6 | function getJsonHeaders() { 7 | return { 8 | Accept: JSON_CONTENT_TYPE, 9 | 'Content-Type': JSON_CONTENT_TYPE, 10 | }; 11 | } 12 | 13 | export async function fetchGet(url: string, 14 | extraHeaders: Record): Promise { 15 | let headers = getJsonHeaders(); 16 | if (extraHeaders) { 17 | headers = Object.assign(headers, extraHeaders); 18 | } 19 | const request = { 20 | method: 'GET', 21 | headers, 22 | }; 23 | const fetchResult = await nodeFetch(url, request); 24 | 25 | if (fetchResult.status !== 200) { 26 | throw new Error(`sending a request to the institute server returned with status code ${fetchResult.status}`); 27 | } 28 | 29 | return fetchResult.json(); 30 | } 31 | 32 | export async function fetchPost(url: string, data: Record, 33 | extraHeaders: Record = {}) { 34 | const request = { 35 | method: 'POST', 36 | headers: { ...getJsonHeaders(), ...extraHeaders }, 37 | body: JSON.stringify(data), 38 | }; 39 | const result = await nodeFetch(url, request); 40 | return result.json(); 41 | } 42 | 43 | export async function fetchGraphql(url: string, query: string, 44 | variables: Record = {}, 45 | extraHeaders: Record = {}): Promise { 46 | const result = await fetchPost(url, { operationName: null, query, variables }, extraHeaders); 47 | if (result.errors?.length) { 48 | throw new Error(result.errors[0].message); 49 | } 50 | return result.data as Promise; 51 | } 52 | 53 | export function fetchGetWithinPage(page: Page, url: string): Promise { 54 | return page.evaluate((url) => { 55 | return new Promise((resolve, reject) => { 56 | fetch(url, { 57 | credentials: 'include', 58 | }).then((result) => { 59 | if (result.status === 204) { 60 | resolve(null); 61 | } else { 62 | resolve(result.json()); 63 | } 64 | }).catch((e) => { 65 | reject(e); 66 | }); 67 | }); 68 | }, url); 69 | } 70 | 71 | export function fetchPostWithinPage(page: Page, url: string, 72 | data: Record, extraHeaders: Record = {}): Promise { 73 | return page.evaluate<(...args: any[]) => Promise>((url: string, data: Record, 74 | extraHeaders: Record) => { 75 | return new Promise((resolve, reject) => { 76 | fetch(url, { 77 | method: 'POST', 78 | body: JSON.stringify(data), 79 | credentials: 'include', 80 | // eslint-disable-next-line prefer-object-spread 81 | headers: Object.assign({ 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' }, extraHeaders), 82 | }).then((result) => { 83 | if (result.status === 204) { 84 | // No content response 85 | resolve(null); 86 | } else { 87 | resolve(result.json()); 88 | } 89 | }).catch((e) => { 90 | reject(e); 91 | }); 92 | }); 93 | }, url, data, extraHeaders); 94 | } 95 | -------------------------------------------------------------------------------- /src/definitions.ts: -------------------------------------------------------------------------------- 1 | // NOTICE: avoid changing exported keys as they are part of the public api 2 | 3 | export const PASSWORD_FIELD = 'password'; 4 | 5 | export enum CompanyTypes { 6 | hapoalim = 'hapoalim', 7 | hapoalimBeOnline = 'hapoalimBeOnline', 8 | beinleumi = 'beinleumi', 9 | union = 'union', 10 | amex = 'amex', 11 | isracard = 'isracard', 12 | visaCal = 'visaCal', 13 | max = 'max', 14 | leumiCard = 'leumiCard', 15 | otsarHahayal = 'otsarHahayal', 16 | discount = 'discount', 17 | mizrahi = 'mizrahi', 18 | leumi = 'leumi', 19 | massad = 'massad', 20 | yahav = 'yahav', 21 | beyahadBishvilha = 'beyahadBishvilha', 22 | oneZero = 'oneZero' 23 | } 24 | 25 | export const SCRAPERS = { 26 | [CompanyTypes.hapoalim]: { 27 | name: 'Bank Hapoalim', 28 | loginFields: ['userCode', PASSWORD_FIELD], 29 | }, 30 | [CompanyTypes.hapoalimBeOnline]: { // TODO remove in Major version 31 | name: 'Bank Hapoalim', 32 | loginFields: ['userCode', PASSWORD_FIELD], 33 | }, 34 | [CompanyTypes.leumi]: { 35 | name: 'Bank Leumi', 36 | loginFields: ['username', PASSWORD_FIELD], 37 | }, 38 | [CompanyTypes.mizrahi]: { 39 | name: 'Mizrahi Bank', 40 | loginFields: ['username', PASSWORD_FIELD], 41 | }, 42 | [CompanyTypes.discount]: { 43 | name: 'Discount Bank', 44 | loginFields: ['id', PASSWORD_FIELD, 'num'], 45 | }, 46 | [CompanyTypes.otsarHahayal]: { 47 | name: 'Bank Otsar Hahayal', 48 | loginFields: ['username', PASSWORD_FIELD], 49 | }, 50 | [CompanyTypes.leumiCard]: { // TODO remove in Major version 51 | name: 'Leumi Card', 52 | loginFields: ['username', PASSWORD_FIELD], 53 | }, 54 | [CompanyTypes.max]: { 55 | name: 'Max', 56 | loginFields: ['username', PASSWORD_FIELD], 57 | }, 58 | [CompanyTypes.visaCal]: { 59 | name: 'Visa Cal', 60 | loginFields: ['username', PASSWORD_FIELD], 61 | }, 62 | [CompanyTypes.isracard]: { 63 | name: 'Isracard', 64 | loginFields: ['id', 'card6Digits', PASSWORD_FIELD], 65 | }, 66 | [CompanyTypes.amex]: { 67 | name: 'Amex', 68 | loginFields: ['id', 'card6Digits', PASSWORD_FIELD], 69 | }, 70 | [CompanyTypes.union]: { 71 | name: 'Union', 72 | loginFields: ['username', PASSWORD_FIELD], 73 | }, 74 | [CompanyTypes.beinleumi]: { 75 | name: 'Beinleumi', 76 | loginFields: ['username', PASSWORD_FIELD], 77 | }, 78 | [CompanyTypes.massad]: { 79 | name: 'Massad', 80 | loginFields: ['username', PASSWORD_FIELD], 81 | }, 82 | [CompanyTypes.yahav]: { 83 | name: 'Bank Yahav', 84 | loginFields: ['username', 'nationalID', PASSWORD_FIELD], 85 | }, 86 | [CompanyTypes.beyahadBishvilha]: { 87 | name: 'Beyahad Bishvilha', 88 | loginFields: ['id', PASSWORD_FIELD], 89 | }, 90 | [CompanyTypes.oneZero]: { 91 | name: 'One Zero', 92 | loginFields: ['email', PASSWORD_FIELD, 'otpCodeRetriever', 'phoneNumber', 'otpLongTermToken'], 93 | }, 94 | }; 95 | 96 | export enum ScraperProgressTypes { 97 | Initializing = 'INITIALIZING', 98 | StartScraping = 'START_SCRAPING', 99 | LoggingIn = 'LOGGING_IN', 100 | LoginSuccess = 'LOGIN_SUCCESS', 101 | LoginFailed = 'LOGIN_FAILED', 102 | ChangePassword = 'CHANGE_PASSWORD', 103 | EndScraping = 'END_SCRAPING', 104 | Terminating = 'TERMINATING', 105 | } 106 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "israeli-bank-scrapers", 3 | "version": "1.0.4", 4 | "private": true, 5 | "description": "Provide scrapers for all major Israeli banks and credit card companies", 6 | "engines": { 7 | "node": ">= 14.14.0", 8 | "npm": ">= 6.14.8" 9 | }, 10 | "main": "lib/index.js", 11 | "types": "lib/index.d.ts", 12 | "scripts": { 13 | "clean": "rimraf lib", 14 | "test": "cross-env BABEL_ENV=test jest", 15 | "test:ci": "ncp src/tests/.tests-config.tpl.js src/tests/.tests-config.js && npm run test", 16 | "lint": "eslint src --ext .ts", 17 | "type-check": "tsc --noEmit", 18 | "dev": "npm run type-check -- --watch", 19 | "build": "npm run lint && npm run clean && npm run build:types && npm run build:js && npm run build:puppeteer-config", 20 | "build:types": "tsc --emitDeclarationOnly", 21 | "build:puppeteer-config": "ncp src/puppeteer-config.json lib/puppeteer-config.json", 22 | "build:js": "babel src --out-dir lib --extensions \".ts\" --source-maps inline --verbose", 23 | "prebuild": "node utils/update-puppeteer-config.js", 24 | "postbuild": "rimraf lib/tests", 25 | "prepare:core": "git reset --hard && node utils/prepare-israeli-bank-scrapers-core.js && npm i --package-lock-only && npm ci && npm run lint -- --fix && npm run build", 26 | "prepare:default": "git reset --hard && npm ci && npm run build", 27 | "reset": "git reset --hard && npm ci" 28 | }, 29 | "husky": { 30 | "hooks": { 31 | "pre-commit": "npm run lint" 32 | } 33 | }, 34 | "repository": { 35 | "type": "git", 36 | "url": "git+https://github.com/eshaham/israeli-bank-scrapers.git" 37 | }, 38 | "keywords": [ 39 | "israel", 40 | "israeli bank", 41 | "israeli bank scraper" 42 | ], 43 | "author": "Elad Shaham", 44 | "license": "MIT", 45 | "bugs": { 46 | "url": "https://github.com/eshaham/israeli-bank-scrapers/issues" 47 | }, 48 | "homepage": "https://github.com/eshaham/israeli-bank-scrapers#readme", 49 | "devDependencies": { 50 | "@babel/cli": "^7.4.4", 51 | "@babel/core": "^7.4.5", 52 | "@babel/plugin-proposal-class-properties": "^7.12.1", 53 | "@babel/preset-env": "^7.4.5", 54 | "@babel/preset-typescript": "^7.9.0", 55 | "@types/debug": "^4.1.7", 56 | "@types/jest": "^25.1.5", 57 | "@types/json2csv": "^5.0.1", 58 | "@types/lodash": "^4.14.149", 59 | "@types/node-fetch": "^2.5.6", 60 | "@types/puppeteer": "^5.4.6", 61 | "@types/source-map-support": "^0.5.1", 62 | "@types/uuid": "^7.0.3", 63 | "@typescript-eslint/eslint-plugin": "^2.27.0", 64 | "@typescript-eslint/parser": "^2.26.0", 65 | "babel-jest": "^25.3.0", 66 | "cross-env": "^6.0.3", 67 | "eslint": "^6.6.0", 68 | "eslint-config-airbnb-base": "^14.0.0", 69 | "eslint-config-airbnb-typescript": "^7.2.1", 70 | "eslint-plugin-import": "^2.20.2", 71 | "fs-extra": "^10.0.0", 72 | "jest": "^25.3.0", 73 | "jscodeshift": "^0.11.0", 74 | "minimist": "^1.2.5", 75 | "ncp": "^2.0.0", 76 | "rimraf": "^3.0.0", 77 | "source-map-support": "^0.5.16", 78 | "ts-jest": "^25.3.0", 79 | "typescript": "^3.9.10" 80 | }, 81 | "dependencies": { 82 | "build-url": "^2.0.0", 83 | "core-js": "^3.1.4", 84 | "debug": "^4.3.2", 85 | "husky": "^4.2.5", 86 | "json2csv": "^4.5.4", 87 | "lodash": "^4.17.10", 88 | "moment": "^2.22.2", 89 | "moment-timezone": "^0.5.37", 90 | "node-fetch": "^2.2.0", 91 | "puppeteer": "^6.0.0", 92 | "uuid": "^3.3.2" 93 | }, 94 | "files": [ 95 | "lib/**/*" 96 | ] 97 | } 98 | -------------------------------------------------------------------------------- /src/scrapers/base-scraper.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events'; 2 | import moment from 'moment-timezone'; 3 | import { TimeoutError } from '../helpers/waiting'; 4 | import { createGenericError, createTimeoutError } from './errors'; 5 | import { 6 | Scraper, 7 | ScraperCredentials, 8 | ScraperGetLongTermTwoFactorTokenResult, 9 | ScraperLoginResult, 10 | ScraperOptions, 11 | ScraperScrapingResult, 12 | ScraperTwoFactorAuthTriggerResult, 13 | } from './interface'; 14 | import { CompanyTypes, ScraperProgressTypes } from '../definitions'; 15 | 16 | const SCRAPE_PROGRESS = 'SCRAPE_PROGRESS'; 17 | 18 | 19 | export class BaseScraper implements Scraper { 20 | private eventEmitter = new EventEmitter(); 21 | 22 | constructor(public options: ScraperOptions) { 23 | } 24 | 25 | // eslint-disable-next-line @typescript-eslint/require-await 26 | async initialize() { 27 | this.emitProgress(ScraperProgressTypes.Initializing); 28 | moment.tz.setDefault('Asia/Jerusalem'); 29 | } 30 | 31 | async scrape(credentials: TCredentials): Promise { 32 | this.emitProgress(ScraperProgressTypes.StartScraping); 33 | await this.initialize(); 34 | 35 | let loginResult; 36 | try { 37 | loginResult = await this.login(credentials); 38 | } catch (e) { 39 | loginResult = e instanceof TimeoutError ? 40 | createTimeoutError(e.message) : 41 | createGenericError(e.message); 42 | } 43 | 44 | let scrapeResult; 45 | if (loginResult.success) { 46 | try { 47 | scrapeResult = await this.fetchData(); 48 | } catch (e) { 49 | scrapeResult = 50 | e instanceof TimeoutError ? 51 | createTimeoutError(e.message) : 52 | createGenericError(e.message); 53 | } 54 | } else { 55 | scrapeResult = loginResult; 56 | } 57 | 58 | try { 59 | const success = scrapeResult && scrapeResult.success === true; 60 | await this.terminate(success); 61 | } catch (e) { 62 | scrapeResult = createGenericError(e.message); 63 | } 64 | this.emitProgress(ScraperProgressTypes.EndScraping); 65 | 66 | return scrapeResult; 67 | } 68 | 69 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 70 | triggerTwoFactorAuth(_phoneNumber: string): Promise { 71 | throw new Error(`triggerOtp() is not created in ${this.options.companyId}`); 72 | } 73 | 74 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 75 | getLongTermTwoFactorToken(_otpCode: string): Promise { 76 | throw new Error(`getPermanentOtpToken() is not created in ${this.options.companyId}`); 77 | } 78 | 79 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 80 | protected async login(_credentials: TCredentials): Promise { 81 | throw new Error(`login() is not created in ${this.options.companyId}`); 82 | } 83 | 84 | // eslint-disable-next-line @typescript-eslint/require-await 85 | protected async fetchData(): Promise { 86 | throw new Error(`fetchData() is not created in ${this.options.companyId}`); 87 | } 88 | 89 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 90 | protected async terminate(_success: boolean) { 91 | this.emitProgress(ScraperProgressTypes.Terminating); 92 | } 93 | 94 | protected emitProgress(type: ScraperProgressTypes) { 95 | this.emit(SCRAPE_PROGRESS, { type }); 96 | } 97 | 98 | protected emit(eventName: string, payload: Record) { 99 | this.eventEmitter.emit(eventName, this.options.companyId, payload); 100 | } 101 | 102 | onProgress(func: (companyId: CompanyTypes, payload: {type: ScraperProgressTypes}) => void) { 103 | this.eventEmitter.on(SCRAPE_PROGRESS, func); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/helpers/elements-interactions.ts: -------------------------------------------------------------------------------- 1 | import { Frame, Page } from 'puppeteer'; 2 | import { waitUntil } from './waiting'; 3 | 4 | async function waitUntilElementFound(page: Page | Frame, elementSelector: string, 5 | onlyVisible = false, timeout?: number) { 6 | await page.waitForSelector(elementSelector, { visible: onlyVisible, timeout }); 7 | } 8 | 9 | async function waitUntilElementDisappear(page: Page, elementSelector: string, timeout?: number) { 10 | await page.waitForSelector(elementSelector, { hidden: true, timeout }); 11 | } 12 | 13 | async function waitUntilIframeFound(page: Page, framePredicate: (frame: Frame) => boolean, description = '', timeout = 30000) { 14 | let frame: Frame | undefined; 15 | await waitUntil(() => { 16 | frame = page 17 | .frames() 18 | .find(framePredicate); 19 | return Promise.resolve(!!frame); 20 | }, description, timeout, 1000); 21 | 22 | if (!frame) { 23 | throw new Error('failed to find iframe'); 24 | } 25 | 26 | return frame; 27 | } 28 | 29 | async function fillInput(pageOrFrame: Page | Frame, inputSelector: string, inputValue: string): Promise { 30 | await pageOrFrame.$eval(inputSelector, (input: Element) => { 31 | const inputElement = input; 32 | // @ts-ignore 33 | inputElement.value = ''; 34 | }); 35 | await pageOrFrame.type(inputSelector, inputValue); 36 | } 37 | 38 | async function setValue(pageOrFrame: Page | Frame, inputSelector: string, inputValue: string): Promise { 39 | await pageOrFrame.$eval(inputSelector, (input: Element, inputValue) => { 40 | const inputElement = input; 41 | // @ts-ignore 42 | inputElement.value = inputValue; 43 | }, [inputValue]); 44 | } 45 | 46 | async function clickButton(page: Page | Frame, buttonSelector: string) { 47 | await page.$eval(buttonSelector, (el) => (el as HTMLElement).click()); 48 | } 49 | 50 | async function clickLink(page: Page, aSelector: string) { 51 | await page.$eval(aSelector, (el: any) => { 52 | if (!el || typeof el.click === 'undefined') { 53 | return; 54 | } 55 | 56 | el.click(); 57 | }); 58 | } 59 | 60 | async function pageEvalAll(page: Page | Frame, selector: string, 61 | defaultResult: any, callback: (elements: Element[], ...args: any) => R, ...args: any[]): Promise { 62 | let result = defaultResult; 63 | try { 64 | result = await page.$$eval(selector, callback, ...args); 65 | } catch (e) { 66 | // TODO temporary workaround to puppeteer@1.5.0 which breaks $$eval bevahvior until they will release a new version. 67 | if (e.message.indexOf('Error: failed to find elements matching selector') !== 0) { 68 | throw e; 69 | } 70 | } 71 | 72 | return result; 73 | } 74 | 75 | async function pageEval(pageOrFrame: Page | Frame, selector: string, 76 | defaultResult: any, callback: (elements: Element, ...args: any) => R, ...args: any[]): Promise { 77 | let result = defaultResult; 78 | try { 79 | result = await pageOrFrame.$eval(selector, callback, ...args); 80 | } catch (e) { 81 | // TODO temporary workaround to puppeteer@1.5.0 which breaks $$eval bevahvior until they will release a new version. 82 | if (e.message.indexOf('Error: failed to find element matching selector') !== 0) { 83 | throw e; 84 | } 85 | } 86 | 87 | return result; 88 | } 89 | 90 | async function elementPresentOnPage(pageOrFrame: Page | Frame, selector: string) { 91 | return await pageOrFrame.$(selector) !== null; 92 | } 93 | 94 | async function dropdownSelect(page: Page, selectSelector: string, value: string) { 95 | await page.select(selectSelector, value); 96 | } 97 | 98 | async function dropdownElements(page: Page, selector: string) { 99 | const options = await page.evaluate((optionSelector) => { 100 | return Array.from(document.querySelectorAll(optionSelector)) 101 | .filter((o) => o.value) 102 | .map((o) => { 103 | return { 104 | name: o.text, 105 | value: o.value, 106 | }; 107 | }); 108 | }, `${selector} > option`); 109 | return options; 110 | } 111 | 112 | export { 113 | waitUntilElementFound, 114 | waitUntilElementDisappear, 115 | waitUntilIframeFound, 116 | fillInput, 117 | clickButton, 118 | clickLink, 119 | dropdownSelect, 120 | dropdownElements, 121 | pageEval, 122 | pageEvalAll, 123 | elementPresentOnPage, 124 | setValue, 125 | }; 126 | -------------------------------------------------------------------------------- /src/scrapers/interface.ts: -------------------------------------------------------------------------------- 1 | import { Browser, Page } from 'puppeteer'; 2 | import { CompanyTypes, ScraperProgressTypes } from '../definitions'; 3 | import { TransactionsAccount } from '../transactions'; 4 | import { ErrorResult, ScraperErrorTypes } from './errors'; 5 | 6 | // TODO: Remove this type when the scraper 'factory' will return concrete scraper types 7 | // Instead of a generic interface (which in turn uses this type) 8 | export type ScraperCredentials = 9 | {userCode: string, password: string} | 10 | {username: string, password: string} | 11 | {id: string, password: string} | 12 | {id: string, password: string, num: string} | 13 | {id: string, password: string, card6Digits: string} | 14 | {username: string, nationalID: string, password: string} | 15 | ({email: string, password: string} & ({ 16 | otpCodeRetriever: () => Promise; 17 | phoneNumber: string; 18 | } | { 19 | otpLongTermToken: string; 20 | })); 21 | 22 | export interface FutureDebit { 23 | amount: number; 24 | amountCurrency: string; 25 | chargeDate?: string; 26 | bankAccountNumber?: string; 27 | } 28 | 29 | export interface ScraperOptions { 30 | /** 31 | * The company you want to scrape 32 | */ 33 | companyId: CompanyTypes; 34 | 35 | /** 36 | * include more debug info about in the output 37 | */ 38 | verbose?: boolean; 39 | 40 | /** 41 | * the date to fetch transactions from (can't be before the minimum allowed time difference for the scraper) 42 | */ 43 | startDate: Date; 44 | 45 | /** 46 | * shows the browser while scraping, good for debugging (default false) 47 | */ 48 | showBrowser?: boolean; 49 | 50 | 51 | /** 52 | * scrape transactions to be processed X months in the future 53 | */ 54 | futureMonthsToScrape?: number; 55 | 56 | /** 57 | * option from init puppeteer browser instance outside the libary scope. you can get 58 | * browser diretly from puppeteer via `puppeteer.launch()` 59 | */ 60 | browser?: any; 61 | 62 | /** 63 | * provide a patch to local chromium to be used by puppeteer. Relevant when using 64 | * `israeli-bank-scrapers-core` library 65 | */ 66 | executablePath?: string; 67 | 68 | /** 69 | * if set to true, all installment transactions will be combine into the first one 70 | */ 71 | combineInstallments?: boolean; 72 | 73 | /** 74 | * additional arguments to pass to the browser instance. The list of flags can be found in 75 | * 76 | * https://developer.mozilla.org/en-US/docs/Mozilla/Command_Line_Options 77 | * https://peter.sh/experiments/chromium-command-line-switches/ 78 | */ 79 | args?: string[]; 80 | 81 | /** 82 | * Maximum navigation time in milliseconds, pass 0 to disable timeout. 83 | * @default 30000 84 | */ 85 | timeout?: number | undefined; 86 | 87 | /** 88 | * adjust the browser instance before it is being used 89 | * 90 | * @param browser 91 | */ 92 | prepareBrowser?: (browser: Browser) => Promise; 93 | 94 | /** 95 | * adjust the page instance before it is being used. 96 | * 97 | * @param page 98 | */ 99 | preparePage?: (page: Page) => Promise; 100 | 101 | /** 102 | * if set, store a screenshot if failed to scrape. Used for debug purposes 103 | */ 104 | storeFailureScreenShotPath?: string; 105 | 106 | /** 107 | * if set, will set the timeout in milliseconds of puppeteer's `page.setDefaultTimeout`. 108 | */ 109 | defaultTimeout?: number; 110 | 111 | /** 112 | * Options for manipulation of output data 113 | */ 114 | outputData?: OutputDataOptions; 115 | 116 | /** 117 | * Perform additional operation for each transaction to get more information (Like category) about it. 118 | * Please note: It will take more time to finish the process. 119 | */ 120 | additionalTransactionInformation?: boolean; 121 | } 122 | 123 | export interface OutputDataOptions { 124 | /** 125 | * if true, the result wouldn't be filtered out by date, and you will return unfiltered scrapped data. 126 | */ 127 | enableTransactionsFilterByDate?: boolean; 128 | } 129 | 130 | export interface ScraperScrapingResult { 131 | success: boolean; 132 | accounts?: TransactionsAccount[]; 133 | futureDebits?: FutureDebit[]; 134 | errorType?: ScraperErrorTypes; 135 | errorMessage?: string; // only on success=false 136 | } 137 | 138 | export interface Scraper { 139 | scrape(credentials: TCredentials): Promise; 140 | onProgress(func: (companyId: CompanyTypes, payload: {type: ScraperProgressTypes}) => void): void; 141 | triggerTwoFactorAuth(phoneNumber: string): Promise; 142 | getLongTermTwoFactorToken(otpCode: string): Promise; 143 | } 144 | 145 | export type ScraperTwoFactorAuthTriggerResult = ErrorResult | { 146 | success: true; 147 | }; 148 | 149 | export type ScraperGetLongTermTwoFactorTokenResult = ErrorResult | { 150 | success: true; 151 | longTermTwoFactorAuthToken: string; 152 | }; 153 | 154 | export interface ScraperLoginResult { 155 | success: boolean; 156 | errorType?: ScraperErrorTypes; 157 | errorMessage?: string; // only on success=false 158 | persistentOtpToken?: string; 159 | } 160 | -------------------------------------------------------------------------------- /src/scrapers/discount.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import moment from 'moment'; 3 | import { Page } from 'puppeteer'; 4 | import { BaseScraperWithBrowser, LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 5 | import { waitUntilElementFound } from '../helpers/elements-interactions'; 6 | import { waitForNavigation } from '../helpers/navigation'; 7 | import { fetchGetWithinPage } from '../helpers/fetch'; 8 | import { 9 | Transaction, TransactionStatuses, TransactionTypes, 10 | } from '../transactions'; 11 | import { ScraperErrorTypes } from './errors'; 12 | import { ScraperScrapingResult, ScraperOptions } from './interface'; 13 | 14 | const BASE_URL = 'https://start.telebank.co.il'; 15 | const DATE_FORMAT = 'YYYYMMDD'; 16 | 17 | interface ScrapedTransaction { 18 | OperationNumber: number; 19 | OperationDate: string; 20 | ValueDate: string; 21 | OperationAmount: number; 22 | OperationDescriptionToDisplay: string; 23 | } 24 | 25 | interface CurrentAccountInfo { 26 | AccountBalance: number; 27 | } 28 | 29 | interface ScrapedAccountData { 30 | UserAccountsData: { 31 | DefaultAccountNumber: string; 32 | }; 33 | } 34 | 35 | interface ScrapedTransactionData { 36 | Error?: { MsgText: string }; 37 | CurrentAccountLastTransactions?: { 38 | OperationEntry: ScrapedTransaction[]; 39 | CurrentAccountInfo: CurrentAccountInfo; 40 | }; 41 | } 42 | 43 | function convertTransactions(txns: ScrapedTransaction[], txnStatus: TransactionStatuses): Transaction[] { 44 | if (!txns) { 45 | return []; 46 | } 47 | return txns.map((txn) => { 48 | return { 49 | type: TransactionTypes.Normal, 50 | identifier: txn.OperationNumber, 51 | date: moment(txn.OperationDate, DATE_FORMAT).toISOString(), 52 | processedDate: moment(txn.ValueDate, DATE_FORMAT).toISOString(), 53 | originalAmount: txn.OperationAmount, 54 | originalCurrency: 'ILS', 55 | chargedAmount: txn.OperationAmount, 56 | description: txn.OperationDescriptionToDisplay, 57 | status: txnStatus, 58 | }; 59 | }); 60 | } 61 | 62 | 63 | async function fetchAccountData(page: Page, options: ScraperOptions): Promise { 64 | const apiSiteUrl = `${BASE_URL}/Titan/gatewayAPI`; 65 | 66 | const accountDataUrl = `${apiSiteUrl}/userAccountsData`; 67 | const accountInfo = await fetchGetWithinPage(page, accountDataUrl); 68 | 69 | if (!accountInfo) { 70 | return { 71 | success: false, 72 | errorType: ScraperErrorTypes.Generic, 73 | errorMessage: 'failed to get account data', 74 | }; 75 | } 76 | const accountNumber = accountInfo.UserAccountsData.DefaultAccountNumber; 77 | 78 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 79 | const startDate = options.startDate || defaultStartMoment.toDate(); 80 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 81 | 82 | const startDateStr = startMoment.format(DATE_FORMAT); 83 | const txnsUrl = `${apiSiteUrl}/lastTransactions/${accountNumber}/Date?IsCategoryDescCode=True&IsTransactionDetails=True&IsEventNames=True&IsFutureTransactionFlag=True&FromDate=${startDateStr}`; 84 | const txnsResult = await fetchGetWithinPage(page, txnsUrl); 85 | if (!txnsResult || txnsResult.Error || 86 | !txnsResult.CurrentAccountLastTransactions) { 87 | return { 88 | success: false, 89 | errorType: ScraperErrorTypes.Generic, 90 | errorMessage: txnsResult && txnsResult.Error ? txnsResult.Error.MsgText : 'unknown error', 91 | }; 92 | } 93 | 94 | const completedTxns = convertTransactions( 95 | txnsResult.CurrentAccountLastTransactions.OperationEntry, 96 | TransactionStatuses.Completed, 97 | ); 98 | const rawFutureTxns = _.get(txnsResult, 'CurrentAccountLastTransactions.FutureTransactionsBlock.FutureTransactionEntry'); 99 | const pendingTxns = convertTransactions(rawFutureTxns, TransactionStatuses.Pending); 100 | 101 | const accountData = { 102 | success: true, 103 | accounts: [{ 104 | accountNumber, 105 | balance: txnsResult.CurrentAccountLastTransactions.CurrentAccountInfo.AccountBalance, 106 | txns: [...completedTxns, ...pendingTxns], 107 | }], 108 | }; 109 | 110 | return accountData; 111 | } 112 | 113 | async function navigateOrErrorLabel(page: Page) { 114 | try { 115 | await waitForNavigation(page); 116 | } catch (e) { 117 | await waitUntilElementFound(page, '#general-error', false, 100); 118 | } 119 | } 120 | 121 | function getPossibleLoginResults(): PossibleLoginResults { 122 | const urls: PossibleLoginResults = {}; 123 | urls[LoginResults.Success] = [`${BASE_URL}/apollo/retail/#/MY_ACCOUNT_HOMEPAGE`]; 124 | urls[LoginResults.InvalidPassword] = [`${BASE_URL}/apollo/core/templates/lobby/masterPage.html#/LOGIN_PAGE`]; 125 | urls[LoginResults.ChangePassword] = [`${BASE_URL}/apollo/core/templates/lobby/masterPage.html#/PWD_RENEW`]; 126 | return urls; 127 | } 128 | 129 | function createLoginFields(credentials: ScraperSpecificCredentials) { 130 | return [ 131 | { selector: '#tzId', value: credentials.id }, 132 | { selector: '#tzPassword', value: credentials.password }, 133 | { selector: '#aidnum', value: credentials.num }, 134 | ]; 135 | } 136 | 137 | type ScraperSpecificCredentials = { id: string, password: string, num: string }; 138 | 139 | class DiscountScraper extends BaseScraperWithBrowser { 140 | getLoginOptions(credentials: ScraperSpecificCredentials) { 141 | return { 142 | loginUrl: `${BASE_URL}/login/#/LOGIN_PAGE`, 143 | checkReadiness: async () => waitUntilElementFound(this.page, '#tzId'), 144 | fields: createLoginFields(credentials), 145 | submitButtonSelector: '.sendBtn', 146 | postAction: async () => navigateOrErrorLabel(this.page), 147 | possibleResults: getPossibleLoginResults(), 148 | }; 149 | } 150 | 151 | async fetchData() { 152 | return fetchAccountData(this.page, this.options); 153 | } 154 | } 155 | 156 | export default DiscountScraper; 157 | -------------------------------------------------------------------------------- /src/scrapers/beyahad-bishvilha.ts: -------------------------------------------------------------------------------- 1 | import { Page } from 'puppeteer'; 2 | import moment from 'moment'; 3 | import { BaseScraperWithBrowser, LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 4 | import { Transaction, TransactionStatuses, TransactionTypes } from '../transactions'; 5 | import { pageEval, pageEvalAll, waitUntilElementFound } from '../helpers/elements-interactions'; 6 | import { getDebug } from '../helpers/debug'; 7 | import { filterOldTransactions } from '../helpers/transactions'; 8 | import { 9 | DOLLAR_CURRENCY, 10 | DOLLAR_CURRENCY_SYMBOL, EURO_CURRENCY, 11 | EURO_CURRENCY_SYMBOL, 12 | SHEKEL_CURRENCY, 13 | SHEKEL_CURRENCY_SYMBOL, 14 | } from '../constants'; 15 | import { ScraperOptions } from './interface'; 16 | 17 | const debug = getDebug('beyahadBishvilha'); 18 | 19 | const DATE_FORMAT = 'DD/MM/YY'; 20 | const LOGIN_URL = 'https://www.hist.org.il/login'; 21 | const SUCCESS_URL = 'https://www.hist.org.il/'; 22 | const CARD_URL = 'https://www.hist.org.il/card/balanceAndUses'; 23 | 24 | interface ScrapedTransaction { 25 | date: string; 26 | description: string; 27 | type: string; 28 | chargedAmount: string; 29 | identifier: string; 30 | } 31 | 32 | function getAmountData(amountStr: string) { 33 | const amountStrCln = amountStr.replace(',', ''); 34 | let currency: string | null = null; 35 | let amount: number | null = null; 36 | if (amountStrCln.includes(SHEKEL_CURRENCY_SYMBOL)) { 37 | amount = parseFloat(amountStrCln.replace(SHEKEL_CURRENCY_SYMBOL, '')); 38 | currency = SHEKEL_CURRENCY; 39 | } else if (amountStrCln.includes(DOLLAR_CURRENCY_SYMBOL)) { 40 | amount = parseFloat(amountStrCln.replace(DOLLAR_CURRENCY_SYMBOL, '')); 41 | currency = DOLLAR_CURRENCY; 42 | } else if (amountStrCln.includes(EURO_CURRENCY_SYMBOL)) { 43 | amount = parseFloat(amountStrCln.replace(EURO_CURRENCY_SYMBOL, '')); 44 | currency = EURO_CURRENCY; 45 | } else { 46 | const parts = amountStrCln.split(' '); 47 | [currency] = parts; 48 | amount = parseFloat(parts[1]); 49 | } 50 | 51 | return { 52 | amount, 53 | currency, 54 | }; 55 | } 56 | 57 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 58 | debug(`convert ${txns.length} raw transactions to official Transaction structure`); 59 | return txns.map((txn) => { 60 | const chargedAmountTuple = getAmountData(txn.chargedAmount || ''); 61 | const txnProcessedDate = moment(txn.date, DATE_FORMAT); 62 | 63 | const result: Transaction = { 64 | type: TransactionTypes.Normal, 65 | status: TransactionStatuses.Completed, 66 | date: txnProcessedDate.toISOString(), 67 | processedDate: txnProcessedDate.toISOString(), 68 | originalAmount: chargedAmountTuple.amount, 69 | originalCurrency: chargedAmountTuple.currency, 70 | chargedAmount: chargedAmountTuple.amount, 71 | chargedCurrency: chargedAmountTuple.currency, 72 | description: txn.description || '', 73 | memo: '', 74 | identifier: txn.identifier, 75 | }; 76 | 77 | return result; 78 | }); 79 | } 80 | 81 | 82 | async function fetchTransactions(page: Page, options: ScraperOptions) { 83 | await page.goto(CARD_URL); 84 | await waitUntilElementFound(page, '.react-loading.hide', false); 85 | const defaultStartMoment = moment().subtract(1, 'years'); 86 | const startDate = options.startDate || defaultStartMoment.toDate(); 87 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 88 | 89 | const accountNumber = await pageEval(page, '.wallet-details div:nth-of-type(2)', null, (element) => { 90 | return (element as any).innerText.replace('מספר כרטיס ', ''); 91 | }); 92 | 93 | const balance = await pageEval(page, '.wallet-details div:nth-of-type(4) > span:nth-of-type(2)', null, (element) => { 94 | return (element as any).innerText; 95 | }); 96 | 97 | debug('fetch raw transactions from page'); 98 | 99 | const rawTransactions: (ScrapedTransaction | null)[] = await pageEvalAll<(ScrapedTransaction | null)[]>(page, '.transaction-container, .transaction-component-container', [], (items) => { 100 | return (items).map((el) => { 101 | const columns: NodeListOf = el.querySelectorAll('.transaction-item > span'); 102 | if (columns.length === 7) { 103 | return { 104 | date: columns[0].innerText, 105 | identifier: columns[1].innerText, 106 | description: columns[3].innerText, 107 | type: columns[5].innerText, 108 | chargedAmount: columns[6].innerText, 109 | }; 110 | } 111 | return null; 112 | }); 113 | }); 114 | debug(`fetched ${rawTransactions.length} raw transactions from page`); 115 | 116 | const accountTransactions = convertTransactions(rawTransactions.filter((item) => !!item) as ScrapedTransaction[]); 117 | 118 | debug('filer out old transactions'); 119 | const txns = (options.outputData?.enableTransactionsFilterByDate ?? true) ? 120 | filterOldTransactions(accountTransactions, startMoment, false) : 121 | accountTransactions; 122 | debug(`found ${txns.length} valid transactions out of ${accountTransactions.length} transactions for account ending with ${accountNumber.substring(accountNumber.length - 2)}`); 123 | 124 | return { 125 | accountNumber, 126 | balance: getAmountData(balance).amount, 127 | txns, 128 | }; 129 | } 130 | 131 | function getPossibleLoginResults(): PossibleLoginResults { 132 | const urls: PossibleLoginResults = {}; 133 | urls[LoginResults.Success] = [SUCCESS_URL]; 134 | urls[LoginResults.ChangePassword] = []; // TODO 135 | urls[LoginResults.InvalidPassword] = []; // TODO 136 | urls[LoginResults.UnknownError] = []; // TODO 137 | return urls; 138 | } 139 | 140 | function createLoginFields(credentials: ScraperSpecificCredentials) { 141 | return [ 142 | { selector: '#loginId', value: credentials.id }, 143 | { selector: '#loginPassword', value: credentials.password }, 144 | ]; 145 | } 146 | 147 | type ScraperSpecificCredentials = { id: string, password: string }; 148 | 149 | class BeyahadBishvilhaScraper extends BaseScraperWithBrowser { 150 | protected getViewPort(): { width: number, height: number } { 151 | return { 152 | width: 1500, 153 | height: 800, 154 | }; 155 | } 156 | 157 | getLoginOptions(credentials: ScraperSpecificCredentials) { 158 | return { 159 | loginUrl: LOGIN_URL, 160 | fields: createLoginFields(credentials), 161 | submitButtonSelector: async () => { 162 | const [button] = await this.page.$x("//button[contains(., 'התחבר')]"); 163 | if (button) { 164 | await button.click(); 165 | } 166 | }, 167 | possibleResults: getPossibleLoginResults(), 168 | }; 169 | } 170 | 171 | async fetchData() { 172 | const account = await fetchTransactions(this.page, this.options); 173 | return { 174 | success: true, 175 | accounts: [account], 176 | }; 177 | } 178 | } 179 | 180 | export default BeyahadBishvilhaScraper; 181 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to Israeli Bank Scrapers 2 | ======== 3 | Hey people, first of all, thanks for taking the time to help improving this project :beers: 4 | 5 | This project needs the help of other developers, since it's impossible getting access to all relevant banks and credit cards. 6 | 7 | # How can I contribute? 8 | Any kind of help is welcome, even if you just discover an issue and don't have the time to invest in helping to fix it. 9 | 10 | ## Filing issues 11 | While there's no specific template for creating a new issue, please take the time to create a clear description so that it is easy to understand the problem. 12 | 13 | ## Testing the scrapers 14 | In order to run tests you need first to create test configuration file `./src/tests/.tests-config.js` from template `./src/tests/.tests-config.tpl.js`. This file will be used by `jest` testing framework. 15 | 16 | > IMPORTANT: Under `src/tests` folder exists `.gitignore` file that ignore the test configuration file thus this file will not be commited to github. Still when you create new PRs make sure that you didn't explicitly added it to the PR. 17 | 18 | This library supports both testing against credit card companies / banks api and also against mock data. Until we will have a good coverage of scrapers test with mock data, the default configuration is set to execute real companies api tests. 19 | 20 | ### Changing tests options 21 | Modify property `options` in the test configuration file. This object is passed as-is to the scraper. 22 | 23 | ### Testing specific companies 24 | Enable any company you wish to test by providing its credetials in the test configuration file under `credentials` property. 25 | 26 | ### Running tests from CLI 27 | > Before running any tests, make sure you created the test configuration file with relevant credentials, 28 | 29 | To run all tests of companies that you provided credentials to: 30 | ``` 31 | npm test 32 | ``` 33 | 34 | To run specific `describe` (a.k.a suite), use the `testNamePattern` arg with the name of the suite. The following will run the all tests under `Leumi legacy scraper` suite. 35 | ``` 36 | npm test -- --testNamePattern="Leumi legacy scraper" 37 | ``` 38 | 39 | To run specific `test`, use the `testNamePattern` arg with suite name following the test name. The following will run test `should expose login fields in scrapers constant` that is part of `Leumi legacy scraper` suite. 40 | ``` 41 | npm test -- --testNamePattern="Leumi legacy scraper should expose login fields in scrapers constant" 42 | ``` 43 | 44 | ### Running tests using IDE 45 | Many IDEs support running jest tests directly from the UI. In webstorm for example a small play icon automatically appears next to each describe/test. 46 | 47 | **IMPORTANT Note** babel is configured to ignore tests by default. You must add an environment variable `BABEL_ENV=test` to the IDE test configuration to allow the tests to work. 48 | 49 | ### save unit test scraper results into file 50 | To save unit test scraper results provide a valid path in test configurations property `excelFilesDist`, for example: 51 | 52 | ``` 53 | { 54 | companyAPI: { 55 | enabled: true, 56 | excelFilesDist: '/Users/xyz/Downloads/Transactions', 57 | 58 | }, 59 | } 60 | ``` 61 | 62 | ### F.A.Q regarding the tests 63 | 64 | #### How can I run tests with CI/CD services? 65 | You can use environment variables instead of a local file to provide the tests configuration. 66 | 67 | copy and adjust the json below with relevant credentials and assign it to environment variable named `TESTS_CONFIG`. Note that this must be a valid json string otherwise it will fail during json parsing. 68 | ``` 69 | { 70 | "options": { 71 | "startDate": "2019-06-01", 72 | "combineInstallments": false, 73 | "showBrowser": true, 74 | "verbose": false, 75 | "args": [] 76 | }, 77 | "credentials": { 78 | "leumi": { "username": "demouser", "password": "demopassword" } 79 | }, 80 | "companyAPI": { 81 | "enabled": true, 82 | "invalidPassword": false 83 | } 84 | } 85 | ``` 86 | 87 | If you wish to try it from cli (mac os), you should either create a one liner json configuration or use cat to provide multiline value: 88 | 89 | ``` 90 | TESTS_CONFIG=`cat < this section is relevant if you are extending class `BaseScraperWithBrowser` 141 | > 142 | Unless you plan to override the entire `login()` function, You can override this function to login regularly in a login form. 143 | 144 | ```typescript 145 | import { LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 146 | 147 | function getPossibleLoginResults(): PossibleLoginResults { 148 | // checkout file `base-scraper-with-browser.ts` for available result types 149 | const urls: PossibleLoginResults = {}; 150 | urls[LoginResults.Success] = []; 151 | urls[LoginResults.InvalidPassword] = []; 152 | urls[LoginResults.ChangePassword] = []; 153 | return urls; 154 | } 155 | 156 | function getLoginOptions(credentials) { 157 | return { 158 | loginUrl: '', 159 | fields: [ 160 | { selector: '', value: credentials.username }, 161 | { selector: ``, value: credentials.password }, 162 | ], 163 | submitButtonSelector: '', 164 | possibleResults: getPossibleLoginResults(), 165 | }; 166 | } 167 | ``` 168 | 169 | ### Overriding fetchData() 170 | You can override this async function however way you want, as long as your return results as `ScaperScrapingResult` (checkout declaration [here](./src/scrapers/base-scraper.ts#L151)). 171 | -------------------------------------------------------------------------------- /src/scrapers/otsar-hahayal.ts: -------------------------------------------------------------------------------- 1 | import moment, { Moment } from 'moment'; 2 | import { Page } from 'puppeteer'; 3 | import { BaseScraperWithBrowser, LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 4 | import { waitForNavigation } from '../helpers/navigation'; 5 | import { 6 | fillInput, 7 | clickButton, 8 | waitUntilElementFound, 9 | pageEvalAll, 10 | elementPresentOnPage, 11 | } from '../helpers/elements-interactions'; 12 | import { SHEKEL_CURRENCY, SHEKEL_CURRENCY_SYMBOL } from '../constants'; 13 | import { Transaction, TransactionStatuses, TransactionTypes } from '../transactions'; 14 | 15 | const BASE_URL = 'https://online.bankotsar.co.il'; 16 | const LONG_DATE_FORMAT = 'DD/MM/YYYY'; 17 | const DATE_FORMAT = 'DD/MM/YY'; 18 | 19 | interface ScrapedTransaction { 20 | balance?: string; 21 | debit?: string; 22 | credit?: string; 23 | memo?: string; 24 | status?: string; 25 | reference?: string; 26 | description?: string; 27 | date: string; 28 | } 29 | 30 | function getPossibleLoginResults(page: Page) { 31 | const urls: PossibleLoginResults = {}; 32 | urls[LoginResults.Success] = [`${BASE_URL}/wps/myportal/FibiMenu/Online`]; 33 | urls[LoginResults.InvalidPassword] = [() => elementPresentOnPage(page, '#validationMsg')]; 34 | // TODO: support change password 35 | /* urls[LOGIN_RESULT.CHANGE_PASSWORD] = [``]; */ 36 | return urls; 37 | } 38 | 39 | function getTransactionsUrl() { 40 | return `${BASE_URL}/wps/myportal/FibiMenu/Online/OnAccountMngment/OnBalanceTrans/PrivateAccountFlow`; 41 | } 42 | 43 | function createLoginFields(credentials: ScraperSpecificCredentials) { 44 | return [ 45 | { selector: '#username', value: credentials.username }, 46 | { selector: '#password', value: credentials.password }, 47 | ]; 48 | } 49 | 50 | function getAmountData(amountStr: string, hasCurrency = false) { 51 | const amountStrCln = amountStr.replace(',', ''); 52 | let currency: string | null = null; 53 | let amount: number | null = null; 54 | if (!hasCurrency) { 55 | amount = parseFloat(amountStrCln); 56 | currency = SHEKEL_CURRENCY; 57 | } else if (amountStrCln.includes(SHEKEL_CURRENCY_SYMBOL)) { 58 | amount = parseFloat(amountStrCln.replace(SHEKEL_CURRENCY_SYMBOL, '')); 59 | currency = SHEKEL_CURRENCY; 60 | } else { 61 | const parts = amountStrCln.split(' '); 62 | amount = parseFloat(parts[0]); 63 | [, currency] = parts; 64 | } 65 | 66 | return { 67 | amount, 68 | currency, 69 | }; 70 | } 71 | 72 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 73 | return txns.map((txn) => { 74 | const dateFormat = 75 | txn.date.length === 8 ? 76 | DATE_FORMAT : 77 | txn.date.length === 10 ? 78 | LONG_DATE_FORMAT : 79 | null; 80 | if (!dateFormat) { 81 | throw new Error('invalid date format'); 82 | } 83 | const txnDate = moment(txn.date, dateFormat).toISOString(); 84 | const credit = getAmountData(txn.credit || '').amount; 85 | const debit = getAmountData(txn.debit || '').amount; 86 | const amount = (Number.isNaN(credit) ? 0 : credit) - (Number.isNaN(debit) ? 0 : debit); 87 | 88 | const result: Transaction = { 89 | type: TransactionTypes.Normal, 90 | status: TransactionStatuses.Completed, 91 | identifier: txn.reference ? parseInt(txn.reference, 10) : undefined, 92 | date: txnDate, 93 | processedDate: txnDate, 94 | originalAmount: amount, 95 | originalCurrency: SHEKEL_CURRENCY, 96 | chargedAmount: amount, 97 | description: txn.description || '', 98 | memo: '', 99 | }; 100 | 101 | return result; 102 | }); 103 | } 104 | 105 | async function parseTransactionPage(page: Page): Promise { 106 | const tdsValues = await pageEvalAll(page, '#dataTable077 tbody tr', [], (trs) => { 107 | return (trs).map((el) => ({ 108 | date: (el.querySelector('.date') as HTMLElement).innerText, 109 | // reference and description have vice-versa class name 110 | description: (el.querySelector('.reference') as HTMLElement).innerText, 111 | reference: (el.querySelector('.details') as HTMLElement).innerText, 112 | credit: (el.querySelector('.credit') as HTMLElement).innerText, 113 | debit: (el.querySelector('.debit') as HTMLElement).innerText, 114 | balance: (el.querySelector('.balance') as HTMLElement).innerText, 115 | })); 116 | }); 117 | 118 | return tdsValues; 119 | } 120 | 121 | async function getAccountSummary(page: Page) { 122 | const balanceElm = await page.$('.current_balance'); 123 | const balanceInnerTextElm = await balanceElm!.getProperty('innerText'); 124 | const balanceText = await balanceInnerTextElm.jsonValue(); 125 | const balanceValue = getAmountData(balanceText as string, true); 126 | // TODO: Find the credit field in bank website (could see it in my account) 127 | return { 128 | balance: Number.isNaN(balanceValue.amount) ? 0 : balanceValue.amount, 129 | creditLimit: 0.0, 130 | creditUtilization: 0.0, 131 | balanceCurrency: balanceValue.currency, 132 | }; 133 | } 134 | 135 | async function fetchTransactionsForAccount(page: Page, startDate: Moment) { 136 | const summary = await getAccountSummary(page); 137 | await waitUntilElementFound(page, 'input#fromDate'); 138 | // Get account number 139 | const branchNum = await page.$eval('.branch_num', (span) => { 140 | return (span as HTMLElement).innerText; 141 | }); 142 | 143 | const accountNmbr = await page.$eval('.acc_num', (span) => { 144 | return (span as HTMLElement).innerText; 145 | }); 146 | const accountNumber = `14-${branchNum}-${accountNmbr}`; 147 | // Search for relavant transaction from startDate 148 | await clickButton(page, '#tabHeader4'); 149 | await fillInput( 150 | page, 151 | 'input#fromDate', 152 | startDate.format('DD/MM/YYYY'), 153 | ); 154 | 155 | await clickButton(page, '#fibi_tab_dates .fibi_btn:nth-child(2)'); 156 | await waitForNavigation(page); 157 | await waitUntilElementFound(page, 'table#dataTable077, #NO_DATA077'); 158 | let hasNextPage = true; 159 | let txns: ScrapedTransaction[] = []; 160 | 161 | const noTransactionElm = await page.$('#NO_DATA077'); 162 | if (noTransactionElm == null) { 163 | // Scape transactions (this maybe spanned on multiple pages) 164 | while (hasNextPage) { 165 | const pageTxns = await parseTransactionPage(page); 166 | txns = txns.concat(pageTxns); 167 | const button = await page.$('#Npage'); 168 | hasNextPage = false; 169 | if (button != null) { 170 | hasNextPage = true; 171 | } 172 | if (hasNextPage) { 173 | await clickButton(page, '#Npage'); 174 | await waitForNavigation(page); 175 | await waitUntilElementFound(page, 'table#dataTable077'); 176 | } 177 | } 178 | } 179 | 180 | return { 181 | accountNumber, 182 | summary, 183 | txns: convertTransactions(txns.slice(1)), // Remove first line which is "opening balance" 184 | }; 185 | } 186 | 187 | async function fetchTransactions(page: Page, startDate: Moment) { 188 | // TODO need to extend to support multiple accounts and foreign accounts 189 | return [await fetchTransactionsForAccount(page, startDate)]; 190 | } 191 | 192 | async function waitForPostLogin(page: Page) { 193 | // TODO check for condition to provide new password 194 | return Promise.race([ 195 | waitUntilElementFound(page, 'div.lotusFrame', true), 196 | waitUntilElementFound(page, '#validationMsg'), 197 | ]); 198 | } 199 | 200 | type ScraperSpecificCredentials = { username: string, password: string }; 201 | 202 | class OtsarHahayalScraper extends BaseScraperWithBrowser { 203 | getLoginOptions(credentials: ScraperSpecificCredentials) { 204 | return { 205 | loginUrl: `${BASE_URL}/MatafLoginService/MatafLoginServlet?bankId=OTSARPRTAL&site=Private&KODSAFA=HE`, 206 | fields: createLoginFields(credentials), 207 | submitButtonSelector: async () => { 208 | await this.page.waitForTimeout(1000); 209 | await clickButton(this.page, '#continueBtn'); 210 | }, 211 | postAction: async () => waitForPostLogin(this.page), 212 | possibleResults: getPossibleLoginResults(this.page), 213 | }; 214 | } 215 | 216 | async fetchData() { 217 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 218 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 219 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 220 | 221 | const url = getTransactionsUrl(); 222 | await this.navigateTo(url); 223 | 224 | const accounts = await fetchTransactions(this.page, startMoment); 225 | 226 | return { 227 | success: true, 228 | accounts, 229 | }; 230 | } 231 | } 232 | 233 | export default OtsarHahayalScraper; 234 | -------------------------------------------------------------------------------- /src/scrapers/mizrahi.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import { Frame, Page, Request } from 'puppeteer'; 3 | import { SHEKEL_CURRENCY } from '../constants'; 4 | import { 5 | pageEvalAll, waitUntilElementDisappear, waitUntilElementFound, waitUntilIframeFound, 6 | } from '../helpers/elements-interactions'; 7 | import { fetchPostWithinPage } from '../helpers/fetch'; 8 | import { waitForUrl } from '../helpers/navigation'; 9 | import { 10 | Transaction, TransactionsAccount, TransactionStatuses, TransactionTypes, 11 | } from '../transactions'; 12 | import { BaseScraperWithBrowser, LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 13 | import { ScraperErrorTypes } from './errors'; 14 | 15 | interface ScrapedTransaction { 16 | RecTypeSpecified: boolean; 17 | MC02PeulaTaaEZ: string; 18 | MC02SchumEZ: number; 19 | MC02AsmahtaMekoritEZ: string; 20 | MC02TnuaTeurEZ: string; 21 | } 22 | 23 | interface ScrapedTransactionsResult { 24 | header: { 25 | success: boolean; 26 | messages: { text: string }[]; 27 | }; 28 | body: { 29 | fields: { 30 | AccountNumber: string; 31 | YitraLeloChekim: string; 32 | }; 33 | table: { 34 | rows: ScrapedTransaction[]; 35 | }; 36 | }; 37 | } 38 | 39 | const BASE_WEBSITE_URL = 'https://www.mizrahi-tefahot.co.il'; 40 | const LOGIN_URL = `${BASE_WEBSITE_URL}/login/index.html#/auth-page-he`; 41 | const BASE_APP_URL = 'https://mto.mizrahi-tefahot.co.il'; 42 | const AFTER_LOGIN_BASE_URL = /https:\/\/mto\.mizrahi-tefahot\.co\.il\/OnlineApp\/.*/; 43 | const OSH_PAGE = '/osh/legacy/legacy-Osh-Main'; 44 | const TRANSACTIONS_PAGE = '/osh/legacy/root-main-osh-p428New'; 45 | const TRANSACTIONS_REQUEST_URLS = [ 46 | `${BASE_APP_URL}/OnlinePilot/api/SkyOSH/get428Index`, 47 | `${BASE_APP_URL}/Online/api/SkyOSH/get428Index`, 48 | ]; 49 | const PENDING_TRANSACTIONS_PAGE = '/osh/legacy/legacy-Osh-p420'; 50 | const PENDING_TRANSACTIONS_IFRAME = 'p420.aspx'; 51 | const CHANGE_PASSWORD_URL = /https:\/\/www\.mizrahi-tefahot\.co\.il\/login\/\w+\/index\.html#\/change-pass/; 52 | const DATE_FORMAT = 'DD/MM/YYYY'; 53 | const MAX_ROWS_PER_REQUEST = 10000000000; 54 | 55 | const usernameSelector = '#emailDesktopHeb'; 56 | const passwordSelector = '#passwordIDDesktopHEB'; 57 | const submitButtonSelector = '.form-desktop button'; 58 | const invalidPasswordSelector = 'a[href*="https://sc.mizrahi-tefahot.co.il/SCServices/SC/P010.aspx"]'; 59 | const afterLoginSelector = '#dropdownBasic'; 60 | const loginSpinnerSelector = 'div.ngx-overlay.loading-foreground'; 61 | const accountDropDownItemSelector = '#AccountPicker .item'; 62 | const pendingTrxIdentifierId = '#ctl00_ContentPlaceHolder2_panel1'; 63 | const checkingAccountTabHebrewName = 'עובר ושב'; 64 | const checkingAccountTabEnglishName = 'Checking Account'; 65 | 66 | 67 | function createLoginFields(credentials: ScraperSpecificCredentials) { 68 | return [ 69 | { selector: usernameSelector, value: credentials.username }, 70 | { selector: passwordSelector, value: credentials.password }, 71 | ]; 72 | } 73 | 74 | function getPossibleLoginResults(page: Page): PossibleLoginResults { 75 | return { 76 | [LoginResults.Success]: [AFTER_LOGIN_BASE_URL, async () => !!(await page.$x(`//a//span[contains(., "${checkingAccountTabHebrewName}") or contains(., "${checkingAccountTabEnglishName}")]`))], 77 | [LoginResults.InvalidPassword]: [async () => !!(await page.$(invalidPasswordSelector))], 78 | [LoginResults.ChangePassword]: [CHANGE_PASSWORD_URL], 79 | }; 80 | } 81 | 82 | function getStartMoment(optionsStartDate: Date) { 83 | const defaultStartMoment = moment().subtract(1, 'years'); 84 | const startDate = optionsStartDate || defaultStartMoment.toDate(); 85 | return moment.max(defaultStartMoment, moment(startDate)); 86 | } 87 | 88 | function createDataFromRequest(request: Request, optionsStartDate: Date) { 89 | const data = JSON.parse(request.postData() || '{}'); 90 | 91 | data.inFromDate = getStartMoment(optionsStartDate).format(DATE_FORMAT); 92 | data.inToDate = moment().format(DATE_FORMAT); 93 | data.table.maxRow = MAX_ROWS_PER_REQUEST; 94 | 95 | return data; 96 | } 97 | 98 | function createHeadersFromRequest(request: Request) { 99 | return { 100 | mizrahixsrftoken: request.headers().mizrahixsrftoken, 101 | 'Content-Type': request.headers()['content-type'], 102 | }; 103 | } 104 | 105 | 106 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 107 | return txns.map((row) => { 108 | const txnDate = moment(row.MC02PeulaTaaEZ, moment.HTML5_FMT.DATETIME_LOCAL_SECONDS) 109 | .toISOString(); 110 | 111 | return { 112 | type: TransactionTypes.Normal, 113 | identifier: row.MC02AsmahtaMekoritEZ ? parseInt(row.MC02AsmahtaMekoritEZ, 10) : undefined, 114 | date: txnDate, 115 | processedDate: txnDate, 116 | originalAmount: row.MC02SchumEZ, 117 | originalCurrency: SHEKEL_CURRENCY, 118 | chargedAmount: row.MC02SchumEZ, 119 | description: row.MC02TnuaTeurEZ, 120 | status: TransactionStatuses.Completed, 121 | }; 122 | }); 123 | } 124 | 125 | async function extractPendingTransactions(page: Frame): Promise { 126 | const pendingTxn = await pageEvalAll(page, 'tr.rgRow', [], (trs) => { 127 | return trs.map((tr) => Array.from(tr.querySelectorAll('td'), (td: HTMLTableDataCellElement) => td.textContent || '')); 128 | }); 129 | 130 | return pendingTxn.map((txn) => { 131 | const date = moment(txn[0], 'DD/MM/YY').toISOString(); 132 | const amount = parseInt(txn[3], 10); 133 | return { 134 | type: TransactionTypes.Normal, 135 | date, 136 | processedDate: date, 137 | originalAmount: amount, 138 | originalCurrency: SHEKEL_CURRENCY, 139 | chargedAmount: amount, 140 | description: txn[1], 141 | status: TransactionStatuses.Pending, 142 | }; 143 | }); 144 | } 145 | 146 | async function postLogin(page: Page) { 147 | await Promise.race([ 148 | waitUntilElementFound(page, afterLoginSelector), 149 | waitUntilElementFound(page, invalidPasswordSelector), 150 | waitForUrl(page, CHANGE_PASSWORD_URL), 151 | ]); 152 | } 153 | 154 | type ScraperSpecificCredentials = { username: string, password: string }; 155 | 156 | class MizrahiScraper extends BaseScraperWithBrowser { 157 | getLoginOptions(credentials: ScraperSpecificCredentials) { 158 | return { 159 | loginUrl: LOGIN_URL, 160 | fields: createLoginFields(credentials), 161 | submitButtonSelector, 162 | checkReadiness: async () => waitUntilElementDisappear(this.page, loginSpinnerSelector), 163 | postAction: async () => postLogin(this.page), 164 | possibleResults: getPossibleLoginResults(this.page), 165 | }; 166 | } 167 | 168 | async fetchData() { 169 | await this.page.$eval('#dropdownBasic, .item', (el) => (el as HTMLElement).click()); 170 | 171 | const numOfAccounts = (await this.page.$$(accountDropDownItemSelector)).length; 172 | 173 | try { 174 | const results: TransactionsAccount[] = []; 175 | 176 | for (let i = 0; i < numOfAccounts; i += 1) { 177 | if (i > 0) { 178 | await this.page.$eval('#dropdownBasic, .item', (el) => (el as HTMLElement).click()); 179 | } 180 | 181 | await this.page.$eval(`${accountDropDownItemSelector}:nth-child(${i + 1})`, (el) => (el as HTMLElement).click()); 182 | results.push((await this.fetchAccount())); 183 | } 184 | 185 | return { 186 | success: true, 187 | accounts: results, 188 | }; 189 | } catch (e) { 190 | return { 191 | success: false, 192 | errorType: ScraperErrorTypes.Generic, 193 | errorMessage: e.message, 194 | }; 195 | } 196 | } 197 | 198 | private async fetchAccount() { 199 | await this.page.$eval(`a[href*="${OSH_PAGE}"]`, (el) => (el as HTMLElement).click()); 200 | await waitUntilElementFound(this.page, `a[href*="${TRANSACTIONS_PAGE}"]`); 201 | await this.page.$eval(`a[href*="${TRANSACTIONS_PAGE}"]`, (el) => (el as HTMLElement).click()); 202 | 203 | const response = await Promise.any(TRANSACTIONS_REQUEST_URLS.map(async (url) => { 204 | const request = await this.page.waitForRequest(url); 205 | const data = createDataFromRequest(request, this.options.startDate); 206 | const headers = createHeadersFromRequest(request); 207 | 208 | return fetchPostWithinPage(this.page, url, data, headers); 209 | })); 210 | 211 | 212 | if (!response || response.header.success === false) { 213 | throw new Error(`Error fetching transaction. Response message: ${response ? response.header.messages[0].text : ''}`); 214 | } 215 | 216 | const relevantRows = response.body.table.rows.filter((row) => row.RecTypeSpecified); 217 | const oshTxn = convertTransactions(relevantRows); 218 | 219 | // workaround for a bug which the bank's API returns transactions before the requested start date 220 | const startMoment = getStartMoment(this.options.startDate); 221 | const oshTxnAfterStartDate = oshTxn.filter((txn) => moment(txn.date).isSameOrAfter(startMoment)); 222 | 223 | await this.page.$eval(`a[href*="${PENDING_TRANSACTIONS_PAGE}"]`, (el) => (el as HTMLElement).click()); 224 | const frame = await waitUntilIframeFound(this.page, (f) => f.url().includes(PENDING_TRANSACTIONS_IFRAME)); 225 | await waitUntilElementFound(frame, pendingTrxIdentifierId); 226 | const pendingTxn = await extractPendingTransactions(frame); 227 | 228 | const allTxn = oshTxnAfterStartDate.concat(pendingTxn); 229 | 230 | return { 231 | accountNumber: response.body.fields.AccountNumber, 232 | txns: allTxn, 233 | balance: +response.body.fields.YitraLeloChekim, 234 | }; 235 | } 236 | } 237 | 238 | export default MizrahiScraper; 239 | -------------------------------------------------------------------------------- /src/scrapers/leumi.ts: -------------------------------------------------------------------------------- 1 | import moment, { Moment } from 'moment'; 2 | import { Page } from 'puppeteer'; 3 | import { BaseScraperWithBrowser, LoginResults, LoginOptions } from './base-scraper-with-browser'; 4 | import { 5 | fillInput, 6 | clickButton, 7 | waitUntilElementFound, 8 | pageEvalAll, 9 | } from '../helpers/elements-interactions'; 10 | import { SHEKEL_CURRENCY } from '../constants'; 11 | import { 12 | TransactionsAccount, Transaction, TransactionStatuses, TransactionTypes, 13 | } from '../transactions'; 14 | import { ScraperScrapingResult } from './interface'; 15 | import { waitForNavigation } from '../helpers/navigation'; 16 | 17 | const BASE_URL = 'https://hb2.bankleumi.co.il'; 18 | const LOGIN_URL = 'https://www.leumi.co.il/'; 19 | const TRANSACTIONS_URL = `${BASE_URL}/eBanking/SO/SPA.aspx#/ts/BusinessAccountTrx?WidgetPar=1`; 20 | const FILTERED_TRANSACTIONS_URL = `${BASE_URL}/ChannelWCF/Broker.svc/ProcessRequest?moduleName=UC_SO_27_GetBusinessAccountTrx`; 21 | 22 | const DATE_FORMAT = 'DD.MM.YY'; 23 | const ACCOUNT_BLOCKED_MSG = 'המנוי חסום'; 24 | const INVALID_PASSWORD_MSG = 'אחד או יותר מפרטי ההזדהות שמסרת שגויים. ניתן לנסות שוב'; 25 | 26 | 27 | function getPossibleLoginResults() { 28 | const urls: LoginOptions['possibleResults'] = { 29 | [LoginResults.Success]: [/ebanking\/SO\/SPA.aspx/i], 30 | [LoginResults.InvalidPassword]: [ 31 | async (options) => { 32 | if (!options || !options.page) { 33 | throw new Error('missing page options argument'); 34 | } 35 | const errorMessage = await pageEvalAll(options.page, 'svg#Capa_1', '', (element) => { 36 | return (element[0]?.parentElement?.children[1] as HTMLDivElement)?.innerText; 37 | }); 38 | 39 | return errorMessage?.startsWith(INVALID_PASSWORD_MSG); 40 | }, 41 | ], 42 | [LoginResults.AccountBlocked]: [ // NOTICE - might not be relevant starting the Leumi re-design during 2022 Sep 43 | async (options) => { 44 | if (!options || !options.page) { 45 | throw new Error('missing page options argument'); 46 | } 47 | const errorMessage = await pageEvalAll(options.page, '.errHeader', '', (label) => { 48 | return (label[0] as HTMLElement)?.innerText; 49 | }); 50 | 51 | return errorMessage?.startsWith(ACCOUNT_BLOCKED_MSG); 52 | }, 53 | ], 54 | [LoginResults.ChangePassword]: ['https://hb2.bankleumi.co.il/authenticate'], // NOTICE - might not be relevant starting the Leumi re-design during 2022 Sep 55 | }; 56 | return urls; 57 | } 58 | 59 | function createLoginFields(credentials: ScraperSpecificCredentials) { 60 | return [ 61 | { selector: 'input[placeholder="שם משתמש"]', value: credentials.username }, 62 | { selector: 'input[placeholder="סיסמה"]', value: credentials.password }, 63 | ]; 64 | } 65 | 66 | function extractTransactionsFromPage(transactions: any[], status: TransactionStatuses): Transaction[] { 67 | if (transactions === null || transactions.length === 0) { 68 | return []; 69 | } 70 | 71 | const result: Transaction[] = transactions.map((rawTransaction) => { 72 | const date = moment(rawTransaction.DateUTC).milliseconds(0).toISOString(); 73 | const newTransaction: Transaction = { 74 | status, 75 | type: TransactionTypes.Normal, 76 | date, 77 | processedDate: date, 78 | description: rawTransaction.Description || '', 79 | identifier: rawTransaction.ReferenceNumberLong, 80 | memo: rawTransaction.AdditionalData || '', 81 | originalCurrency: SHEKEL_CURRENCY, 82 | chargedAmount: rawTransaction.Amount, 83 | originalAmount: rawTransaction.Amount, 84 | }; 85 | 86 | return newTransaction; 87 | }); 88 | 89 | return result; 90 | } 91 | 92 | function hangProcess(timeout: number) { 93 | return new Promise((resolve) => { 94 | setTimeout(() => { 95 | resolve(); 96 | }, timeout); 97 | }); 98 | } 99 | 100 | async function clickByXPath(page: Page, xpath: string): Promise { 101 | await page.waitForXPath(xpath, { timeout: 30000, visible: true }); 102 | const elm = await page.$x(xpath); 103 | await elm[0].click(); 104 | } 105 | 106 | function removeSpecialCharacters(str: string): string { 107 | return str.replace(/[^0-9/-]/g, ''); 108 | } 109 | 110 | async function fetchTransactionsForAccount(page: Page, startDate: Moment, accountId: string): Promise { 111 | // DEVELOPER NOTICE the account number received from the server is being altered at 112 | // runtime for some accounts after 1-2 seconds so we need to hang the process for a short while. 113 | await hangProcess(4000); 114 | 115 | await waitUntilElementFound(page, 'button[title="חיפוש מתקדם"]', true); 116 | await clickButton(page, 'button[title="חיפוש מתקדם"]'); 117 | await waitUntilElementFound(page, 'bll-radio-button', true); 118 | await clickButton(page, 'bll-radio-button:not([checked])'); 119 | 120 | await waitUntilElementFound(page, 'input[formcontrolname="txtInputFrom"]', true); 121 | 122 | await fillInput( 123 | page, 124 | 'input[formcontrolname="txtInputFrom"]', 125 | startDate.format(DATE_FORMAT), 126 | ); 127 | 128 | // we must blur the from control otherwise the search will use the previous value 129 | await page.focus("button[aria-label='סנן']"); 130 | 131 | await clickButton(page, "button[aria-label='סנן']"); 132 | const finalResponse = await page.waitForResponse((response) => { 133 | return response.url() === FILTERED_TRANSACTIONS_URL && 134 | response.request().method() === 'POST'; 135 | }); 136 | 137 | const responseJson: any = await finalResponse.json(); 138 | 139 | const accountNumber = accountId.replace('/', '_').replace(/[^\d-_]/g, ''); 140 | 141 | const response = JSON.parse(responseJson.jsonResp); 142 | 143 | const pendingTransactions = response.TodayTransactionsItems; 144 | const transactions = response.HistoryTransactionsItems; 145 | const balance = response.BalanceDisplay ? parseFloat(response.BalanceDisplay) : undefined; 146 | 147 | const pendingTxns = extractTransactionsFromPage(pendingTransactions, TransactionStatuses.Pending); 148 | const completedTxns = extractTransactionsFromPage(transactions, TransactionStatuses.Completed); 149 | const txns = [ 150 | ...pendingTxns, 151 | ...completedTxns, 152 | ]; 153 | 154 | return { 155 | accountNumber, 156 | balance, 157 | txns, 158 | }; 159 | } 160 | 161 | async function fetchTransactions(page: Page, startDate: Moment): Promise { 162 | const accounts: TransactionsAccount[] = []; 163 | 164 | // DEVELOPER NOTICE the account number received from the server is being altered at 165 | // runtime for some accounts after 1-2 seconds so we need to hang the process for a short while. 166 | await hangProcess(4000); 167 | 168 | const accountsIds = await page.evaluate(() => Array.from(document.querySelectorAll('app-masked-number-combo span.display-number-li'), (e) => e.textContent)) as string[]; 169 | 170 | // due to a bug, the altered value might include undesired signs like & that should be removed 171 | 172 | if (!accountsIds.length) { 173 | throw new Error('Failed to extract or parse the account number'); 174 | } 175 | 176 | for (const accountId of accountsIds) { 177 | if (accountsIds.length > 1) { 178 | // get list of accounts and check accountId 179 | await clickByXPath(page, '//*[contains(@class, "number") and contains(@class, "combo-inner")]'); 180 | await clickByXPath(page, `//span[contains(text(), '${accountId}')]`); 181 | } 182 | 183 | accounts.push(await fetchTransactionsForAccount(page, startDate, removeSpecialCharacters(accountId))); 184 | } 185 | 186 | return accounts; 187 | } 188 | 189 | 190 | async function navigateToLogin(page: Page): Promise { 191 | const loginButtonSelector = '#enter_your_account a'; 192 | await waitUntilElementFound(page, loginButtonSelector); 193 | await clickButton(page, loginButtonSelector); 194 | await waitForNavigation(page, { waitUntil: 'networkidle2' }); 195 | await Promise.all([ 196 | waitUntilElementFound(page, 'input[placeholder="שם משתמש"]', true), 197 | waitUntilElementFound(page, 'input[placeholder="סיסמה"]', true), 198 | waitUntilElementFound(page, 'button[type="submit"]', true), 199 | ]); 200 | } 201 | 202 | async function waitForPostLogin(page: Page): Promise { 203 | await Promise.race([ 204 | waitUntilElementFound(page, 'a[title="דלג לחשבון"]', true, 60000), 205 | waitUntilElementFound(page, 'div.main-content', true, 60000), 206 | page.waitForXPath(`//div[contains(string(),"${INVALID_PASSWORD_MSG}")]`), 207 | waitUntilElementFound(page, 'form[action="/changepassword"]', true, 60000), // not sure if they kept this one 208 | ]); 209 | } 210 | 211 | type ScraperSpecificCredentials = { username: string, password: string}; 212 | 213 | class LeumiScraper extends BaseScraperWithBrowser { 214 | getLoginOptions(credentials: ScraperSpecificCredentials) { 215 | return { 216 | loginUrl: LOGIN_URL, 217 | fields: createLoginFields(credentials), 218 | submitButtonSelector: "button[type='submit']", 219 | checkReadiness: async () => navigateToLogin(this.page), 220 | postAction: async () => waitForPostLogin(this.page), 221 | possibleResults: getPossibleLoginResults(), 222 | }; 223 | } 224 | 225 | async fetchData(): Promise { 226 | const minimumStartMoment = moment().subtract(3, 'years').add(1, 'day'); 227 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 228 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 229 | const startMoment = moment.max(minimumStartMoment, moment(startDate)); 230 | 231 | await this.navigateTo(TRANSACTIONS_URL); 232 | 233 | const accounts = await fetchTransactions(this.page, startMoment); 234 | 235 | return { 236 | success: true, 237 | accounts, 238 | }; 239 | } 240 | } 241 | 242 | export default LeumiScraper; 243 | -------------------------------------------------------------------------------- /src/scrapers/hapoalim.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import uuid4 from 'uuid/v4'; 3 | 4 | import { Page } from 'puppeteer'; 5 | import { BaseScraperWithBrowser, LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 6 | import { waitForRedirect } from '../helpers/navigation'; 7 | import { waitUntil } from '../helpers/waiting'; 8 | import { fetchGetWithinPage, fetchPostWithinPage } from '../helpers/fetch'; 9 | import { 10 | TransactionsAccount, Transaction, TransactionStatuses, TransactionTypes, 11 | } from '../transactions'; 12 | import { getDebug } from '../helpers/debug'; 13 | import { ScraperOptions } from './interface'; 14 | 15 | const debug = getDebug('hapoalim'); 16 | 17 | const DATE_FORMAT = 'YYYYMMDD'; 18 | 19 | // eslint-disable-next-line @typescript-eslint/no-namespace 20 | declare namespace window { 21 | const bnhpApp: any; 22 | } 23 | 24 | interface ScrapedTransaction { 25 | serialNumber?: number; 26 | activityDescription?: string; 27 | eventAmount: number; 28 | valueDate?: string; 29 | eventDate?: string; 30 | referenceNumber?: number; 31 | ScrapedTransaction?: string; 32 | eventActivityTypeCode: number; 33 | currentBalance: number; 34 | pfmDetails: string; 35 | beneficiaryDetailsData?: { 36 | partyHeadline?: string; 37 | partyName?: string; 38 | messageHeadline?: string; 39 | messageDetail?: string; 40 | }; 41 | } 42 | 43 | interface ScrapedPfmTransaction { 44 | transactionNumber: number; 45 | } 46 | 47 | type FetchedAccountData = { 48 | bankNumber: string; 49 | accountNumber: string; 50 | branchNumber: string; 51 | accountClosingReasonCode: number; 52 | }[]; 53 | 54 | type FetchedAccountTransactionsData = { 55 | transactions: ScrapedTransaction[]; 56 | }; 57 | 58 | type BalanceAndCreditLimit = { 59 | creditLimitAmount: number; 60 | creditLimitDescription: string; 61 | creditLimitUtilizationAmount: number; 62 | creditLimitUtilizationExistanceCode: number; 63 | creditLimitUtilizationPercent: number; 64 | currentAccountLimitsAmount: number; 65 | currentBalance: number; 66 | withdrawalBalance: number; 67 | }; 68 | 69 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 70 | return txns.map((txn) => { 71 | const isOutbound = txn.eventActivityTypeCode === 2; 72 | 73 | let memo = ''; 74 | if (txn.beneficiaryDetailsData) { 75 | const { 76 | partyHeadline, 77 | partyName, 78 | messageHeadline, 79 | messageDetail, 80 | } = txn.beneficiaryDetailsData; 81 | const memoLines: string[] = []; 82 | if (partyHeadline) { 83 | memoLines.push(partyHeadline); 84 | } 85 | 86 | if (partyName) { 87 | memoLines.push(`${partyName}.`); 88 | } 89 | 90 | if (messageHeadline) { 91 | memoLines.push(messageHeadline); 92 | } 93 | 94 | if (messageDetail) { 95 | memoLines.push(`${messageDetail}.`); 96 | } 97 | 98 | if (memoLines.length) { 99 | memo = memoLines.join(' '); 100 | } 101 | } 102 | 103 | const result: Transaction = { 104 | type: TransactionTypes.Normal, 105 | identifier: txn.referenceNumber, 106 | date: moment(txn.eventDate, DATE_FORMAT).toISOString(), 107 | processedDate: moment(txn.valueDate, DATE_FORMAT).toISOString(), 108 | originalAmount: isOutbound ? -txn.eventAmount : txn.eventAmount, 109 | originalCurrency: 'ILS', 110 | chargedAmount: isOutbound ? -txn.eventAmount : txn.eventAmount, 111 | description: txn.activityDescription || '', 112 | status: txn.serialNumber === 0 ? TransactionStatuses.Pending : TransactionStatuses.Completed, 113 | memo, 114 | }; 115 | 116 | return result; 117 | }); 118 | } 119 | 120 | async function getRestContext(page: Page) { 121 | await waitUntil(() => { 122 | return page.evaluate(() => !!window.bnhpApp); 123 | }, 'waiting for app data load'); 124 | 125 | const result = await page.evaluate(() => { 126 | return window.bnhpApp.restContext; 127 | }); 128 | 129 | return result.slice(1); 130 | } 131 | 132 | async function fetchPoalimXSRFWithinPage(page: Page, url: string, pageUuid: string): Promise { 133 | const cookies = await page.cookies(); 134 | const XSRFCookie = cookies.find((cookie) => cookie.name === 'XSRF-TOKEN'); 135 | const headers: Record = {}; 136 | if (XSRFCookie != null) { 137 | headers['X-XSRF-TOKEN'] = XSRFCookie.value; 138 | } 139 | headers.pageUuid = pageUuid; 140 | headers.uuid = uuid4(); 141 | headers['Content-Type'] = 'application/json;charset=UTF-8'; 142 | return fetchPostWithinPage(page, url, [], headers); 143 | } 144 | 145 | async function getExtraScrap(txnsResult: FetchedAccountTransactionsData, baseUrl: string, page: Page, accountNumber: string): Promise { 146 | const promises = txnsResult.transactions.map(async (transaction: ScrapedTransaction): Promise => { 147 | const { pfmDetails, serialNumber } = transaction; 148 | if (serialNumber !== 0) { 149 | const url = `${baseUrl}${pfmDetails}&accountId=${accountNumber}&lang=he`; 150 | const extraTransactionDetails = await fetchGetWithinPage(page, url) || []; 151 | if (extraTransactionDetails && extraTransactionDetails.length) { 152 | const { transactionNumber } = extraTransactionDetails[0]; 153 | if (transactionNumber) { 154 | return { ...transaction, referenceNumber: transactionNumber }; 155 | } 156 | } 157 | } 158 | return transaction; 159 | }); 160 | const res = await Promise.all(promises); 161 | return { transactions: res }; 162 | } 163 | 164 | async function getAccountTransactions(baseUrl: string, apiSiteUrl: string, page: Page, accountNumber: string, startDate: string, endDate: string, additionalTransactionInformation = false) { 165 | const txnsUrl = `${apiSiteUrl}/current-account/transactions?accountId=${accountNumber}&numItemsPerPage=1000&retrievalEndDate=${endDate}&retrievalStartDate=${startDate}&sortCode=1`; 166 | const txnsResult = await fetchPoalimXSRFWithinPage(page, txnsUrl, '/current-account/transactions'); 167 | 168 | const finalResult = 169 | additionalTransactionInformation && txnsResult?.transactions.length ? 170 | await getExtraScrap(txnsResult, baseUrl, page, accountNumber) : 171 | txnsResult; 172 | 173 | return convertTransactions(finalResult?.transactions ?? []); 174 | } 175 | 176 | async function getAccountBalance(apiSiteUrl: string, page: Page, accountNumber: string) { 177 | const balanceAndCreditLimitUrl = `${apiSiteUrl}/current-account/composite/balanceAndCreditLimit?accountId=${accountNumber}&view=details&lang=he`; 178 | const balanceAndCreditLimit = await fetchGetWithinPage(page, balanceAndCreditLimitUrl); 179 | 180 | return balanceAndCreditLimit?.currentBalance; 181 | } 182 | 183 | async function fetchAccountData(page: Page, baseUrl: string, options: ScraperOptions) { 184 | const restContext = await getRestContext(page); 185 | const apiSiteUrl = `${baseUrl}/${restContext}`; 186 | const accountDataUrl = `${baseUrl}/ServerServices/general/accounts`; 187 | 188 | debug('fetching accounts data'); 189 | const accountsInfo = await fetchGetWithinPage(page, accountDataUrl) || []; 190 | debug('got %d accounts, fetching txns and balance', accountsInfo.length); 191 | 192 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 193 | const startDate = options.startDate || defaultStartMoment.toDate(); 194 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 195 | const { additionalTransactionInformation } = options; 196 | 197 | const startDateStr = startMoment.format(DATE_FORMAT); 198 | const endDateStr = moment().format(DATE_FORMAT); 199 | 200 | const accounts: TransactionsAccount[] = []; 201 | 202 | for (const account of accountsInfo) { 203 | let balance: number | undefined; 204 | const accountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`; 205 | 206 | const isActiveAccount = account.accountClosingReasonCode === 0; 207 | if (isActiveAccount) { 208 | balance = await getAccountBalance(apiSiteUrl, page, accountNumber); 209 | } else { 210 | debug('Skipping balance for a closed account, balance will be undefined'); 211 | } 212 | 213 | const txns = await getAccountTransactions( 214 | baseUrl, 215 | apiSiteUrl, 216 | page, 217 | accountNumber, 218 | startDateStr, 219 | endDateStr, 220 | additionalTransactionInformation, 221 | ); 222 | 223 | accounts.push({ 224 | accountNumber, 225 | balance, 226 | txns, 227 | }); 228 | } 229 | 230 | const accountData = { 231 | success: true, 232 | accounts, 233 | }; 234 | debug('fetching ended'); 235 | return accountData; 236 | } 237 | 238 | function getPossibleLoginResults(baseUrl: string) { 239 | const urls: PossibleLoginResults = {}; 240 | urls[LoginResults.Success] = [ 241 | `${baseUrl}/portalserver/HomePage`, 242 | `${baseUrl}/ng-portals-bt/rb/he/homepage`, 243 | `${baseUrl}/ng-portals/rb/he/homepage`]; 244 | urls[LoginResults.InvalidPassword] = [`${baseUrl}/AUTHENTICATE/LOGON?flow=AUTHENTICATE&state=LOGON&errorcode=1.6&callme=false`]; 245 | urls[LoginResults.ChangePassword] = [ 246 | `${baseUrl}/MCP/START?flow=MCP&state=START&expiredDate=null`, 247 | /\/ABOUTTOEXPIRE\/START/i, 248 | ]; 249 | return urls; 250 | } 251 | 252 | function createLoginFields(credentials: ScraperSpecificCredentials) { 253 | return [ 254 | { selector: '#userCode', value: credentials.userCode }, 255 | { selector: '#password', value: credentials.password }, 256 | ]; 257 | } 258 | 259 | type ScraperSpecificCredentials = { userCode: string, password: string }; 260 | 261 | class HapoalimScraper extends BaseScraperWithBrowser { 262 | // eslint-disable-next-line class-methods-use-this 263 | get baseUrl() { 264 | return 'https://login.bankhapoalim.co.il'; 265 | } 266 | 267 | getLoginOptions(credentials: ScraperSpecificCredentials) { 268 | return { 269 | loginUrl: `${this.baseUrl}/cgi-bin/poalwwwc?reqName=getLogonPage`, 270 | fields: createLoginFields(credentials), 271 | submitButtonSelector: '.login-btn', 272 | postAction: async () => waitForRedirect(this.page), 273 | possibleResults: getPossibleLoginResults(this.baseUrl), 274 | }; 275 | } 276 | 277 | async fetchData() { 278 | return fetchAccountData(this.page, this.baseUrl, this.options); 279 | } 280 | } 281 | 282 | export default HapoalimScraper; 283 | -------------------------------------------------------------------------------- /src/scrapers/yahav.ts: -------------------------------------------------------------------------------- 1 | import moment, { Moment } from 'moment'; 2 | import { Page } from 'puppeteer'; 3 | import { SHEKEL_CURRENCY } from '../constants'; 4 | import { 5 | clickButton, elementPresentOnPage, 6 | pageEvalAll, waitUntilElementDisappear, waitUntilElementFound, 7 | } from '../helpers/elements-interactions'; 8 | import { waitForNavigation } from '../helpers/navigation'; 9 | import { 10 | Transaction, TransactionsAccount, 11 | TransactionStatuses, TransactionTypes, 12 | } from '../transactions'; 13 | import { 14 | BaseScraperWithBrowser, 15 | LoginResults, 16 | PossibleLoginResults, 17 | } from './base-scraper-with-browser'; 18 | 19 | const LOGIN_URL = 'https://login.yahav.co.il/login/'; 20 | const BASE_URL = 'https://digital.yahav.co.il/BaNCSDigitalUI/app/index.html#/'; 21 | const INVALID_DETAILS_SELECTOR = '.ui-dialog-buttons'; 22 | const CHANGE_PASSWORD_OLD_PASS = 'input#ef_req_parameter_old_credential'; 23 | const BASE_WELCOME_URL = `${BASE_URL}main/home`; 24 | 25 | const ACCOUNT_ID_SELECTOR = '.dropdown-dir .selected-item-top'; 26 | const ACCOUNT_DETAILS_SELECTOR = '.account-details'; 27 | const DATE_FORMAT = 'DD/MM/YYYY'; 28 | 29 | const USER_ELEM = '#username'; 30 | const PASSWD_ELEM = '#password'; 31 | const NATIONALID_ELEM = '#pinno'; 32 | const SUBMIT_LOGIN_SELECTOR = '.btn'; 33 | 34 | interface ScrapedTransaction { 35 | credit: string; 36 | debit: string; 37 | date: string; 38 | reference?: string; 39 | description: string; 40 | memo: string; 41 | status: TransactionStatuses; 42 | } 43 | 44 | function getPossibleLoginResults(page: Page): PossibleLoginResults { 45 | // checkout file `base-scraper-with-browser.ts` for available result types 46 | const urls: PossibleLoginResults = {}; 47 | urls[LoginResults.Success] = [ 48 | `${BASE_WELCOME_URL}`, 49 | ]; 50 | urls[LoginResults.InvalidPassword] = [async () => { 51 | return elementPresentOnPage(page, `${INVALID_DETAILS_SELECTOR}`); 52 | }]; 53 | 54 | urls[LoginResults.ChangePassword] = [async () => { 55 | return elementPresentOnPage(page, `${CHANGE_PASSWORD_OLD_PASS}`); 56 | }]; 57 | 58 | return urls; 59 | } 60 | 61 | async function getAccountID(page: Page) { 62 | const selectedSnifAccount = await page.$eval(`${ACCOUNT_ID_SELECTOR}`, (option) => { 63 | return (option as HTMLElement).innerText; 64 | }); 65 | 66 | return selectedSnifAccount; 67 | } 68 | 69 | function getAmountData(amountStr: string) { 70 | const amountStrCopy = amountStr.replace(',', ''); 71 | return parseFloat(amountStrCopy); 72 | } 73 | 74 | function getTxnAmount(txn: ScrapedTransaction) { 75 | const credit = getAmountData(txn.credit); 76 | const debit = getAmountData(txn.debit); 77 | return (Number.isNaN(credit) ? 0 : credit) - (Number.isNaN(debit) ? 0 : debit); 78 | } 79 | 80 | type TransactionsTr = { id: string, innerDivs: string[] }; 81 | 82 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 83 | return txns.map((txn) => { 84 | const convertedDate = moment(txn.date, DATE_FORMAT).toISOString(); 85 | const convertedAmount = getTxnAmount(txn); 86 | return { 87 | type: TransactionTypes.Normal, 88 | identifier: txn.reference ? parseInt(txn.reference, 10) : undefined, 89 | date: convertedDate, 90 | processedDate: convertedDate, 91 | originalAmount: convertedAmount, 92 | originalCurrency: SHEKEL_CURRENCY, 93 | chargedAmount: convertedAmount, 94 | status: txn.status, 95 | description: txn.description, 96 | memo: txn.memo, 97 | }; 98 | }); 99 | } 100 | 101 | function handleTransactionRow(txns: ScrapedTransaction[], txnRow: TransactionsTr) { 102 | const div = txnRow.innerDivs; 103 | 104 | // Remove anything except digits. 105 | const regex = /\D+/gm; 106 | 107 | const tx: ScrapedTransaction = { 108 | date: div[1], 109 | reference: div[2].replace(regex, ''), 110 | memo: '', 111 | description: div[3], 112 | debit: div[4], 113 | credit: div[5], 114 | status: TransactionStatuses.Completed, 115 | }; 116 | 117 | txns.push(tx); 118 | } 119 | 120 | async function getAccountTransactions(page: Page): Promise { 121 | // Wait for transactions. 122 | await waitUntilElementFound(page, '.under-line-txn-table-header', true); 123 | 124 | const txns: ScrapedTransaction[] = []; 125 | const transactionsDivs = await pageEvalAll(page, '.list-item-holder .entire-content-ctr', [], (divs) => { 126 | return (divs as HTMLElement[]).map((div) => ({ 127 | id: (div).getAttribute('id') || '', 128 | innerDivs: Array.from(div.getElementsByTagName('div')).map((div) => (div as HTMLElement).innerText), 129 | })); 130 | }); 131 | 132 | for (const txnRow of transactionsDivs) { 133 | handleTransactionRow(txns, txnRow); 134 | } 135 | 136 | return convertTransactions(txns); 137 | } 138 | 139 | // Manipulate the calendar drop down to choose the txs start date. 140 | async function searchByDates(page: Page, startDate: Moment) { 141 | // Get the day number from startDate. 1-31 (usually 1) 142 | const startDateDay = startDate.format('D'); 143 | const startDateMonth = startDate.format('M'); 144 | const startDateYear = startDate.format('Y'); 145 | 146 | // Open the calendar date picker 147 | const dateFromPick = 'div.date-options-cell:nth-child(7) > date-picker:nth-child(1) > div:nth-child(1) > span:nth-child(2)'; 148 | await waitUntilElementFound(page, dateFromPick, true); 149 | await clickButton(page, dateFromPick); 150 | 151 | // Wait until first day appear. 152 | await waitUntilElementFound(page, '.pmu-days > div:nth-child(1)', true); 153 | 154 | // Open Months options. 155 | const monthFromPick = '.pmu-month'; 156 | await waitUntilElementFound(page, monthFromPick, true); 157 | await clickButton(page, monthFromPick); 158 | await waitUntilElementFound(page, '.pmu-months > div:nth-child(1)', true); 159 | 160 | // Open Year options. 161 | // Use same selector... Yahav knows why... 162 | await waitUntilElementFound(page, monthFromPick, true); 163 | await clickButton(page, monthFromPick); 164 | await waitUntilElementFound(page, '.pmu-years > div:nth-child(1)', true); 165 | 166 | // Select year from a 12 year grid. 167 | for (let i = 1; i < 13; i += 1) { 168 | const selector = `.pmu-years > div:nth-child(${i})`; 169 | const year = await page.$eval(selector, (y) => { 170 | return (y as HTMLElement).innerText; 171 | }); 172 | if (startDateYear === year) { 173 | await clickButton(page, selector); 174 | break; 175 | } 176 | } 177 | 178 | // Select Month. 179 | await waitUntilElementFound(page, '.pmu-months > div:nth-child(1)', true); 180 | // The first element (1) is January. 181 | const monthSelector = `.pmu-months > div:nth-child(${startDateMonth})`; 182 | await clickButton(page, monthSelector); 183 | 184 | // Select Day. 185 | // The calendar grid shows 7 days and 6 weeks = 42 days. 186 | // In theory, the first day of the month will be in the first row. 187 | // Let's check everything just in case... 188 | for (let i = 1; i < 42; i += 1) { 189 | const selector = `.pmu-days > div:nth-child(${i})`; 190 | const day = await page.$eval(selector, (d) => { 191 | return (d as HTMLElement).innerText; 192 | }); 193 | 194 | if (startDateDay === day) { 195 | await clickButton(page, selector); 196 | break; 197 | } 198 | } 199 | } 200 | 201 | 202 | async function fetchAccountData(page: Page, startDate: Moment, accountID: string): Promise { 203 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 204 | await searchByDates(page, startDate); 205 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 206 | const txns = await getAccountTransactions(page); 207 | 208 | return { 209 | accountNumber: accountID, 210 | txns, 211 | }; 212 | } 213 | 214 | async function fetchAccounts(page: Page, startDate: Moment): Promise { 215 | const accounts: TransactionsAccount[] = []; 216 | 217 | // TODO: get more accounts. Not sure is supported. 218 | const accountID = await getAccountID(page); 219 | const accountData = await fetchAccountData(page, startDate, accountID); 220 | accounts.push(accountData); 221 | 222 | return accounts; 223 | } 224 | 225 | async function waitReadinessForAll(page: Page) { 226 | await waitUntilElementFound(page, `${USER_ELEM}`, true); 227 | await waitUntilElementFound(page, `${PASSWD_ELEM}`, true); 228 | await waitUntilElementFound(page, `${NATIONALID_ELEM}`, true); 229 | await waitUntilElementFound(page, `${SUBMIT_LOGIN_SELECTOR}`, true); 230 | } 231 | 232 | async function redirectOrDialog(page: Page) { 233 | // Click on bank messages if any. 234 | await waitForNavigation(page); 235 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 236 | const hasMessage = await elementPresentOnPage(page, '.messaging-links-container'); 237 | if (hasMessage) { 238 | await clickButton(page, '.link-1'); 239 | } 240 | 241 | const promise1 = page.waitForSelector(ACCOUNT_DETAILS_SELECTOR, { timeout: 30000 }); 242 | const promise2 = page.waitForSelector(CHANGE_PASSWORD_OLD_PASS, { timeout: 30000 }); 243 | const promises = [promise1, promise2]; 244 | 245 | await Promise.race(promises); 246 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 247 | } 248 | 249 | type ScraperSpecificCredentials = {username: string, password: string, nationalID: string}; 250 | 251 | class YahavScraper extends BaseScraperWithBrowser { 252 | getLoginOptions(credentials: ScraperSpecificCredentials) { 253 | return { 254 | loginUrl: `${LOGIN_URL}`, 255 | fields: [ 256 | { selector: `${USER_ELEM}`, value: credentials.username }, 257 | { selector: `${PASSWD_ELEM}`, value: credentials.password }, 258 | { selector: `${NATIONALID_ELEM}`, value: credentials.nationalID }, 259 | ], 260 | submitButtonSelector: `${SUBMIT_LOGIN_SELECTOR}`, 261 | checkReadiness: async () => waitReadinessForAll(this.page), 262 | postAction: async () => redirectOrDialog(this.page), 263 | possibleResults: getPossibleLoginResults(this.page), 264 | }; 265 | } 266 | 267 | async fetchData() { 268 | // Goto statements page 269 | await waitUntilElementFound(this.page, ACCOUNT_DETAILS_SELECTOR, true); 270 | await clickButton(this.page, ACCOUNT_DETAILS_SELECTOR); 271 | await waitUntilElementFound(this.page, '.statement-options .selected-item-top', true); 272 | 273 | const defaultStartMoment = moment().subtract(3, 'months').add(1, 'day'); 274 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 275 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 276 | 277 | const accounts = await fetchAccounts(this.page, startMoment); 278 | 279 | return { 280 | success: true, 281 | accounts, 282 | }; 283 | } 284 | } 285 | 286 | export default YahavScraper; 287 | -------------------------------------------------------------------------------- /src/scrapers/base-scraper-with-browser.ts: -------------------------------------------------------------------------------- 1 | import puppeteer, { Browser, Frame, Page } from 'puppeteer'; 2 | 3 | import { 4 | BaseScraper, 5 | 6 | } from './base-scraper'; 7 | import { getCurrentUrl, waitForNavigation } from '../helpers/navigation'; 8 | import { clickButton, fillInput, waitUntilElementFound } from '../helpers/elements-interactions'; 9 | import { getDebug } from '../helpers/debug'; 10 | import { ScraperErrorTypes } from './errors'; 11 | import { ScraperScrapingResult, ScraperCredentials } from './interface'; 12 | import { ScraperProgressTypes } from '../definitions'; 13 | 14 | const VIEWPORT_WIDTH = 1024; 15 | const VIEWPORT_HEIGHT = 768; 16 | const OK_STATUS = 200; 17 | 18 | const debug = getDebug('base-scraper-with-browser'); 19 | 20 | enum LoginBaseResults { 21 | Success = 'SUCCESS', 22 | UnknownError = 'UNKNOWN_ERROR' 23 | } 24 | 25 | const { 26 | Timeout, Generic, General, ...rest 27 | } = ScraperErrorTypes; 28 | export const LoginResults = { 29 | ...rest, 30 | ...LoginBaseResults, 31 | }; 32 | 33 | export type LoginResults = Exclude | LoginBaseResults; 37 | 38 | export type PossibleLoginResults = { 39 | [key in LoginResults]?: (string | RegExp | ((options?: { page?: Page}) => Promise))[] 40 | }; 41 | 42 | export interface LoginOptions { 43 | loginUrl: string; 44 | checkReadiness?: () => Promise; 45 | fields: {selector: string, value: string}[]; 46 | submitButtonSelector: string | (() => Promise); 47 | preAction?: () => Promise; 48 | postAction?: () => Promise; 49 | possibleResults: PossibleLoginResults; 50 | userAgent?: string; 51 | } 52 | 53 | async function getKeyByValue(object: PossibleLoginResults, value: string, page: Page): Promise { 54 | const keys = Object.keys(object); 55 | for (const key of keys) { 56 | // @ts-ignore 57 | const conditions = object[key]; 58 | 59 | for (const condition of conditions) { 60 | let result = false; 61 | 62 | if (condition instanceof RegExp) { 63 | result = condition.test(value); 64 | } else if (typeof condition === 'function') { 65 | result = await condition({ page, value }); 66 | } else { 67 | result = value.toLowerCase() === condition.toLowerCase(); 68 | } 69 | 70 | if (result) { 71 | // @ts-ignore 72 | return Promise.resolve(key); 73 | } 74 | } 75 | } 76 | 77 | return Promise.resolve(LoginResults.UnknownError); 78 | } 79 | 80 | function createGeneralError(): ScraperScrapingResult { 81 | return { 82 | success: false, 83 | errorType: ScraperErrorTypes.General, 84 | }; 85 | } 86 | 87 | class BaseScraperWithBrowser extends BaseScraper { 88 | // NOTICE - it is discouraged to use bang (!) in general. It is used here because 89 | // all the classes that inherit from this base assume is it mandatory. 90 | protected browser!: Browser; 91 | 92 | // NOTICE - it is discouraged to use bang (!) in general. It is used here because 93 | // all the classes that inherit from this base assume is it mandatory. 94 | protected page!: Page; 95 | 96 | protected getViewPort() { 97 | return { 98 | width: VIEWPORT_WIDTH, 99 | height: VIEWPORT_HEIGHT, 100 | }; 101 | } 102 | 103 | async initialize() { 104 | await super.initialize(); 105 | debug('initialize scraper'); 106 | this.emitProgress(ScraperProgressTypes.Initializing); 107 | 108 | let env: Record | undefined; 109 | if (this.options.verbose) { 110 | env = { DEBUG: '*', ...process.env }; 111 | } 112 | 113 | if (typeof this.options.browser !== 'undefined' && this.options.browser !== null) { 114 | debug('use custom browser instance provided in options'); 115 | this.browser = this.options.browser; 116 | } else { 117 | const executablePath = this.options.executablePath || undefined; 118 | const args = this.options.args || []; 119 | const { timeout } = this.options; 120 | 121 | const headless = !this.options.showBrowser; 122 | debug(`launch a browser with headless mode = ${headless}`); 123 | this.browser = await puppeteer.launch({ 124 | env, 125 | headless, 126 | executablePath, 127 | args, 128 | timeout, 129 | }); 130 | } 131 | 132 | if (this.options.prepareBrowser) { 133 | debug('execute \'prepareBrowser\' interceptor provided in options'); 134 | await this.options.prepareBrowser(this.browser); 135 | } 136 | 137 | if (!this.browser) { 138 | debug('failed to initiate a browser, exit'); 139 | return; 140 | } 141 | 142 | const pages = await this.browser.pages(); 143 | if (pages.length) { 144 | debug('browser has already pages open, use the first one'); 145 | [this.page] = pages; 146 | } else { 147 | debug('create a new browser page'); 148 | this.page = await this.browser.newPage(); 149 | } 150 | 151 | if (this.options.defaultTimeout) { 152 | this.page.setDefaultTimeout(this.options.defaultTimeout); 153 | } 154 | 155 | if (this.options.preparePage) { 156 | debug('execute \'preparePage\' interceptor provided in options'); 157 | await this.options.preparePage(this.page); 158 | } 159 | 160 | const viewport = this.getViewPort(); 161 | debug(`set viewport to width ${viewport.width}, height ${viewport.height}`); 162 | await this.page.setViewport({ 163 | width: viewport.width, 164 | height: viewport.height, 165 | }); 166 | 167 | this.page.on('requestfailed', (request) => { 168 | debug('Request failed: %s %s', request.failure()?.errorText, request.url()); 169 | }); 170 | } 171 | 172 | async navigateTo(url: string, page?: Page, timeout?: number): Promise { 173 | const pageToUse = page || this.page; 174 | 175 | if (!pageToUse) { 176 | return; 177 | } 178 | 179 | const options = { ...(timeout === null ? null : { timeout }) }; 180 | const response = await pageToUse.goto(url, options); 181 | 182 | // note: response will be null when navigating to same url while changing the hash part. the condition below will always accept null as valid result. 183 | if (response !== null && (response === undefined || response.status() !== OK_STATUS)) { 184 | throw new Error(`Error while trying to navigate to url ${url}`); 185 | } 186 | } 187 | 188 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 189 | getLoginOptions(_credentials: ScraperCredentials): LoginOptions { 190 | throw new Error(`getLoginOptions() is not created in ${this.options.companyId}`); 191 | } 192 | 193 | async fillInputs(pageOrFrame: Page | Frame, fields: { selector: string, value: string}[]): Promise { 194 | const modified = [...fields]; 195 | const input = modified.shift(); 196 | 197 | if (!input) { 198 | return; 199 | } 200 | await fillInput(pageOrFrame, input.selector, input.value); 201 | if (modified.length) { 202 | await this.fillInputs(pageOrFrame, modified); 203 | } 204 | } 205 | 206 | async login(credentials: ScraperCredentials): Promise { 207 | if (!credentials || !this.page) { 208 | return createGeneralError(); 209 | } 210 | 211 | debug('execute login process'); 212 | const loginOptions = this.getLoginOptions(credentials); 213 | 214 | if (loginOptions.userAgent) { 215 | debug('set custom user agent provided in options'); 216 | await this.page.setUserAgent(loginOptions.userAgent); 217 | } 218 | 219 | debug('navigate to login url'); 220 | await this.navigateTo(loginOptions.loginUrl); 221 | if (loginOptions.checkReadiness) { 222 | debug('execute \'checkReadiness\' interceptor provided in login options'); 223 | await loginOptions.checkReadiness(); 224 | } else if (typeof loginOptions.submitButtonSelector === 'string') { 225 | debug('wait until submit button is available'); 226 | await waitUntilElementFound(this.page, loginOptions.submitButtonSelector); 227 | } 228 | 229 | let loginFrameOrPage: (Page | Frame | null) = this.page; 230 | if (loginOptions.preAction) { 231 | debug('execute \'preAction\' interceptor provided in login options'); 232 | loginFrameOrPage = await loginOptions.preAction() || this.page; 233 | } 234 | 235 | debug('fill login components input with relevant values'); 236 | await this.fillInputs(loginFrameOrPage, loginOptions.fields); 237 | debug('click on login submit button'); 238 | if (typeof loginOptions.submitButtonSelector === 'string') { 239 | await clickButton(loginFrameOrPage, loginOptions.submitButtonSelector); 240 | } else { 241 | await loginOptions.submitButtonSelector(); 242 | } 243 | this.emitProgress(ScraperProgressTypes.LoggingIn); 244 | 245 | if (loginOptions.postAction) { 246 | debug('execute \'postAction\' interceptor provided in login options'); 247 | await loginOptions.postAction(); 248 | } else { 249 | debug('wait for page navigation'); 250 | await waitForNavigation(this.page); 251 | } 252 | 253 | debug('check login result'); 254 | const current = await getCurrentUrl(this.page, true); 255 | const loginResult = await getKeyByValue(loginOptions.possibleResults, current, this.page); 256 | debug(`handle login results ${loginResult}`); 257 | return this.handleLoginResult(loginResult); 258 | } 259 | 260 | async terminate(_success: boolean) { 261 | debug(`terminating browser with success = ${_success}`); 262 | this.emitProgress(ScraperProgressTypes.Terminating); 263 | 264 | if (!_success && !!this.options.storeFailureScreenShotPath) { 265 | debug(`create a snapshot before terminated in ${this.options.storeFailureScreenShotPath}`); 266 | await this.page.screenshot({ 267 | path: this.options.storeFailureScreenShotPath, 268 | fullPage: true, 269 | }); 270 | } 271 | 272 | if (!this.browser) { 273 | return; 274 | } 275 | 276 | await this.browser.close(); 277 | } 278 | 279 | private handleLoginResult(loginResult: LoginResults) { 280 | switch (loginResult) { 281 | case LoginResults.Success: 282 | this.emitProgress(ScraperProgressTypes.LoginSuccess); 283 | return { success: true }; 284 | case LoginResults.InvalidPassword: 285 | case LoginResults.UnknownError: 286 | this.emitProgress(ScraperProgressTypes.LoginFailed); 287 | return { 288 | success: false, 289 | errorType: loginResult === LoginResults.InvalidPassword ? ScraperErrorTypes.InvalidPassword : 290 | ScraperErrorTypes.General, 291 | errorMessage: `Login failed with ${loginResult} error`, 292 | }; 293 | case LoginResults.ChangePassword: 294 | this.emitProgress(ScraperProgressTypes.ChangePassword); 295 | return { 296 | success: false, 297 | errorType: ScraperErrorTypes.ChangePassword, 298 | }; 299 | default: 300 | throw new Error(`unexpected login result "${loginResult}"`); 301 | } 302 | } 303 | } 304 | 305 | export { BaseScraperWithBrowser }; 306 | -------------------------------------------------------------------------------- /src/scrapers/one-zero.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment/moment'; 2 | import { 3 | ScraperGetLongTermTwoFactorTokenResult, ScraperLoginResult, 4 | ScraperScrapingResult, ScraperTwoFactorAuthTriggerResult, 5 | } from './interface'; 6 | import { getDebug } from '../helpers/debug'; 7 | import { fetchGraphql, fetchPost } from '../helpers/fetch'; 8 | import { createGenericError, ScraperErrorTypes } from './errors'; 9 | import { GET_CUSTOMER, GET_MOVEMENTS } from './one-zero-queries'; 10 | import { 11 | Transaction as ScrapingTransaction, 12 | TransactionsAccount, 13 | TransactionStatuses, 14 | TransactionTypes, 15 | } from '../transactions'; 16 | import { BaseScraper } from './base-scraper'; 17 | 18 | const HEBREW_WORDS_REGEX = /[\u0590-\u05FF][\u0590-\u05FF"'\-_ /\\]*[\u0590-\u05FF]/g; 19 | 20 | const debug = getDebug('one-zero'); 21 | 22 | type Account = { 23 | accountId: string; 24 | }; 25 | 26 | type Portfolio = { 27 | accounts: Array; 28 | portfolioId: string; 29 | portfolioNum: string; 30 | }; 31 | 32 | type Customer = { 33 | customerId: string; 34 | portfolios?: Array | null; 35 | }; 36 | 37 | export type Category = { 38 | categoryId: number; 39 | dataSource: string; 40 | subCategoryId?: number | null; 41 | }; 42 | 43 | export type Recurrence = { 44 | dataSource: string; 45 | isRecurrent: boolean; 46 | }; 47 | 48 | type TransactionEnrichment = { 49 | categories?: Category[] | null; 50 | recurrences?: Recurrence[] | null; 51 | }; 52 | 53 | type Transaction = { 54 | enrichment?: TransactionEnrichment | null; 55 | // TODO: Get installments information here 56 | // transactionDetails: TransactionDetails; 57 | }; 58 | 59 | type Movement = { 60 | accountId: string; 61 | bankCurrencyAmount: string; 62 | bookingDate: string; 63 | conversionRate: string; 64 | creditDebit: string; 65 | description: string; 66 | isReversed: boolean; 67 | movementAmount: string; 68 | movementCurrency: string; 69 | movementId: string; 70 | movementReversedId?: string|null; 71 | movementTimestamp: string; 72 | movementType: string; 73 | portfolioId: string; 74 | runningBalance: string; 75 | transaction?: Transaction | null; 76 | valueDate: string; 77 | }; 78 | 79 | type QueryPagination = { hasMore: boolean, cursor: string }; 80 | 81 | const IDENTITY_SERVER_URL = 'https://identity.tfd-bank.com/v1/'; 82 | 83 | const GRAPHQL_API_URL = 'https://mobile.tfd-bank.com/mobile-graph/graphql'; 84 | 85 | 86 | type ScraperSpecificCredentials = {email: string, password: string} & ({ 87 | otpCodeRetriever: () => Promise; 88 | phoneNumber: string; 89 | } | { 90 | otpLongTermToken: string; 91 | }); 92 | 93 | export default class OneZeroScraper extends BaseScraper { 94 | private otpContext?: string; 95 | 96 | private accessToken?: string; 97 | 98 | 99 | async triggerTwoFactorAuth(phoneNumber: string): Promise { 100 | if (!phoneNumber.startsWith('+')) { 101 | return createGenericError('A full international phone number starting with + and a three digit country code is required'); 102 | } 103 | 104 | debug('Fetching device token'); 105 | const deviceTokenResponse = await fetchPost(`${IDENTITY_SERVER_URL}/devices/token`, { 106 | extClientId: 'mobile', 107 | os: 'Android', 108 | }); 109 | 110 | const { resultData: { deviceToken } } = deviceTokenResponse; 111 | 112 | debug(`Sending OTP to phone number ${phoneNumber}`); 113 | 114 | const otpPrepareResponse = await fetchPost(`${IDENTITY_SERVER_URL}/otp/prepare`, { 115 | factorValue: phoneNumber, 116 | deviceToken, 117 | otpChannel: 'SMS_OTP', 118 | }); 119 | 120 | const { resultData: { otpContext } } = otpPrepareResponse; 121 | 122 | this.otpContext = otpContext; 123 | 124 | return { 125 | success: true, 126 | }; 127 | } 128 | 129 | public async getLongTermTwoFactorToken(otpCode: string): Promise { 130 | if (!this.otpContext) { 131 | return createGenericError('triggerOtp was not called before calling getPermenantOtpToken()'); 132 | } 133 | 134 | debug('Requesting OTP token'); 135 | const otpVerifyResponse = await fetchPost(`${IDENTITY_SERVER_URL}/otp/verify`, { 136 | otpContext: this.otpContext, 137 | otpCode, 138 | }); 139 | 140 | const { resultData: { otpToken } } = otpVerifyResponse; 141 | return { success: true, longTermTwoFactorAuthToken: otpToken }; 142 | } 143 | 144 | private async resolveOtpToken( 145 | credentials: ScraperSpecificCredentials, 146 | ): Promise { 147 | if ('otpLongTermToken' in credentials) { 148 | if (!credentials.otpLongTermToken) { 149 | return createGenericError('Invalid otpLongTermToken'); 150 | } 151 | return { success: true, longTermTwoFactorAuthToken: credentials.otpLongTermToken }; 152 | } 153 | 154 | if (!credentials.otpCodeRetriever) { 155 | return { 156 | success: false, 157 | errorType: ScraperErrorTypes.TwoFactorRetrieverMissing, 158 | errorMessage: 'otpCodeRetriever is required when otpPermanentToken is not provided', 159 | }; 160 | } 161 | 162 | if (!credentials.phoneNumber) { 163 | return createGenericError('phoneNumber is required when providing a otpCodeRetriever callback'); 164 | } 165 | 166 | debug('Triggering user supplied otpCodeRetriever callback'); 167 | const triggerResult = await this.triggerTwoFactorAuth(credentials.phoneNumber); 168 | 169 | if (!triggerResult.success) { 170 | return triggerResult; 171 | } 172 | 173 | const otpCode = await credentials.otpCodeRetriever(); 174 | 175 | const otpTokenResult = await this.getLongTermTwoFactorToken(otpCode); 176 | if (!otpTokenResult.success) { 177 | return otpTokenResult; 178 | } 179 | 180 | return { success: true, longTermTwoFactorAuthToken: otpTokenResult.longTermTwoFactorAuthToken }; 181 | } 182 | 183 | async login(credentials: ScraperSpecificCredentials): 184 | Promise { 185 | const otpTokenResult = await this.resolveOtpToken(credentials); 186 | if (!otpTokenResult.success) { 187 | return otpTokenResult; 188 | } 189 | 190 | 191 | debug('Requesting id token'); 192 | const getIdTokenResponse = await fetchPost(`${IDENTITY_SERVER_URL}/getIdToken`, { 193 | otpSmsToken: otpTokenResult.longTermTwoFactorAuthToken, 194 | email: credentials.email, 195 | pass: credentials.password, 196 | pinCode: '', 197 | }); 198 | 199 | const { resultData: { idToken } } = getIdTokenResponse; 200 | 201 | 202 | debug('Requesting session token'); 203 | 204 | const getSessionTokenResponse = await fetchPost(`${IDENTITY_SERVER_URL}/sessions/token`, { 205 | idToken, 206 | pass: credentials.password, 207 | }); 208 | 209 | const { resultData: { accessToken } } = getSessionTokenResponse; 210 | 211 | this.accessToken = accessToken; 212 | 213 | return { 214 | success: true, 215 | persistentOtpToken: otpTokenResult.longTermTwoFactorAuthToken, 216 | }; 217 | } 218 | 219 | private async fetchPortfolioMovements(portfolio: Portfolio, startDate: Date): Promise { 220 | // TODO: Find out if we need the other accounts, there seems to always be one 221 | const account = portfolio.accounts[0]; 222 | let cursor = null; 223 | const movements = []; 224 | 225 | 226 | while (!movements.length || new Date(movements[0].movementTimestamp) >= startDate) { 227 | debug(`Fetching transactions for account ${portfolio.portfolioNum}...`); 228 | const { movements: { movements: newMovements, pagination } }: 229 | {movements: { movements: Movement[], pagination: QueryPagination }} = 230 | await fetchGraphql( 231 | GRAPHQL_API_URL, 232 | GET_MOVEMENTS, { 233 | portfolioId: portfolio.portfolioId, 234 | accountId: account.accountId, 235 | language: 'HEBREW', 236 | pagination: { 237 | cursor, 238 | limit: 50, 239 | }, 240 | }, 241 | { authorization: `Bearer ${this.accessToken}` }, 242 | ); 243 | 244 | movements.unshift(...newMovements); 245 | cursor = pagination.cursor; 246 | if (!pagination.hasMore) { 247 | break; 248 | } 249 | } 250 | 251 | movements.sort((x, y) => new Date(x.movementTimestamp).valueOf() - new Date(y.movementTimestamp).valueOf()); 252 | 253 | const matchingMovements = movements.filter((movement) => new Date(movement.movementTimestamp) >= startDate); 254 | return { 255 | accountNumber: portfolio.portfolioNum, 256 | balance: !movements.length ? 0 : parseFloat(movements[movements.length - 1].runningBalance), 257 | txns: matchingMovements.map((movement): ScrapingTransaction => { 258 | const hasInstallments = movement.transaction?.enrichment?.recurrences?.some((x) => x.isRecurrent); 259 | const modifier = movement.creditDebit === 'DEBIT' ? -1 : 1; 260 | return ({ 261 | date: movement.valueDate, 262 | chargedAmount: (+movement.movementAmount) * modifier, 263 | chargedCurrency: movement.movementCurrency, 264 | originalAmount: (+movement.movementAmount) * modifier, 265 | originalCurrency: movement.movementCurrency, 266 | description: this.sanitizeHebrew(movement.description), 267 | processedDate: movement.movementTimestamp, 268 | status: TransactionStatuses.Completed, 269 | type: hasInstallments ? TransactionTypes.Installments : TransactionTypes.Normal, 270 | }); 271 | }), 272 | }; 273 | } 274 | 275 | /** 276 | * one zero hebrew strings are reversed with a unicode control character that forces display in LTR order 277 | * We need to remove the unicode control character, and then reverse hebrew substrings inside the string 278 | */ 279 | private sanitizeHebrew(text: string) { 280 | if (!text.includes('\u202d')) { 281 | return text.trim(); 282 | } 283 | 284 | const plainString = text.replace(/\u202d/gi, '').trim(); 285 | const hebrewSubStringsRanges = [...plainString.matchAll(HEBREW_WORDS_REGEX)]; 286 | const rangesToReverse = hebrewSubStringsRanges.map((text) => ( 287 | { start: text.index!, end: text.index! + text[0].length })); 288 | const out = []; 289 | let index = 0; 290 | 291 | for (const { start, end } of rangesToReverse) { 292 | out.push(...plainString.substring(index, start)); 293 | index += (start - index); 294 | const reversed = [...plainString.substring(start, end)].reverse(); 295 | out.push(...reversed); 296 | index += (end - start); 297 | } 298 | 299 | out.push(...plainString.substring(index, plainString.length)); 300 | 301 | return out.join(''); 302 | } 303 | 304 | async fetchData(): Promise { 305 | if (!this.accessToken) { 306 | return createGenericError('login() was not called'); 307 | } 308 | 309 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 310 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 311 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 312 | 313 | debug('Fetching account list'); 314 | const result = await fetchGraphql<{ customer: Customer[] }>(GRAPHQL_API_URL, GET_CUSTOMER, {}, { authorization: `Bearer ${this.accessToken}` }); 315 | const portfolios = result.customer.flatMap((customer) => (customer.portfolios || [])); 316 | 317 | return { 318 | success: true, 319 | accounts: await Promise.all(portfolios.map( 320 | (portfolio) => this.fetchPortfolioMovements(portfolio, startMoment.toDate()), 321 | )), 322 | }; 323 | } 324 | } 325 | -------------------------------------------------------------------------------- /src/scrapers/union-bank.ts: -------------------------------------------------------------------------------- 1 | import moment, { Moment } from 'moment'; 2 | import { Page } from 'puppeteer'; 3 | import { BaseScraperWithBrowser, LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 4 | import { 5 | dropdownSelect, 6 | dropdownElements, 7 | fillInput, 8 | clickButton, 9 | waitUntilElementFound, 10 | pageEvalAll, 11 | elementPresentOnPage, 12 | } from '../helpers/elements-interactions'; 13 | import { waitForNavigation } from '../helpers/navigation'; 14 | import { SHEKEL_CURRENCY } from '../constants'; 15 | import { 16 | TransactionsAccount, Transaction, TransactionStatuses, 17 | TransactionTypes, 18 | } from '../transactions'; 19 | 20 | const BASE_URL = 'https://hb.unionbank.co.il'; 21 | const TRANSACTIONS_URL = `${BASE_URL}/eBanking/Accounts/ExtendedActivity.aspx#/`; 22 | const DATE_FORMAT = 'DD/MM/YY'; 23 | const NO_TRANSACTION_IN_DATE_RANGE_TEXT = 'לא קיימות תנועות מתאימות על פי הסינון שהוגדר'; 24 | const DATE_HEADER = 'תאריך'; 25 | const DESCRIPTION_HEADER = 'תיאור'; 26 | const REFERENCE_HEADER = 'אסמכתא'; 27 | const DEBIT_HEADER = 'חובה'; 28 | const CREDIT_HEADER = 'זכות'; 29 | const PENDING_TRANSACTIONS_TABLE_ID = 'trTodayActivityNapaTableUpper'; 30 | const COMPLETED_TRANSACTIONS_TABLE_ID = 'ctlActivityTable'; 31 | const ERROR_MESSAGE_CLASS = 'errInfo'; 32 | const ACCOUNTS_DROPDOWN_SELECTOR = 'select#ddlAccounts_m_ddl'; 33 | 34 | function getPossibleLoginResults() { 35 | const urls: PossibleLoginResults = {}; 36 | urls[LoginResults.Success] = [/eBanking\/Accounts/]; 37 | urls[LoginResults.InvalidPassword] = [/InternalSite\/CustomUpdate\/leumi\/LoginPage.ASP/]; 38 | return urls; 39 | } 40 | 41 | function createLoginFields(credentials: ScraperSpecificCredentials) { 42 | return [ 43 | { selector: '#uid', value: credentials.username }, 44 | { selector: '#password', value: credentials.password }, 45 | ]; 46 | } 47 | 48 | function getAmountData(amountStr: string) { 49 | const amountStrCopy = amountStr.replace(',', ''); 50 | return parseFloat(amountStrCopy); 51 | } 52 | 53 | interface ScrapedTransaction { 54 | credit: string; 55 | debit: string; 56 | date: string; 57 | reference?: string; 58 | description: string; 59 | memo: string; 60 | status: TransactionStatuses; 61 | } 62 | 63 | function getTxnAmount(txn: ScrapedTransaction) { 64 | const credit = getAmountData(txn.credit); 65 | const debit = getAmountData(txn.debit); 66 | return (Number.isNaN(credit) ? 0 : credit) - (Number.isNaN(debit) ? 0 : debit); 67 | } 68 | 69 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 70 | return txns.map((txn) => { 71 | const convertedDate = moment(txn.date, DATE_FORMAT).toISOString(); 72 | const convertedAmount = getTxnAmount(txn); 73 | return { 74 | type: TransactionTypes.Normal, 75 | identifier: txn.reference ? parseInt(txn.reference, 10) : undefined, 76 | date: convertedDate, 77 | processedDate: convertedDate, 78 | originalAmount: convertedAmount, 79 | originalCurrency: SHEKEL_CURRENCY, 80 | chargedAmount: convertedAmount, 81 | status: txn.status, 82 | description: txn.description, 83 | memo: txn.memo, 84 | }; 85 | }); 86 | } 87 | 88 | type TransactionsTr = { id: string, innerTds: TransactionsTrTds }; 89 | type TransactionTableHeaders = Record; 90 | type TransactionsTrTds = string[]; 91 | 92 | function getTransactionDate(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 93 | return (tds[txnsTableHeaders[DATE_HEADER]] || '').trim(); 94 | } 95 | 96 | function getTransactionDescription(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 97 | return (tds[txnsTableHeaders[DESCRIPTION_HEADER]] || '').trim(); 98 | } 99 | 100 | function getTransactionReference(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 101 | return (tds[txnsTableHeaders[REFERENCE_HEADER]] || '').trim(); 102 | } 103 | 104 | function getTransactionDebit(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 105 | return (tds[txnsTableHeaders[DEBIT_HEADER]] || '').trim(); 106 | } 107 | 108 | function getTransactionCredit(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 109 | return (tds[txnsTableHeaders[CREDIT_HEADER]] || '').trim(); 110 | } 111 | 112 | function extractTransactionDetails(txnRow: TransactionsTr, txnsTableHeaders: TransactionTableHeaders, txnStatus: TransactionStatuses): ScrapedTransaction { 113 | const tds = txnRow.innerTds; 114 | return { 115 | status: txnStatus, 116 | date: getTransactionDate(tds, txnsTableHeaders), 117 | description: getTransactionDescription(tds, txnsTableHeaders), 118 | reference: getTransactionReference(tds, txnsTableHeaders), 119 | debit: getTransactionDebit(tds, txnsTableHeaders), 120 | credit: getTransactionCredit(tds, txnsTableHeaders), 121 | memo: '', 122 | }; 123 | } 124 | 125 | function isExpandedDescRow(txnRow: TransactionsTr) { 126 | return txnRow.id === 'rowAdded'; 127 | } 128 | 129 | /* eslint-disable no-param-reassign */ 130 | function editLastTransactionDesc(txnRow: TransactionsTr, lastTxn: ScrapedTransaction): ScrapedTransaction { 131 | lastTxn.description = `${lastTxn.description} ${txnRow.innerTds[0]}`; 132 | return lastTxn; 133 | } 134 | 135 | function handleTransactionRow(txns: ScrapedTransaction[], txnsTableHeaders: TransactionTableHeaders, txnRow: TransactionsTr, txnType: TransactionStatuses) { 136 | if (isExpandedDescRow(txnRow)) { 137 | const lastTransaction = txns.pop(); 138 | if (lastTransaction) { 139 | txns.push(editLastTransactionDesc(txnRow, lastTransaction)); 140 | } else { 141 | throw new Error('internal union-bank error'); 142 | } 143 | } else { 144 | txns.push(extractTransactionDetails(txnRow, txnsTableHeaders, txnType)); 145 | } 146 | } 147 | 148 | async function getTransactionsTableHeaders(page: Page, tableTypeId: string) { 149 | const headersMap: Record = []; 150 | const headersObjs = await pageEvalAll(page, `#WorkSpaceBox #${tableTypeId} tr[class='header'] th`, null, (ths) => { 151 | return ths.map((th, index) => ({ 152 | text: (th as HTMLElement).innerText.trim(), 153 | index, 154 | })); 155 | }); 156 | 157 | for (const headerObj of headersObjs) { 158 | headersMap[headerObj.text] = headerObj.index; 159 | } 160 | return headersMap; 161 | } 162 | 163 | async function extractTransactionsFromTable(page: Page, tableTypeId: string, txnType: TransactionStatuses): Promise { 164 | const txns: ScrapedTransaction[] = []; 165 | const transactionsTableHeaders = await getTransactionsTableHeaders(page, tableTypeId); 166 | 167 | const transactionsRows = await pageEvalAll(page, `#WorkSpaceBox #${tableTypeId} tr[class]:not([class='header'])`, [], (trs) => { 168 | return (trs as HTMLElement[]).map((tr) => ({ 169 | id: (tr).getAttribute('id') || '', 170 | innerTds: Array.from(tr.getElementsByTagName('td')).map((td) => (td as HTMLElement).innerText), 171 | })); 172 | }); 173 | 174 | for (const txnRow of transactionsRows) { 175 | handleTransactionRow(txns, transactionsTableHeaders, txnRow, txnType); 176 | } 177 | return txns; 178 | } 179 | 180 | async function isNoTransactionInDateRangeError(page: Page) { 181 | const hasErrorInfoElement = await elementPresentOnPage(page, `.${ERROR_MESSAGE_CLASS}`); 182 | if (hasErrorInfoElement) { 183 | const errorText = await page.$eval(`.${ERROR_MESSAGE_CLASS}`, (errorElement) => { 184 | return (errorElement as HTMLElement).innerText; 185 | }); 186 | return errorText.trim() === NO_TRANSACTION_IN_DATE_RANGE_TEXT; 187 | } 188 | return false; 189 | } 190 | 191 | async function chooseAccount(page: Page, accountId: string) { 192 | const hasDropDownList = await elementPresentOnPage(page, ACCOUNTS_DROPDOWN_SELECTOR); 193 | if (hasDropDownList) { 194 | await dropdownSelect(page, ACCOUNTS_DROPDOWN_SELECTOR, accountId); 195 | } 196 | } 197 | 198 | async function searchByDates(page: Page, startDate: Moment) { 199 | await dropdownSelect(page, 'select#ddlTransactionPeriod', '004'); 200 | await waitUntilElementFound(page, 'select#ddlTransactionPeriod'); 201 | await fillInput( 202 | page, 203 | 'input#dtFromDate_textBox', 204 | startDate.format(DATE_FORMAT), 205 | ); 206 | await clickButton(page, 'input#btnDisplayDates'); 207 | await waitForNavigation(page); 208 | } 209 | 210 | async function getAccountNumber(page: Page) { 211 | const selectedSnifAccount = await page.$eval('#ddlAccounts_m_ddl option[selected="selected"]', (option) => { 212 | return (option as HTMLElement).innerText; 213 | }); 214 | 215 | return selectedSnifAccount.replace('/', '_'); 216 | } 217 | 218 | async function expandTransactionsTable(page: Page) { 219 | const hasExpandAllButton = await elementPresentOnPage(page, "a[id*='lnkCtlExpandAll']"); 220 | if (hasExpandAllButton) { 221 | await clickButton(page, "a[id*='lnkCtlExpandAll']"); 222 | } 223 | } 224 | 225 | async function scrapeTransactionsFromTable(page: Page): Promise { 226 | const pendingTxns = await extractTransactionsFromTable(page, PENDING_TRANSACTIONS_TABLE_ID, 227 | TransactionStatuses.Pending); 228 | const completedTxns = await extractTransactionsFromTable(page, COMPLETED_TRANSACTIONS_TABLE_ID, 229 | TransactionStatuses.Completed); 230 | const txns = [ 231 | ...pendingTxns, 232 | ...completedTxns, 233 | ]; 234 | return convertTransactions(txns); 235 | } 236 | 237 | async function getAccountTransactions(page: Page): Promise { 238 | await Promise.race([ 239 | waitUntilElementFound(page, `#${COMPLETED_TRANSACTIONS_TABLE_ID}`, false), 240 | waitUntilElementFound(page, `.${ERROR_MESSAGE_CLASS}`, false), 241 | ]); 242 | 243 | const noTransactionInRangeError = await isNoTransactionInDateRangeError(page); 244 | if (noTransactionInRangeError) { 245 | return []; 246 | } 247 | 248 | await expandTransactionsTable(page); 249 | return scrapeTransactionsFromTable(page); 250 | } 251 | 252 | async function fetchAccountData(page: Page, startDate: Moment, accountId: string): Promise { 253 | await chooseAccount(page, accountId); 254 | await searchByDates(page, startDate); 255 | const accountNumber = await getAccountNumber(page); 256 | const txns = await getAccountTransactions(page); 257 | return { 258 | accountNumber, 259 | txns, 260 | }; 261 | } 262 | 263 | async function fetchAccounts(page: Page, startDate: Moment) { 264 | const accounts: TransactionsAccount[] = []; 265 | const accountsList = await dropdownElements(page, ACCOUNTS_DROPDOWN_SELECTOR); 266 | for (const account of accountsList) { 267 | if (account.value !== '-1') { // Skip "All accounts" option 268 | const accountData = await fetchAccountData(page, startDate, account.value); 269 | accounts.push(accountData); 270 | } 271 | } 272 | return accounts; 273 | } 274 | 275 | async function waitForPostLogin(page: Page) { 276 | return Promise.race([ 277 | waitUntilElementFound(page, '#signoff', true), 278 | waitUntilElementFound(page, '#restore', true), 279 | ]); 280 | } 281 | 282 | type ScraperSpecificCredentials = { username: string, password: string }; 283 | 284 | class UnionBankScraper extends BaseScraperWithBrowser { 285 | getLoginOptions(credentials: ScraperSpecificCredentials) { 286 | return { 287 | loginUrl: `${BASE_URL}`, 288 | fields: createLoginFields(credentials), 289 | submitButtonSelector: '#enter', 290 | postAction: async () => waitForPostLogin(this.page), 291 | possibleResults: getPossibleLoginResults(), 292 | }; 293 | } 294 | 295 | async fetchData() { 296 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 297 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 298 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 299 | 300 | await this.navigateTo(TRANSACTIONS_URL); 301 | 302 | const accounts = await fetchAccounts(this.page, startMoment); 303 | 304 | return { 305 | success: true, 306 | accounts, 307 | }; 308 | } 309 | } 310 | 311 | export default UnionBankScraper; 312 | -------------------------------------------------------------------------------- /src/scrapers/one-zero-queries.ts: -------------------------------------------------------------------------------- 1 | export const GET_CUSTOMER = ` 2 | query GetCustomer { 3 | customer { 4 | __typename 5 | customerId 6 | userId 7 | idType 8 | idNumber 9 | hebrewFirstName 10 | hebrewLastName 11 | latinFirstName 12 | latinLastName 13 | dateOfBirth 14 | lastLoginDate 15 | userEmail 16 | gender 17 | portfolioRelations { 18 | __typename 19 | customerId 20 | customerRole 21 | portfolioId 22 | initiator 23 | relationToInitiator 24 | status 25 | } 26 | portfolios { 27 | __typename 28 | ...Portfolio 29 | } 30 | status 31 | } 32 | } 33 | fragment Portfolio on Portfolio { 34 | __typename 35 | accounts { 36 | __typename 37 | accountId 38 | accountType 39 | closingDate 40 | currency 41 | openingDate 42 | status 43 | subType 44 | } 45 | activationDate 46 | bank 47 | baseCurrency 48 | branch 49 | club 50 | clubDescription 51 | iban 52 | imageURL 53 | isJointAccount 54 | partnerName { 55 | __typename 56 | partnerFirstName 57 | partnerLastName 58 | } 59 | portfolioId 60 | portfolioNum 61 | portfolioType 62 | status 63 | subType 64 | onboardingCompleted 65 | } 66 | `; 67 | 68 | export const GET_MOVEMENTS = `query GetMovements( 69 | $portfolioId: String! 70 | $accountId: String! 71 | $pagination: PaginationInput! 72 | $language: BffLanguage! 73 | ) { 74 | movements( 75 | portfolioId: $portfolioId 76 | accountId: $accountId 77 | pagination: $pagination 78 | language: $language 79 | ) { 80 | __typename 81 | ...MovementsFragment 82 | } 83 | } 84 | fragment TransactionInstrumentAmountFragment on TransactionInstrumentAmount { 85 | __typename 86 | instrumentAmount 87 | instrumentSymbol 88 | instrumentType 89 | } 90 | fragment CounterPartyReferenceFragment on CounterPartyReference { 91 | __typename 92 | bankId 93 | bic 94 | branchCode 95 | id 96 | name 97 | type 98 | } 99 | fragment BaseTransactionFragment on BaseTransaction { 100 | __typename 101 | accountId 102 | betweenOwnAccounts 103 | bookDate 104 | calculatedStatus 105 | chargeAmount { 106 | __typename 107 | ...TransactionInstrumentAmountFragment 108 | } 109 | clearingSystem 110 | counterParty { 111 | __typename 112 | ...CounterPartyReferenceFragment 113 | } 114 | currentPaymentNumber 115 | direction 116 | domainType 117 | isReversal 118 | method 119 | originalAmount { 120 | __typename 121 | ...TransactionInstrumentAmountFragment 122 | } 123 | portfolioId 124 | totalPaymentsCount 125 | transactionId 126 | transactionType 127 | valueDate 128 | } 129 | fragment CategoryFragment on Category { 130 | __typename 131 | categoryId 132 | dataSource 133 | subCategoryId 134 | } 135 | fragment RecurrenceFragment on Recurrence { 136 | __typename 137 | dataSource 138 | isRecurrent 139 | } 140 | fragment TransactionEnrichmentFragment on TransactionEnrichment { 141 | __typename 142 | categories { 143 | __typename 144 | ...CategoryFragment 145 | } 146 | recurrences { 147 | __typename 148 | ...RecurrenceFragment 149 | } 150 | } 151 | fragment TransactionEventMetadataFragment on TransactionEventMetadata { 152 | __typename 153 | correlationId 154 | processingOrder 155 | } 156 | fragment CounterPartyTransferData on CounterPartyTransfer { 157 | __typename 158 | accountId 159 | bank_id 160 | branch_code 161 | counter_party_name 162 | } 163 | fragment BankTransferDetailsData on BankTransferDetails { 164 | __typename 165 | ... on CashBlockTransfer { 166 | counterParty { 167 | __typename 168 | ...CounterPartyTransferData 169 | } 170 | transferDescriptionKey 171 | } 172 | ... on RTGSReturnTransfer { 173 | transferDescriptionKey 174 | } 175 | ... on RTGSTransfer { 176 | transferDescriptionKey 177 | } 178 | ... on SwiftReturnTransfer { 179 | transferConversionRate 180 | transferDescriptionKey 181 | } 182 | ... on SwiftTransfer { 183 | transferConversionRate 184 | transferDescriptionKey 185 | } 186 | ... on Transfer { 187 | counterParty { 188 | __typename 189 | ...CounterPartyTransferData 190 | } 191 | transferDescriptionKey 192 | } 193 | } 194 | fragment CategoryData on Category { 195 | __typename 196 | categoryId 197 | dataSource 198 | subCategoryId 199 | } 200 | fragment RecurrenceData on Recurrence { 201 | __typename 202 | dataSource 203 | isRecurrent 204 | } 205 | fragment CardDetailsData on CardDetails { 206 | __typename 207 | ... on CardCharge { 208 | book_date 209 | cardDescriptionKey 210 | } 211 | ... on CardChargeFCY { 212 | book_date 213 | cardConversionRate 214 | cardDescriptionKey 215 | cardFCYAmount 216 | cardFCYCurrency 217 | } 218 | ... on CardMonthlySettlement { 219 | cardDescriptionKey 220 | } 221 | ... on CardRefund { 222 | cardDescriptionKey 223 | } 224 | ... on CashBlockCardCharge { 225 | cardDescriptionKey 226 | } 227 | } 228 | fragment CashDetailsData on CashDetails { 229 | __typename 230 | ... on CashWithdrawal { 231 | cashDescriptionKey 232 | } 233 | ... on CashWithdrawalFCY { 234 | FCYAmount 235 | FCYCurrency 236 | cashDescriptionKey 237 | conversionRate 238 | } 239 | } 240 | fragment ChequesDetailsData on ChequesDetails { 241 | __typename 242 | ... on CashBlockChequeDeposit { 243 | bookDate 244 | chequesDescriptionKey 245 | } 246 | ... on ChequeDeposit { 247 | bookDate 248 | chequesDescriptionKey 249 | } 250 | ... on ChequeReturn { 251 | bookDate 252 | chequeReturnReason 253 | chequesDescriptionKey 254 | } 255 | ... on ChequeWithdrawal { 256 | chequesDescriptionKey 257 | } 258 | } 259 | fragment DefaultDetailsData on DefaultDetails { 260 | __typename 261 | ... on DefaultWithTransaction { 262 | defaultDescriptionKey 263 | } 264 | ... on DefaultWithoutTransaction { 265 | categories { 266 | __typename 267 | ...CategoryData 268 | } 269 | defaultDescriptionKey 270 | } 271 | } 272 | fragment FeeDetailsData on FeeDetails { 273 | __typename 274 | ... on GeneralFee { 275 | feeDescriptionKey 276 | } 277 | } 278 | fragment LoanDetailsData on LoanDetails { 279 | __typename 280 | ... on FullPrePayment { 281 | loanDescriptionKey 282 | } 283 | ... on Initiate { 284 | loanDescriptionKey 285 | } 286 | ... on MonthlyPayment { 287 | loanDescriptionKey 288 | loanPaymentNumber 289 | loanTotalPaymentsCount 290 | } 291 | ... on PartialPrePayment { 292 | loanDescriptionKey 293 | } 294 | } 295 | fragment MandateDetailsData on MandateDetails { 296 | __typename 297 | ... on MandatePayment { 298 | mandateDescriptionKey 299 | } 300 | ... on MandateReturnPayment { 301 | mandateDescriptionKey 302 | } 303 | } 304 | fragment SavingsDetailsData on SavingsDetails { 305 | __typename 306 | ... on FullSavingsWithdrawal { 307 | savingsDescriptionKey 308 | } 309 | ... on MonthlySavingsDeposit { 310 | savingsDepositNumber 311 | savingsDescriptionKey 312 | savingsTotalDepositCount 313 | } 314 | ... on PartialSavingsWithdrawal { 315 | savingsDescriptionKey 316 | } 317 | ... on SavingsClosing { 318 | savingsDescriptionKey 319 | } 320 | ... on SavingsDeposit { 321 | savingsDescriptionKey 322 | } 323 | ... on SavingsInterest { 324 | savingsDescriptionKey 325 | } 326 | ... on SavingsPenalty { 327 | savingsDescriptionKey 328 | } 329 | ... on SavingsTax { 330 | savingsDescriptionKey 331 | } 332 | } 333 | fragment SubscriptionDetailsData on SubscriptionDetails { 334 | __typename 335 | ... on SubscriptionPayment { 336 | subscriptionDescriptionKey 337 | } 338 | ... on SubscriptionReturnPayment { 339 | subscriptionDescriptionKey 340 | } 341 | } 342 | fragment TransactionsDetailsData on TransactionDetails { 343 | __typename 344 | ... on BankTransfer { 345 | bank_transfer_details { 346 | __typename 347 | ...BankTransferDetailsData 348 | } 349 | book_date 350 | categories { 351 | __typename 352 | ...CategoryData 353 | } 354 | recurrences { 355 | __typename 356 | ...RecurrenceData 357 | } 358 | value_date 359 | } 360 | ... on Card { 361 | card_details { 362 | __typename 363 | ...CardDetailsData 364 | } 365 | categories { 366 | __typename 367 | ...CategoryData 368 | } 369 | recurrences { 370 | __typename 371 | ...RecurrenceData 372 | } 373 | value_date 374 | } 375 | ... on Cash { 376 | cash_details { 377 | __typename 378 | ...CashDetailsData 379 | } 380 | categories { 381 | __typename 382 | ...CategoryData 383 | } 384 | recurrences { 385 | __typename 386 | ...RecurrenceData 387 | } 388 | value_date 389 | } 390 | ... on Cheques { 391 | categories { 392 | __typename 393 | ...CategoryData 394 | } 395 | chequesDetails { 396 | __typename 397 | ...ChequesDetailsData 398 | } 399 | recurrences { 400 | __typename 401 | ...RecurrenceData 402 | } 403 | valueDate 404 | referenceNumber 405 | frontImageUrl 406 | backImageUrl 407 | } 408 | ... on Default { 409 | default_details { 410 | __typename 411 | ...DefaultDetailsData 412 | } 413 | recurrences { 414 | __typename 415 | ...RecurrenceData 416 | } 417 | value_date 418 | } 419 | ... on Fee { 420 | categories { 421 | __typename 422 | ...CategoryData 423 | } 424 | fee_details { 425 | __typename 426 | ...FeeDetailsData 427 | } 428 | value_date 429 | } 430 | ... on Loans { 431 | categories { 432 | __typename 433 | ...CategoryData 434 | } 435 | loan_details { 436 | __typename 437 | ...LoanDetailsData 438 | } 439 | recurrences { 440 | __typename 441 | ...RecurrenceData 442 | } 443 | value_date 444 | } 445 | ... on Mandate { 446 | categories { 447 | __typename 448 | ...CategoryData 449 | } 450 | mandate_details { 451 | __typename 452 | ...MandateDetailsData 453 | } 454 | recurrences { 455 | __typename 456 | ...RecurrenceData 457 | } 458 | value_date 459 | } 460 | ... on Savings { 461 | categories { 462 | __typename 463 | ...CategoryData 464 | } 465 | recurrences { 466 | __typename 467 | ...RecurrenceData 468 | } 469 | savings_details { 470 | __typename 471 | ...SavingsDetailsData 472 | } 473 | value_date 474 | } 475 | ... on SubscriptionTransaction { 476 | categories { 477 | __typename 478 | ...CategoryData 479 | } 480 | recurrences { 481 | __typename 482 | ...RecurrenceData 483 | } 484 | subscription_details { 485 | __typename 486 | ...SubscriptionDetailsData 487 | } 488 | value_date 489 | } 490 | } 491 | fragment TransactionFragment on Transaction { 492 | __typename 493 | baseTransaction { 494 | __typename 495 | ...BaseTransactionFragment 496 | } 497 | enrichment { 498 | __typename 499 | ...TransactionEnrichmentFragment 500 | } 501 | metadata { 502 | __typename 503 | ...TransactionEventMetadataFragment 504 | } 505 | referenceNumber 506 | transactionDetails { 507 | __typename 508 | ...TransactionsDetailsData 509 | } 510 | } 511 | fragment MovementFragment on Movement { 512 | __typename 513 | accountId 514 | bankCurrencyAmount 515 | bookingDate 516 | conversionRate 517 | creditDebit 518 | description 519 | isReversed 520 | linkTransaction { 521 | __typename 522 | ...TransactionFragment 523 | } 524 | movementAmount 525 | movementCurrency 526 | movementId 527 | movementReversedId 528 | movementTimestamp 529 | movementType 530 | portfolioId 531 | runningBalance 532 | transaction { 533 | __typename 534 | ...TransactionFragment 535 | } 536 | valueDate 537 | } 538 | fragment PaginationFragment on Pagination { 539 | __typename 540 | cursor 541 | hasMore 542 | } 543 | fragment MovementsFragment on Movements { 544 | __typename 545 | isRunningBalanceInSync 546 | movements { 547 | __typename 548 | ...MovementFragment 549 | } 550 | pagination { 551 | __typename 552 | ...PaginationFragment 553 | } 554 | }`; 555 | -------------------------------------------------------------------------------- /src/scrapers/max.ts: -------------------------------------------------------------------------------- 1 | import buildUrl from 'build-url'; 2 | import moment, { Moment } from 'moment'; 3 | import { Page } from 'puppeteer'; 4 | import { fetchGetWithinPage } from '../helpers/fetch'; 5 | import { BaseScraperWithBrowser, LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 6 | import { waitForRedirect } from '../helpers/navigation'; 7 | import { waitUntilElementFound, elementPresentOnPage, clickButton } from '../helpers/elements-interactions'; 8 | import getAllMonthMoments from '../helpers/dates'; 9 | import { fixInstallments, sortTransactionsByDate, filterOldTransactions } from '../helpers/transactions'; 10 | import { Transaction, TransactionStatuses, TransactionTypes } from '../transactions'; 11 | import { getDebug } from '../helpers/debug'; 12 | import { ScraperOptions } from './interface'; 13 | 14 | const debug = getDebug('max'); 15 | 16 | interface ScrapedTransaction { 17 | shortCardNumber: string; 18 | paymentDate?: string; 19 | purchaseDate: string; 20 | actualPaymentAmount: string; 21 | originalCurrency: string; 22 | originalAmount: number; 23 | planName: string; 24 | comments: string; 25 | merchantName: string; 26 | categoryId: number; 27 | dealData?: { 28 | arn: string; 29 | }; 30 | } 31 | 32 | const BASE_ACTIONS_URL = 'https://online.max.co.il'; 33 | const BASE_API_ACTIONS_URL = 'https://onlinelcapi.max.co.il'; 34 | const BASE_WELCOME_URL = 'https://www.max.co.il'; 35 | 36 | const LOGIN_URL = `${BASE_WELCOME_URL}/homepage/welcome`; 37 | const PASSWORD_EXPIRED_URL = `${BASE_ACTIONS_URL}/Anonymous/Login/PasswordExpired.aspx`; 38 | const SUCCESS_URL = `${BASE_WELCOME_URL}/homepage/personal`; 39 | 40 | const NORMAL_TYPE_NAME = 'רגילה'; 41 | const ATM_TYPE_NAME = 'חיוב עסקות מיידי'; 42 | const INTERNET_SHOPPING_TYPE_NAME = 'אינטרנט/חו"ל'; 43 | const INSTALLMENTS_TYPE_NAME = 'תשלומים'; 44 | const MONTHLY_CHARGE_TYPE_NAME = 'חיוב חודשי'; 45 | const ONE_MONTH_POSTPONED_TYPE_NAME = 'דחוי חודש'; 46 | const MONTHLY_POSTPONED_TYPE_NAME = 'דחוי לחיוב החודשי'; 47 | const MONTHLY_PAYMENT_TYPE_NAME = 'תשלום חודשי'; 48 | const FUTURE_PURCHASE_FINANCING = 'מימון לרכישה עתידית'; 49 | const MONTHLY_POSTPONED_INSTALLMENTS_TYPE_NAME = 'דחוי חודש תשלומים'; 50 | const THIRTY_DAYS_PLUS_TYPE_NAME = 'עסקת 30 פלוס'; 51 | const TWO_MONTHS_POSTPONED_TYPE_NAME = 'דחוי חודשיים'; 52 | const MONTHLY_CHARGE_PLUS_INTEREST_TYPE_NAME = 'חודשי + ריבית'; 53 | const CREDIT_TYPE_NAME = 'קרדיט'; 54 | const ACCUMULATING_BASKET = 'סל מצטבר'; 55 | const POSTPONED_TRANSACTION_INSTALLMENTS = 'פריסת העסקה הדחויה'; 56 | const REPLACEMENT_CARD = 'כרטיס חליפי'; 57 | const EARLY_REPAYMENT = 'פרעון מוקדם'; 58 | const MONTHLY_CARD_FEE = 'דמי כרטיס'; 59 | 60 | const INVALID_DETAILS_SELECTOR = '#popupWrongDetails'; 61 | const LOGIN_ERROR_SELECTOR = '#popupCardHoldersLoginError'; 62 | 63 | const categories = new Map(); 64 | 65 | function redirectOrDialog(page: Page) { 66 | return Promise.race([ 67 | waitForRedirect(page, 20000, false, [BASE_WELCOME_URL, `${BASE_WELCOME_URL}/`]), 68 | waitUntilElementFound(page, INVALID_DETAILS_SELECTOR, true), 69 | waitUntilElementFound(page, LOGIN_ERROR_SELECTOR, true), 70 | ]); 71 | } 72 | 73 | function getTransactionsUrl(monthMoment: Moment) { 74 | const month = monthMoment.month() + 1; 75 | const year = monthMoment.year(); 76 | const date = `${year}-${month}-01`; 77 | 78 | /** 79 | * url explanation: 80 | * userIndex: -1 for all account owners 81 | * cardIndex: -1 for all cards under the account 82 | * all other query params are static, beside the date which changes for request per month 83 | */ 84 | return buildUrl(BASE_API_ACTIONS_URL, { 85 | path: `/api/registered/transactionDetails/getTransactionsAndGraphs?filterData={"userIndex":-1,"cardIndex":-1,"monthView":true,"date":"${date}","dates":{"startDate":"0","endDate":"0"},"bankAccount":{"bankAccountIndex":-1,"cards":null}}&firstCallCardIndex=-1`, 86 | }); 87 | } 88 | 89 | interface FetchCategoryResult { 90 | result? : Array<{ 91 | id: number; 92 | name: string; 93 | }>; 94 | } 95 | 96 | async function loadCategories(page: Page) { 97 | debug('Loading categories'); 98 | const res = await fetchGetWithinPage(page, `${BASE_API_ACTIONS_URL}/api/contents/getCategories`); 99 | if (res && Array.isArray(res.result)) { 100 | debug(`${res.result.length} categories loaded`); 101 | res.result?.forEach(({ id, name }) => categories.set(id, name)); 102 | } 103 | } 104 | 105 | function getTransactionType(txnTypeStr: string) { 106 | const cleanedUpTxnTypeStr = txnTypeStr.replace('\t', ' ').trim(); 107 | switch (cleanedUpTxnTypeStr) { 108 | case ATM_TYPE_NAME: 109 | case NORMAL_TYPE_NAME: 110 | case MONTHLY_CHARGE_TYPE_NAME: 111 | case ONE_MONTH_POSTPONED_TYPE_NAME: 112 | case MONTHLY_POSTPONED_TYPE_NAME: 113 | case FUTURE_PURCHASE_FINANCING: 114 | case MONTHLY_PAYMENT_TYPE_NAME: 115 | case MONTHLY_POSTPONED_INSTALLMENTS_TYPE_NAME: 116 | case THIRTY_DAYS_PLUS_TYPE_NAME: 117 | case TWO_MONTHS_POSTPONED_TYPE_NAME: 118 | case ACCUMULATING_BASKET: 119 | case INTERNET_SHOPPING_TYPE_NAME: 120 | case MONTHLY_CHARGE_PLUS_INTEREST_TYPE_NAME: 121 | case POSTPONED_TRANSACTION_INSTALLMENTS: 122 | case REPLACEMENT_CARD: 123 | case EARLY_REPAYMENT: 124 | case MONTHLY_CARD_FEE: 125 | return TransactionTypes.Normal; 126 | case INSTALLMENTS_TYPE_NAME: 127 | case CREDIT_TYPE_NAME: 128 | return TransactionTypes.Installments; 129 | default: 130 | throw new Error(`Unknown transaction type ${cleanedUpTxnTypeStr}`); 131 | } 132 | } 133 | 134 | function getInstallmentsInfo(comments: string) { 135 | if (!comments) { 136 | return undefined; 137 | } 138 | const matches = comments.match(/\d+/g); 139 | if (!matches || matches.length < 2) { 140 | return undefined; 141 | } 142 | 143 | return { 144 | number: parseInt(matches[0], 10), 145 | total: parseInt(matches[1], 10), 146 | }; 147 | } 148 | function mapTransaction(rawTransaction: ScrapedTransaction): Transaction { 149 | const isPending = rawTransaction.paymentDate === null; 150 | const processedDate = moment(isPending ? 151 | rawTransaction.purchaseDate : 152 | rawTransaction.paymentDate).toISOString(); 153 | const status = isPending ? TransactionStatuses.Pending : TransactionStatuses.Completed; 154 | 155 | const installments = getInstallmentsInfo(rawTransaction.comments); 156 | const identifier = installments ? 157 | `${rawTransaction.dealData?.arn}_${installments.number}` : 158 | rawTransaction.dealData?.arn; 159 | 160 | return { 161 | type: getTransactionType(rawTransaction.planName), 162 | date: moment(rawTransaction.purchaseDate).toISOString(), 163 | processedDate, 164 | originalAmount: -rawTransaction.originalAmount, 165 | originalCurrency: rawTransaction.originalCurrency, 166 | chargedAmount: -rawTransaction.actualPaymentAmount, 167 | description: rawTransaction.merchantName.trim(), 168 | memo: rawTransaction.comments, 169 | category: categories.get(rawTransaction?.categoryId), 170 | installments, 171 | identifier, 172 | status, 173 | }; 174 | } 175 | interface ScrapedTransactionsResult{ 176 | result?: { 177 | transactions: ScrapedTransaction[]; 178 | }; 179 | } 180 | 181 | async function fetchTransactionsForMonth(page: Page, monthMoment: Moment) { 182 | const url = getTransactionsUrl(monthMoment); 183 | 184 | const data = await fetchGetWithinPage(page, url); 185 | const transactionsByAccount: Record = {}; 186 | 187 | if (!data || !data.result) return transactionsByAccount; 188 | 189 | data.result.transactions 190 | // Filter out non-transactions without a plan type, e.g. summary rows 191 | .filter((transaction) => !!transaction.planName) 192 | .forEach((transaction: ScrapedTransaction) => { 193 | if (!transactionsByAccount[transaction.shortCardNumber]) { 194 | transactionsByAccount[transaction.shortCardNumber] = []; 195 | } 196 | 197 | const mappedTransaction = mapTransaction(transaction); 198 | transactionsByAccount[transaction.shortCardNumber].push(mappedTransaction); 199 | }); 200 | 201 | return transactionsByAccount; 202 | } 203 | 204 | function addResult(allResults: Record, result: Record) { 205 | const clonedResults: Record = { ...allResults }; 206 | Object.keys(result).forEach((accountNumber) => { 207 | if (!clonedResults[accountNumber]) { 208 | clonedResults[accountNumber] = []; 209 | } 210 | clonedResults[accountNumber].push(...result[accountNumber]); 211 | }); 212 | return clonedResults; 213 | } 214 | 215 | function prepareTransactions(txns: Transaction[], startMoment: moment.Moment, combineInstallments: boolean, enableTransactionsFilterByDate: boolean) { 216 | let clonedTxns = Array.from(txns); 217 | if (!combineInstallments) { 218 | clonedTxns = fixInstallments(clonedTxns); 219 | } 220 | clonedTxns = sortTransactionsByDate(clonedTxns); 221 | clonedTxns = enableTransactionsFilterByDate ? 222 | filterOldTransactions(clonedTxns, startMoment, combineInstallments || false) : 223 | clonedTxns; 224 | return clonedTxns; 225 | } 226 | 227 | async function fetchTransactions(page: Page, options: ScraperOptions) { 228 | const futureMonthsToScrape = options.futureMonthsToScrape ?? 1; 229 | const defaultStartMoment = moment().subtract(1, 'years'); 230 | const startDate = options.startDate || defaultStartMoment.toDate(); 231 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 232 | const allMonths = getAllMonthMoments(startMoment, futureMonthsToScrape); 233 | 234 | await loadCategories(page); 235 | 236 | let allResults: Record = {}; 237 | for (let i = 0; i < allMonths.length; i += 1) { 238 | const result = await fetchTransactionsForMonth(page, allMonths[i]); 239 | allResults = addResult(allResults, result); 240 | } 241 | 242 | Object.keys(allResults).forEach((accountNumber) => { 243 | let txns = allResults[accountNumber]; 244 | txns = prepareTransactions(txns, startMoment, options.combineInstallments || false, 245 | (options.outputData?.enableTransactionsFilterByDate ?? true)); 246 | allResults[accountNumber] = txns; 247 | }); 248 | 249 | return allResults; 250 | } 251 | 252 | function getPossibleLoginResults(page: Page): PossibleLoginResults { 253 | const urls: PossibleLoginResults = {}; 254 | urls[LoginResults.Success] = [SUCCESS_URL]; 255 | urls[LoginResults.ChangePassword] = [PASSWORD_EXPIRED_URL]; 256 | urls[LoginResults.InvalidPassword] = [async () => { 257 | return elementPresentOnPage(page, INVALID_DETAILS_SELECTOR); 258 | }]; 259 | urls[LoginResults.UnknownError] = [async () => { 260 | return elementPresentOnPage(page, LOGIN_ERROR_SELECTOR); 261 | }]; 262 | return urls; 263 | } 264 | 265 | function createLoginFields(credentials: ScraperSpecificCredentials) { 266 | return [ 267 | { selector: '#user-name', value: credentials.username }, 268 | { selector: '#password', value: credentials.password }, 269 | ]; 270 | } 271 | 272 | type ScraperSpecificCredentials = {username: string, password: string}; 273 | 274 | class MaxScraper extends BaseScraperWithBrowser { 275 | getLoginOptions(credentials: ScraperSpecificCredentials) { 276 | return { 277 | loginUrl: LOGIN_URL, 278 | fields: createLoginFields(credentials), 279 | submitButtonSelector: '#login-password #send-code', 280 | preAction: async () => { 281 | if (await elementPresentOnPage(this.page, '#closePopup')) { 282 | await clickButton(this.page, '#closePopup'); 283 | } 284 | await clickButton(this.page, '.personal-area > a.go-to-personal-area'); 285 | await waitUntilElementFound(this.page, '#login-password-link', true); 286 | await clickButton(this.page, '#login-password-link'); 287 | await waitUntilElementFound(this.page, '#login-password.tab-pane.active app-user-login-form', true); 288 | }, 289 | checkReadiness: async () => { 290 | await waitUntilElementFound(this.page, '.personal-area > a.go-to-personal-area', true); 291 | }, 292 | postAction: async () => redirectOrDialog(this.page), 293 | possibleResults: getPossibleLoginResults(this.page), 294 | }; 295 | } 296 | 297 | async fetchData() { 298 | const results = await fetchTransactions(this.page, this.options); 299 | const accounts = Object.keys(results).map((accountNumber) => { 300 | return { 301 | accountNumber, 302 | txns: results[accountNumber], 303 | }; 304 | }); 305 | 306 | return { 307 | success: true, 308 | accounts, 309 | }; 310 | } 311 | } 312 | 313 | export default MaxScraper; 314 | --------------------------------------------------------------------------------