├── .nvmrc ├── tests ├── .gitignore └── readme.md ├── .npmrc ├── logo.png ├── src ├── tests │ ├── .gitignore │ ├── .eslintrc.js │ ├── jest-setup.ts │ ├── .tests-config.tpl.js │ └── tests-utils.ts ├── assertNever.ts ├── helpers │ ├── debug.ts │ ├── storage.ts │ ├── browser.ts │ ├── dates.ts │ ├── navigation.ts │ ├── transactions.ts │ ├── waiting.ts │ ├── fetch.ts │ └── elements-interactions.ts ├── scrapers │ ├── amex.ts │ ├── isracard.ts │ ├── mercantile.ts │ ├── pagi.ts │ ├── beinleumi.ts │ ├── massad.ts │ ├── otsar-hahayal.ts │ ├── factory.test.ts │ ├── errors.ts │ ├── behatsdaa.test.ts │ ├── pagi.test.ts │ ├── leumi.test.ts │ ├── union-bank.test.ts │ ├── beinleumi.test.ts │ ├── hapoalim.test.ts │ ├── amex.test.ts │ ├── otsar-hahayal.test.ts │ ├── base-scraper-with-browser.test.ts │ ├── yahav.test.ts │ ├── discount.test.ts │ ├── beyahad-bishvilha.test.ts │ ├── mercantile.test.ts │ ├── isracard.test.ts │ ├── visa-cal.test.ts │ ├── one-zero.test.ts │ ├── max.test.ts │ ├── factory.ts │ ├── mizrahi.test.ts │ ├── base-scraper.ts │ ├── behatsdaa.ts │ ├── discount.ts │ ├── interface.ts │ ├── beyahad-bishvilha.ts │ ├── leumi.ts │ ├── hapoalim.ts │ ├── yahav.ts │ ├── one-zero.ts │ ├── union-bank.ts │ ├── base-scraper-with-browser.ts │ └── one-zero-queries.ts ├── constants.ts ├── index.ts ├── transactions.ts └── definitions.ts ├── .husky └── pre-commit ├── tsconfig.build.json ├── .babelrc.js ├── .github ├── workflows │ ├── lint.yml │ ├── pr-title.yml │ ├── nodeCI.yml │ ├── stale.yml │ └── release.yml ├── stale.yml └── component_owners.yml ├── utils ├── core-utils.js ├── jscodeshift │ ├── puppeteer-imports.js │ └── index.js ├── pre-publish.js └── prepare-israeli-bank-scrapers-core.js ├── .prettierrc.js ├── jest.config.js ├── tsconfig.json ├── LICENSE ├── .gitignore ├── .eslintrc.js ├── package.json └── CONTRIBUTING.md /.nvmrc: -------------------------------------------------------------------------------- 1 | v18.19.0 2 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | **/* 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry = https://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eshaham/israeli-bank-scrapers/HEAD/logo.png -------------------------------------------------------------------------------- /src/tests/.gitignore: -------------------------------------------------------------------------------- 1 | .*.* 2 | .* 3 | !.tests-config.tpl.js 4 | snapshots/** 5 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npm run lint 5 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": [ 4 | "**/*.test.ts", 5 | "src/tests/**/*" 6 | ] 7 | } -------------------------------------------------------------------------------- /src/assertNever.ts: -------------------------------------------------------------------------------- 1 | export function assertNever(x: never, error = ''): never { 2 | throw new Error(error || `Unexpected object: ${x as any}`); 3 | } 4 | -------------------------------------------------------------------------------- /src/helpers/debug.ts: -------------------------------------------------------------------------------- 1 | import debug from 'debug'; 2 | 3 | export function getDebug(name: string) { 4 | return debug(`israeli-bank-scrapers:${name}`); 5 | } 6 | -------------------------------------------------------------------------------- /.babelrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [['@babel/preset-env', { targets: { node: '18' } }], '@babel/preset-typescript'], 3 | ignore: ['**/*.test.(js,ts)', 'tests/**/*', 'src/tests/**/*'], 4 | }; 5 | -------------------------------------------------------------------------------- /src/tests/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | rules: { 3 | 'import/no-extraneous-dependencies': 0, 4 | 'import/no-dynamic-require': 0, 5 | 'global-require': 0, 6 | 'no-console': 0, 7 | }, 8 | }; 9 | -------------------------------------------------------------------------------- /tests/readme.md: -------------------------------------------------------------------------------- 1 | > Don't remove file .gitignore otherwise people might commit their credentials 2 | 3 | 1. This folder is obslete. It was moved to src/tests. 4 | 2. If you already have file `.tests-config.js`, move it to `src/tests` and change its extension to `ts`. 5 | 6 | -------------------------------------------------------------------------------- /src/tests/jest-setup.ts: -------------------------------------------------------------------------------- 1 | import * as SourceMap from 'source-map-support'; 2 | import { extendAsyncTimeout, getTestsConfig } from './tests-utils'; 3 | 4 | SourceMap.install(); 5 | // Try to get test configuration object, no need to do anything beside that 6 | getTestsConfig(); 7 | extendAsyncTimeout(); 8 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lint: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | 13 | - name: Run npm install 14 | run: npm install 15 | 16 | - name: Run lint 17 | run: npm run lint 18 | 19 | -------------------------------------------------------------------------------- /src/helpers/storage.ts: -------------------------------------------------------------------------------- 1 | import { type Page } from 'puppeteer'; 2 | 3 | export async function getFromSessionStorage(page: Page, key: string): Promise { 4 | const strData = await page.evaluate((k: string) => { 5 | return sessionStorage.getItem(k); 6 | }, key); 7 | 8 | if (!strData) return null; 9 | 10 | return JSON.parse(strData) as T; 11 | } 12 | -------------------------------------------------------------------------------- /utils/core-utils.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = function checkIfCoreVariation() { 4 | const packagePath = path.join(__dirname, '..', 'package.json'); 5 | // eslint-disable-next-line import/no-dynamic-require,global-require 6 | const packageJson = require(packagePath); 7 | 8 | return (packageJson.name === 'israeli-bank-scrapers-core'); 9 | } 10 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | printWidth: 120, // Matches existing max-len rule 3 | semi: true, 4 | singleQuote: true, // Common in airbnb style 5 | trailingComma: 'all', 6 | bracketSpacing: true, 7 | endOfLine: 'auto', // Handle linebreak-style automatically 8 | tabWidth: 2, 9 | useTabs: false, 10 | arrowParens: 'avoid', 11 | parser: 'typescript', // Since this is a TypeScript project 12 | }; -------------------------------------------------------------------------------- /src/scrapers/amex.ts: -------------------------------------------------------------------------------- 1 | import IsracardAmexBaseScraper from './base-isracard-amex'; 2 | import { type ScraperOptions } from './interface'; 3 | 4 | const BASE_URL = 'https://he.americanexpress.co.il'; 5 | const COMPANY_CODE = '77'; 6 | 7 | class AmexScraper extends IsracardAmexBaseScraper { 8 | constructor(options: ScraperOptions) { 9 | super(options, BASE_URL, COMPANY_CODE); 10 | } 11 | } 12 | 13 | export default AmexScraper; 14 | -------------------------------------------------------------------------------- /src/scrapers/isracard.ts: -------------------------------------------------------------------------------- 1 | import IsracardAmexBaseScraper from './base-isracard-amex'; 2 | import { type ScraperOptions } from './interface'; 3 | 4 | const BASE_URL = 'https://digital.isracard.co.il'; 5 | const COMPANY_CODE = '11'; 6 | 7 | class IsracardScraper extends IsracardAmexBaseScraper { 8 | constructor(options: ScraperOptions) { 9 | super(options, BASE_URL, COMPANY_CODE); 10 | } 11 | } 12 | 13 | export default IsracardScraper; 14 | -------------------------------------------------------------------------------- /src/scrapers/mercantile.ts: -------------------------------------------------------------------------------- 1 | import DiscountScraper from './discount'; 2 | 3 | type ScraperSpecificCredentials = { id: string; password: string; num: string }; 4 | class MercantileScraper extends DiscountScraper { 5 | getLoginOptions(credentials: ScraperSpecificCredentials) { 6 | return { 7 | ...super.getLoginOptions(credentials), 8 | loginUrl: 'https://start.telebank.co.il/login/?bank=m', 9 | }; 10 | } 11 | } 12 | 13 | export default MercantileScraper; 14 | -------------------------------------------------------------------------------- /src/scrapers/pagi.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiGroupBaseScraper from './base-beinleumi-group'; 2 | 3 | class PagiScraper extends BeinleumiGroupBaseScraper { 4 | BASE_URL = 'https://online.pagi.co.il/'; 5 | 6 | LOGIN_URL = `${this.BASE_URL}/MatafLoginService/MatafLoginServlet?bankId=PAGIPORTAL&site=Private&KODSAFA=HE`; 7 | 8 | TRANSACTIONS_URL = `${this.BASE_URL}/wps/myportal/FibiMenu/Online/OnAccountMngment/OnBalanceTrans/PrivateAccountFlow`; 9 | } 10 | 11 | export default PagiScraper; 12 | -------------------------------------------------------------------------------- /src/scrapers/beinleumi.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiGroupBaseScraper from './base-beinleumi-group'; 2 | 3 | class BeinleumiScraper extends BeinleumiGroupBaseScraper { 4 | BASE_URL = 'https://online.fibi.co.il'; 5 | 6 | LOGIN_URL = `${this.BASE_URL}/MatafLoginService/MatafLoginServlet?bankId=FIBIPORTAL&site=Private&KODSAFA=HE`; 7 | 8 | TRANSACTIONS_URL = `${this.BASE_URL}/wps/myportal/FibiMenu/Online/OnAccountMngment/OnBalanceTrans/PrivateAccountFlow`; 9 | } 10 | 11 | export default BeinleumiScraper; 12 | -------------------------------------------------------------------------------- /src/scrapers/massad.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiGroupBaseScraper from './base-beinleumi-group'; 2 | 3 | class MassadScraper extends BeinleumiGroupBaseScraper { 4 | BASE_URL = 'https://online.bankmassad.co.il'; 5 | 6 | LOGIN_URL = `${this.BASE_URL}/MatafLoginService/MatafLoginServlet?bankId=MASADPRTAL&site=Private&KODSAFA=HE`; 7 | 8 | TRANSACTIONS_URL = `${this.BASE_URL}/wps/myportal/FibiMenu/Online/OnAccountMngment/OnBalanceTrans/PrivateAccountFlow`; 9 | } 10 | 11 | export default MassadScraper; 12 | -------------------------------------------------------------------------------- /src/scrapers/otsar-hahayal.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiGroupBaseScraper from './base-beinleumi-group'; 2 | 3 | class OtsarHahayalScraper extends BeinleumiGroupBaseScraper { 4 | BASE_URL = 'https://online.bankotsar.co.il'; 5 | 6 | LOGIN_URL = `${this.BASE_URL}/MatafLoginService/MatafLoginServlet?bankId=OTSARPRTAL&site=Private&KODSAFA=HE`; 7 | 8 | TRANSACTIONS_URL = `${this.BASE_URL}/wps/myportal/FibiMenu/Online/OnAccountMngment/OnBalanceTrans/PrivateAccountFlow`; 9 | } 10 | 11 | export default OtsarHahayalScraper; 12 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | // For a detailed explanation regarding each configuration property, visit: 2 | // https://jestjs.io/docs/en/configuration.html 3 | /** @type {import('jest').Config} */ 4 | const config = { 5 | preset: 'ts-jest', 6 | clearMocks: true, 7 | coverageDirectory: 'coverage', 8 | rootDir: './src', 9 | transform: { 10 | '^.+\\.ts$': ['ts-jest'], 11 | }, 12 | setupFilesAfterEnv: [ 13 | './tests/jest-setup.ts', 14 | ], 15 | testEnvironment: 'node', 16 | }; 17 | 18 | module.exports = config; -------------------------------------------------------------------------------- /utils/jscodeshift/puppeteer-imports.js: -------------------------------------------------------------------------------- 1 | 2 | const transform = (file, api) => { 3 | const j = api.jscodeshift; 4 | 5 | const root = j(file.source); 6 | root 7 | .find(j.ImportDeclaration) 8 | .find(j.Literal) 9 | .replaceWith(nodePath => { 10 | const { node } = nodePath; 11 | 12 | if (!node.value || node.value !== 'puppeteer') { 13 | return node; 14 | } 15 | 16 | node.value = 'puppeteer-core'; 17 | 18 | return node; 19 | }); 20 | 21 | return root.toSource(); 22 | }; 23 | 24 | module.exports = transform; -------------------------------------------------------------------------------- /src/scrapers/factory.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/unbound-method */ 2 | import { CompanyTypes } from '../definitions'; 3 | import createScraper from './factory'; 4 | 5 | describe('Factory', () => { 6 | test('should return a scraper instance', () => { 7 | const scraper = createScraper({ 8 | companyId: CompanyTypes.hapoalim, 9 | startDate: new Date(), 10 | }); 11 | expect(scraper).toBeDefined(); 12 | 13 | expect(scraper.scrape).toBeInstanceOf(Function); 14 | expect(scraper.onProgress).toBeInstanceOf(Function); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | export const SHEKEL_CURRENCY_SYMBOL = '₪'; 2 | export const SHEKEL_CURRENCY_KEYWORD = 'ש"ח'; 3 | export const ALT_SHEKEL_CURRENCY = 'NIS'; 4 | export const SHEKEL_CURRENCY = 'ILS'; 5 | 6 | export const DOLLAR_CURRENCY_SYMBOL = '$'; 7 | export const DOLLAR_CURRENCY = 'USD'; 8 | 9 | export const EURO_CURRENCY_SYMBOL = '€'; 10 | export const EURO_CURRENCY = 'EUR'; 11 | 12 | export const ISO_DATE_FORMAT = 'YYYY-MM-DD[T]HH:mm:ss.SSS[Z]'; 13 | 14 | export const ISO_DATE_REGEX = 15 | /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T([0-1][0-9]|2[0-3])(:[0-5][0-9]){2}\.[0-9]{3}Z$/; 16 | -------------------------------------------------------------------------------- /utils/jscodeshift/index.js: -------------------------------------------------------------------------------- 1 | const { resolve } = require("path"); 2 | const { spawnSync } = require("child_process"); 3 | const { run: jscodeshift } = require("jscodeshift/src/Runner"); 4 | const path = require("node:path"); 5 | 6 | module.exports = async function () { 7 | 8 | const transformPath = path.resolve(__dirname, "./puppeteer-imports.js"); 9 | const paths = [path.resolve(__dirname, "../../src")]; 10 | const options = { 11 | extensions: "ts", 12 | parser: "ts", 13 | }; 14 | 15 | const res = await jscodeshift(transformPath, paths, options); 16 | console.log(res); 17 | }; 18 | -------------------------------------------------------------------------------- /src/helpers/browser.ts: -------------------------------------------------------------------------------- 1 | import { type Page } from 'puppeteer'; 2 | 3 | export async function maskHeadlessUserAgent(page: Page): Promise { 4 | const userAgent = await page.evaluate(() => navigator.userAgent); 5 | await page.setUserAgent(userAgent.replace('HeadlessChrome/', 'Chrome/')); 6 | } 7 | 8 | /** 9 | * Priorities for request interception. The higher the number, the higher the priority. 10 | * We want to let others to have the ability to override our interception logic therefore we hardcode them. 11 | */ 12 | export const interceptionPriorities = { 13 | abort: 1000, 14 | continue: 10, 15 | }; 16 | -------------------------------------------------------------------------------- /.github/workflows/pr-title.yml: -------------------------------------------------------------------------------- 1 | name: Validate Conventional Commit title 2 | 3 | on: 4 | pull_request: 5 | types: [opened, edited, synchronize, reopened] 6 | 7 | jobs: 8 | validate: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: install commitlint 12 | run: npm install -g @commitlint/cli @commitlint/config-angular 13 | - name: config commitlint 14 | run: | 15 | echo "module.exports = {extends: ['@commitlint/config-angular']}" > commitlint.config.js 16 | - name: validate PR title 17 | run: | 18 | echo "${{ github.event.pull_request.title }}" | commitlint -------------------------------------------------------------------------------- /src/helpers/dates.ts: -------------------------------------------------------------------------------- 1 | import moment, { type Moment } from 'moment'; 2 | 3 | export default function getAllMonthMoments(startMoment: Moment | string, futureMonths?: number) { 4 | let monthMoment = moment(startMoment).startOf('month'); 5 | 6 | const allMonths: Moment[] = []; 7 | let lastMonth = moment().startOf('month'); 8 | if (futureMonths && futureMonths > 0) { 9 | lastMonth = lastMonth.add(futureMonths, 'month'); 10 | } 11 | while (monthMoment.isSameOrBefore(lastMonth)) { 12 | allMonths.push(monthMoment); 13 | monthMoment = moment(monthMoment).add(1, 'month'); 14 | } 15 | 16 | return allMonths; 17 | } 18 | -------------------------------------------------------------------------------- /utils/pre-publish.js: -------------------------------------------------------------------------------- 1 | const fsExtra = require('fs-extra'); 2 | const path = require('path'); 3 | const argv = require('minimist')(process.argv.slice(2), { string: 'version' }); 4 | 5 | const version = argv.version; 6 | 7 | if (!version) { 8 | console.error(`missing argument 'version'`); 9 | process.exit(1); 10 | return; 11 | } 12 | 13 | const packageJSONPath = path.resolve(__dirname, '../package.json'); 14 | 15 | const packageJSON = fsExtra.readJSONSync(packageJSONPath); 16 | 17 | 18 | packageJSON.version = version; 19 | packageJSON.private = false; 20 | 21 | fsExtra.writeJSONSync(packageJSONPath, packageJSON, { spaces: 2 }) 22 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { CompanyTypes, SCRAPERS } from './definitions'; 2 | export { default as createScraper } from './scrapers/factory'; 3 | 4 | // Note: the typo ScaperScrapingResult & ScraperLoginResult (sic) are exported here for backward compatibility 5 | export { 6 | ScraperLoginResult as ScaperLoginResult, 7 | ScraperScrapingResult as ScaperScrapingResult, 8 | Scraper, 9 | ScraperCredentials, 10 | ScraperLoginResult, 11 | ScraperOptions, 12 | ScraperScrapingResult, 13 | } from './scrapers/interface'; 14 | 15 | export { default as OneZeroScraper } from './scrapers/one-zero'; 16 | 17 | export function getPuppeteerConfig() { 18 | return { chromiumRevision: '1250580' }; // https://github.com/puppeteer/puppeteer/releases/tag/puppeteer-core-v22.5.0 19 | } 20 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | - dependencies 10 | # Label to use when marking an issue as stale 11 | staleLabel: wontfix 12 | # Comment to post when marking an issue as stale. Set to `false` to disable 13 | markComment: > 14 | This issue has been automatically marked as stale because it has not had 15 | recent activity. It will be closed if no further activity occurs. Thank you 16 | for your contributions. 17 | # Comment to post when closing a stale issue. Set to `false` to disable 18 | closeComment: false 19 | -------------------------------------------------------------------------------- /.github/workflows/nodeCI.yml: -------------------------------------------------------------------------------- 1 | name: Node CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | if: github.ref != 'refs/heads/master' 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest, windows-latest, macOS-latest] 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Setup Node.js 15 | uses: actions/setup-node@v4 16 | with: 17 | node-version-file: .nvmrc 18 | - uses: browser-actions/setup-chrome@v1 19 | - name: npm install and test 20 | env: 21 | PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: ${{ runner.os == 'macOS-latest' && 'true' || 'false' }} 22 | run: | 23 | npm ci 24 | npm run test:ci 25 | - name: Verify prepare:core 26 | run: npm run prepare:core 27 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: 'Close stale issues and PRs' 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: '30 1 * * *' 6 | 7 | jobs: 8 | stale: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/stale@v9 12 | with: 13 | ascending: true 14 | days-before-close: 14 15 | stale-issue-message: 'Issue has been marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.' 16 | stale-pr-message: 'Pull request has been marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.' 17 | close-issue-reason: 'not_planned' 18 | exempt-issue-labels: 'not-stale' 19 | exempt-pr-labels: 'not-stale' -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2022", 4 | "module": "commonjs", 5 | "allowJs": false, 6 | "skipLibCheck": true, 7 | "declaration": true, 8 | "outDir": "lib", 9 | "rootDir": "./src", 10 | "strict": true, 11 | "noImplicitAny": true, 12 | "resolveJsonModule": true, 13 | "noImplicitThis": true, 14 | "noUnusedLocals": true, 15 | "noUnusedParameters": true, 16 | "noImplicitReturns": true, 17 | "noFallthroughCasesInSwitch": true, 18 | "moduleResolution": "node", 19 | "allowSyntheticDefaultImports": true, 20 | "esModuleInterop": true, 21 | "forceConsistentCasingInFileNames": true, 22 | "inlineSourceMap": true, 23 | "inlineSources": true 24 | }, 25 | "include": ["src/**/*",".eslintrc.js","jest.config.js"], 26 | "exclude": ["node_modules"] 27 | } 28 | -------------------------------------------------------------------------------- /src/scrapers/errors.ts: -------------------------------------------------------------------------------- 1 | export enum ScraperErrorTypes { 2 | TwoFactorRetrieverMissing = 'TWO_FACTOR_RETRIEVER_MISSING', 3 | InvalidPassword = 'INVALID_PASSWORD', 4 | ChangePassword = 'CHANGE_PASSWORD', 5 | Timeout = 'TIMEOUT', 6 | AccountBlocked = 'ACCOUNT_BLOCKED', 7 | Generic = 'GENERIC', 8 | General = 'GENERAL_ERROR', 9 | } 10 | 11 | export type ErrorResult = { 12 | success: false; 13 | errorType: ScraperErrorTypes; 14 | errorMessage: string; 15 | }; 16 | 17 | function createErrorResult(errorType: ScraperErrorTypes, errorMessage: string): ErrorResult { 18 | return { 19 | success: false, 20 | errorType, 21 | errorMessage, 22 | }; 23 | } 24 | 25 | export function createTimeoutError(errorMessage: string): ErrorResult { 26 | return createErrorResult(ScraperErrorTypes.Timeout, errorMessage); 27 | } 28 | 29 | export function createGenericError(errorMessage: string): ErrorResult { 30 | return createErrorResult(ScraperErrorTypes.Generic, errorMessage); 31 | } 32 | -------------------------------------------------------------------------------- /utils/prepare-israeli-bank-scrapers-core.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const checkIfCoreVariation = require('./core-utils'); 4 | const transformImports = require('./jscodeshift'); 5 | 6 | function updatePackageJson() { 7 | const packagePath = path.join(__dirname, '..', 'package.json'); 8 | // eslint-disable-next-line import/no-dynamic-require,global-require 9 | const json = require(packagePath); 10 | 11 | json.dependencies['puppeteer-core'] = json.dependencies.puppeteer; 12 | delete json.dependencies.puppeteer; 13 | 14 | json.name = 'israeli-bank-scrapers-core'; 15 | fs.writeFileSync(packagePath, JSON.stringify(json, null, ' ')); 16 | 17 | console.log('change package.json name to \'israeli-bank-scrapers-core\' and use \'puppeteer-core\''); 18 | } 19 | 20 | (async function () { 21 | if (checkIfCoreVariation()) { 22 | console.log('library is already in core variation'); 23 | process.exit(1); 24 | } 25 | 26 | updatePackageJson(); 27 | await transformImports(); 28 | }()); 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Elad Shaham 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/transactions.ts: -------------------------------------------------------------------------------- 1 | export interface TransactionsAccount { 2 | accountNumber: string; 3 | balance?: number; 4 | txns: Transaction[]; 5 | } 6 | 7 | export enum TransactionTypes { 8 | Normal = 'normal', 9 | Installments = 'installments', 10 | } 11 | 12 | export enum TransactionStatuses { 13 | Completed = 'completed', 14 | Pending = 'pending', 15 | } 16 | 17 | export interface TransactionInstallments { 18 | /** 19 | * the current installment number 20 | */ 21 | number: number; 22 | 23 | /** 24 | * the total number of installments 25 | */ 26 | total: number; 27 | } 28 | 29 | export interface Transaction { 30 | type: TransactionTypes; 31 | /** 32 | * sometimes called Asmachta 33 | */ 34 | identifier?: string | number; 35 | /** 36 | * ISO date string 37 | */ 38 | date: string; 39 | /** 40 | * ISO date string 41 | */ 42 | processedDate: string; 43 | originalAmount: number; 44 | originalCurrency: string; 45 | chargedAmount: number; 46 | chargedCurrency?: string; 47 | description: string; 48 | memo?: string; 49 | status: TransactionStatuses; 50 | installments?: TransactionInstallments; 51 | category?: string; 52 | } 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # NPM package related 61 | lib/ 62 | 63 | # IDEs 64 | .idea 65 | .vscode 66 | .DS_Store 67 | -------------------------------------------------------------------------------- /.github/component_owners.yml: -------------------------------------------------------------------------------- 1 | # https://github.com/dyladan/component-owners 2 | 3 | components: 4 | src/scrapers/base-beinleumi-group.ts: 5 | # Beinleumi and Massad are using this base scraper 6 | - nissant 7 | src/scrapers/base-isracard-amex.ts: 8 | # Isracard and Amex are using this base scraper 9 | - baruchiro 10 | - galbarm 11 | - daniel-hauser 12 | 13 | src/scrapers/amex.ts: 14 | - daniel-hauser 15 | src/scrapers/behatsdaa.ts: 16 | - daniel-hauser 17 | src/scrapers/beinleumi.ts: 18 | - nissant 19 | # src/scrapers/beyahad-bishvilha.ts: 20 | src/scrapers/discount.ts: 21 | - galbarm 22 | src/scrapers/hapoalim.ts: 23 | - galbarm 24 | - daniel-hauser 25 | src/scrapers/isracard.ts: 26 | - baruchiro 27 | - galbarm 28 | - daniel-hauser 29 | src/scrapers/leumi.ts: 30 | - dimdimi4 31 | # src/scrapers/massad.ts: 32 | src/scrapers/max.ts: 33 | - baruchiro 34 | - galbarm 35 | - daniel-hauser 36 | - dimdimi4 37 | src/scrapers/marcentile.ts: 38 | - kfirarad 39 | - EzzatQ 40 | src/scrapers/mizrahi.ts: 41 | - baruchiro 42 | # src/scrapers/one-zero.ts: 43 | # src/scrapers/otzar-hahayal.ts: 44 | # src/scrapers/union-bank.ts: 45 | src/scrapers/visa-cal.ts: 46 | - baruchiro 47 | - galbarm 48 | - daniel-hauser 49 | src/scrapers/yahav.ts: 50 | - gczobel 51 | -------------------------------------------------------------------------------- /src/helpers/navigation.ts: -------------------------------------------------------------------------------- 1 | import { type Frame, type Page, type WaitForOptions } from 'puppeteer'; 2 | import { waitUntil } from './waiting'; 3 | 4 | export async function waitForNavigation(pageOrFrame: Page | Frame, options?: WaitForOptions) { 5 | await pageOrFrame.waitForNavigation(options); 6 | } 7 | 8 | export async function waitForNavigationAndDomLoad(page: Page) { 9 | await waitForNavigation(page, { waitUntil: 'domcontentloaded' }); 10 | } 11 | 12 | export function getCurrentUrl(pageOrFrame: Page | Frame, clientSide = false) { 13 | if (clientSide) { 14 | return pageOrFrame.evaluate(() => window.location.href); 15 | } 16 | 17 | return pageOrFrame.url(); 18 | } 19 | 20 | export async function waitForRedirect( 21 | pageOrFrame: Page | Frame, 22 | timeout = 20000, 23 | clientSide = false, 24 | ignoreList: string[] = [], 25 | ) { 26 | const initial = await getCurrentUrl(pageOrFrame, clientSide); 27 | 28 | await waitUntil( 29 | async () => { 30 | const current = await getCurrentUrl(pageOrFrame, clientSide); 31 | return current !== initial && !ignoreList.includes(current); 32 | }, 33 | `waiting for redirect from ${initial}`, 34 | timeout, 35 | 1000, 36 | ); 37 | } 38 | 39 | export async function waitForUrl(pageOrFrame: Page | Frame, url: string | RegExp, timeout = 20000, clientSide = false) { 40 | await waitUntil( 41 | async () => { 42 | const current = await getCurrentUrl(pageOrFrame, clientSide); 43 | return url instanceof RegExp ? url.test(current) : url === current; 44 | }, 45 | `waiting for url to be ${url}`, 46 | timeout, 47 | 1000, 48 | ); 49 | } 50 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | rules: { 4 | 'quotes': ['error', 'single', { avoidEscape: true }], 5 | 'import/prefer-default-export': 0, 6 | 'no-nested-ternary': 0, 7 | 'class-methods-use-this': 0, 8 | 'arrow-body-style': 0, 9 | 'no-shadow': 0, 10 | 'no-await-in-loop': 0, 11 | 'no-restricted-syntax': ['error', 'ForInStatement', 'LabeledStatement', 'WithStatement'], 12 | '@typescript-eslint/explicit-function-return-type': 0, 13 | '@typescript-eslint/no-explicit-any': 0, 14 | '@typescript-eslint/ban-ts-ignore': 0, 15 | '@typescript-eslint/no-non-null-assertion': 0, 16 | '@typescript-eslint/no-unsafe-member-access': 0, 17 | '@typescript-eslint/no-unsafe-call': 0, 18 | '@typescript-eslint/no-unsafe-assignment': 0, 19 | '@typescript-eslint/no-unsafe-argument': 0, 20 | '@typescript-eslint/no-unsafe-return': 0, 21 | '@typescript-eslint/ban-ts-comment': 0, 22 | '@typescript-eslint/restrict-template-expressions': [ 23 | 'error', 24 | { 25 | allowNever: true, 26 | }, 27 | ], 28 | '@typescript-eslint/consistent-type-imports': [ 29 | 'error', 30 | { 31 | fixStyle: 'inline-type-imports', 32 | }, 33 | ], 34 | }, 35 | globals: { 36 | document: true, 37 | window: true, 38 | fetch: true, 39 | Headers: true, 40 | }, 41 | env: { 42 | jest: true, 43 | node: true, 44 | es2022: true, 45 | }, 46 | parserOptions: { 47 | project: './tsconfig.json', 48 | ecmaVersion: 2022, 49 | sourceType: 'module', 50 | }, 51 | extends: [ 52 | 'airbnb-typescript/base', 53 | 'plugin:@typescript-eslint/eslint-recommended', 54 | 'plugin:@typescript-eslint/recommended', 55 | 'plugin:@typescript-eslint/recommended-requiring-type-checking', 56 | 'plugin:import/errors', 57 | 'prettier', 58 | ], 59 | }; 60 | -------------------------------------------------------------------------------- /src/tests/.tests-config.tpl.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const startDate = new Date(); 4 | startDate.setMonth(startDate.getMonth() - 1); 5 | 6 | module.exports = { 7 | options: { 8 | // options object that is passed to the scrapers. see more in readme.md file 9 | startDate, 10 | combineInstallments: false, 11 | showBrowser: true, 12 | verbose: false, 13 | args: [], 14 | storeFailureScreenShotPath: false, // path.resolve(__dirname, 'snapshots/failure.jpg') 15 | }, 16 | credentials: { 17 | // commented companies will be skipped automatically, uncomment those you wish to test 18 | // hapoalim: { userCode: '', password: '' }, 19 | // leumi: { username: '', password: '' }, 20 | // hapoalimBeOnline: { userCode: '', password: '' }, 21 | // discount: { id: '', password: '', num: '' }, 22 | // otsarHahayal: { username: '', password: '' }, 23 | // max: { username: '', password: '' }, 24 | // visaCal: { username: '', password: '' }, 25 | // isracard: { id: '', password: '', card6Digits: '' }, 26 | // amex: { id: '', card6Digits: '', password: ''}, 27 | // mizrahi: { username: '', password: ''}, 28 | // union: {username:'',password:''} 29 | // beinleumi: { username: '', password: ''}, 30 | // yahav: {username: '', nationalID: '', password: ''} 31 | // beyahadBishvilha: { id: '', password: ''}, 32 | // behatsdaa: { id: '', password: ''}, 33 | // oneZero: { email: '', password: '', otpCode: '', otpToken: null }, 34 | // pagi: { username: '', password: ''}, 35 | }, 36 | companyAPI: { 37 | // enable companyAPI to execute tests against the real companies api 38 | enabled: true, 39 | excelFilesDist: '', // optional - provide exists directory path to save scraper results (csv format) 40 | invalidPassword: false, // enable to execute tests that execute with invalid credentials 41 | }, 42 | }; 43 | -------------------------------------------------------------------------------- /src/helpers/transactions.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import moment, { type Moment } from 'moment'; 3 | import { TransactionTypes, type Transaction } from '../transactions'; 4 | 5 | function isNormalTransaction(txn: any): boolean { 6 | return txn && txn.type === TransactionTypes.Normal; 7 | } 8 | 9 | function isInstallmentTransaction(txn: any): boolean { 10 | return txn && txn.type === TransactionTypes.Installments; 11 | } 12 | 13 | function isNonInitialInstallmentTransaction(txn: Transaction): boolean { 14 | return isInstallmentTransaction(txn) && !!txn.installments && txn.installments.number > 1; 15 | } 16 | 17 | function isInitialInstallmentTransaction(txn: Transaction): boolean { 18 | return isInstallmentTransaction(txn) && !!txn.installments && txn.installments.number === 1; 19 | } 20 | 21 | export function fixInstallments(txns: Transaction[]): Transaction[] { 22 | return txns.map((txn: Transaction) => { 23 | const clonedTxn = { ...txn }; 24 | 25 | if ( 26 | isInstallmentTransaction(clonedTxn) && 27 | isNonInitialInstallmentTransaction(clonedTxn) && 28 | clonedTxn.installments 29 | ) { 30 | const dateMoment = moment(clonedTxn.date); 31 | const actualDateMoment = dateMoment.add(clonedTxn.installments.number - 1, 'month'); 32 | clonedTxn.date = actualDateMoment.toISOString(); 33 | } 34 | return clonedTxn; 35 | }); 36 | } 37 | 38 | export function sortTransactionsByDate(txns: Transaction[]) { 39 | return _.sortBy(txns, ['date']); 40 | } 41 | 42 | export function filterOldTransactions(txns: Transaction[], startMoment: Moment, combineInstallments: boolean) { 43 | return txns.filter(txn => { 44 | const combineNeededAndInitialOrNormal = 45 | combineInstallments && (isNormalTransaction(txn) || isInitialInstallmentTransaction(txn)); 46 | return ( 47 | (!combineInstallments && startMoment.isSameOrBefore(txn.date)) || 48 | (combineNeededAndInitialOrNormal && startMoment.isSameOrBefore(txn.date)) 49 | ); 50 | }); 51 | } 52 | -------------------------------------------------------------------------------- /src/helpers/waiting.ts: -------------------------------------------------------------------------------- 1 | import type { Falsy } from 'utility-types'; 2 | 3 | export class TimeoutError extends Error {} 4 | 5 | export const SECOND = 1000; 6 | 7 | type WaitUntilReturn = T extends Falsy ? never : Promise>; 8 | 9 | function timeoutPromise(ms: number, promise: Promise, description: string): Promise { 10 | const timeout = new Promise((_, reject) => { 11 | const id = setTimeout(() => { 12 | clearTimeout(id); 13 | const error = new TimeoutError(description); 14 | reject(error); 15 | }, ms); 16 | }); 17 | 18 | return Promise.race([ 19 | promise, 20 | // casting to avoid type error- safe since this promise will always reject 21 | timeout as Promise, 22 | ]); 23 | } 24 | 25 | /** 26 | * Wait until a promise resolves with a truthy value or reject after a timeout 27 | */ 28 | export function waitUntil( 29 | asyncTest: () => Promise, 30 | description = '', 31 | timeout = 10000, 32 | interval = 100, 33 | ): WaitUntilReturn { 34 | const promise = new Promise>((resolve, reject) => { 35 | function wait() { 36 | asyncTest() 37 | .then(value => { 38 | if (value) { 39 | resolve(value); 40 | } else { 41 | setTimeout(wait, interval); 42 | } 43 | }) 44 | .catch(() => { 45 | reject(); 46 | }); 47 | } 48 | wait(); 49 | }); 50 | return timeoutPromise(timeout, promise, description) as WaitUntilReturn; 51 | } 52 | 53 | export function raceTimeout(ms: number, promise: Promise) { 54 | return timeoutPromise(ms, promise, 'timeout').catch(err => { 55 | if (!(err instanceof TimeoutError)) throw err; 56 | }); 57 | } 58 | 59 | export function runSerial(actions: (() => Promise)[]): Promise { 60 | return actions.reduce((m, a) => m.then(async x => [...x, await a()]), Promise.resolve(new Array())); 61 | } 62 | 63 | export function sleep(ms: number) { 64 | return new Promise(resolve => setTimeout(resolve, ms)); 65 | } 66 | -------------------------------------------------------------------------------- /src/scrapers/behatsdaa.test.ts: -------------------------------------------------------------------------------- 1 | import { CompanyTypes, SCRAPERS } from '../definitions'; 2 | import { exportTransactions, extendAsyncTimeout, getTestsConfig, maybeTestCompanyAPI } from '../tests/tests-utils'; 3 | import { LoginResults } from './base-scraper-with-browser'; 4 | import BehatsdaaScraper from './behatsdaa'; 5 | 6 | const testsConfig = getTestsConfig(); 7 | 8 | describe('Behatsdaa scraper', () => { 9 | beforeAll(() => { 10 | extendAsyncTimeout(); 11 | }); 12 | 13 | it('should expose login fields in scrapers constant', () => { 14 | expect(SCRAPERS[CompanyTypes.behatsdaa]).toBeDefined(); 15 | expect(SCRAPERS[CompanyTypes.behatsdaa].loginFields).toContain('id'); 16 | expect(SCRAPERS[CompanyTypes.behatsdaa].loginFields).toContain('password'); 17 | }); 18 | 19 | maybeTestCompanyAPI(CompanyTypes.behatsdaa, config => config.companyAPI.invalidPassword)( 20 | 'should fail on invalid user/password"', 21 | async () => { 22 | const scraper = new BehatsdaaScraper({ 23 | ...testsConfig.options, 24 | companyId: CompanyTypes.behatsdaa, 25 | }); 26 | 27 | const result = await scraper.scrape({ id: 'foofoofoo', password: 'barbarbar' }); 28 | 29 | expect(result).toBeDefined(); 30 | expect(result.success).toBeFalsy(); 31 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 32 | }, 33 | ); 34 | 35 | maybeTestCompanyAPI(CompanyTypes.behatsdaa)('should scrape transactions', async () => { 36 | const scraper = new BehatsdaaScraper({ 37 | ...testsConfig.options, 38 | companyId: CompanyTypes.behatsdaa, 39 | }); 40 | 41 | const result = await scraper.scrape(testsConfig.credentials[CompanyTypes.behatsdaa]); 42 | expect(result).toBeDefined(); 43 | expect(result.errorMessage).toBeFalsy(); 44 | expect(result.errorType).toBeFalsy(); 45 | expect(result.success).toBeTruthy(); 46 | expect(result.accounts).toBeDefined(); 47 | expect(result.accounts).toHaveLength(1); 48 | exportTransactions(CompanyTypes.behatsdaa, result.accounts || []); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /src/scrapers/pagi.test.ts: -------------------------------------------------------------------------------- 1 | import { SCRAPERS } from '../definitions'; 2 | import { exportTransactions, extendAsyncTimeout, getTestsConfig, maybeTestCompanyAPI } from '../tests/tests-utils'; 3 | import { LoginResults } from './base-scraper-with-browser'; 4 | import PagiScraper from './pagi'; 5 | 6 | const COMPANY_ID = 'pagi'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Pagi legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | test('should expose login fields in scrapers constant', () => { 14 | expect(SCRAPERS.pagi).toBeDefined(); 15 | expect(SCRAPERS.pagi.loginFields).toContain('username'); 16 | expect(SCRAPERS.pagi.loginFields).toContain('password'); 17 | }); 18 | 19 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 20 | 'should fail on invalid user/password"', 21 | async () => { 22 | const options = { 23 | ...testsConfig.options, 24 | companyId: COMPANY_ID, 25 | }; 26 | 27 | const scraper = new PagiScraper(options); 28 | 29 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 30 | 31 | expect(result).toBeDefined(); 32 | expect(result.success).toBeFalsy(); 33 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 34 | }, 35 | ); 36 | 37 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 38 | const options = { 39 | ...testsConfig.options, 40 | companyId: COMPANY_ID, 41 | }; 42 | 43 | const scraper = new PagiScraper(options); 44 | const result = await scraper.scrape(testsConfig.credentials.pagi); 45 | expect(result).toBeDefined(); 46 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 47 | expect(error).toBe(''); 48 | expect(result.success).toBeTruthy(); 49 | 50 | exportTransactions(COMPANY_ID, result.accounts || []); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/scrapers/leumi.test.ts: -------------------------------------------------------------------------------- 1 | import LeumiScraper from './leumi'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'leumi'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Leumi legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.leumi).toBeDefined(); 16 | expect(SCRAPERS.leumi.loginFields).toContain('username'); 17 | expect(SCRAPERS.leumi.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password"', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new LeumiScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new LeumiScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.leumi); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/union-bank.test.ts: -------------------------------------------------------------------------------- 1 | import UnionBankScraper from './union-bank'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'union'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Union', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.union).toBeDefined(); 16 | expect(SCRAPERS.union.loginFields).toContain('username'); 17 | expect(SCRAPERS.union.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password"', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new UnionBankScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new UnionBankScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.union); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/beinleumi.test.ts: -------------------------------------------------------------------------------- 1 | import BeinleumiScraper from './beinleumi'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'beinleumi'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Beinleumi', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.beinleumi).toBeDefined(); 16 | expect(SCRAPERS.beinleumi.loginFields).toContain('username'); 17 | expect(SCRAPERS.beinleumi.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new BeinleumiScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new BeinleumiScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.beinleumi); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/hapoalim.test.ts: -------------------------------------------------------------------------------- 1 | import HapoalimScraper from './hapoalim'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'hapoalim'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Hapoalim legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.hapoalim).toBeDefined(); 16 | expect(SCRAPERS.hapoalim.loginFields).toContain('userCode'); 17 | expect(SCRAPERS.hapoalim.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password"', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new HapoalimScraper(options); 29 | 30 | const result = await scraper.scrape({ userCode: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new HapoalimScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.hapoalim); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/amex.test.ts: -------------------------------------------------------------------------------- 1 | import AMEXScraper from './amex'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'amex'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('AMEX legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.amex).toBeDefined(); 16 | expect(SCRAPERS.amex.loginFields).toContain('id'); 17 | expect(SCRAPERS.amex.loginFields).toContain('card6Digits'); 18 | expect(SCRAPERS.amex.loginFields).toContain('password'); 19 | }); 20 | 21 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 22 | 'should fail on invalid user/password"', 23 | async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new AMEXScraper(options); 30 | 31 | const result = await scraper.scrape({ id: 'e10s12', card6Digits: '123456', password: '3f3ss3d' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }, 37 | ); 38 | 39 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 40 | const options = { 41 | ...testsConfig.options, 42 | companyId: COMPANY_ID, 43 | }; 44 | 45 | const scraper = new AMEXScraper(options); 46 | const result = await scraper.scrape(testsConfig.credentials.amex); 47 | expect(result).toBeDefined(); 48 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 49 | expect(error).toBe(''); 50 | expect(result.success).toBeTruthy(); 51 | 52 | exportTransactions(COMPANY_ID, result.accounts || []); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/scrapers/otsar-hahayal.test.ts: -------------------------------------------------------------------------------- 1 | import OtsarHahayalScraper from './otsar-hahayal'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'otsarHahayal'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('OtsarHahayal legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.otsarHahayal).toBeDefined(); 16 | expect(SCRAPERS.otsarHahayal.loginFields).toContain('username'); 17 | expect(SCRAPERS.otsarHahayal.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password"', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new OtsarHahayalScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new OtsarHahayalScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.otsarHahayal); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/base-scraper-with-browser.test.ts: -------------------------------------------------------------------------------- 1 | import { extendAsyncTimeout, getTestsConfig } from '../tests/tests-utils'; 2 | import { BaseScraperWithBrowser } from './base-scraper-with-browser'; 3 | 4 | const testsConfig = getTestsConfig(); 5 | 6 | function isNoSandbox(browser: any) { 7 | // eslint-disable-next-line no-underscore-dangle 8 | const args = browser._process.spawnargs; 9 | return args.includes('--no-sandbox'); 10 | } 11 | 12 | describe('Base scraper with browser', () => { 13 | beforeAll(() => { 14 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 15 | }); 16 | 17 | xtest('should pass custom args to scraper if provided', async () => { 18 | const options = { 19 | ...testsConfig.options, 20 | companyId: 'test', 21 | showBrowser: false, 22 | args: [], 23 | }; 24 | 25 | // avoid false-positive result by confirming that --no-sandbox is not a default flag provided by puppeteer 26 | let baseScraperWithBrowser = new BaseScraperWithBrowser(options); 27 | try { 28 | await baseScraperWithBrowser.initialize(); 29 | // @ts-ignore 30 | expect(baseScraperWithBrowser.browser).toBeDefined(); 31 | // @ts-ignore 32 | expect(isNoSandbox(baseScraperWithBrowser.browser)).toBe(false); 33 | await baseScraperWithBrowser.terminate(true); 34 | } catch (e) { 35 | await baseScraperWithBrowser.terminate(false); 36 | throw e; 37 | } 38 | 39 | // set --no-sandbox flag and expect it to be passed by puppeteer.lunch to the new created browser instance 40 | options.args = ['--no-sandbox', '--disable-gpu', '--window-size=1920x1080']; 41 | baseScraperWithBrowser = new BaseScraperWithBrowser(options); 42 | try { 43 | await baseScraperWithBrowser.initialize(); 44 | // @ts-ignore 45 | expect(baseScraperWithBrowser.browser).toBeDefined(); 46 | // @ts-ignore 47 | expect(isNoSandbox(baseScraperWithBrowser.browser)).toBe(true); 48 | await baseScraperWithBrowser.terminate(true); 49 | } catch (e) { 50 | await baseScraperWithBrowser.terminate(false); 51 | throw e; 52 | } 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/scrapers/yahav.test.ts: -------------------------------------------------------------------------------- 1 | import YahavScraper from './yahav'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'yahav'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Yahav scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.yahav).toBeDefined(); 16 | expect(SCRAPERS.yahav.loginFields).toContain('username'); 17 | expect(SCRAPERS.yahav.loginFields).toContain('password'); 18 | expect(SCRAPERS.yahav.loginFields).toContain('nationalID'); 19 | }); 20 | 21 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 22 | 'should fail on invalid user/password"', 23 | async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new YahavScraper(options); 30 | 31 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d', nationalID: '12345679' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }, 37 | ); 38 | 39 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 40 | const options = { 41 | ...testsConfig.options, 42 | companyId: COMPANY_ID, 43 | }; 44 | 45 | const scraper = new YahavScraper(options); 46 | const result = await scraper.scrape(testsConfig.credentials.yahav); 47 | expect(result).toBeDefined(); 48 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 49 | expect(error).toBe(''); 50 | expect(result.success).toBeTruthy(); 51 | 52 | exportTransactions(COMPANY_ID, result.accounts || []); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/scrapers/discount.test.ts: -------------------------------------------------------------------------------- 1 | import DiscountScraper from './discount'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'discount'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Discount legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.discount).toBeDefined(); 16 | expect(SCRAPERS.discount.loginFields).toContain('id'); 17 | expect(SCRAPERS.discount.loginFields).toContain('password'); 18 | expect(SCRAPERS.discount.loginFields).toContain('num'); 19 | }); 20 | 21 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 22 | 'should fail on invalid user/password"', 23 | async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new DiscountScraper(options); 30 | 31 | const result = await scraper.scrape({ id: 'e10s12', password: '3f3ss3d', num: '1234' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }, 37 | ); 38 | 39 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 40 | const options = { 41 | ...testsConfig.options, 42 | companyId: COMPANY_ID, 43 | }; 44 | 45 | const scraper = new DiscountScraper(options); 46 | const result = await scraper.scrape(testsConfig.credentials.discount); 47 | expect(result).toBeDefined(); 48 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 49 | expect(error).toBe(''); 50 | expect(result.success).toBeTruthy(); 51 | 52 | exportTransactions(COMPANY_ID, result.accounts || []); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/scrapers/beyahad-bishvilha.test.ts: -------------------------------------------------------------------------------- 1 | import BeyahadBishvilhaScraper from './beyahad-bishvilha'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'beyahadBishvilha'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Beyahad Bishvilha scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.beyahadBishvilha).toBeDefined(); 16 | expect(SCRAPERS.beyahadBishvilha.loginFields).toContain('id'); 17 | expect(SCRAPERS.beyahadBishvilha.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password"', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new BeyahadBishvilhaScraper(options); 29 | 30 | const result = await scraper.scrape({ id: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new BeyahadBishvilhaScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.beyahadBishvilha); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /src/scrapers/mercantile.test.ts: -------------------------------------------------------------------------------- 1 | import MercantileScraper from './mercantile'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'mercantile'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Mercantile legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.mercantile).toBeDefined(); 16 | expect(SCRAPERS.mercantile.loginFields).toContain('id'); 17 | expect(SCRAPERS.mercantile.loginFields).toContain('password'); 18 | expect(SCRAPERS.mercantile.loginFields).toContain('num'); 19 | }); 20 | 21 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 22 | 'should fail on invalid user/password"', 23 | async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new MercantileScraper(options); 30 | 31 | const result = await scraper.scrape(testsConfig.credentials.mercantile); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }, 37 | ); 38 | 39 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 40 | const options = { 41 | ...testsConfig.options, 42 | companyId: COMPANY_ID, 43 | }; 44 | 45 | const scraper = new MercantileScraper(options); 46 | const result = await scraper.scrape(testsConfig.credentials.mercantile); 47 | expect(result).toBeDefined(); 48 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 49 | expect(error).toBe(''); 50 | expect(result.success).toBeTruthy(); 51 | 52 | exportTransactions(COMPANY_ID, result.accounts || []); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/scrapers/isracard.test.ts: -------------------------------------------------------------------------------- 1 | import IsracardScraper from './isracard'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'isracard'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Isracard legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.isracard).toBeDefined(); 16 | expect(SCRAPERS.isracard.loginFields).toContain('id'); 17 | expect(SCRAPERS.isracard.loginFields).toContain('card6Digits'); 18 | expect(SCRAPERS.isracard.loginFields).toContain('password'); 19 | }); 20 | 21 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 22 | 'should fail on invalid user/password"', 23 | async () => { 24 | const options = { 25 | ...testsConfig.options, 26 | companyId: COMPANY_ID, 27 | }; 28 | 29 | const scraper = new IsracardScraper(options); 30 | 31 | const result = await scraper.scrape({ id: 'e10s12', password: '3f3ss3d', card6Digits: '123456' }); 32 | 33 | expect(result).toBeDefined(); 34 | expect(result.success).toBeFalsy(); 35 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 36 | }, 37 | ); 38 | 39 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 40 | const options = { 41 | ...testsConfig.options, 42 | companyId: COMPANY_ID, 43 | }; 44 | 45 | const scraper = new IsracardScraper(options); 46 | const result = await scraper.scrape(testsConfig.credentials.isracard); 47 | expect(result).toBeDefined(); 48 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 49 | expect(error).toBe(''); 50 | expect(result.success).toBeTruthy(); 51 | 52 | exportTransactions(COMPANY_ID, result.accounts || []); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/scrapers/visa-cal.test.ts: -------------------------------------------------------------------------------- 1 | import { SCRAPERS } from '../definitions'; 2 | import { exportTransactions, extendAsyncTimeout, getTestsConfig, maybeTestCompanyAPI } from '../tests/tests-utils'; 3 | import { LoginResults } from './base-scraper-with-browser'; 4 | import VisaCalScraper from './visa-cal'; 5 | 6 | const COMPANY_ID = 'visaCal'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('VisaCal legacy scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.visaCal).toBeDefined(); 16 | expect(SCRAPERS.visaCal.loginFields).toContain('username'); 17 | expect(SCRAPERS.visaCal.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password"', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new VisaCalScraper(options); 29 | 30 | const result = await scraper.scrape({ username: '971sddksmsl', password: '3f3ssdkSD3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new VisaCalScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.visaCal); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | // uncomment to test multiple accounts 51 | // expect(result?.accounts?.length).toEqual(2) 52 | exportTransactions(COMPANY_ID, result.accounts || []); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - master 6 | - next 7 | jobs: 8 | release: 9 | name: Release 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | - name: Setup Node.js 15 | uses: actions/setup-node@v4 16 | with: 17 | node-version-file: .nvmrc 18 | registry-url: https://registry.npmjs.org/ 19 | - name: prepare default version 20 | run: npm run prepare:default 21 | - name: Release to Github 22 | uses: cycjimmy/semantic-release-action@v4 23 | id: semantic # Need an `id` for output variables 24 | with: 25 | semantic_version: 16.0.2 26 | branches: | 27 | [ 28 | '+([0-9])?(.{+([0-9]),x}).x', 29 | 'master', 30 | 'next', 31 | 'next-major', 32 | { 33 | name: 'hotfix', 34 | prerelease: true 35 | }, 36 | { 37 | name: 'beta', 38 | prerelease: true 39 | }, 40 | { 41 | name: 'alpha', 42 | prerelease: true 43 | } 44 | ] 45 | env: 46 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 47 | - name: Prepare release of default to NPM 48 | if: steps.semantic.outputs.new_release_published == 'true' 49 | run: | 50 | node utils/pre-publish.js --version ${{ steps.semantic.outputs.new_release_version }} 51 | - name: Publish to NPM 52 | if: steps.semantic.outputs.new_release_published == 'true' 53 | run: npm publish --access public --ignore-scripts 54 | env: 55 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 56 | - name: Prepare release of core to NPM 57 | if: steps.semantic.outputs.new_release_published == 'true' 58 | run: | 59 | npm run prepare:core 60 | node utils/pre-publish.js --version ${{ steps.semantic.outputs.new_release_version }} 61 | - name: Publish to NPM 62 | if: steps.semantic.outputs.new_release_published == 'true' 63 | run: npm publish --access public --ignore-scripts 64 | env: 65 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 66 | -------------------------------------------------------------------------------- /src/scrapers/one-zero.test.ts: -------------------------------------------------------------------------------- 1 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 2 | import { SCRAPERS } from '../definitions'; 3 | import { LoginResults } from './base-scraper-with-browser'; 4 | import OneZeroScraper from './one-zero'; 5 | 6 | const COMPANY_ID = 'oneZero'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('OneZero scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.oneZero).toBeDefined(); 16 | expect(SCRAPERS.oneZero.loginFields).toContain('email'); 17 | expect(SCRAPERS.oneZero.loginFields).toContain('password'); 18 | expect(SCRAPERS.oneZero.loginFields).toContain('otpCodeRetriever'); 19 | expect(SCRAPERS.oneZero.loginFields).toContain('phoneNumber'); 20 | expect(SCRAPERS.oneZero.loginFields).toContain('otpLongTermToken'); 21 | }); 22 | 23 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 24 | 'should fail on invalid user/password"', 25 | async () => { 26 | const options = { 27 | ...testsConfig.options, 28 | companyId: COMPANY_ID, 29 | }; 30 | 31 | const scraper = new OneZeroScraper(options); 32 | 33 | const result = await scraper.scrape({ 34 | email: 'e10s12@gmail.com', 35 | password: '3f3ss3d', 36 | otpLongTermToken: '11111', 37 | }); 38 | 39 | expect(result).toBeDefined(); 40 | expect(result.success).toBeFalsy(); 41 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 42 | }, 43 | ); 44 | 45 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 46 | const options = { 47 | ...testsConfig.options, 48 | companyId: COMPANY_ID, 49 | }; 50 | 51 | const scraper = new OneZeroScraper(options); 52 | const result = await scraper.scrape(testsConfig.credentials.oneZero); 53 | expect(result).toBeDefined(); 54 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 55 | expect(error).toBe(''); 56 | expect(result.success).toBeTruthy(); 57 | 58 | exportTransactions(COMPANY_ID, result.accounts || []); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /src/scrapers/max.test.ts: -------------------------------------------------------------------------------- 1 | import MaxScraper, { getMemo } from './max'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { LoginResults } from './base-scraper-with-browser'; 5 | 6 | const COMPANY_ID = 'max'; // TODO this property should be hard-coded in the provider 7 | const testsConfig = getTestsConfig(); 8 | 9 | describe('Max scraper', () => { 10 | beforeAll(() => { 11 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 12 | }); 13 | 14 | test('should expose login fields in scrapers constant', () => { 15 | expect(SCRAPERS.max).toBeDefined(); 16 | expect(SCRAPERS.max.loginFields).toContain('username'); 17 | expect(SCRAPERS.max.loginFields).toContain('password'); 18 | }); 19 | 20 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 21 | 'should fail on invalid user/password"', 22 | async () => { 23 | const options = { 24 | ...testsConfig.options, 25 | companyId: COMPANY_ID, 26 | }; 27 | 28 | const scraper = new MaxScraper(options); 29 | 30 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 31 | 32 | expect(result).toBeDefined(); 33 | expect(result.success).toBeFalsy(); 34 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 35 | }, 36 | ); 37 | 38 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions"', async () => { 39 | const options = { 40 | ...testsConfig.options, 41 | companyId: COMPANY_ID, 42 | }; 43 | 44 | const scraper = new MaxScraper(options); 45 | const result = await scraper.scrape(testsConfig.credentials.max); 46 | expect(result).toBeDefined(); 47 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 48 | expect(error).toBe(''); 49 | expect(result.success).toBeTruthy(); 50 | 51 | exportTransactions(COMPANY_ID, result.accounts || []); 52 | }); 53 | }); 54 | 55 | describe('getMemo', () => { 56 | type TransactionForMemoTest = Parameters[0]; 57 | test.each<[TransactionForMemoTest, string]>([ 58 | [{ comments: '' }, ''], 59 | [{ comments: 'comment without funds' }, 'comment without funds'], 60 | [{ comments: '', fundsTransferReceiverOrTransfer: 'Daniel H' }, 'Daniel H'], 61 | [{ comments: '', fundsTransferReceiverOrTransfer: 'Daniel', fundsTransferComment: 'Foo bar' }, 'Daniel: Foo bar'], 62 | ])('%o should create memo: %s', (transaction, expected) => { 63 | const memo = getMemo(transaction); 64 | expect(memo).toBe(expected); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /src/scrapers/factory.ts: -------------------------------------------------------------------------------- 1 | import { assertNever } from '../assertNever'; 2 | import { CompanyTypes } from '../definitions'; 3 | import AmexScraper from './amex'; 4 | import BehatsdaaScraper from './behatsdaa'; 5 | import BeinleumiScraper from './beinleumi'; 6 | import BeyahadBishvilhaScraper from './beyahad-bishvilha'; 7 | import DiscountScraper from './discount'; 8 | import HapoalimScraper from './hapoalim'; 9 | import { type Scraper, type ScraperCredentials, type ScraperOptions } from './interface'; 10 | import IsracardScraper from './isracard'; 11 | import LeumiScraper from './leumi'; 12 | import MassadScraper from './massad'; 13 | import MaxScraper from './max'; 14 | import MercantileScraper from './mercantile'; 15 | import MizrahiScraper from './mizrahi'; 16 | import OneZeroScraper from './one-zero'; 17 | import OtsarHahayalScraper from './otsar-hahayal'; 18 | import PagiScraper from './pagi'; 19 | import UnionBankScraper from './union-bank'; 20 | import VisaCalScraper from './visa-cal'; 21 | import YahavScraper from './yahav'; 22 | 23 | export default function createScraper(options: ScraperOptions): Scraper { 24 | switch (options.companyId) { 25 | case CompanyTypes.hapoalim: 26 | return new HapoalimScraper(options); 27 | case CompanyTypes.leumi: 28 | return new LeumiScraper(options); 29 | case CompanyTypes.beyahadBishvilha: 30 | return new BeyahadBishvilhaScraper(options); 31 | case CompanyTypes.mizrahi: 32 | return new MizrahiScraper(options); 33 | case CompanyTypes.discount: 34 | return new DiscountScraper(options); 35 | case CompanyTypes.mercantile: 36 | return new MercantileScraper(options); 37 | case CompanyTypes.otsarHahayal: 38 | return new OtsarHahayalScraper(options); 39 | case CompanyTypes.visaCal: 40 | return new VisaCalScraper(options); 41 | case CompanyTypes.max: 42 | return new MaxScraper(options); 43 | case CompanyTypes.isracard: 44 | return new IsracardScraper(options); 45 | case CompanyTypes.amex: 46 | return new AmexScraper(options); 47 | case CompanyTypes.union: 48 | return new UnionBankScraper(options); 49 | case CompanyTypes.beinleumi: 50 | return new BeinleumiScraper(options); 51 | case CompanyTypes.massad: 52 | return new MassadScraper(options); 53 | case CompanyTypes.yahav: 54 | return new YahavScraper(options); 55 | case CompanyTypes.oneZero: 56 | return new OneZeroScraper(options); 57 | case CompanyTypes.behatsdaa: 58 | return new BehatsdaaScraper(options); 59 | case CompanyTypes.pagi: 60 | return new PagiScraper(options); 61 | default: 62 | return assertNever(options.companyId, `unknown company id ${options.companyId}`); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/scrapers/mizrahi.test.ts: -------------------------------------------------------------------------------- 1 | import MizrahiScraper from './mizrahi'; 2 | import { maybeTestCompanyAPI, extendAsyncTimeout, getTestsConfig, exportTransactions } from '../tests/tests-utils'; 3 | import { SCRAPERS } from '../definitions'; 4 | import { ISO_DATE_REGEX } from '../constants'; 5 | import { LoginResults } from './base-scraper-with-browser'; 6 | import { type TransactionsAccount } from '../transactions'; 7 | import debug from 'debug'; 8 | import { type ScraperOptions } from './interface'; 9 | 10 | debug.enable('israeli-bank-scrapers:mizrahi'); 11 | 12 | const COMPANY_ID = 'mizrahi'; // TODO this property should be hard-coded in the provider 13 | const testsConfig = getTestsConfig(); 14 | 15 | describe('Mizrahi scraper', () => { 16 | beforeAll(() => { 17 | extendAsyncTimeout(); // The default timeout is 5 seconds per async test, this function extends the timeout value 18 | }); 19 | 20 | test('should expose login fields in scrapers constant', () => { 21 | expect(SCRAPERS.mizrahi).toBeDefined(); 22 | expect(SCRAPERS.mizrahi.loginFields).toContain('username'); 23 | expect(SCRAPERS.mizrahi.loginFields).toContain('password'); 24 | }); 25 | 26 | maybeTestCompanyAPI(COMPANY_ID, config => config.companyAPI.invalidPassword)( 27 | 'should fail on invalid user/password', 28 | async () => { 29 | const options = { 30 | ...testsConfig.options, 31 | companyId: COMPANY_ID, 32 | }; 33 | 34 | const scraper = new MizrahiScraper(options); 35 | 36 | const result = await scraper.scrape({ username: 'e10s12', password: '3f3ss3d' }); 37 | 38 | expect(result).toBeDefined(); 39 | expect(result.success).toBeFalsy(); 40 | expect(result.errorType).toBe(LoginResults.InvalidPassword); 41 | }, 42 | ); 43 | 44 | maybeTestCompanyAPI(COMPANY_ID)('should scrape transactions', async () => { 45 | const options: ScraperOptions = { 46 | ...testsConfig.options, 47 | optInFeatures: [ 48 | 'mizrahi:pendingIfHasGenericDescription', 49 | 'mizrahi:pendingIfHasGenericDescriptionWithDate', 50 | 'mizrahi:pendingIfTodayTransaction', 51 | ], 52 | companyId: COMPANY_ID, 53 | }; 54 | 55 | const scraper = new MizrahiScraper(options); 56 | const result = await scraper.scrape(testsConfig.credentials.mizrahi); 57 | expect(result).toBeDefined(); 58 | const error = `${result.errorType || ''} ${result.errorMessage || ''}`.trim(); 59 | expect(error).toBe(''); 60 | expect(result.success).toBeTruthy(); 61 | expect(result.accounts).toBeDefined(); 62 | expect((result.accounts as any).length).toBeGreaterThan(0); 63 | const account: TransactionsAccount = (result as any).accounts[0]; 64 | expect(account.accountNumber).not.toBe(''); 65 | expect(account.txns[0].date).toMatch(ISO_DATE_REGEX); 66 | 67 | exportTransactions(COMPANY_ID, result.accounts || []); 68 | }); 69 | }); 70 | -------------------------------------------------------------------------------- /src/tests/tests-utils.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import { Parser } from '@json2csv/plainjs'; 3 | import moment from 'moment'; 4 | import path from 'path'; 5 | import { type TransactionsAccount } from '../transactions'; 6 | 7 | let testsConfig: Record; 8 | let configurationLoaded = false; 9 | 10 | const MISSING_ERROR_MESSAGE = 11 | 'Missing test environment configuration. To troubleshoot this issue open CONTRIBUTING.md file and read the "F.A.Q regarding the tests" section.'; 12 | 13 | export function getTestsConfig() { 14 | if (configurationLoaded) { 15 | if (!testsConfig) { 16 | throw new Error(MISSING_ERROR_MESSAGE); 17 | } 18 | 19 | return testsConfig; 20 | } 21 | 22 | configurationLoaded = true; 23 | 24 | try { 25 | const environmentConfig = process.env.TESTS_CONFIG; 26 | if (environmentConfig) { 27 | testsConfig = JSON.parse(environmentConfig); 28 | return testsConfig; 29 | } 30 | } catch (e) { 31 | throw new Error(`failed to parse environment variable 'TESTS_CONFIG' with error '${(e as Error).message}'`); 32 | } 33 | 34 | try { 35 | const configPath = path.join(__dirname, '.tests-config.js'); 36 | testsConfig = require(configPath); 37 | return testsConfig; 38 | } catch (e) { 39 | console.error(e); 40 | throw new Error(MISSING_ERROR_MESSAGE); 41 | } 42 | } 43 | 44 | export function maybeTestCompanyAPI(scraperId: string, filter?: (config: any) => boolean) { 45 | if (!configurationLoaded) { 46 | getTestsConfig(); 47 | } 48 | return testsConfig && 49 | testsConfig.companyAPI.enabled && 50 | testsConfig.credentials[scraperId] && 51 | (!filter || filter(testsConfig)) 52 | ? test 53 | : test.skip; 54 | } 55 | 56 | export function extendAsyncTimeout(timeout = 120000) { 57 | jest.setTimeout(timeout); 58 | } 59 | 60 | export function exportTransactions(fileName: string, accounts: TransactionsAccount[]) { 61 | const config = getTestsConfig(); 62 | 63 | if ( 64 | !config.companyAPI.enabled || 65 | !config.companyAPI.excelFilesDist || 66 | !fs.existsSync(config.companyAPI.excelFilesDist) 67 | ) { 68 | return; 69 | } 70 | 71 | let data: any = []; 72 | 73 | for (let i = 0; i < accounts.length; i += 1) { 74 | const account = accounts[i]; 75 | 76 | data = [ 77 | ...data, 78 | ...account.txns.map(txn => { 79 | return { 80 | account: account.accountNumber, 81 | balance: `account balance: ${account.balance}`, 82 | ...txn, 83 | date: moment(txn.date).format('DD/MM/YYYY'), 84 | processedDate: moment(txn.processedDate).format('DD/MM/YYYY'), 85 | }; 86 | }), 87 | ]; 88 | } 89 | 90 | if (data.length === 0) { 91 | data = [ 92 | { 93 | comment: 'no transaction found for requested time frame', 94 | }, 95 | ]; 96 | } 97 | 98 | const parser = new Parser({ withBOM: true }); 99 | const csv = parser.parse(data); 100 | const filePath = `${path.join(config.companyAPI.excelFilesDist, fileName)}.csv`; 101 | fs.writeFileSync(filePath, csv); 102 | } 103 | -------------------------------------------------------------------------------- /src/definitions.ts: -------------------------------------------------------------------------------- 1 | // NOTICE: avoid changing exported keys as they are part of the public api 2 | 3 | export const PASSWORD_FIELD = 'password'; 4 | 5 | export enum CompanyTypes { 6 | hapoalim = 'hapoalim', 7 | beinleumi = 'beinleumi', 8 | union = 'union', 9 | amex = 'amex', 10 | isracard = 'isracard', 11 | visaCal = 'visaCal', 12 | max = 'max', 13 | otsarHahayal = 'otsarHahayal', 14 | discount = 'discount', 15 | mercantile = 'mercantile', 16 | mizrahi = 'mizrahi', 17 | leumi = 'leumi', 18 | massad = 'massad', 19 | yahav = 'yahav', 20 | behatsdaa = 'behatsdaa', 21 | beyahadBishvilha = 'beyahadBishvilha', 22 | oneZero = 'oneZero', 23 | pagi = 'pagi', 24 | } 25 | 26 | export const SCRAPERS = { 27 | [CompanyTypes.hapoalim]: { 28 | name: 'Bank Hapoalim', 29 | loginFields: ['userCode', PASSWORD_FIELD], 30 | }, 31 | [CompanyTypes.leumi]: { 32 | name: 'Bank Leumi', 33 | loginFields: ['username', PASSWORD_FIELD], 34 | }, 35 | [CompanyTypes.mizrahi]: { 36 | name: 'Mizrahi Bank', 37 | loginFields: ['username', PASSWORD_FIELD], 38 | }, 39 | [CompanyTypes.discount]: { 40 | name: 'Discount Bank', 41 | loginFields: ['id', PASSWORD_FIELD, 'num'], 42 | }, 43 | [CompanyTypes.mercantile]: { 44 | name: 'Mercantile Bank', 45 | loginFields: ['id', PASSWORD_FIELD, 'num'], 46 | }, 47 | [CompanyTypes.otsarHahayal]: { 48 | name: 'Bank Otsar Hahayal', 49 | loginFields: ['username', PASSWORD_FIELD], 50 | }, 51 | [CompanyTypes.max]: { 52 | name: 'Max', 53 | loginFields: ['username', PASSWORD_FIELD], 54 | }, 55 | [CompanyTypes.visaCal]: { 56 | name: 'Visa Cal', 57 | loginFields: ['username', PASSWORD_FIELD], 58 | }, 59 | [CompanyTypes.isracard]: { 60 | name: 'Isracard', 61 | loginFields: ['id', 'card6Digits', PASSWORD_FIELD], 62 | }, 63 | [CompanyTypes.amex]: { 64 | name: 'Amex', 65 | loginFields: ['id', 'card6Digits', PASSWORD_FIELD], 66 | }, 67 | [CompanyTypes.union]: { 68 | name: 'Union', 69 | loginFields: ['username', PASSWORD_FIELD], 70 | }, 71 | [CompanyTypes.beinleumi]: { 72 | name: 'Beinleumi', 73 | loginFields: ['username', PASSWORD_FIELD], 74 | }, 75 | [CompanyTypes.massad]: { 76 | name: 'Massad', 77 | loginFields: ['username', PASSWORD_FIELD], 78 | }, 79 | [CompanyTypes.yahav]: { 80 | name: 'Bank Yahav', 81 | loginFields: ['username', 'nationalID', PASSWORD_FIELD], 82 | }, 83 | [CompanyTypes.beyahadBishvilha]: { 84 | name: 'Beyahad Bishvilha', 85 | loginFields: ['id', PASSWORD_FIELD], 86 | }, 87 | [CompanyTypes.oneZero]: { 88 | name: 'One Zero', 89 | loginFields: ['email', PASSWORD_FIELD, 'otpCodeRetriever', 'phoneNumber', 'otpLongTermToken'], 90 | }, 91 | [CompanyTypes.behatsdaa]: { 92 | name: 'Behatsdaa', 93 | loginFields: ['id', PASSWORD_FIELD], 94 | }, 95 | [CompanyTypes.pagi]: { 96 | name: 'Pagi', 97 | loginFields: ['username', PASSWORD_FIELD], 98 | }, 99 | }; 100 | 101 | export enum ScraperProgressTypes { 102 | Initializing = 'INITIALIZING', 103 | StartScraping = 'START_SCRAPING', 104 | LoggingIn = 'LOGGING_IN', 105 | LoginSuccess = 'LOGIN_SUCCESS', 106 | LoginFailed = 'LOGIN_FAILED', 107 | ChangePassword = 'CHANGE_PASSWORD', 108 | EndScraping = 'END_SCRAPING', 109 | Terminating = 'TERMINATING', 110 | } 111 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "israeli-bank-scrapers", 3 | "version": "1.0.4", 4 | "private": true, 5 | "description": "Provide scrapers for all major Israeli banks and credit card companies", 6 | "engines": { 7 | "node": ">= 18.19.0" 8 | }, 9 | "main": "lib/index.js", 10 | "types": "lib/index.d.ts", 11 | "scripts": { 12 | "clean": "rimraf lib", 13 | "test": "jest", 14 | "test:ci": "ncp src/tests/.tests-config.tpl.js src/tests/.tests-config.js && npm run test", 15 | "lint": "eslint src --ext .ts && npm run format:check", 16 | "lint:fix": "eslint src --ext .ts --fix && npm run format", 17 | "format": "prettier --write \"src/**/*.{ts,tsx,js,jsx,json,md}\"", 18 | "format:check": "prettier --check \"src/**/*.{ts,tsx,js,jsx,json,md}\"", 19 | "type-check": "tsc --noEmit", 20 | "dev": "npm run type-check -- --watch", 21 | "build": "npm run lint && npm run clean && npm run build:types && npm run build:js", 22 | "build:types": "tsc --emitDeclarationOnly", 23 | "build:js": "babel src --out-dir lib --extensions \".ts\" --source-maps inline --verbose", 24 | "postbuild": "rimraf lib/tests", 25 | "prepare:core": "git reset --hard && node utils/prepare-israeli-bank-scrapers-core.js && npm i --package-lock-only && npm ci && npm run lint:fix && npm run build", 26 | "prepare:default": "git reset --hard && npm ci && npm run build", 27 | "reset": "git reset --hard && npm ci" 28 | }, 29 | "repository": { 30 | "type": "git", 31 | "url": "git+https://github.com/eshaham/israeli-bank-scrapers.git" 32 | }, 33 | "keywords": [ 34 | "israel", 35 | "israeli bank", 36 | "israeli bank scraper" 37 | ], 38 | "author": "Elad Shaham", 39 | "license": "MIT", 40 | "bugs": { 41 | "url": "https://github.com/eshaham/israeli-bank-scrapers/issues" 42 | }, 43 | "homepage": "https://github.com/eshaham/israeli-bank-scrapers#readme", 44 | "devDependencies": { 45 | "@babel/cli": "^7.4.4", 46 | "@babel/core": "^7.4.5", 47 | "@babel/preset-env": "^7.4.5", 48 | "@babel/preset-typescript": "^7.9.0", 49 | "@json2csv/plainjs": "^7.0.6", 50 | "@types/debug": "^4.1.7", 51 | "@types/jest": "^29.5.12", 52 | "@types/lodash": "^4.14.149", 53 | "@types/node-fetch": "^2.5.6", 54 | "@types/source-map-support": "^0.5.1", 55 | "@types/uuid": "^9.0.1", 56 | "@typescript-eslint/eslint-plugin": "^7.12.0", 57 | "@typescript-eslint/parser": "^7.12.0", 58 | "cross-env": "^6.0.3", 59 | "eslint": "^8.57.0", 60 | "eslint-config-airbnb-base": "^15.0.0", 61 | "eslint-config-airbnb-typescript": "^18.0.0", 62 | "eslint-config-prettier": "^10.1.5", 63 | "eslint-plugin-import": "^2.29.1", 64 | "fs-extra": "^10.0.0", 65 | "husky": "^8.0.3", 66 | "jest": "^29.7.0", 67 | "jscodeshift": "^0.16.1", 68 | "minimist": "^1.2.5", 69 | "ncp": "^2.0.0", 70 | "prettier": "^3.5.3", 71 | "prettier-eslint": "^16.4.2", 72 | "rimraf": "^3.0.0", 73 | "source-map-support": "^0.5.16", 74 | "ts-jest": "^29.1.4", 75 | "typescript": "^4.7.4" 76 | }, 77 | "dependencies": { 78 | "core-js": "^3.1.4", 79 | "debug": "^4.3.2", 80 | "lodash": "^4.17.10", 81 | "moment": "^2.22.2", 82 | "moment-timezone": "^0.5.37", 83 | "node-fetch": "^2.2.0", 84 | "puppeteer": "22.15.0", 85 | "utility-types": "^3.11.0", 86 | "uuid": "^9.0.1" 87 | }, 88 | "files": [ 89 | "lib/**/*" 90 | ] 91 | } 92 | -------------------------------------------------------------------------------- /src/scrapers/base-scraper.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events'; 2 | import moment from 'moment-timezone'; 3 | import { type CompanyTypes, ScraperProgressTypes } from '../definitions'; 4 | import { TimeoutError } from '../helpers/waiting'; 5 | import { createGenericError, createTimeoutError } from './errors'; 6 | import { 7 | type Scraper, 8 | type ScraperCredentials, 9 | type ScraperGetLongTermTwoFactorTokenResult, 10 | type ScraperLoginResult, 11 | type ScraperOptions, 12 | type ScraperScrapingResult, 13 | type ScraperTwoFactorAuthTriggerResult, 14 | } from './interface'; 15 | 16 | const SCRAPE_PROGRESS = 'SCRAPE_PROGRESS'; 17 | 18 | export class BaseScraper implements Scraper { 19 | private eventEmitter = new EventEmitter(); 20 | 21 | constructor(public options: ScraperOptions) {} 22 | 23 | // eslint-disable-next-line @typescript-eslint/require-await 24 | async initialize() { 25 | this.emitProgress(ScraperProgressTypes.Initializing); 26 | moment.tz.setDefault('Asia/Jerusalem'); 27 | } 28 | 29 | async scrape(credentials: TCredentials): Promise { 30 | this.emitProgress(ScraperProgressTypes.StartScraping); 31 | await this.initialize(); 32 | 33 | let loginResult; 34 | try { 35 | loginResult = await this.login(credentials); 36 | } catch (e) { 37 | loginResult = 38 | e instanceof TimeoutError ? createTimeoutError((e as Error).message) : createGenericError((e as Error).message); 39 | } 40 | 41 | let scrapeResult; 42 | if (loginResult.success) { 43 | try { 44 | scrapeResult = await this.fetchData(); 45 | } catch (e) { 46 | scrapeResult = 47 | e instanceof TimeoutError 48 | ? createTimeoutError((e as Error).message) 49 | : createGenericError((e as Error).message); 50 | } 51 | } else { 52 | scrapeResult = loginResult; 53 | } 54 | 55 | try { 56 | const success = scrapeResult && scrapeResult.success === true; 57 | await this.terminate(success); 58 | } catch (e) { 59 | scrapeResult = createGenericError((e as Error).message); 60 | } 61 | this.emitProgress(ScraperProgressTypes.EndScraping); 62 | 63 | return scrapeResult; 64 | } 65 | 66 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 67 | triggerTwoFactorAuth(_phoneNumber: string): Promise { 68 | throw new Error(`triggerOtp() is not created in ${this.options.companyId}`); 69 | } 70 | 71 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 72 | getLongTermTwoFactorToken(_otpCode: string): Promise { 73 | throw new Error(`getPermanentOtpToken() is not created in ${this.options.companyId}`); 74 | } 75 | 76 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 77 | protected async login(_credentials: TCredentials): Promise { 78 | throw new Error(`login() is not created in ${this.options.companyId}`); 79 | } 80 | 81 | // eslint-disable-next-line @typescript-eslint/require-await 82 | protected async fetchData(): Promise { 83 | throw new Error(`fetchData() is not created in ${this.options.companyId}`); 84 | } 85 | 86 | // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await 87 | protected async terminate(_success: boolean) { 88 | this.emitProgress(ScraperProgressTypes.Terminating); 89 | } 90 | 91 | protected emitProgress(type: ScraperProgressTypes) { 92 | this.emit(SCRAPE_PROGRESS, { type }); 93 | } 94 | 95 | protected emit(eventName: string, payload: Record) { 96 | this.eventEmitter.emit(eventName, this.options.companyId, payload); 97 | } 98 | 99 | onProgress(func: (companyId: CompanyTypes, payload: { type: ScraperProgressTypes }) => void) { 100 | this.eventEmitter.on(SCRAPE_PROGRESS, func); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/helpers/fetch.ts: -------------------------------------------------------------------------------- 1 | import nodeFetch from 'node-fetch'; 2 | import { type Page } from 'puppeteer'; 3 | 4 | const JSON_CONTENT_TYPE = 'application/json'; 5 | 6 | function getJsonHeaders() { 7 | return { 8 | Accept: JSON_CONTENT_TYPE, 9 | 'Content-Type': JSON_CONTENT_TYPE, 10 | }; 11 | } 12 | 13 | export async function fetchGet(url: string, extraHeaders: Record): Promise { 14 | let headers = getJsonHeaders(); 15 | if (extraHeaders) { 16 | headers = Object.assign(headers, extraHeaders); 17 | } 18 | const request = { 19 | method: 'GET', 20 | headers, 21 | }; 22 | const fetchResult = await nodeFetch(url, request); 23 | 24 | if (fetchResult.status !== 200) { 25 | throw new Error(`sending a request to the institute server returned with status code ${fetchResult.status}`); 26 | } 27 | 28 | return fetchResult.json(); 29 | } 30 | 31 | export async function fetchPost(url: string, data: Record, extraHeaders: Record = {}) { 32 | const request = { 33 | method: 'POST', 34 | headers: { ...getJsonHeaders(), ...extraHeaders }, 35 | body: JSON.stringify(data), 36 | }; 37 | const result = await nodeFetch(url, request); 38 | return result.json(); 39 | } 40 | 41 | export async function fetchGraphql( 42 | url: string, 43 | query: string, 44 | variables: Record = {}, 45 | extraHeaders: Record = {}, 46 | ): Promise { 47 | const result = await fetchPost(url, { operationName: null, query, variables }, extraHeaders); 48 | if (result.errors?.length) { 49 | throw new Error(result.errors[0].message); 50 | } 51 | return result.data as Promise; 52 | } 53 | 54 | export async function fetchGetWithinPage( 55 | page: Page, 56 | url: string, 57 | ignoreErrors = false, 58 | ): Promise { 59 | const [result, status] = await page.evaluate(async innerUrl => { 60 | let response: Response | undefined; 61 | try { 62 | response = await fetch(innerUrl, { credentials: 'include' }); 63 | if (response.status === 204) { 64 | return [null, response.status] as const; 65 | } 66 | return [await response.text(), response.status] as const; 67 | } catch (e) { 68 | throw new Error( 69 | `fetchGetWithinPage error: ${e instanceof Error ? `${e.message}\n${e.stack}` : String(e)}, url: ${innerUrl}, status: ${response?.status}`, 70 | ); 71 | } 72 | }, url); 73 | if (result !== null) { 74 | try { 75 | return JSON.parse(result); 76 | } catch (e) { 77 | if (!ignoreErrors) { 78 | throw new Error( 79 | `fetchGetWithinPage parse error: ${e instanceof Error ? `${e.message}\n${e.stack}` : String(e)}, url: ${url}, result: ${result}, status: ${status}`, 80 | ); 81 | } 82 | } 83 | } 84 | return null; 85 | } 86 | 87 | export async function fetchPostWithinPage( 88 | page: Page, 89 | url: string, 90 | data: Record, 91 | extraHeaders: Record = {}, 92 | ignoreErrors = false, 93 | ): Promise { 94 | const result = await page.evaluate( 95 | async (innerUrl: string, innerData: Record, innerExtraHeaders: Record) => { 96 | const response = await fetch(innerUrl, { 97 | method: 'POST', 98 | body: JSON.stringify(innerData), 99 | credentials: 'include', 100 | // eslint-disable-next-line prefer-object-spread 101 | headers: Object.assign( 102 | { 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' }, 103 | innerExtraHeaders, 104 | ), 105 | }); 106 | if (response.status === 204) { 107 | return null; 108 | } 109 | return response.text(); 110 | }, 111 | url, 112 | data, 113 | extraHeaders, 114 | ); 115 | 116 | try { 117 | if (result !== null) { 118 | return JSON.parse(result); 119 | } 120 | } catch (e) { 121 | if (!ignoreErrors) { 122 | throw new Error( 123 | `fetchPostWithinPage parse error: ${e instanceof Error ? `${e.message}\n${e.stack}` : String(e)}, url: ${url}, data: ${JSON.stringify(data)}, extraHeaders: ${JSON.stringify(extraHeaders)}, result: ${result}`, 124 | ); 125 | } 126 | } 127 | return null; 128 | } 129 | -------------------------------------------------------------------------------- /src/helpers/elements-interactions.ts: -------------------------------------------------------------------------------- 1 | import { type Frame, type Page } from 'puppeteer'; 2 | import { waitUntil } from './waiting'; 3 | 4 | async function waitUntilElementFound( 5 | page: Page | Frame, 6 | elementSelector: string, 7 | onlyVisible = false, 8 | timeout?: number, 9 | ) { 10 | await page.waitForSelector(elementSelector, { visible: onlyVisible, timeout }); 11 | } 12 | 13 | async function waitUntilElementDisappear(page: Page, elementSelector: string, timeout?: number) { 14 | await page.waitForSelector(elementSelector, { hidden: true, timeout }); 15 | } 16 | 17 | async function waitUntilIframeFound( 18 | page: Page, 19 | framePredicate: (frame: Frame) => boolean, 20 | description = '', 21 | timeout = 30000, 22 | ) { 23 | let frame: Frame | undefined; 24 | await waitUntil( 25 | () => { 26 | frame = page.frames().find(framePredicate); 27 | return Promise.resolve(!!frame); 28 | }, 29 | description, 30 | timeout, 31 | 1000, 32 | ); 33 | 34 | if (!frame) { 35 | throw new Error('failed to find iframe'); 36 | } 37 | 38 | return frame; 39 | } 40 | 41 | async function fillInput(pageOrFrame: Page | Frame, inputSelector: string, inputValue: string): Promise { 42 | await pageOrFrame.$eval(inputSelector, (input: Element) => { 43 | const inputElement = input; 44 | // @ts-ignore 45 | inputElement.value = ''; 46 | }); 47 | await pageOrFrame.type(inputSelector, inputValue); 48 | } 49 | 50 | async function setValue(pageOrFrame: Page | Frame, inputSelector: string, inputValue: string): Promise { 51 | await pageOrFrame.$eval( 52 | inputSelector, 53 | (input: Element, value) => { 54 | const inputElement = input; 55 | // @ts-ignore 56 | inputElement.value = value; 57 | }, 58 | [inputValue], 59 | ); 60 | } 61 | 62 | async function clickButton(page: Page | Frame, buttonSelector: string) { 63 | await page.$eval(buttonSelector, el => (el as HTMLElement).click()); 64 | } 65 | 66 | async function clickLink(page: Page, aSelector: string) { 67 | await page.$eval(aSelector, (el: any) => { 68 | if (!el || typeof el.click === 'undefined') { 69 | return; 70 | } 71 | 72 | el.click(); 73 | }); 74 | } 75 | 76 | async function pageEvalAll( 77 | page: Page | Frame, 78 | selector: string, 79 | defaultResult: any, 80 | callback: (elements: Element[], ...args: any) => R, 81 | ...args: any[] 82 | ): Promise { 83 | let result = defaultResult; 84 | try { 85 | await page.waitForFunction(() => document.readyState === 'complete'); 86 | result = await page.$$eval(selector, callback, ...args); 87 | } catch (e) { 88 | // TODO temporary workaround to puppeteer@1.5.0 which breaks $$eval bevahvior until they will release a new version. 89 | if (!(e as Error).message.startsWith('Error: failed to find elements matching selector')) { 90 | throw e; 91 | } 92 | } 93 | 94 | return result; 95 | } 96 | 97 | async function pageEval( 98 | pageOrFrame: Page | Frame, 99 | selector: string, 100 | defaultResult: any, 101 | callback: (elements: Element, ...args: any) => R, 102 | ...args: any[] 103 | ): Promise { 104 | let result = defaultResult; 105 | try { 106 | await pageOrFrame.waitForFunction(() => document.readyState === 'complete'); 107 | result = await pageOrFrame.$eval(selector, callback, ...args); 108 | } catch (e) { 109 | // TODO temporary workaround to puppeteer@1.5.0 which breaks $$eval bevahvior until they will release a new version. 110 | if (!(e as Error).message.startsWith('Error: failed to find element matching selector')) { 111 | throw e; 112 | } 113 | } 114 | 115 | return result; 116 | } 117 | 118 | async function elementPresentOnPage(pageOrFrame: Page | Frame, selector: string) { 119 | return (await pageOrFrame.$(selector)) !== null; 120 | } 121 | 122 | async function dropdownSelect(page: Page, selectSelector: string, value: string) { 123 | await page.select(selectSelector, value); 124 | } 125 | 126 | async function dropdownElements(page: Page, selector: string) { 127 | const options = await page.evaluate(optionSelector => { 128 | return Array.from(document.querySelectorAll(optionSelector)) 129 | .filter(o => o.value) 130 | .map(o => { 131 | return { 132 | name: o.text, 133 | value: o.value, 134 | }; 135 | }); 136 | }, `${selector} > option`); 137 | return options; 138 | } 139 | 140 | export { 141 | clickButton, 142 | clickLink, 143 | dropdownElements, 144 | dropdownSelect, 145 | elementPresentOnPage, 146 | fillInput, 147 | pageEval, 148 | pageEvalAll, 149 | setValue, 150 | waitUntilElementDisappear, 151 | waitUntilElementFound, 152 | waitUntilIframeFound, 153 | }; 154 | -------------------------------------------------------------------------------- /src/scrapers/behatsdaa.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import { getDebug } from '../helpers/debug'; 3 | import { waitUntilElementFound } from '../helpers/elements-interactions'; 4 | import { fetchPostWithinPage } from '../helpers/fetch'; 5 | import { sleep } from '../helpers/waiting'; 6 | import { type Transaction, TransactionStatuses, TransactionTypes } from '../transactions'; 7 | import { BaseScraperWithBrowser, type LoginOptions, LoginResults } from './base-scraper-with-browser'; 8 | import { type ScraperScrapingResult } from './interface'; 9 | 10 | const BASE_URL = 'https://www.behatsdaa.org.il'; 11 | const LOGIN_URL = `${BASE_URL}/login`; 12 | const PURCHASE_HISTORY_URL = 'https://back.behatsdaa.org.il/api/purchases/purchaseHistory'; 13 | 14 | const debug = getDebug('behatsdaa'); 15 | 16 | type ScraperSpecificCredentials = { id: string; password: string }; 17 | 18 | type Variant = { 19 | name: string; 20 | variantName: string; 21 | customerPrice: number; 22 | orderDate: string; // ISO timestamp with no timezone 23 | tTransactionID: string; 24 | }; 25 | 26 | type PurchaseHistoryResponse = { 27 | data?: { 28 | errorDescription?: string; 29 | memberId: string; 30 | variants: Variant[]; 31 | }; 32 | errorDescription?: string; 33 | }; 34 | 35 | function variantToTransaction(variant: Variant): Transaction { 36 | // The price is positive, make it negative as it's an expense 37 | const originalAmount = -variant.customerPrice; 38 | return { 39 | type: TransactionTypes.Normal, 40 | identifier: variant.tTransactionID, 41 | date: moment(variant.orderDate).format('YYYY-MM-DD'), 42 | processedDate: moment(variant.orderDate).format('YYYY-MM-DD'), 43 | originalAmount, 44 | originalCurrency: 'ILS', 45 | chargedAmount: originalAmount, 46 | chargedCurrency: 'ILS', 47 | description: variant.name, 48 | status: TransactionStatuses.Completed, 49 | memo: variant.variantName, 50 | }; 51 | } 52 | 53 | class BehatsdaaScraper extends BaseScraperWithBrowser { 54 | public getLoginOptions(credentials: ScraperSpecificCredentials): LoginOptions { 55 | return { 56 | loginUrl: LOGIN_URL, 57 | fields: [ 58 | { selector: '#loginId', value: credentials.id }, 59 | { selector: '#loginPassword', value: credentials.password }, 60 | ], 61 | checkReadiness: async () => { 62 | await Promise.all([ 63 | waitUntilElementFound(this.page, '#loginPassword'), 64 | waitUntilElementFound(this.page, '#loginId'), 65 | ]); 66 | }, 67 | possibleResults: { 68 | [LoginResults.Success]: [`${BASE_URL}/`], 69 | [LoginResults.InvalidPassword]: ['.custom-input-error-label'], 70 | }, 71 | submitButtonSelector: async () => { 72 | await sleep(1000); 73 | debug('Trying to find submit button'); 74 | const button = await this.page.$('xpath=//button[contains(., "התחברות")]'); 75 | if (button) { 76 | debug('Submit button found'); 77 | await button.click(); 78 | } else { 79 | debug('Submit button not found'); 80 | } 81 | }, 82 | }; 83 | } 84 | 85 | async fetchData(): Promise { 86 | const token = await this.page.evaluate(() => window.localStorage.getItem('userToken')); 87 | if (!token) { 88 | debug('Token not found in local storage'); 89 | return { 90 | success: false, 91 | errorMessage: 'TokenNotFound', 92 | }; 93 | } 94 | 95 | const body = { 96 | FromDate: moment(this.options.startDate).format('YYYY-MM-DDTHH:mm:ss'), 97 | ToDate: moment().format('YYYY-MM-DDTHH:mm:ss'), 98 | BenefitStatusId: null, 99 | }; 100 | 101 | debug('Fetching data'); 102 | 103 | const res = await fetchPostWithinPage(this.page, PURCHASE_HISTORY_URL, body, { 104 | authorization: `Bearer ${token}`, 105 | 'Content-Type': 'application/json', 106 | organizationid: '20', 107 | }); 108 | 109 | debug('Data fetched'); 110 | 111 | if (res?.errorDescription || res?.data?.errorDescription) { 112 | debug('Error fetching data', res.errorDescription || res.data?.errorDescription); 113 | return { success: false, errorMessage: res.errorDescription }; 114 | } 115 | 116 | if (!res?.data) { 117 | debug('No data found'); 118 | return { success: false, errorMessage: 'NoData' }; 119 | } 120 | 121 | debug('Data fetched successfully'); 122 | return { 123 | success: true, 124 | accounts: [ 125 | { 126 | accountNumber: res.data.memberId, 127 | txns: res.data.variants.map(variantToTransaction), 128 | }, 129 | ], 130 | }; 131 | } 132 | } 133 | 134 | export default BehatsdaaScraper; 135 | -------------------------------------------------------------------------------- /src/scrapers/discount.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import moment from 'moment'; 3 | import { type Page } from 'puppeteer'; 4 | import { waitUntilElementFound } from '../helpers/elements-interactions'; 5 | import { fetchGetWithinPage } from '../helpers/fetch'; 6 | import { waitForNavigation } from '../helpers/navigation'; 7 | import { type Transaction, TransactionStatuses, TransactionTypes } from '../transactions'; 8 | import { BaseScraperWithBrowser, LoginResults, type PossibleLoginResults } from './base-scraper-with-browser'; 9 | import { ScraperErrorTypes } from './errors'; 10 | import { type ScraperOptions, type ScraperScrapingResult } from './interface'; 11 | 12 | const BASE_URL = 'https://start.telebank.co.il'; 13 | const DATE_FORMAT = 'YYYYMMDD'; 14 | 15 | interface ScrapedTransaction { 16 | OperationNumber: number; 17 | OperationDate: string; 18 | ValueDate: string; 19 | OperationAmount: number; 20 | OperationDescriptionToDisplay: string; 21 | } 22 | 23 | interface CurrentAccountInfo { 24 | AccountBalance: number; 25 | } 26 | 27 | interface ScrapedAccountData { 28 | UserAccountsData: { 29 | DefaultAccountNumber: string; 30 | UserAccounts: Array<{ 31 | NewAccountInfo: { 32 | AccountID: string; 33 | }; 34 | }>; 35 | }; 36 | } 37 | 38 | interface ScrapedTransactionData { 39 | Error?: { MsgText: string }; 40 | CurrentAccountLastTransactions?: { 41 | OperationEntry: ScrapedTransaction[]; 42 | CurrentAccountInfo: CurrentAccountInfo; 43 | FutureTransactionsBlock: { 44 | FutureTransactionEntry: ScrapedTransaction[]; 45 | }; 46 | }; 47 | } 48 | 49 | function convertTransactions(txns: ScrapedTransaction[], txnStatus: TransactionStatuses): Transaction[] { 50 | if (!txns) { 51 | return []; 52 | } 53 | return txns.map(txn => { 54 | return { 55 | type: TransactionTypes.Normal, 56 | identifier: txn.OperationNumber, 57 | date: moment(txn.OperationDate, DATE_FORMAT).toISOString(), 58 | processedDate: moment(txn.ValueDate, DATE_FORMAT).toISOString(), 59 | originalAmount: txn.OperationAmount, 60 | originalCurrency: 'ILS', 61 | chargedAmount: txn.OperationAmount, 62 | description: txn.OperationDescriptionToDisplay, 63 | status: txnStatus, 64 | }; 65 | }); 66 | } 67 | 68 | async function fetchAccountData(page: Page, options: ScraperOptions): Promise { 69 | const apiSiteUrl = `${BASE_URL}/Titan/gatewayAPI`; 70 | 71 | const accountDataUrl = `${apiSiteUrl}/userAccountsData`; 72 | const accountInfo = await fetchGetWithinPage(page, accountDataUrl); 73 | 74 | if (!accountInfo) { 75 | return { 76 | success: false, 77 | errorType: ScraperErrorTypes.Generic, 78 | errorMessage: 'failed to get account data', 79 | }; 80 | } 81 | 82 | const defaultStartMoment = moment().subtract(1, 'years').add(2, 'day'); 83 | const startDate = options.startDate || defaultStartMoment.toDate(); 84 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 85 | 86 | const startDateStr = startMoment.format(DATE_FORMAT); 87 | 88 | const accounts: string[] = accountInfo.UserAccountsData.UserAccounts.map(acc => acc.NewAccountInfo.AccountID); 89 | const accountsData: Array<{ accountNumber: string; balance: number; txns: Transaction[] }> = []; 90 | 91 | for (const accountNumber of accounts) { 92 | const txnsUrl = `${apiSiteUrl}/lastTransactions/${accountNumber}/Date?IsCategoryDescCode=True&IsTransactionDetails=True&IsEventNames=True&IsFutureTransactionFlag=True&FromDate=${startDateStr}`; 93 | const txnsResult = await fetchGetWithinPage(page, txnsUrl); 94 | if (!txnsResult || txnsResult.Error || !txnsResult.CurrentAccountLastTransactions) { 95 | return { 96 | success: false, 97 | errorType: ScraperErrorTypes.Generic, 98 | errorMessage: txnsResult && txnsResult.Error ? txnsResult.Error.MsgText : 'unknown error', 99 | }; 100 | } 101 | 102 | const accountCompletedTxns = convertTransactions( 103 | txnsResult.CurrentAccountLastTransactions.OperationEntry, 104 | TransactionStatuses.Completed, 105 | ); 106 | const rawFutureTxns = _.get( 107 | txnsResult, 108 | 'CurrentAccountLastTransactions.FutureTransactionsBlock.FutureTransactionEntry', 109 | ) as ScrapedTransaction[]; 110 | const accountPendingTxns = convertTransactions(rawFutureTxns, TransactionStatuses.Pending); 111 | 112 | accountsData.push({ 113 | accountNumber, 114 | balance: txnsResult.CurrentAccountLastTransactions.CurrentAccountInfo.AccountBalance, 115 | txns: [...accountCompletedTxns, ...accountPendingTxns], 116 | }); 117 | } 118 | 119 | const accountData = { 120 | success: true, 121 | accounts: accountsData, 122 | }; 123 | 124 | return accountData; 125 | } 126 | 127 | async function navigateOrErrorLabel(page: Page) { 128 | try { 129 | await waitForNavigation(page); 130 | } catch (e) { 131 | await waitUntilElementFound(page, '#general-error', false, 100); 132 | } 133 | } 134 | 135 | function getPossibleLoginResults(): PossibleLoginResults { 136 | const urls: PossibleLoginResults = {}; 137 | urls[LoginResults.Success] = [ 138 | `${BASE_URL}/apollo/retail/#/MY_ACCOUNT_HOMEPAGE`, 139 | `${BASE_URL}/apollo/retail2/#/MY_ACCOUNT_HOMEPAGE`, 140 | ]; 141 | urls[LoginResults.InvalidPassword] = [`${BASE_URL}/apollo/core/templates/lobby/masterPage.html#/LOGIN_PAGE`]; 142 | urls[LoginResults.ChangePassword] = [`${BASE_URL}/apollo/core/templates/lobby/masterPage.html#/PWD_RENEW`]; 143 | return urls; 144 | } 145 | 146 | function createLoginFields(credentials: ScraperSpecificCredentials) { 147 | return [ 148 | { selector: '#tzId', value: credentials.id }, 149 | { selector: '#tzPassword', value: credentials.password }, 150 | { selector: '#aidnum', value: credentials.num }, 151 | ]; 152 | } 153 | 154 | type ScraperSpecificCredentials = { id: string; password: string; num: string }; 155 | 156 | class DiscountScraper extends BaseScraperWithBrowser { 157 | getLoginOptions(credentials: ScraperSpecificCredentials) { 158 | return { 159 | loginUrl: `${BASE_URL}/login/#/LOGIN_PAGE`, 160 | checkReadiness: async () => waitUntilElementFound(this.page, '#tzId'), 161 | fields: createLoginFields(credentials), 162 | submitButtonSelector: '.sendBtn', 163 | postAction: async () => navigateOrErrorLabel(this.page), 164 | possibleResults: getPossibleLoginResults(), 165 | }; 166 | } 167 | 168 | async fetchData() { 169 | return fetchAccountData(this.page, this.options); 170 | } 171 | } 172 | 173 | export default DiscountScraper; 174 | -------------------------------------------------------------------------------- /src/scrapers/interface.ts: -------------------------------------------------------------------------------- 1 | import { type BrowserContext, type Browser, type Page } from 'puppeteer'; 2 | import { type CompanyTypes, type ScraperProgressTypes } from '../definitions'; 3 | import { type TransactionsAccount } from '../transactions'; 4 | import { type ErrorResult, type ScraperErrorTypes } from './errors'; 5 | 6 | // TODO: Remove this type when the scraper 'factory' will return concrete scraper types 7 | // Instead of a generic interface (which in turn uses this type) 8 | export type ScraperCredentials = 9 | | { userCode: string; password: string } 10 | | { username: string; password: string } 11 | | { id: string; password: string } 12 | | { id: string; password: string; num: string } 13 | | { id: string; password: string; card6Digits: string } 14 | | { username: string; nationalID: string; password: string } 15 | | ({ email: string; password: string } & ( 16 | | { 17 | otpCodeRetriever: () => Promise; 18 | phoneNumber: string; 19 | } 20 | | { 21 | otpLongTermToken: string; 22 | } 23 | )); 24 | 25 | export type OptInFeatures = 26 | | 'isracard-amex:skipAdditionalTransactionInformation' 27 | | 'mizrahi:pendingIfNoIdentifier' 28 | | 'mizrahi:pendingIfHasGenericDescription' 29 | | 'mizrahi:pendingIfTodayTransaction'; 30 | 31 | export interface FutureDebit { 32 | amount: number; 33 | amountCurrency: string; 34 | chargeDate?: string; 35 | bankAccountNumber?: string; 36 | } 37 | 38 | interface ExternalBrowserOptions { 39 | /** 40 | * An externally created browser instance. 41 | * you can get a browser directly from puppeteer via `puppeteer.launch()` 42 | * 43 | * Note: The browser will be closed by the library after the scraper finishes unless `skipCloseBrowser` is set to true 44 | */ 45 | browser: Browser; 46 | 47 | /** 48 | * If true, the browser will not be closed by the library after the scraper finishes 49 | */ 50 | skipCloseBrowser?: boolean; 51 | } 52 | 53 | interface ExternalBrowserContextOptions { 54 | /** 55 | * An externally managed browser context. This is useful when you want to manage the browser 56 | */ 57 | browserContext: BrowserContext; 58 | } 59 | 60 | interface DefaultBrowserOptions { 61 | /** 62 | * shows the browser while scraping, good for debugging (default false) 63 | */ 64 | showBrowser?: boolean; 65 | 66 | /** 67 | * provide a patch to local chromium to be used by puppeteer. Relevant when using 68 | * `israeli-bank-scrapers-core` library 69 | */ 70 | executablePath?: string; 71 | 72 | /** 73 | * additional arguments to pass to the browser instance. The list of flags can be found in 74 | * 75 | * https://developer.mozilla.org/en-US/docs/Mozilla/Command_Line_Options 76 | * https://peter.sh/experiments/chromium-command-line-switches/ 77 | */ 78 | args?: string[]; 79 | 80 | /** 81 | * Maximum navigation time in milliseconds, pass 0 to disable timeout. 82 | * @default 30000 83 | */ 84 | timeout?: number; 85 | 86 | /** 87 | * adjust the browser instance before it is being used 88 | * 89 | * @param browser 90 | */ 91 | prepareBrowser?: (browser: Browser) => Promise; 92 | } 93 | 94 | type ScraperBrowserOptions = ExternalBrowserOptions | ExternalBrowserContextOptions | DefaultBrowserOptions; 95 | 96 | export type ScraperOptions = ScraperBrowserOptions & { 97 | /** 98 | * The company you want to scrape 99 | */ 100 | companyId: CompanyTypes; 101 | 102 | /** 103 | * include more debug info about in the output 104 | */ 105 | verbose?: boolean; 106 | 107 | /** 108 | * the date to fetch transactions from (can't be before the minimum allowed time difference for the scraper) 109 | */ 110 | startDate: Date; 111 | 112 | /** 113 | * scrape transactions to be processed X months in the future 114 | */ 115 | futureMonthsToScrape?: number; 116 | 117 | /** 118 | * if set to true, all installment transactions will be combine into the first one 119 | */ 120 | combineInstallments?: boolean; 121 | 122 | /** 123 | * adjust the page instance before it is being used. 124 | * 125 | * @param page 126 | */ 127 | preparePage?: (page: Page) => Promise; 128 | 129 | /** 130 | * if set, store a screenshot if failed to scrape. Used for debug purposes 131 | */ 132 | storeFailureScreenShotPath?: string; 133 | 134 | /** 135 | * if set, will set the timeout in milliseconds of puppeteer's `page.setDefaultTimeout`. 136 | */ 137 | defaultTimeout?: number; 138 | 139 | /** 140 | * Options for manipulation of output data 141 | */ 142 | outputData?: OutputDataOptions; 143 | 144 | /** 145 | * Perform additional operation for each transaction to get more information (Like category) about it. 146 | * Please note: It will take more time to finish the process. 147 | */ 148 | additionalTransactionInformation?: boolean; 149 | 150 | /** 151 | * Adjust the viewport size of the browser page. 152 | * If not set, the default viewport size of 1024x768 will be used. 153 | */ 154 | viewportSize?: { 155 | width: number; 156 | height: number; 157 | }; 158 | 159 | /** 160 | * The number of times to retry the navigation in case of a failure (default 0) 161 | */ 162 | navigationRetryCount?: number; 163 | 164 | /** 165 | * Opt-in features for the scrapers, allowing safe rollout of new breaking changes. 166 | */ 167 | optInFeatures?: Array; 168 | }; 169 | 170 | export interface OutputDataOptions { 171 | /** 172 | * if true, the result wouldn't be filtered out by date, and you will return unfiltered scrapped data. 173 | */ 174 | enableTransactionsFilterByDate?: boolean; 175 | } 176 | 177 | export interface ScraperScrapingResult { 178 | success: boolean; 179 | accounts?: TransactionsAccount[]; 180 | futureDebits?: FutureDebit[]; 181 | errorType?: ScraperErrorTypes; 182 | errorMessage?: string; // only on success=false 183 | } 184 | 185 | export interface Scraper { 186 | scrape(credentials: TCredentials): Promise; 187 | onProgress(func: (companyId: CompanyTypes, payload: { type: ScraperProgressTypes }) => void): void; 188 | triggerTwoFactorAuth(phoneNumber: string): Promise; 189 | getLongTermTwoFactorToken(otpCode: string): Promise; 190 | } 191 | 192 | export type ScraperTwoFactorAuthTriggerResult = 193 | | ErrorResult 194 | | { 195 | success: true; 196 | }; 197 | 198 | export type ScraperGetLongTermTwoFactorTokenResult = 199 | | ErrorResult 200 | | { 201 | success: true; 202 | longTermTwoFactorAuthToken: string; 203 | }; 204 | 205 | export interface ScraperLoginResult { 206 | success: boolean; 207 | errorType?: ScraperErrorTypes; 208 | errorMessage?: string; // only on success=false 209 | persistentOtpToken?: string; 210 | } 211 | -------------------------------------------------------------------------------- /src/scrapers/beyahad-bishvilha.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import { type Page } from 'puppeteer'; 3 | import { 4 | DOLLAR_CURRENCY, 5 | DOLLAR_CURRENCY_SYMBOL, 6 | EURO_CURRENCY, 7 | EURO_CURRENCY_SYMBOL, 8 | SHEKEL_CURRENCY, 9 | SHEKEL_CURRENCY_SYMBOL, 10 | } from '../constants'; 11 | import { getDebug } from '../helpers/debug'; 12 | import { pageEval, pageEvalAll, waitUntilElementFound } from '../helpers/elements-interactions'; 13 | import { filterOldTransactions } from '../helpers/transactions'; 14 | import { TransactionStatuses, TransactionTypes, type Transaction } from '../transactions'; 15 | import { BaseScraperWithBrowser, LoginResults, type PossibleLoginResults } from './base-scraper-with-browser'; 16 | import { type ScraperOptions } from './interface'; 17 | 18 | const debug = getDebug('beyahadBishvilha'); 19 | 20 | const DATE_FORMAT = 'DD/MM/YY'; 21 | const LOGIN_URL = 'https://www.hist.org.il/login'; 22 | const SUCCESS_URL = 'https://www.hist.org.il/'; 23 | const CARD_URL = 'https://www.hist.org.il/card/balanceAndUses'; 24 | 25 | interface ScrapedTransaction { 26 | date: string; 27 | description: string; 28 | type: string; 29 | chargedAmount: string; 30 | identifier: string; 31 | } 32 | 33 | function getAmountData(amountStr: string) { 34 | const amountStrCln = amountStr.replace(',', ''); 35 | let currency: string | null = null; 36 | let amount: number | null = null; 37 | if (amountStrCln.includes(SHEKEL_CURRENCY_SYMBOL)) { 38 | amount = parseFloat(amountStrCln.replace(SHEKEL_CURRENCY_SYMBOL, '')); 39 | currency = SHEKEL_CURRENCY; 40 | } else if (amountStrCln.includes(DOLLAR_CURRENCY_SYMBOL)) { 41 | amount = parseFloat(amountStrCln.replace(DOLLAR_CURRENCY_SYMBOL, '')); 42 | currency = DOLLAR_CURRENCY; 43 | } else if (amountStrCln.includes(EURO_CURRENCY_SYMBOL)) { 44 | amount = parseFloat(amountStrCln.replace(EURO_CURRENCY_SYMBOL, '')); 45 | currency = EURO_CURRENCY; 46 | } else { 47 | const parts = amountStrCln.split(' '); 48 | [currency] = parts; 49 | amount = parseFloat(parts[1]); 50 | } 51 | 52 | return { 53 | amount, 54 | currency, 55 | }; 56 | } 57 | 58 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 59 | debug(`convert ${txns.length} raw transactions to official Transaction structure`); 60 | return txns.map(txn => { 61 | const chargedAmountTuple = getAmountData(txn.chargedAmount || ''); 62 | const txnProcessedDate = moment(txn.date, DATE_FORMAT); 63 | 64 | const result: Transaction = { 65 | type: TransactionTypes.Normal, 66 | status: TransactionStatuses.Completed, 67 | date: txnProcessedDate.toISOString(), 68 | processedDate: txnProcessedDate.toISOString(), 69 | originalAmount: chargedAmountTuple.amount, 70 | originalCurrency: chargedAmountTuple.currency, 71 | chargedAmount: chargedAmountTuple.amount, 72 | chargedCurrency: chargedAmountTuple.currency, 73 | description: txn.description || '', 74 | memo: '', 75 | identifier: txn.identifier, 76 | }; 77 | 78 | return result; 79 | }); 80 | } 81 | 82 | async function fetchTransactions(page: Page, options: ScraperOptions) { 83 | await page.goto(CARD_URL); 84 | await waitUntilElementFound(page, '.react-loading.hide', false); 85 | const defaultStartMoment = moment().subtract(1, 'years'); 86 | const startDate = options.startDate || defaultStartMoment.toDate(); 87 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 88 | 89 | const accountNumber = await pageEval(page, '.wallet-details div:nth-of-type(2)', null, element => { 90 | return (element as any).innerText.replace('מספר כרטיס ', ''); 91 | }); 92 | 93 | const balance = await pageEval(page, '.wallet-details div:nth-of-type(4) > span:nth-of-type(2)', null, element => { 94 | return (element as any).innerText; 95 | }); 96 | 97 | debug('fetch raw transactions from page'); 98 | 99 | const rawTransactions: (ScrapedTransaction | null)[] = await pageEvalAll<(ScrapedTransaction | null)[]>( 100 | page, 101 | '.transaction-container, .transaction-component-container', 102 | [], 103 | items => { 104 | return items.map(el => { 105 | const columns: NodeListOf = el.querySelectorAll('.transaction-item > span'); 106 | if (columns.length === 7) { 107 | return { 108 | date: columns[0].innerText, 109 | identifier: columns[1].innerText, 110 | description: columns[3].innerText, 111 | type: columns[5].innerText, 112 | chargedAmount: columns[6].innerText, 113 | }; 114 | } 115 | return null; 116 | }); 117 | }, 118 | ); 119 | debug(`fetched ${rawTransactions.length} raw transactions from page`); 120 | 121 | const accountTransactions = convertTransactions(rawTransactions.filter(item => !!item) as ScrapedTransaction[]); 122 | 123 | debug('filer out old transactions'); 124 | const txns = 125 | (options.outputData?.enableTransactionsFilterByDate ?? true) 126 | ? filterOldTransactions(accountTransactions, startMoment, false) 127 | : accountTransactions; 128 | debug( 129 | `found ${txns.length} valid transactions out of ${accountTransactions.length} transactions for account ending with ${accountNumber.substring(accountNumber.length - 2)}`, 130 | ); 131 | 132 | return { 133 | accountNumber, 134 | balance: getAmountData(balance).amount, 135 | txns, 136 | }; 137 | } 138 | 139 | function getPossibleLoginResults(): PossibleLoginResults { 140 | const urls: PossibleLoginResults = {}; 141 | urls[LoginResults.Success] = [SUCCESS_URL]; 142 | urls[LoginResults.ChangePassword] = []; // TODO 143 | urls[LoginResults.InvalidPassword] = []; // TODO 144 | urls[LoginResults.UnknownError] = []; // TODO 145 | return urls; 146 | } 147 | 148 | function createLoginFields(credentials: ScraperSpecificCredentials) { 149 | return [ 150 | { selector: '#loginId', value: credentials.id }, 151 | { selector: '#loginPassword', value: credentials.password }, 152 | ]; 153 | } 154 | 155 | type ScraperSpecificCredentials = { id: string; password: string }; 156 | 157 | class BeyahadBishvilhaScraper extends BaseScraperWithBrowser { 158 | protected getViewPort(): { width: number; height: number } { 159 | return { 160 | width: 1500, 161 | height: 800, 162 | }; 163 | } 164 | 165 | getLoginOptions(credentials: ScraperSpecificCredentials) { 166 | return { 167 | loginUrl: LOGIN_URL, 168 | fields: createLoginFields(credentials), 169 | submitButtonSelector: async () => { 170 | const button = await this.page.$('xpath//button[contains(., "התחבר")]'); 171 | if (button) { 172 | await button.click(); 173 | } 174 | }, 175 | possibleResults: getPossibleLoginResults(), 176 | }; 177 | } 178 | 179 | async fetchData() { 180 | const account = await fetchTransactions(this.page, this.options); 181 | return { 182 | success: true, 183 | accounts: [account], 184 | }; 185 | } 186 | } 187 | 188 | export default BeyahadBishvilhaScraper; 189 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to Israeli Bank Scrapers 2 | ======== 3 | Hey people, first of all, thanks for taking the time to help improving this project :beers: 4 | 5 | This project needs the help of other developers, since it's impossible getting access to all relevant banks and credit cards. 6 | 7 | # How can I contribute? 8 | Any kind of help is welcome, even if you just discover an issue and don't have the time to invest in helping to fix it. 9 | 10 | ## Filing issues 11 | While there's no specific template for creating a new issue, please take the time to create a clear description so that it is easy to understand the problem. 12 | 13 | ## Testing the scrapers 14 | In order to run tests you need first to create test configuration file `./src/tests/.tests-config.js` from template `./src/tests/.tests-config.tpl.js`. This file will be used by `jest` testing framework. 15 | 16 | > IMPORTANT: Under `src/tests` folder exists `.gitignore` file that ignore the test configuration file thus this file will not be commited to github. Still when you create new PRs make sure that you didn't explicitly added it to the PR. 17 | 18 | This library supports both testing against credit card companies / banks api and also against mock data. Until we will have a good coverage of scrapers test with mock data, the default configuration is set to execute real companies api tests. 19 | 20 | ### Changing tests options 21 | Modify property `options` in the test configuration file. This object is passed as-is to the scraper. 22 | 23 | ### Testing specific companies 24 | Enable any company you wish to test by providing its credetials in the test configuration file under `credentials` property. 25 | 26 | ### Running tests from CLI 27 | > Before running any tests, make sure you created the test configuration file with relevant credentials, 28 | 29 | To run all tests of companies that you provided credentials to: 30 | ``` 31 | npm test 32 | ``` 33 | 34 | To run specific `describe` (a.k.a suite), use the `testNamePattern` arg with the name of the suite. The following will run the all tests under `Leumi legacy scraper` suite. 35 | ``` 36 | npm test -- --testNamePattern="Leumi legacy scraper" 37 | ``` 38 | 39 | To run specific `test`, use the `testNamePattern` arg with suite name following the test name. The following will run test `should expose login fields in scrapers constant` that is part of `Leumi legacy scraper` suite. 40 | ``` 41 | npm test -- --testNamePattern="Leumi legacy scraper should expose login fields in scrapers constant" 42 | ``` 43 | 44 | ### Running tests using IDE 45 | Many IDEs support running jest tests directly from the UI. In webstorm for example a small play icon automatically appears next to each describe/test. 46 | 47 | **IMPORTANT Note** babel is configured to ignore tests by default. You must add an environment variable `BABEL_ENV=test` to the IDE test configuration to allow the tests to work. 48 | 49 | ### save unit test scraper results into file 50 | To save unit test scraper results provide a valid path in test configurations property `excelFilesDist`, for example: 51 | 52 | ``` 53 | { 54 | companyAPI: { 55 | enabled: true, 56 | excelFilesDist: '/Users/xyz/Downloads/Transactions', 57 | 58 | }, 59 | } 60 | ``` 61 | 62 | ### F.A.Q regarding the tests 63 | 64 | #### How can I run tests with CI/CD services? 65 | You can use environment variables instead of a local file to provide the tests configuration. 66 | 67 | copy and adjust the json below with relevant credentials and assign it to environment variable named `TESTS_CONFIG`. Note that this must be a valid json string otherwise it will fail during json parsing. 68 | ``` 69 | { 70 | "options": { 71 | "startDate": "2019-06-01", 72 | "combineInstallments": false, 73 | "showBrowser": true, 74 | "verbose": false, 75 | "args": [] 76 | }, 77 | "credentials": { 78 | "leumi": { "username": "demouser", "password": "demopassword" } 79 | }, 80 | "companyAPI": { 81 | "enabled": true, 82 | "invalidPassword": false 83 | } 84 | } 85 | ``` 86 | 87 | If you wish to try it from cli (mac os), you should either create a one liner json configuration or use cat to provide multiline value: 88 | 89 | ``` 90 | TESTS_CONFIG=`cat < this section is relevant if you are extending class `BaseScraperWithBrowser` 141 | > 142 | Unless you plan to override the entire `login()` function, You can override this function to login regularly in a login form. 143 | 144 | ```typescript 145 | import { LoginResults, PossibleLoginResults } from './base-scraper-with-browser'; 146 | 147 | function getPossibleLoginResults(): PossibleLoginResults { 148 | // checkout file `base-scraper-with-browser.ts` for available result types 149 | const urls: PossibleLoginResults = {}; 150 | urls[LoginResults.Success] = []; 151 | urls[LoginResults.InvalidPassword] = []; 152 | urls[LoginResults.ChangePassword] = []; 153 | return urls; 154 | } 155 | 156 | function getLoginOptions(credentials) { 157 | return { 158 | loginUrl: '', 159 | fields: [ 160 | { selector: '', value: credentials.username }, 161 | { selector: ``, value: credentials.password }, 162 | ], 163 | submitButtonSelector: '', 164 | possibleResults: getPossibleLoginResults(), 165 | }; 166 | } 167 | ``` 168 | 169 | ### Overriding fetchData() 170 | You can override this async function however way you want, as long as your return results as `ScaperScrapingResult` (checkout declaration [here](./src/scrapers/base-scraper.ts#L151)). 171 | -------------------------------------------------------------------------------- /src/scrapers/leumi.ts: -------------------------------------------------------------------------------- 1 | import moment, { type Moment } from 'moment'; 2 | import { type Page } from 'puppeteer'; 3 | import { SHEKEL_CURRENCY } from '../constants'; 4 | import { getDebug } from '../helpers/debug'; 5 | import { clickButton, fillInput, pageEval, pageEvalAll, waitUntilElementFound } from '../helpers/elements-interactions'; 6 | import { waitForNavigation } from '../helpers/navigation'; 7 | import { TransactionStatuses, TransactionTypes, type Transaction, type TransactionsAccount } from '../transactions'; 8 | import { BaseScraperWithBrowser, LoginResults, type LoginOptions } from './base-scraper-with-browser'; 9 | import { type ScraperScrapingResult } from './interface'; 10 | 11 | const debug = getDebug('leumi'); 12 | const BASE_URL = 'https://hb2.bankleumi.co.il'; 13 | const LOGIN_URL = 'https://www.leumi.co.il/'; 14 | const TRANSACTIONS_URL = `${BASE_URL}/eBanking/SO/SPA.aspx#/ts/BusinessAccountTrx?WidgetPar=1`; 15 | const FILTERED_TRANSACTIONS_URL = `${BASE_URL}/ChannelWCF/Broker.svc/ProcessRequest?moduleName=UC_SO_27_GetBusinessAccountTrx`; 16 | 17 | const DATE_FORMAT = 'DD.MM.YY'; 18 | const ACCOUNT_BLOCKED_MSG = 'המנוי חסום'; 19 | const INVALID_PASSWORD_MSG = 'אחד או יותר מפרטי ההזדהות שמסרת שגויים. ניתן לנסות שוב'; 20 | 21 | function getPossibleLoginResults() { 22 | const urls: LoginOptions['possibleResults'] = { 23 | [LoginResults.Success]: [/ebanking\/SO\/SPA.aspx/i], 24 | [LoginResults.InvalidPassword]: [ 25 | async options => { 26 | if (!options || !options.page) { 27 | throw new Error('missing page options argument'); 28 | } 29 | const errorMessage = await pageEvalAll(options.page, 'svg#Capa_1', '', element => { 30 | return (element[0]?.parentElement?.children[1] as HTMLDivElement)?.innerText; 31 | }); 32 | 33 | return errorMessage?.startsWith(INVALID_PASSWORD_MSG); 34 | }, 35 | ], 36 | [LoginResults.AccountBlocked]: [ 37 | // NOTICE - might not be relevant starting the Leumi re-design during 2022 Sep 38 | async options => { 39 | if (!options || !options.page) { 40 | throw new Error('missing page options argument'); 41 | } 42 | const errorMessage = await pageEvalAll(options.page, '.errHeader', '', label => { 43 | return (label[0] as HTMLElement)?.innerText; 44 | }); 45 | 46 | return errorMessage?.startsWith(ACCOUNT_BLOCKED_MSG); 47 | }, 48 | ], 49 | [LoginResults.ChangePassword]: ['https://hb2.bankleumi.co.il/authenticate'], // NOTICE - might not be relevant starting the Leumi re-design during 2022 Sep 50 | }; 51 | return urls; 52 | } 53 | 54 | function createLoginFields(credentials: ScraperSpecificCredentials) { 55 | return [ 56 | { selector: 'input[placeholder="שם משתמש"]', value: credentials.username }, 57 | { selector: 'input[placeholder="סיסמה"]', value: credentials.password }, 58 | ]; 59 | } 60 | 61 | function extractTransactionsFromPage(transactions: any[], status: TransactionStatuses): Transaction[] { 62 | if (transactions === null || transactions.length === 0) { 63 | return []; 64 | } 65 | 66 | const result: Transaction[] = transactions.map(rawTransaction => { 67 | const date = moment(rawTransaction.DateUTC).milliseconds(0).toISOString(); 68 | const newTransaction: Transaction = { 69 | status, 70 | type: TransactionTypes.Normal, 71 | date, 72 | processedDate: date, 73 | description: rawTransaction.Description || '', 74 | identifier: rawTransaction.ReferenceNumberLong, 75 | memo: rawTransaction.AdditionalData || '', 76 | originalCurrency: SHEKEL_CURRENCY, 77 | chargedAmount: rawTransaction.Amount, 78 | originalAmount: rawTransaction.Amount, 79 | }; 80 | 81 | return newTransaction; 82 | }); 83 | 84 | return result; 85 | } 86 | 87 | function hangProcess(timeout: number) { 88 | return new Promise(resolve => { 89 | setTimeout(() => { 90 | resolve(); 91 | }, timeout); 92 | }); 93 | } 94 | 95 | async function clickByXPath(page: Page, xpath: string): Promise { 96 | await page.waitForSelector(xpath, { timeout: 30000, visible: true }); 97 | const elm = await page.$$(xpath); 98 | await elm[0].click(); 99 | } 100 | 101 | function removeSpecialCharacters(str: string): string { 102 | return str.replace(/[^0-9/-]/g, ''); 103 | } 104 | 105 | async function fetchTransactionsForAccount( 106 | page: Page, 107 | startDate: Moment, 108 | accountId: string, 109 | ): Promise { 110 | // DEVELOPER NOTICE the account number received from the server is being altered at 111 | // runtime for some accounts after 1-2 seconds so we need to hang the process for a short while. 112 | await hangProcess(4000); 113 | 114 | await waitUntilElementFound(page, 'button[title="חיפוש מתקדם"]', true); 115 | await clickButton(page, 'button[title="חיפוש מתקדם"]'); 116 | await waitUntilElementFound(page, 'bll-radio-button', true); 117 | await clickButton(page, 'bll-radio-button:not([checked])'); 118 | 119 | await waitUntilElementFound(page, 'input[formcontrolname="txtInputFrom"]', true); 120 | 121 | await fillInput(page, 'input[formcontrolname="txtInputFrom"]', startDate.format(DATE_FORMAT)); 122 | 123 | // we must blur the from control otherwise the search will use the previous value 124 | await page.focus("button[aria-label='סנן']"); 125 | 126 | await clickButton(page, "button[aria-label='סנן']"); 127 | const finalResponse = await page.waitForResponse(response => { 128 | return response.url() === FILTERED_TRANSACTIONS_URL && response.request().method() === 'POST'; 129 | }); 130 | 131 | const responseJson: any = await finalResponse.json(); 132 | 133 | const accountNumber = accountId.replace('/', '_').replace(/[^\d-_]/g, ''); 134 | 135 | const response = JSON.parse(responseJson.jsonResp); 136 | 137 | const pendingTransactions = response.TodayTransactionsItems; 138 | const transactions = response.HistoryTransactionsItems; 139 | const balance = response.BalanceDisplay ? parseFloat(response.BalanceDisplay) : undefined; 140 | 141 | const pendingTxns = extractTransactionsFromPage(pendingTransactions, TransactionStatuses.Pending); 142 | const completedTxns = extractTransactionsFromPage(transactions, TransactionStatuses.Completed); 143 | const txns = [...pendingTxns, ...completedTxns]; 144 | 145 | return { 146 | accountNumber, 147 | balance, 148 | txns, 149 | }; 150 | } 151 | 152 | async function fetchTransactions(page: Page, startDate: Moment): Promise { 153 | const accounts: TransactionsAccount[] = []; 154 | 155 | // DEVELOPER NOTICE the account number received from the server is being altered at 156 | // runtime for some accounts after 1-2 seconds so we need to hang the process for a short while. 157 | await hangProcess(4000); 158 | 159 | const accountsIds = (await page.evaluate(() => 160 | Array.from(document.querySelectorAll('app-masked-number-combo span.display-number-li'), e => e.textContent), 161 | )) as string[]; 162 | 163 | // due to a bug, the altered value might include undesired signs like & that should be removed 164 | 165 | if (!accountsIds.length) { 166 | throw new Error('Failed to extract or parse the account number'); 167 | } 168 | 169 | for (const accountId of accountsIds) { 170 | if (accountsIds.length > 1) { 171 | // get list of accounts and check accountId 172 | await clickByXPath(page, 'xpath///*[contains(@class, "number") and contains(@class, "combo-inner")]'); 173 | await clickByXPath(page, `xpath///span[contains(text(), '${accountId}')]`); 174 | } 175 | 176 | accounts.push(await fetchTransactionsForAccount(page, startDate, removeSpecialCharacters(accountId))); 177 | } 178 | 179 | return accounts; 180 | } 181 | 182 | async function navigateToLogin(page: Page): Promise { 183 | const loginButtonSelector = '.enter-account a[originaltitle="כניסה לחשבונך"]'; 184 | debug('wait for homepage to click on login button'); 185 | await waitUntilElementFound(page, loginButtonSelector); 186 | debug('navigate to login page'); 187 | const loginUrl = await pageEval(page, loginButtonSelector, null, element => { 188 | return (element as any).href; 189 | }); 190 | debug(`navigating to page (${loginUrl})`); 191 | await page.goto(loginUrl); 192 | debug('waiting for page to be loaded (networkidle2)'); 193 | await waitForNavigation(page, { waitUntil: 'networkidle2' }); 194 | debug('waiting for components of login to enter credentials'); 195 | await Promise.all([ 196 | waitUntilElementFound(page, 'input[placeholder="שם משתמש"]', true), 197 | waitUntilElementFound(page, 'input[placeholder="סיסמה"]', true), 198 | waitUntilElementFound(page, 'button[type="submit"]', true), 199 | ]); 200 | } 201 | 202 | async function waitForPostLogin(page: Page): Promise { 203 | await Promise.race([ 204 | waitUntilElementFound(page, 'a[title="דלג לחשבון"]', true, 60000), 205 | waitUntilElementFound(page, 'div.main-content', false, 60000), 206 | page.waitForSelector(`xpath//div[contains(string(),"${INVALID_PASSWORD_MSG}")]`), 207 | waitUntilElementFound(page, 'form[action="/changepassword"]', true, 60000), // not sure if they kept this one 208 | ]); 209 | } 210 | 211 | type ScraperSpecificCredentials = { username: string; password: string }; 212 | 213 | class LeumiScraper extends BaseScraperWithBrowser { 214 | getLoginOptions(credentials: ScraperSpecificCredentials) { 215 | return { 216 | loginUrl: LOGIN_URL, 217 | fields: createLoginFields(credentials), 218 | submitButtonSelector: "button[type='submit']", 219 | checkReadiness: async () => navigateToLogin(this.page), 220 | postAction: async () => waitForPostLogin(this.page), 221 | possibleResults: getPossibleLoginResults(), 222 | }; 223 | } 224 | 225 | async fetchData(): Promise { 226 | const minimumStartMoment = moment().subtract(3, 'years').add(1, 'day'); 227 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 228 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 229 | const startMoment = moment.max(minimumStartMoment, moment(startDate)); 230 | 231 | await this.navigateTo(TRANSACTIONS_URL); 232 | 233 | const accounts = await fetchTransactions(this.page, startMoment); 234 | 235 | return { 236 | success: true, 237 | accounts, 238 | }; 239 | } 240 | } 241 | 242 | export default LeumiScraper; 243 | -------------------------------------------------------------------------------- /src/scrapers/hapoalim.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import { type Page } from 'puppeteer'; 3 | import { v4 as uuid4 } from 'uuid'; 4 | import { getDebug } from '../helpers/debug'; 5 | import { fetchGetWithinPage, fetchPostWithinPage } from '../helpers/fetch'; 6 | import { waitForRedirect } from '../helpers/navigation'; 7 | import { waitUntil } from '../helpers/waiting'; 8 | import { type Transaction, TransactionStatuses, TransactionTypes, type TransactionsAccount } from '../transactions'; 9 | import { BaseScraperWithBrowser, LoginResults, type PossibleLoginResults } from './base-scraper-with-browser'; 10 | import { type ScraperOptions } from './interface'; 11 | 12 | const debug = getDebug('hapoalim'); 13 | 14 | const DATE_FORMAT = 'YYYYMMDD'; 15 | 16 | // eslint-disable-next-line @typescript-eslint/no-namespace 17 | declare namespace window { 18 | const bnhpApp: any; 19 | } 20 | 21 | interface ScrapedTransaction { 22 | serialNumber?: number; 23 | activityDescription?: string; 24 | eventAmount: number; 25 | valueDate?: string; 26 | eventDate?: string; 27 | referenceNumber?: number; 28 | ScrapedTransaction?: string; 29 | eventActivityTypeCode: number; 30 | currentBalance: number; 31 | pfmDetails: string; 32 | beneficiaryDetailsData?: { 33 | partyHeadline?: string; 34 | partyName?: string; 35 | messageHeadline?: string; 36 | messageDetail?: string; 37 | }; 38 | } 39 | 40 | interface ScrapedPfmTransaction { 41 | transactionNumber: number; 42 | } 43 | 44 | type FetchedAccountData = { 45 | bankNumber: string; 46 | accountNumber: string; 47 | branchNumber: string; 48 | accountClosingReasonCode: number; 49 | }[]; 50 | 51 | type FetchedAccountTransactionsData = { 52 | transactions: ScrapedTransaction[]; 53 | }; 54 | 55 | type BalanceAndCreditLimit = { 56 | creditLimitAmount: number; 57 | creditLimitDescription: string; 58 | creditLimitUtilizationAmount: number; 59 | creditLimitUtilizationExistanceCode: number; 60 | creditLimitUtilizationPercent: number; 61 | currentAccountLimitsAmount: number; 62 | currentBalance: number; 63 | withdrawalBalance: number; 64 | }; 65 | 66 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 67 | return txns.map(txn => { 68 | const isOutbound = txn.eventActivityTypeCode === 2; 69 | 70 | let memo = ''; 71 | if (txn.beneficiaryDetailsData) { 72 | const { partyHeadline, partyName, messageHeadline, messageDetail } = txn.beneficiaryDetailsData; 73 | const memoLines: string[] = []; 74 | if (partyHeadline) { 75 | memoLines.push(partyHeadline); 76 | } 77 | 78 | if (partyName) { 79 | memoLines.push(`${partyName}.`); 80 | } 81 | 82 | if (messageHeadline) { 83 | memoLines.push(messageHeadline); 84 | } 85 | 86 | if (messageDetail) { 87 | memoLines.push(`${messageDetail}.`); 88 | } 89 | 90 | if (memoLines.length) { 91 | memo = memoLines.join(' '); 92 | } 93 | } 94 | 95 | const result: Transaction = { 96 | type: TransactionTypes.Normal, 97 | identifier: txn.referenceNumber, 98 | date: moment(txn.eventDate, DATE_FORMAT).toISOString(), 99 | processedDate: moment(txn.valueDate, DATE_FORMAT).toISOString(), 100 | originalAmount: isOutbound ? -txn.eventAmount : txn.eventAmount, 101 | originalCurrency: 'ILS', 102 | chargedAmount: isOutbound ? -txn.eventAmount : txn.eventAmount, 103 | description: txn.activityDescription || '', 104 | status: txn.serialNumber === 0 ? TransactionStatuses.Pending : TransactionStatuses.Completed, 105 | memo, 106 | }; 107 | 108 | return result; 109 | }); 110 | } 111 | 112 | async function getRestContext(page: Page) { 113 | await waitUntil(() => { 114 | return page.evaluate(() => !!window.bnhpApp); 115 | }, 'waiting for app data load'); 116 | 117 | const result = await page.evaluate(() => { 118 | return window.bnhpApp.restContext; 119 | }); 120 | 121 | return result.slice(1); 122 | } 123 | 124 | async function fetchPoalimXSRFWithinPage( 125 | page: Page, 126 | url: string, 127 | pageUuid: string, 128 | ): Promise { 129 | const cookies = await page.cookies(); 130 | const XSRFCookie = cookies.find(cookie => cookie.name === 'XSRF-TOKEN'); 131 | const headers: Record = {}; 132 | if (XSRFCookie != null) { 133 | headers['X-XSRF-TOKEN'] = XSRFCookie.value; 134 | } 135 | headers.pageUuid = pageUuid; 136 | headers.uuid = uuid4(); 137 | headers['Content-Type'] = 'application/json;charset=UTF-8'; 138 | return fetchPostWithinPage(page, url, [], headers); 139 | } 140 | 141 | async function getExtraScrap( 142 | txnsResult: FetchedAccountTransactionsData, 143 | baseUrl: string, 144 | page: Page, 145 | accountNumber: string, 146 | ): Promise { 147 | const promises = txnsResult.transactions.map(async (transaction: ScrapedTransaction): Promise => { 148 | const { pfmDetails, serialNumber } = transaction; 149 | if (serialNumber !== 0) { 150 | const url = `${baseUrl}${pfmDetails}&accountId=${accountNumber}&lang=he`; 151 | const extraTransactionDetails = (await fetchGetWithinPage(page, url)) || []; 152 | if (extraTransactionDetails && extraTransactionDetails.length) { 153 | const { transactionNumber } = extraTransactionDetails[0]; 154 | if (transactionNumber) { 155 | return { ...transaction, referenceNumber: transactionNumber }; 156 | } 157 | } 158 | } 159 | return transaction; 160 | }); 161 | const res = await Promise.all(promises); 162 | return { transactions: res }; 163 | } 164 | 165 | async function getAccountTransactions( 166 | baseUrl: string, 167 | apiSiteUrl: string, 168 | page: Page, 169 | accountNumber: string, 170 | startDate: string, 171 | endDate: string, 172 | additionalTransactionInformation = false, 173 | ) { 174 | const txnsUrl = `${apiSiteUrl}/current-account/transactions?accountId=${accountNumber}&numItemsPerPage=1000&retrievalEndDate=${endDate}&retrievalStartDate=${startDate}&sortCode=1`; 175 | const txnsResult = await fetchPoalimXSRFWithinPage(page, txnsUrl, '/current-account/transactions'); 176 | 177 | const finalResult = 178 | additionalTransactionInformation && txnsResult?.transactions.length 179 | ? await getExtraScrap(txnsResult, baseUrl, page, accountNumber) 180 | : txnsResult; 181 | 182 | return convertTransactions(finalResult?.transactions ?? []); 183 | } 184 | 185 | async function getAccountBalance(apiSiteUrl: string, page: Page, accountNumber: string) { 186 | const balanceAndCreditLimitUrl = `${apiSiteUrl}/current-account/composite/balanceAndCreditLimit?accountId=${accountNumber}&view=details&lang=he`; 187 | const balanceAndCreditLimit = await fetchGetWithinPage(page, balanceAndCreditLimitUrl); 188 | 189 | return balanceAndCreditLimit?.currentBalance; 190 | } 191 | 192 | async function fetchAccountData(page: Page, baseUrl: string, options: ScraperOptions) { 193 | const restContext = await getRestContext(page); 194 | const apiSiteUrl = `${baseUrl}/${restContext}`; 195 | const accountDataUrl = `${baseUrl}/ServerServices/general/accounts`; 196 | 197 | debug('fetching accounts data'); 198 | const accountsInfo = (await fetchGetWithinPage(page, accountDataUrl)) || []; 199 | debug('got %d accounts, fetching txns and balance', accountsInfo.length); 200 | 201 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 202 | const startDate = options.startDate || defaultStartMoment.toDate(); 203 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 204 | const { additionalTransactionInformation } = options; 205 | 206 | const startDateStr = startMoment.format(DATE_FORMAT); 207 | const endDateStr = moment().format(DATE_FORMAT); 208 | 209 | const accounts: TransactionsAccount[] = []; 210 | 211 | for (const account of accountsInfo) { 212 | let balance: number | undefined; 213 | const accountNumber = `${account.bankNumber}-${account.branchNumber}-${account.accountNumber}`; 214 | 215 | const isActiveAccount = account.accountClosingReasonCode === 0; 216 | if (isActiveAccount) { 217 | balance = await getAccountBalance(apiSiteUrl, page, accountNumber); 218 | } else { 219 | debug('Skipping balance for a closed account, balance will be undefined'); 220 | } 221 | 222 | const txns = await getAccountTransactions( 223 | baseUrl, 224 | apiSiteUrl, 225 | page, 226 | accountNumber, 227 | startDateStr, 228 | endDateStr, 229 | additionalTransactionInformation, 230 | ); 231 | 232 | accounts.push({ 233 | accountNumber, 234 | balance, 235 | txns, 236 | }); 237 | } 238 | 239 | const accountData = { 240 | success: true, 241 | accounts, 242 | }; 243 | debug('fetching ended'); 244 | return accountData; 245 | } 246 | 247 | function getPossibleLoginResults(baseUrl: string) { 248 | const urls: PossibleLoginResults = {}; 249 | urls[LoginResults.Success] = [ 250 | `${baseUrl}/portalserver/HomePage`, 251 | `${baseUrl}/ng-portals-bt/rb/he/homepage`, 252 | `${baseUrl}/ng-portals/rb/he/homepage`, 253 | ]; 254 | urls[LoginResults.InvalidPassword] = [ 255 | `${baseUrl}/AUTHENTICATE/LOGON?flow=AUTHENTICATE&state=LOGON&errorcode=1.6&callme=false`, 256 | ]; 257 | urls[LoginResults.ChangePassword] = [ 258 | `${baseUrl}/MCP/START?flow=MCP&state=START&expiredDate=null`, 259 | /\/ABOUTTOEXPIRE\/START/i, 260 | ]; 261 | return urls; 262 | } 263 | 264 | function createLoginFields(credentials: ScraperSpecificCredentials) { 265 | return [ 266 | { selector: '#userCode', value: credentials.userCode }, 267 | { selector: '#password', value: credentials.password }, 268 | ]; 269 | } 270 | 271 | type ScraperSpecificCredentials = { userCode: string; password: string }; 272 | 273 | class HapoalimScraper extends BaseScraperWithBrowser { 274 | // eslint-disable-next-line class-methods-use-this 275 | get baseUrl() { 276 | return 'https://login.bankhapoalim.co.il'; 277 | } 278 | 279 | getLoginOptions(credentials: ScraperSpecificCredentials) { 280 | return { 281 | loginUrl: `${this.baseUrl}/cgi-bin/poalwwwc?reqName=getLogonPage`, 282 | fields: createLoginFields(credentials), 283 | submitButtonSelector: '.login-btn', 284 | postAction: async () => waitForRedirect(this.page), 285 | possibleResults: getPossibleLoginResults(this.baseUrl), 286 | }; 287 | } 288 | 289 | async fetchData() { 290 | return fetchAccountData(this.page, this.baseUrl, this.options); 291 | } 292 | } 293 | 294 | export default HapoalimScraper; 295 | -------------------------------------------------------------------------------- /src/scrapers/yahav.ts: -------------------------------------------------------------------------------- 1 | import moment, { type Moment } from 'moment'; 2 | import { type Page } from 'puppeteer'; 3 | import { SHEKEL_CURRENCY } from '../constants'; 4 | import { 5 | clickButton, 6 | elementPresentOnPage, 7 | pageEvalAll, 8 | waitUntilElementDisappear, 9 | waitUntilElementFound, 10 | } from '../helpers/elements-interactions'; 11 | import { waitForNavigation } from '../helpers/navigation'; 12 | import { TransactionStatuses, TransactionTypes, type Transaction, type TransactionsAccount } from '../transactions'; 13 | import { BaseScraperWithBrowser, LoginResults, type PossibleLoginResults } from './base-scraper-with-browser'; 14 | 15 | const LOGIN_URL = 'https://login.yahav.co.il/login/'; 16 | const BASE_URL = 'https://digital.yahav.co.il/BaNCSDigitalUI/app/index.html#/'; 17 | const INVALID_DETAILS_SELECTOR = '.ui-dialog-buttons'; 18 | const CHANGE_PASSWORD_OLD_PASS = 'input#ef_req_parameter_old_credential'; 19 | const BASE_WELCOME_URL = `${BASE_URL}main/home`; 20 | 21 | const ACCOUNT_ID_SELECTOR = 'span.portfolio-value[ng-if="mainController.data.portfolioList.length === 1"]'; 22 | const ACCOUNT_DETAILS_SELECTOR = '.account-details'; 23 | const DATE_FORMAT = 'DD/MM/YYYY'; 24 | 25 | const USER_ELEM = '#username'; 26 | const PASSWD_ELEM = '#password'; 27 | const NATIONALID_ELEM = '#pinno'; 28 | const SUBMIT_LOGIN_SELECTOR = '.btn'; 29 | 30 | interface ScrapedTransaction { 31 | credit: string; 32 | debit: string; 33 | date: string; 34 | reference?: string; 35 | description: string; 36 | memo: string; 37 | status: TransactionStatuses; 38 | } 39 | 40 | function getPossibleLoginResults(page: Page): PossibleLoginResults { 41 | // checkout file `base-scraper-with-browser.ts` for available result types 42 | const urls: PossibleLoginResults = {}; 43 | urls[LoginResults.Success] = [`${BASE_WELCOME_URL}`]; 44 | urls[LoginResults.InvalidPassword] = [ 45 | async () => { 46 | return elementPresentOnPage(page, `${INVALID_DETAILS_SELECTOR}`); 47 | }, 48 | ]; 49 | 50 | urls[LoginResults.ChangePassword] = [ 51 | async () => { 52 | return elementPresentOnPage(page, `${CHANGE_PASSWORD_OLD_PASS}`); 53 | }, 54 | ]; 55 | 56 | return urls; 57 | } 58 | 59 | async function getAccountID(page: Page): Promise { 60 | try { 61 | const selectedSnifAccount = await page.$eval(ACCOUNT_ID_SELECTOR, (element: Element) => { 62 | return element.textContent as string; 63 | }); 64 | 65 | return selectedSnifAccount; 66 | } catch (error) { 67 | const errorMessage = error instanceof Error ? error.message : String(error); 68 | throw new Error( 69 | `Failed to retrieve account ID. Possible outdated selector '${ACCOUNT_ID_SELECTOR}: ${errorMessage}`, 70 | ); 71 | } 72 | } 73 | 74 | function getAmountData(amountStr: string) { 75 | const amountStrCopy = amountStr.replace(',', ''); 76 | return parseFloat(amountStrCopy); 77 | } 78 | 79 | function getTxnAmount(txn: ScrapedTransaction) { 80 | const credit = getAmountData(txn.credit); 81 | const debit = getAmountData(txn.debit); 82 | return (Number.isNaN(credit) ? 0 : credit) - (Number.isNaN(debit) ? 0 : debit); 83 | } 84 | 85 | type TransactionsTr = { id: string; innerDivs: string[] }; 86 | 87 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 88 | return txns.map(txn => { 89 | const convertedDate = moment(txn.date, DATE_FORMAT).toISOString(); 90 | const convertedAmount = getTxnAmount(txn); 91 | return { 92 | type: TransactionTypes.Normal, 93 | identifier: txn.reference ? parseInt(txn.reference, 10) : undefined, 94 | date: convertedDate, 95 | processedDate: convertedDate, 96 | originalAmount: convertedAmount, 97 | originalCurrency: SHEKEL_CURRENCY, 98 | chargedAmount: convertedAmount, 99 | status: txn.status, 100 | description: txn.description, 101 | memo: txn.memo, 102 | }; 103 | }); 104 | } 105 | 106 | function handleTransactionRow(txns: ScrapedTransaction[], txnRow: TransactionsTr) { 107 | const div = txnRow.innerDivs; 108 | 109 | // Remove anything except digits. 110 | const regex = /\D+/gm; 111 | 112 | const tx: ScrapedTransaction = { 113 | date: div[1], 114 | reference: div[2].replace(regex, ''), 115 | memo: '', 116 | description: div[3], 117 | debit: div[4], 118 | credit: div[5], 119 | status: TransactionStatuses.Completed, 120 | }; 121 | 122 | txns.push(tx); 123 | } 124 | 125 | async function getAccountTransactions(page: Page): Promise { 126 | // Wait for transactions. 127 | await waitUntilElementFound(page, '.under-line-txn-table-header', true); 128 | 129 | const txns: ScrapedTransaction[] = []; 130 | const transactionsDivs = await pageEvalAll( 131 | page, 132 | '.list-item-holder .entire-content-ctr', 133 | [], 134 | divs => { 135 | return (divs as HTMLElement[]).map(div => ({ 136 | id: div.getAttribute('id') || '', 137 | innerDivs: Array.from(div.getElementsByTagName('div')).map(el => (el as HTMLElement).innerText), 138 | })); 139 | }, 140 | ); 141 | 142 | for (const txnRow of transactionsDivs) { 143 | handleTransactionRow(txns, txnRow); 144 | } 145 | 146 | return convertTransactions(txns); 147 | } 148 | 149 | // Manipulate the calendar drop down to choose the txs start date. 150 | async function searchByDates(page: Page, startDate: Moment) { 151 | // Get the day number from startDate. 1-31 (usually 1) 152 | const startDateDay = startDate.format('D'); 153 | const startDateMonth = startDate.format('M'); 154 | const startDateYear = startDate.format('Y'); 155 | 156 | // Open the calendar date picker 157 | const dateFromPick = 158 | 'div.date-options-cell:nth-child(7) > date-picker:nth-child(1) > div:nth-child(1) > span:nth-child(2)'; 159 | await waitUntilElementFound(page, dateFromPick, true); 160 | await clickButton(page, dateFromPick); 161 | 162 | // Wait until first day appear. 163 | await waitUntilElementFound(page, '.pmu-days > div:nth-child(1)', true); 164 | 165 | // Open Months options. 166 | const monthFromPick = '.pmu-month'; 167 | await waitUntilElementFound(page, monthFromPick, true); 168 | await clickButton(page, monthFromPick); 169 | await waitUntilElementFound(page, '.pmu-months > div:nth-child(1)', true); 170 | 171 | // Open Year options. 172 | // Use same selector... Yahav knows why... 173 | await waitUntilElementFound(page, monthFromPick, true); 174 | await clickButton(page, monthFromPick); 175 | await waitUntilElementFound(page, '.pmu-years > div:nth-child(1)', true); 176 | 177 | // Select year from a 12 year grid. 178 | for (let i = 1; i < 13; i += 1) { 179 | const selector = `.pmu-years > div:nth-child(${i})`; 180 | const year = await page.$eval(selector, y => { 181 | return (y as HTMLElement).innerText; 182 | }); 183 | if (startDateYear === year) { 184 | await clickButton(page, selector); 185 | break; 186 | } 187 | } 188 | 189 | // Select Month. 190 | await waitUntilElementFound(page, '.pmu-months > div:nth-child(1)', true); 191 | // The first element (1) is January. 192 | const monthSelector = `.pmu-months > div:nth-child(${startDateMonth})`; 193 | await clickButton(page, monthSelector); 194 | 195 | // Select Day. 196 | // The calendar grid shows 7 days and 6 weeks = 42 days. 197 | // In theory, the first day of the month will be in the first row. 198 | // Let's check everything just in case... 199 | for (let i = 1; i < 42; i += 1) { 200 | const selector = `.pmu-days > div:nth-child(${i})`; 201 | const day = await page.$eval(selector, d => { 202 | return (d as HTMLElement).innerText; 203 | }); 204 | 205 | if (startDateDay === day) { 206 | await clickButton(page, selector); 207 | break; 208 | } 209 | } 210 | } 211 | 212 | async function fetchAccountData(page: Page, startDate: Moment, accountID: string): Promise { 213 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 214 | await searchByDates(page, startDate); 215 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 216 | const txns = await getAccountTransactions(page); 217 | 218 | return { 219 | accountNumber: accountID, 220 | txns, 221 | }; 222 | } 223 | 224 | async function fetchAccounts(page: Page, startDate: Moment): Promise { 225 | const accounts: TransactionsAccount[] = []; 226 | 227 | // TODO: get more accounts. Not sure is supported. 228 | const accountID = await getAccountID(page); 229 | const accountData = await fetchAccountData(page, startDate, accountID); 230 | accounts.push(accountData); 231 | 232 | return accounts; 233 | } 234 | 235 | async function waitReadinessForAll(page: Page) { 236 | await waitUntilElementFound(page, `${USER_ELEM}`, true); 237 | await waitUntilElementFound(page, `${PASSWD_ELEM}`, true); 238 | await waitUntilElementFound(page, `${NATIONALID_ELEM}`, true); 239 | await waitUntilElementFound(page, `${SUBMIT_LOGIN_SELECTOR}`, true); 240 | } 241 | 242 | async function redirectOrDialog(page: Page) { 243 | // Click on bank messages if any. 244 | await waitForNavigation(page); 245 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 246 | const hasMessage = await elementPresentOnPage(page, '.messaging-links-container'); 247 | if (hasMessage) { 248 | await clickButton(page, '.link-1'); 249 | } 250 | 251 | const promise1 = page.waitForSelector(ACCOUNT_DETAILS_SELECTOR, { timeout: 30000 }); 252 | const promise2 = page.waitForSelector(CHANGE_PASSWORD_OLD_PASS, { timeout: 30000 }); 253 | const promises = [promise1, promise2]; 254 | 255 | await Promise.race(promises); 256 | await waitUntilElementDisappear(page, '.loading-bar-spinner'); 257 | } 258 | 259 | type ScraperSpecificCredentials = { username: string; password: string; nationalID: string }; 260 | 261 | class YahavScraper extends BaseScraperWithBrowser { 262 | getLoginOptions(credentials: ScraperSpecificCredentials) { 263 | return { 264 | loginUrl: `${LOGIN_URL}`, 265 | fields: [ 266 | { selector: `${USER_ELEM}`, value: credentials.username }, 267 | { selector: `${PASSWD_ELEM}`, value: credentials.password }, 268 | { selector: `${NATIONALID_ELEM}`, value: credentials.nationalID }, 269 | ], 270 | submitButtonSelector: `${SUBMIT_LOGIN_SELECTOR}`, 271 | checkReadiness: async () => waitReadinessForAll(this.page), 272 | postAction: async () => redirectOrDialog(this.page), 273 | possibleResults: getPossibleLoginResults(this.page), 274 | }; 275 | } 276 | 277 | async fetchData() { 278 | // Goto statements page 279 | await waitUntilElementFound(this.page, ACCOUNT_DETAILS_SELECTOR, true); 280 | await clickButton(this.page, ACCOUNT_DETAILS_SELECTOR); 281 | await waitUntilElementFound(this.page, '.statement-options .selected-item-top', true); 282 | 283 | const defaultStartMoment = moment().subtract(3, 'months').add(1, 'day'); 284 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 285 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 286 | 287 | const accounts = await fetchAccounts(this.page, startMoment); 288 | 289 | return { 290 | success: true, 291 | accounts, 292 | }; 293 | } 294 | } 295 | 296 | export default YahavScraper; 297 | -------------------------------------------------------------------------------- /src/scrapers/one-zero.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment/moment'; 2 | import { getDebug } from '../helpers/debug'; 3 | import { fetchGraphql, fetchPost } from '../helpers/fetch'; 4 | import { 5 | type Transaction as ScrapingTransaction, 6 | TransactionStatuses, 7 | TransactionTypes, 8 | type TransactionsAccount, 9 | } from '../transactions'; 10 | import { BaseScraper } from './base-scraper'; 11 | import { ScraperErrorTypes, createGenericError } from './errors'; 12 | import { 13 | type ScraperGetLongTermTwoFactorTokenResult, 14 | type ScraperLoginResult, 15 | type ScraperScrapingResult, 16 | type ScraperTwoFactorAuthTriggerResult, 17 | } from './interface'; 18 | import { GET_CUSTOMER, GET_MOVEMENTS } from './one-zero-queries'; 19 | 20 | const HEBREW_WORDS_REGEX = /[\u0590-\u05FF][\u0590-\u05FF"'\-_ /\\]*[\u0590-\u05FF]/g; 21 | 22 | const debug = getDebug('one-zero'); 23 | 24 | type Account = { 25 | accountId: string; 26 | }; 27 | 28 | type Portfolio = { 29 | accounts: Array; 30 | portfolioId: string; 31 | portfolioNum: string; 32 | }; 33 | 34 | type Customer = { 35 | customerId: string; 36 | portfolios?: Array | null; 37 | }; 38 | 39 | export type Category = { 40 | categoryId: number; 41 | dataSource: string; 42 | subCategoryId?: number | null; 43 | }; 44 | 45 | export type Recurrence = { 46 | dataSource: string; 47 | isRecurrent: boolean; 48 | }; 49 | 50 | type TransactionEnrichment = { 51 | categories?: Category[] | null; 52 | recurrences?: Recurrence[] | null; 53 | }; 54 | 55 | type Transaction = { 56 | enrichment?: TransactionEnrichment | null; 57 | // TODO: Get installments information here 58 | // transactionDetails: TransactionDetails; 59 | }; 60 | 61 | type Movement = { 62 | accountId: string; 63 | bankCurrencyAmount: string; 64 | bookingDate: string; 65 | conversionRate: string; 66 | creditDebit: string; 67 | description: string; 68 | isReversed: boolean; 69 | movementAmount: string; 70 | movementCurrency: string; 71 | movementId: string; 72 | movementReversedId?: string | null; 73 | movementTimestamp: string; 74 | movementType: string; 75 | portfolioId: string; 76 | runningBalance: string; 77 | transaction?: Transaction | null; 78 | valueDate: string; 79 | }; 80 | 81 | type QueryPagination = { hasMore: boolean; cursor: string }; 82 | 83 | const IDENTITY_SERVER_URL = 'https://identity.tfd-bank.com/v1/'; 84 | 85 | const GRAPHQL_API_URL = 'https://mobile.tfd-bank.com/mobile-graph/graphql'; 86 | 87 | type ScraperSpecificCredentials = { email: string; password: string } & ( 88 | | { 89 | otpCodeRetriever: () => Promise; 90 | phoneNumber: string; 91 | } 92 | | { 93 | otpLongTermToken: string; 94 | } 95 | ); 96 | 97 | export default class OneZeroScraper extends BaseScraper { 98 | private otpContext?: string; 99 | 100 | private accessToken?: string; 101 | 102 | async triggerTwoFactorAuth(phoneNumber: string): Promise { 103 | if (!phoneNumber.startsWith('+')) { 104 | return createGenericError( 105 | 'A full international phone number starting with + and a three digit country code is required', 106 | ); 107 | } 108 | 109 | debug('Fetching device token'); 110 | const deviceTokenResponse = await fetchPost(`${IDENTITY_SERVER_URL}/devices/token`, { 111 | extClientId: 'mobile', 112 | os: 'Android', 113 | }); 114 | 115 | const { 116 | resultData: { deviceToken }, 117 | } = deviceTokenResponse; 118 | 119 | debug(`Sending OTP to phone number ${phoneNumber}`); 120 | 121 | const otpPrepareResponse = await fetchPost(`${IDENTITY_SERVER_URL}/otp/prepare`, { 122 | factorValue: phoneNumber, 123 | deviceToken, 124 | otpChannel: 'SMS_OTP', 125 | }); 126 | 127 | const { 128 | resultData: { otpContext }, 129 | } = otpPrepareResponse; 130 | 131 | this.otpContext = otpContext; 132 | 133 | return { 134 | success: true, 135 | }; 136 | } 137 | 138 | public async getLongTermTwoFactorToken(otpCode: string): Promise { 139 | if (!this.otpContext) { 140 | return createGenericError('triggerOtp was not called before calling getPermenantOtpToken()'); 141 | } 142 | 143 | debug('Requesting OTP token'); 144 | const otpVerifyResponse = await fetchPost(`${IDENTITY_SERVER_URL}/otp/verify`, { 145 | otpContext: this.otpContext, 146 | otpCode, 147 | }); 148 | 149 | const { 150 | resultData: { otpToken }, 151 | } = otpVerifyResponse; 152 | return { success: true, longTermTwoFactorAuthToken: otpToken }; 153 | } 154 | 155 | private async resolveOtpToken( 156 | credentials: ScraperSpecificCredentials, 157 | ): Promise { 158 | if ('otpLongTermToken' in credentials) { 159 | if (!credentials.otpLongTermToken) { 160 | return createGenericError('Invalid otpLongTermToken'); 161 | } 162 | return { success: true, longTermTwoFactorAuthToken: credentials.otpLongTermToken }; 163 | } 164 | 165 | if (!credentials.otpCodeRetriever) { 166 | return { 167 | success: false, 168 | errorType: ScraperErrorTypes.TwoFactorRetrieverMissing, 169 | errorMessage: 'otpCodeRetriever is required when otpPermanentToken is not provided', 170 | }; 171 | } 172 | 173 | if (!credentials.phoneNumber) { 174 | return createGenericError('phoneNumber is required when providing a otpCodeRetriever callback'); 175 | } 176 | 177 | debug('Triggering user supplied otpCodeRetriever callback'); 178 | const triggerResult = await this.triggerTwoFactorAuth(credentials.phoneNumber); 179 | 180 | if (!triggerResult.success) { 181 | return triggerResult; 182 | } 183 | 184 | const otpCode = await credentials.otpCodeRetriever(); 185 | 186 | const otpTokenResult = await this.getLongTermTwoFactorToken(otpCode); 187 | if (!otpTokenResult.success) { 188 | return otpTokenResult; 189 | } 190 | 191 | return { success: true, longTermTwoFactorAuthToken: otpTokenResult.longTermTwoFactorAuthToken }; 192 | } 193 | 194 | async login(credentials: ScraperSpecificCredentials): Promise { 195 | const otpTokenResult = await this.resolveOtpToken(credentials); 196 | if (!otpTokenResult.success) { 197 | return otpTokenResult; 198 | } 199 | 200 | debug('Requesting id token'); 201 | const getIdTokenResponse = await fetchPost(`${IDENTITY_SERVER_URL}/getIdToken`, { 202 | otpSmsToken: otpTokenResult.longTermTwoFactorAuthToken, 203 | email: credentials.email, 204 | pass: credentials.password, 205 | pinCode: '', 206 | }); 207 | 208 | const { 209 | resultData: { idToken }, 210 | } = getIdTokenResponse; 211 | 212 | debug('Requesting session token'); 213 | 214 | const getSessionTokenResponse = await fetchPost(`${IDENTITY_SERVER_URL}/sessions/token`, { 215 | idToken, 216 | pass: credentials.password, 217 | }); 218 | 219 | const { 220 | resultData: { accessToken }, 221 | } = getSessionTokenResponse; 222 | 223 | this.accessToken = accessToken; 224 | 225 | return { 226 | success: true, 227 | persistentOtpToken: otpTokenResult.longTermTwoFactorAuthToken, 228 | }; 229 | } 230 | 231 | private async fetchPortfolioMovements(portfolio: Portfolio, startDate: Date): Promise { 232 | // TODO: Find out if we need the other accounts, there seems to always be one 233 | const account = portfolio.accounts[0]; 234 | let cursor = null; 235 | const movements = []; 236 | 237 | while (!movements.length || new Date(movements[0].movementTimestamp) >= startDate) { 238 | debug(`Fetching transactions for account ${portfolio.portfolioNum}...`); 239 | const { 240 | movements: { movements: newMovements, pagination }, 241 | }: { movements: { movements: Movement[]; pagination: QueryPagination } } = await fetchGraphql( 242 | GRAPHQL_API_URL, 243 | GET_MOVEMENTS, 244 | { 245 | portfolioId: portfolio.portfolioId, 246 | accountId: account.accountId, 247 | language: 'HEBREW', 248 | pagination: { 249 | cursor, 250 | limit: 50, 251 | }, 252 | }, 253 | { authorization: `Bearer ${this.accessToken}` }, 254 | ); 255 | 256 | movements.unshift(...newMovements); 257 | cursor = pagination.cursor; 258 | if (!pagination.hasMore) { 259 | break; 260 | } 261 | } 262 | 263 | movements.sort((x, y) => new Date(x.movementTimestamp).valueOf() - new Date(y.movementTimestamp).valueOf()); 264 | 265 | const matchingMovements = movements.filter(movement => new Date(movement.movementTimestamp) >= startDate); 266 | return { 267 | accountNumber: portfolio.portfolioNum, 268 | balance: !movements.length ? 0 : parseFloat(movements[movements.length - 1].runningBalance), 269 | txns: matchingMovements.map((movement): ScrapingTransaction => { 270 | const hasInstallments = movement.transaction?.enrichment?.recurrences?.some(x => x.isRecurrent); 271 | const modifier = movement.creditDebit === 'DEBIT' ? -1 : 1; 272 | return { 273 | identifier: movement.movementId, 274 | date: movement.valueDate, 275 | chargedAmount: +movement.movementAmount * modifier, 276 | chargedCurrency: movement.movementCurrency, 277 | originalAmount: +movement.movementAmount * modifier, 278 | originalCurrency: movement.movementCurrency, 279 | description: this.sanitizeHebrew(movement.description), 280 | processedDate: movement.movementTimestamp, 281 | status: TransactionStatuses.Completed, 282 | type: hasInstallments ? TransactionTypes.Installments : TransactionTypes.Normal, 283 | }; 284 | }), 285 | }; 286 | } 287 | 288 | /** 289 | * one zero hebrew strings are reversed with a unicode control character that forces display in LTR order 290 | * We need to remove the unicode control character, and then reverse hebrew substrings inside the string 291 | */ 292 | private sanitizeHebrew(text: string) { 293 | if (!text.includes('\u202d')) { 294 | return text.trim(); 295 | } 296 | 297 | const plainString = text.replace(/\u202d/gi, '').trim(); 298 | const hebrewSubStringsRanges = [...plainString.matchAll(HEBREW_WORDS_REGEX)]; 299 | const rangesToReverse = hebrewSubStringsRanges.map(str => ({ start: str.index!, end: str.index! + str[0].length })); 300 | const out = []; 301 | let index = 0; 302 | 303 | for (const { start, end } of rangesToReverse) { 304 | out.push(...plainString.substring(index, start)); 305 | index += start - index; 306 | const reversed = [...plainString.substring(start, end)].reverse(); 307 | out.push(...reversed); 308 | index += end - start; 309 | } 310 | 311 | out.push(...plainString.substring(index, plainString.length)); 312 | 313 | return out.join(''); 314 | } 315 | 316 | async fetchData(): Promise { 317 | if (!this.accessToken) { 318 | return createGenericError('login() was not called'); 319 | } 320 | 321 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 322 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 323 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 324 | 325 | debug('Fetching account list'); 326 | const result = await fetchGraphql<{ customer: Customer[] }>( 327 | GRAPHQL_API_URL, 328 | GET_CUSTOMER, 329 | {}, 330 | { authorization: `Bearer ${this.accessToken}` }, 331 | ); 332 | const portfolios = result.customer.flatMap(customer => customer.portfolios || []); 333 | 334 | return { 335 | success: true, 336 | accounts: await Promise.all( 337 | portfolios.map(portfolio => this.fetchPortfolioMovements(portfolio, startMoment.toDate())), 338 | ), 339 | }; 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /src/scrapers/union-bank.ts: -------------------------------------------------------------------------------- 1 | import moment, { type Moment } from 'moment'; 2 | import { type Page } from 'puppeteer'; 3 | import { SHEKEL_CURRENCY } from '../constants'; 4 | import { 5 | clickButton, 6 | dropdownElements, 7 | dropdownSelect, 8 | elementPresentOnPage, 9 | fillInput, 10 | pageEvalAll, 11 | waitUntilElementFound, 12 | } from '../helpers/elements-interactions'; 13 | import { waitForNavigation } from '../helpers/navigation'; 14 | import { TransactionStatuses, TransactionTypes, type Transaction, type TransactionsAccount } from '../transactions'; 15 | import { BaseScraperWithBrowser, LoginResults, type PossibleLoginResults } from './base-scraper-with-browser'; 16 | 17 | const BASE_URL = 'https://hb.unionbank.co.il'; 18 | const TRANSACTIONS_URL = `${BASE_URL}/eBanking/Accounts/ExtendedActivity.aspx#/`; 19 | const DATE_FORMAT = 'DD/MM/YY'; 20 | const NO_TRANSACTION_IN_DATE_RANGE_TEXT = 'לא קיימות תנועות מתאימות על פי הסינון שהוגדר'; 21 | const DATE_HEADER = 'תאריך'; 22 | const DESCRIPTION_HEADER = 'תיאור'; 23 | const REFERENCE_HEADER = 'אסמכתא'; 24 | const DEBIT_HEADER = 'חובה'; 25 | const CREDIT_HEADER = 'זכות'; 26 | const PENDING_TRANSACTIONS_TABLE_ID = 'trTodayActivityNapaTableUpper'; 27 | const COMPLETED_TRANSACTIONS_TABLE_ID = 'ctlActivityTable'; 28 | const ERROR_MESSAGE_CLASS = 'errInfo'; 29 | const ACCOUNTS_DROPDOWN_SELECTOR = 'select#ddlAccounts_m_ddl'; 30 | 31 | function getPossibleLoginResults() { 32 | const urls: PossibleLoginResults = {}; 33 | urls[LoginResults.Success] = [/eBanking\/Accounts/]; 34 | urls[LoginResults.InvalidPassword] = [/InternalSite\/CustomUpdate\/leumi\/LoginPage.ASP/]; 35 | return urls; 36 | } 37 | 38 | function createLoginFields(credentials: ScraperSpecificCredentials) { 39 | return [ 40 | { selector: '#uid', value: credentials.username }, 41 | { selector: '#password', value: credentials.password }, 42 | ]; 43 | } 44 | 45 | function getAmountData(amountStr: string) { 46 | const amountStrCopy = amountStr.replace(',', ''); 47 | return parseFloat(amountStrCopy); 48 | } 49 | 50 | interface ScrapedTransaction { 51 | credit: string; 52 | debit: string; 53 | date: string; 54 | reference?: string; 55 | description: string; 56 | memo: string; 57 | status: TransactionStatuses; 58 | } 59 | 60 | function getTxnAmount(txn: ScrapedTransaction) { 61 | const credit = getAmountData(txn.credit); 62 | const debit = getAmountData(txn.debit); 63 | return (Number.isNaN(credit) ? 0 : credit) - (Number.isNaN(debit) ? 0 : debit); 64 | } 65 | 66 | function convertTransactions(txns: ScrapedTransaction[]): Transaction[] { 67 | return txns.map(txn => { 68 | const convertedDate = moment(txn.date, DATE_FORMAT).toISOString(); 69 | const convertedAmount = getTxnAmount(txn); 70 | return { 71 | type: TransactionTypes.Normal, 72 | identifier: txn.reference ? parseInt(txn.reference, 10) : undefined, 73 | date: convertedDate, 74 | processedDate: convertedDate, 75 | originalAmount: convertedAmount, 76 | originalCurrency: SHEKEL_CURRENCY, 77 | chargedAmount: convertedAmount, 78 | status: txn.status, 79 | description: txn.description, 80 | memo: txn.memo, 81 | }; 82 | }); 83 | } 84 | 85 | type TransactionsTr = { id: string; innerTds: TransactionsTrTds }; 86 | type TransactionTableHeaders = Record; 87 | type TransactionsTrTds = string[]; 88 | 89 | function getTransactionDate(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 90 | return (tds[txnsTableHeaders[DATE_HEADER]] || '').trim(); 91 | } 92 | 93 | function getTransactionDescription(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 94 | return (tds[txnsTableHeaders[DESCRIPTION_HEADER]] || '').trim(); 95 | } 96 | 97 | function getTransactionReference(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 98 | return (tds[txnsTableHeaders[REFERENCE_HEADER]] || '').trim(); 99 | } 100 | 101 | function getTransactionDebit(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 102 | return (tds[txnsTableHeaders[DEBIT_HEADER]] || '').trim(); 103 | } 104 | 105 | function getTransactionCredit(tds: TransactionsTrTds, txnsTableHeaders: TransactionTableHeaders) { 106 | return (tds[txnsTableHeaders[CREDIT_HEADER]] || '').trim(); 107 | } 108 | 109 | function extractTransactionDetails( 110 | txnRow: TransactionsTr, 111 | txnsTableHeaders: TransactionTableHeaders, 112 | txnStatus: TransactionStatuses, 113 | ): ScrapedTransaction { 114 | const tds = txnRow.innerTds; 115 | return { 116 | status: txnStatus, 117 | date: getTransactionDate(tds, txnsTableHeaders), 118 | description: getTransactionDescription(tds, txnsTableHeaders), 119 | reference: getTransactionReference(tds, txnsTableHeaders), 120 | debit: getTransactionDebit(tds, txnsTableHeaders), 121 | credit: getTransactionCredit(tds, txnsTableHeaders), 122 | memo: '', 123 | }; 124 | } 125 | 126 | function isExpandedDescRow(txnRow: TransactionsTr) { 127 | return txnRow.id === 'rowAdded'; 128 | } 129 | 130 | /* eslint-disable no-param-reassign */ 131 | function editLastTransactionDesc(txnRow: TransactionsTr, lastTxn: ScrapedTransaction): ScrapedTransaction { 132 | lastTxn.description = `${lastTxn.description} ${txnRow.innerTds[0]}`; 133 | return lastTxn; 134 | } 135 | 136 | function handleTransactionRow( 137 | txns: ScrapedTransaction[], 138 | txnsTableHeaders: TransactionTableHeaders, 139 | txnRow: TransactionsTr, 140 | txnType: TransactionStatuses, 141 | ) { 142 | if (isExpandedDescRow(txnRow)) { 143 | const lastTransaction = txns.pop(); 144 | if (lastTransaction) { 145 | txns.push(editLastTransactionDesc(txnRow, lastTransaction)); 146 | } else { 147 | throw new Error('internal union-bank error'); 148 | } 149 | } else { 150 | txns.push(extractTransactionDetails(txnRow, txnsTableHeaders, txnType)); 151 | } 152 | } 153 | 154 | async function getTransactionsTableHeaders(page: Page, tableTypeId: string) { 155 | const headersMap: Record = []; 156 | const headersObjs = await pageEvalAll(page, `#WorkSpaceBox #${tableTypeId} tr[class='header'] th`, null, ths => { 157 | return ths.map((th, index) => ({ 158 | text: (th as HTMLElement).innerText.trim(), 159 | index, 160 | })); 161 | }); 162 | 163 | for (const headerObj of headersObjs) { 164 | headersMap[headerObj.text] = headerObj.index; 165 | } 166 | return headersMap; 167 | } 168 | 169 | async function extractTransactionsFromTable( 170 | page: Page, 171 | tableTypeId: string, 172 | txnType: TransactionStatuses, 173 | ): Promise { 174 | const txns: ScrapedTransaction[] = []; 175 | const transactionsTableHeaders = await getTransactionsTableHeaders(page, tableTypeId); 176 | 177 | const transactionsRows = await pageEvalAll( 178 | page, 179 | `#WorkSpaceBox #${tableTypeId} tr[class]:not([class='header'])`, 180 | [], 181 | trs => { 182 | return (trs as HTMLElement[]).map(tr => ({ 183 | id: tr.getAttribute('id') || '', 184 | innerTds: Array.from(tr.getElementsByTagName('td')).map(td => (td as HTMLElement).innerText), 185 | })); 186 | }, 187 | ); 188 | 189 | for (const txnRow of transactionsRows) { 190 | handleTransactionRow(txns, transactionsTableHeaders, txnRow, txnType); 191 | } 192 | return txns; 193 | } 194 | 195 | async function isNoTransactionInDateRangeError(page: Page) { 196 | const hasErrorInfoElement = await elementPresentOnPage(page, `.${ERROR_MESSAGE_CLASS}`); 197 | if (hasErrorInfoElement) { 198 | const errorText = await page.$eval(`.${ERROR_MESSAGE_CLASS}`, errorElement => { 199 | return (errorElement as HTMLElement).innerText; 200 | }); 201 | return errorText.trim() === NO_TRANSACTION_IN_DATE_RANGE_TEXT; 202 | } 203 | return false; 204 | } 205 | 206 | async function chooseAccount(page: Page, accountId: string) { 207 | const hasDropDownList = await elementPresentOnPage(page, ACCOUNTS_DROPDOWN_SELECTOR); 208 | if (hasDropDownList) { 209 | await dropdownSelect(page, ACCOUNTS_DROPDOWN_SELECTOR, accountId); 210 | } 211 | } 212 | 213 | async function searchByDates(page: Page, startDate: Moment) { 214 | await dropdownSelect(page, 'select#ddlTransactionPeriod', '004'); 215 | await waitUntilElementFound(page, 'select#ddlTransactionPeriod'); 216 | await fillInput(page, 'input#dtFromDate_textBox', startDate.format(DATE_FORMAT)); 217 | await clickButton(page, 'input#btnDisplayDates'); 218 | await waitForNavigation(page); 219 | } 220 | 221 | async function getAccountNumber(page: Page) { 222 | const selectedSnifAccount = await page.$eval('#ddlAccounts_m_ddl option[selected="selected"]', option => { 223 | return (option as HTMLElement).innerText; 224 | }); 225 | 226 | return selectedSnifAccount.replace('/', '_'); 227 | } 228 | 229 | async function expandTransactionsTable(page: Page) { 230 | const hasExpandAllButton = await elementPresentOnPage(page, "a[id*='lnkCtlExpandAll']"); 231 | if (hasExpandAllButton) { 232 | await clickButton(page, "a[id*='lnkCtlExpandAll']"); 233 | } 234 | } 235 | 236 | async function scrapeTransactionsFromTable(page: Page): Promise { 237 | const pendingTxns = await extractTransactionsFromTable( 238 | page, 239 | PENDING_TRANSACTIONS_TABLE_ID, 240 | TransactionStatuses.Pending, 241 | ); 242 | const completedTxns = await extractTransactionsFromTable( 243 | page, 244 | COMPLETED_TRANSACTIONS_TABLE_ID, 245 | TransactionStatuses.Completed, 246 | ); 247 | const txns = [...pendingTxns, ...completedTxns]; 248 | return convertTransactions(txns); 249 | } 250 | 251 | async function getAccountTransactions(page: Page): Promise { 252 | await Promise.race([ 253 | waitUntilElementFound(page, `#${COMPLETED_TRANSACTIONS_TABLE_ID}`, false), 254 | waitUntilElementFound(page, `.${ERROR_MESSAGE_CLASS}`, false), 255 | ]); 256 | 257 | const noTransactionInRangeError = await isNoTransactionInDateRangeError(page); 258 | if (noTransactionInRangeError) { 259 | return []; 260 | } 261 | 262 | await expandTransactionsTable(page); 263 | return scrapeTransactionsFromTable(page); 264 | } 265 | 266 | async function fetchAccountData(page: Page, startDate: Moment, accountId: string): Promise { 267 | await chooseAccount(page, accountId); 268 | await searchByDates(page, startDate); 269 | const accountNumber = await getAccountNumber(page); 270 | const txns = await getAccountTransactions(page); 271 | return { 272 | accountNumber, 273 | txns, 274 | }; 275 | } 276 | 277 | async function fetchAccounts(page: Page, startDate: Moment) { 278 | const accounts: TransactionsAccount[] = []; 279 | const accountsList = await dropdownElements(page, ACCOUNTS_DROPDOWN_SELECTOR); 280 | for (const account of accountsList) { 281 | if (account.value !== '-1') { 282 | // Skip "All accounts" option 283 | const accountData = await fetchAccountData(page, startDate, account.value); 284 | accounts.push(accountData); 285 | } 286 | } 287 | return accounts; 288 | } 289 | 290 | async function waitForPostLogin(page: Page) { 291 | return Promise.race([waitUntilElementFound(page, '#signoff', true), waitUntilElementFound(page, '#restore', true)]); 292 | } 293 | 294 | type ScraperSpecificCredentials = { username: string; password: string }; 295 | 296 | class UnionBankScraper extends BaseScraperWithBrowser { 297 | getLoginOptions(credentials: ScraperSpecificCredentials) { 298 | return { 299 | loginUrl: `${BASE_URL}`, 300 | fields: createLoginFields(credentials), 301 | submitButtonSelector: '#enter', 302 | postAction: async () => waitForPostLogin(this.page), 303 | possibleResults: getPossibleLoginResults(), 304 | }; 305 | } 306 | 307 | async fetchData() { 308 | const defaultStartMoment = moment().subtract(1, 'years').add(1, 'day'); 309 | const startDate = this.options.startDate || defaultStartMoment.toDate(); 310 | const startMoment = moment.max(defaultStartMoment, moment(startDate)); 311 | 312 | await this.navigateTo(TRANSACTIONS_URL); 313 | 314 | const accounts = await fetchAccounts(this.page, startMoment); 315 | 316 | return { 317 | success: true, 318 | accounts, 319 | }; 320 | } 321 | } 322 | 323 | export default UnionBankScraper; 324 | -------------------------------------------------------------------------------- /src/scrapers/base-scraper-with-browser.ts: -------------------------------------------------------------------------------- 1 | import puppeteer, { type Frame, type Page, type PuppeteerLifeCycleEvent } from 'puppeteer'; 2 | import { ScraperProgressTypes } from '../definitions'; 3 | import { getDebug } from '../helpers/debug'; 4 | import { clickButton, fillInput, waitUntilElementFound } from '../helpers/elements-interactions'; 5 | import { getCurrentUrl, waitForNavigation } from '../helpers/navigation'; 6 | import { BaseScraper } from './base-scraper'; 7 | import { ScraperErrorTypes } from './errors'; 8 | import { type ScraperCredentials, type ScraperScrapingResult } from './interface'; 9 | 10 | const debug = getDebug('base-scraper-with-browser'); 11 | 12 | enum LoginBaseResults { 13 | Success = 'SUCCESS', 14 | UnknownError = 'UNKNOWN_ERROR', 15 | } 16 | 17 | const { Timeout, Generic, General, ...rest } = ScraperErrorTypes; 18 | export const LoginResults = { 19 | ...rest, 20 | ...LoginBaseResults, 21 | }; 22 | 23 | // eslint-disable-next-line @typescript-eslint/no-redeclare 24 | export type LoginResults = 25 | | Exclude 26 | | LoginBaseResults; 27 | 28 | export type PossibleLoginResults = { 29 | [key in LoginResults]?: (string | RegExp | ((options?: { page?: Page }) => Promise))[]; 30 | }; 31 | 32 | export interface LoginOptions { 33 | loginUrl: string; 34 | checkReadiness?: () => Promise; 35 | fields: { selector: string; value: string }[]; 36 | submitButtonSelector: string | (() => Promise); 37 | preAction?: () => Promise; 38 | postAction?: () => Promise; 39 | possibleResults: PossibleLoginResults; 40 | userAgent?: string; 41 | waitUntil?: PuppeteerLifeCycleEvent; 42 | } 43 | 44 | async function getKeyByValue(object: PossibleLoginResults, value: string, page: Page): Promise { 45 | const keys = Object.keys(object); 46 | for (const key of keys) { 47 | // @ts-ignore 48 | const conditions = object[key]; 49 | 50 | for (const condition of conditions) { 51 | let result = false; 52 | 53 | if (condition instanceof RegExp) { 54 | result = condition.test(value); 55 | } else if (typeof condition === 'function') { 56 | result = await condition({ page, value }); 57 | } else { 58 | result = value.toLowerCase() === condition.toLowerCase(); 59 | } 60 | 61 | if (result) { 62 | // @ts-ignore 63 | return Promise.resolve(key); 64 | } 65 | } 66 | } 67 | 68 | return Promise.resolve(LoginResults.UnknownError); 69 | } 70 | 71 | function createGeneralError(): ScraperScrapingResult { 72 | return { 73 | success: false, 74 | errorType: ScraperErrorTypes.General, 75 | }; 76 | } 77 | 78 | async function safeCleanup(cleanup: () => Promise) { 79 | try { 80 | await cleanup(); 81 | } catch (e) { 82 | debug(`Cleanup function failed: ${(e as Error).message}`); 83 | } 84 | } 85 | 86 | class BaseScraperWithBrowser extends BaseScraper { 87 | private cleanups: Array<() => Promise> = []; 88 | 89 | private defaultViewportSize = { 90 | width: 1024, 91 | height: 768, 92 | }; 93 | 94 | // NOTICE - it is discouraged to use bang (!) in general. It is used here because 95 | // all the classes that inherit from this base assume is it mandatory. 96 | protected page!: Page; 97 | 98 | protected getViewPort() { 99 | return this.options.viewportSize ?? this.defaultViewportSize; 100 | } 101 | 102 | async initialize() { 103 | await super.initialize(); 104 | debug('initialize scraper'); 105 | this.emitProgress(ScraperProgressTypes.Initializing); 106 | 107 | const page = await this.initializePage(); 108 | await page.setCacheEnabled(false); // Clear cache and avoid 300's response status 109 | 110 | if (!page) { 111 | debug('failed to initiate a browser page, exit'); 112 | return; 113 | } 114 | 115 | this.page = page; 116 | 117 | this.cleanups.push(() => page.close()); 118 | 119 | if (this.options.defaultTimeout) { 120 | this.page.setDefaultTimeout(this.options.defaultTimeout); 121 | } 122 | 123 | if (this.options.preparePage) { 124 | debug("execute 'preparePage' interceptor provided in options"); 125 | await this.options.preparePage(this.page); 126 | } 127 | 128 | const viewport = this.getViewPort(); 129 | debug(`set viewport to width ${viewport.width}, height ${viewport.height}`); 130 | await this.page.setViewport({ 131 | width: viewport.width, 132 | height: viewport.height, 133 | }); 134 | 135 | this.page.on('requestfailed', request => { 136 | debug('Request failed: %s %s', request.failure()?.errorText, request.url()); 137 | }); 138 | } 139 | 140 | private async initializePage() { 141 | debug('initialize browser page'); 142 | if ('browserContext' in this.options) { 143 | debug('Using the browser context provided in options'); 144 | return this.options.browserContext.newPage(); 145 | } 146 | 147 | if ('browser' in this.options) { 148 | debug('Using the browser instance provided in options'); 149 | const { browser } = this.options; 150 | 151 | /** 152 | * For backward compatibility, we will close the browser even if we didn't create it 153 | */ 154 | if (!this.options.skipCloseBrowser) { 155 | this.cleanups.push(async () => { 156 | debug('closing the browser'); 157 | await browser.close(); 158 | }); 159 | } 160 | 161 | return browser.newPage(); 162 | } 163 | 164 | const { timeout, args, executablePath, showBrowser } = this.options; 165 | 166 | const headless = !showBrowser; 167 | debug(`launch a browser with headless mode = ${headless}`); 168 | 169 | const browser = await puppeteer.launch({ 170 | env: this.options.verbose ? { DEBUG: '*', ...process.env } : undefined, 171 | headless, 172 | executablePath, 173 | args, 174 | timeout, 175 | }); 176 | 177 | this.cleanups.push(async () => { 178 | debug('closing the browser'); 179 | await browser.close(); 180 | }); 181 | 182 | if (this.options.prepareBrowser) { 183 | debug("execute 'prepareBrowser' interceptor provided in options"); 184 | await this.options.prepareBrowser(browser); 185 | } 186 | 187 | debug('create a new browser page'); 188 | return browser.newPage(); 189 | } 190 | 191 | async navigateTo( 192 | url: string, 193 | waitUntil: PuppeteerLifeCycleEvent | undefined = 'load', 194 | retries = this.options.navigationRetryCount ?? 0, 195 | ): Promise { 196 | const response = await this.page?.goto(url, { waitUntil }); 197 | if (response === null) { 198 | // note: response will be null when navigating to same url while changing the hash part. 199 | // the condition below will always accept null as valid result. 200 | return; 201 | } 202 | 203 | if (!response) { 204 | throw new Error(`Error while trying to navigate to url ${url}, response is undefined`); 205 | } 206 | 207 | if (!response.ok()) { 208 | const status = response.status(); 209 | if (retries > 0) { 210 | debug(`Failed to navigate to url ${url}, status code: ${status}, retrying ${retries} more times`); 211 | await this.navigateTo(url, waitUntil, retries - 1); 212 | } else { 213 | throw new Error(`Failed to navigate to url ${url}, status code: ${status}`); 214 | } 215 | } 216 | } 217 | 218 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 219 | getLoginOptions(_credentials: ScraperCredentials): LoginOptions { 220 | throw new Error(`getLoginOptions() is not created in ${this.options.companyId}`); 221 | } 222 | 223 | async fillInputs(pageOrFrame: Page | Frame, fields: { selector: string; value: string }[]): Promise { 224 | const modified = [...fields]; 225 | const input = modified.shift(); 226 | 227 | if (!input) { 228 | return; 229 | } 230 | await fillInput(pageOrFrame, input.selector, input.value); 231 | if (modified.length) { 232 | await this.fillInputs(pageOrFrame, modified); 233 | } 234 | } 235 | 236 | async login(credentials: ScraperCredentials): Promise { 237 | if (!credentials || !this.page) { 238 | return createGeneralError(); 239 | } 240 | 241 | debug('execute login process'); 242 | const loginOptions = this.getLoginOptions(credentials); 243 | 244 | if (loginOptions.userAgent) { 245 | debug('set custom user agent provided in options'); 246 | await this.page.setUserAgent(loginOptions.userAgent); 247 | } 248 | 249 | debug('navigate to login url'); 250 | await this.navigateTo(loginOptions.loginUrl, loginOptions.waitUntil); 251 | if (loginOptions.checkReadiness) { 252 | debug("execute 'checkReadiness' interceptor provided in login options"); 253 | await loginOptions.checkReadiness(); 254 | } else if (typeof loginOptions.submitButtonSelector === 'string') { 255 | debug('wait until submit button is available'); 256 | await waitUntilElementFound(this.page, loginOptions.submitButtonSelector); 257 | } 258 | 259 | let loginFrameOrPage: Page | Frame | null = this.page; 260 | if (loginOptions.preAction) { 261 | debug("execute 'preAction' interceptor provided in login options"); 262 | loginFrameOrPage = (await loginOptions.preAction()) || this.page; 263 | } 264 | 265 | debug('fill login components input with relevant values'); 266 | await this.fillInputs(loginFrameOrPage, loginOptions.fields); 267 | debug('click on login submit button'); 268 | if (typeof loginOptions.submitButtonSelector === 'string') { 269 | await clickButton(loginFrameOrPage, loginOptions.submitButtonSelector); 270 | } else { 271 | await loginOptions.submitButtonSelector(); 272 | } 273 | this.emitProgress(ScraperProgressTypes.LoggingIn); 274 | 275 | if (loginOptions.postAction) { 276 | debug("execute 'postAction' interceptor provided in login options"); 277 | await loginOptions.postAction(); 278 | } else { 279 | debug('wait for page navigation'); 280 | await waitForNavigation(this.page); 281 | } 282 | 283 | debug('check login result'); 284 | const current = await getCurrentUrl(this.page, true); 285 | const loginResult = await getKeyByValue(loginOptions.possibleResults, current, this.page); 286 | debug(`handle login results ${loginResult}`); 287 | return this.handleLoginResult(loginResult); 288 | } 289 | 290 | async terminate(_success: boolean) { 291 | debug(`terminating browser with success = ${_success}`); 292 | this.emitProgress(ScraperProgressTypes.Terminating); 293 | 294 | if (!_success && !!this.options.storeFailureScreenShotPath) { 295 | debug(`create a snapshot before terminated in ${this.options.storeFailureScreenShotPath}`); 296 | await this.page.screenshot({ 297 | path: this.options.storeFailureScreenShotPath, 298 | fullPage: true, 299 | }); 300 | } 301 | 302 | await Promise.all(this.cleanups.reverse().map(safeCleanup)); 303 | this.cleanups = []; 304 | } 305 | 306 | private handleLoginResult(loginResult: LoginResults) { 307 | switch (loginResult) { 308 | case LoginResults.Success: 309 | this.emitProgress(ScraperProgressTypes.LoginSuccess); 310 | return { success: true }; 311 | case LoginResults.InvalidPassword: 312 | case LoginResults.UnknownError: 313 | this.emitProgress(ScraperProgressTypes.LoginFailed); 314 | return { 315 | success: false, 316 | errorType: 317 | loginResult === LoginResults.InvalidPassword 318 | ? ScraperErrorTypes.InvalidPassword 319 | : ScraperErrorTypes.General, 320 | errorMessage: `Login failed with ${loginResult} error`, 321 | }; 322 | case LoginResults.ChangePassword: 323 | this.emitProgress(ScraperProgressTypes.ChangePassword); 324 | return { 325 | success: false, 326 | errorType: ScraperErrorTypes.ChangePassword, 327 | }; 328 | default: 329 | throw new Error(`unexpected login result "${loginResult}"`); 330 | } 331 | } 332 | } 333 | 334 | export { BaseScraperWithBrowser }; 335 | -------------------------------------------------------------------------------- /src/scrapers/one-zero-queries.ts: -------------------------------------------------------------------------------- 1 | export const GET_CUSTOMER = ` 2 | query GetCustomer { 3 | customer { 4 | __typename 5 | customerId 6 | userId 7 | idType 8 | idNumber 9 | hebrewFirstName 10 | hebrewLastName 11 | latinFirstName 12 | latinLastName 13 | dateOfBirth 14 | lastLoginDate 15 | userEmail 16 | gender 17 | portfolioRelations { 18 | __typename 19 | customerId 20 | customerRole 21 | portfolioId 22 | initiator 23 | relationToInitiator 24 | status 25 | } 26 | portfolios { 27 | __typename 28 | ...Portfolio 29 | } 30 | status 31 | } 32 | } 33 | fragment Portfolio on Portfolio { 34 | __typename 35 | accounts { 36 | __typename 37 | accountId 38 | accountType 39 | closingDate 40 | currency 41 | openingDate 42 | status 43 | subType 44 | } 45 | activationDate 46 | bank 47 | baseCurrency 48 | branch 49 | club 50 | clubDescription 51 | iban 52 | imageURL 53 | isJointAccount 54 | partnerName { 55 | __typename 56 | partnerFirstName 57 | partnerLastName 58 | } 59 | portfolioId 60 | portfolioNum 61 | portfolioType 62 | status 63 | subType 64 | onboardingCompleted 65 | } 66 | `; 67 | 68 | export const GET_MOVEMENTS = `query GetMovements( 69 | $portfolioId: String! 70 | $accountId: String! 71 | $pagination: PaginationInput! 72 | $language: BffLanguage! 73 | ) { 74 | movements( 75 | portfolioId: $portfolioId 76 | accountId: $accountId 77 | pagination: $pagination 78 | language: $language 79 | ) { 80 | __typename 81 | ...MovementsFragment 82 | } 83 | } 84 | fragment TransactionInstrumentAmountFragment on TransactionInstrumentAmount { 85 | __typename 86 | instrumentAmount 87 | instrumentSymbol 88 | instrumentType 89 | } 90 | fragment CounterPartyReferenceFragment on CounterPartyReference { 91 | __typename 92 | bankId 93 | bic 94 | branchCode 95 | id 96 | name 97 | type 98 | } 99 | fragment BaseTransactionFragment on BaseTransaction { 100 | __typename 101 | accountId 102 | betweenOwnAccounts 103 | bookDate 104 | calculatedStatus 105 | chargeAmount { 106 | __typename 107 | ...TransactionInstrumentAmountFragment 108 | } 109 | clearingSystem 110 | counterParty { 111 | __typename 112 | ...CounterPartyReferenceFragment 113 | } 114 | currentPaymentNumber 115 | direction 116 | domainType 117 | isReversal 118 | method 119 | originalAmount { 120 | __typename 121 | ...TransactionInstrumentAmountFragment 122 | } 123 | portfolioId 124 | totalPaymentsCount 125 | transactionId 126 | transactionType 127 | valueDate 128 | } 129 | fragment CategoryFragment on Category { 130 | __typename 131 | categoryId 132 | dataSource 133 | subCategoryId 134 | } 135 | fragment RecurrenceFragment on Recurrence { 136 | __typename 137 | dataSource 138 | isRecurrent 139 | } 140 | fragment TransactionEnrichmentFragment on TransactionEnrichment { 141 | __typename 142 | categories { 143 | __typename 144 | ...CategoryFragment 145 | } 146 | recurrences { 147 | __typename 148 | ...RecurrenceFragment 149 | } 150 | } 151 | fragment TransactionEventMetadataFragment on TransactionEventMetadata { 152 | __typename 153 | correlationId 154 | processingOrder 155 | } 156 | fragment CounterPartyTransferData on CounterPartyTransfer { 157 | __typename 158 | accountId 159 | bank_id 160 | branch_code 161 | counter_party_name 162 | } 163 | fragment BankTransferDetailsData on BankTransferDetails { 164 | __typename 165 | ... on CashBlockTransfer { 166 | counterParty { 167 | __typename 168 | ...CounterPartyTransferData 169 | } 170 | transferDescriptionKey 171 | } 172 | ... on RTGSReturnTransfer { 173 | transferDescriptionKey 174 | } 175 | ... on RTGSTransfer { 176 | transferDescriptionKey 177 | } 178 | ... on SwiftReturnTransfer { 179 | transferConversionRate 180 | transferDescriptionKey 181 | } 182 | ... on SwiftTransfer { 183 | transferConversionRate 184 | transferDescriptionKey 185 | } 186 | ... on Transfer { 187 | counterParty { 188 | __typename 189 | ...CounterPartyTransferData 190 | } 191 | transferDescriptionKey 192 | } 193 | } 194 | fragment CategoryData on Category { 195 | __typename 196 | categoryId 197 | dataSource 198 | subCategoryId 199 | } 200 | fragment RecurrenceData on Recurrence { 201 | __typename 202 | dataSource 203 | isRecurrent 204 | } 205 | fragment CardDetailsData on CardDetails { 206 | __typename 207 | ... on CardCharge { 208 | book_date 209 | cardDescriptionKey 210 | } 211 | ... on CardChargeFCY { 212 | book_date 213 | cardConversionRate 214 | cardDescriptionKey 215 | cardFCYAmount 216 | cardFCYCurrency 217 | } 218 | ... on CardMonthlySettlement { 219 | cardDescriptionKey 220 | } 221 | ... on CardRefund { 222 | cardDescriptionKey 223 | } 224 | ... on CashBlockCardCharge { 225 | cardDescriptionKey 226 | } 227 | } 228 | fragment CashDetailsData on CashDetails { 229 | __typename 230 | ... on CashWithdrawal { 231 | cashDescriptionKey 232 | } 233 | ... on CashWithdrawalFCY { 234 | FCYAmount 235 | FCYCurrency 236 | cashDescriptionKey 237 | conversionRate 238 | } 239 | } 240 | fragment ChequesDetailsData on ChequesDetails { 241 | __typename 242 | ... on CashBlockChequeDeposit { 243 | bookDate 244 | chequesDescriptionKey 245 | } 246 | ... on ChequeDeposit { 247 | bookDate 248 | chequesDescriptionKey 249 | } 250 | ... on ChequeReturn { 251 | bookDate 252 | chequeReturnReason 253 | chequesDescriptionKey 254 | } 255 | ... on ChequeWithdrawal { 256 | chequesDescriptionKey 257 | } 258 | } 259 | fragment DefaultDetailsData on DefaultDetails { 260 | __typename 261 | ... on DefaultWithTransaction { 262 | defaultDescriptionKey 263 | } 264 | ... on DefaultWithoutTransaction { 265 | categories { 266 | __typename 267 | ...CategoryData 268 | } 269 | defaultDescriptionKey 270 | } 271 | } 272 | fragment FeeDetailsData on FeeDetails { 273 | __typename 274 | ... on GeneralFee { 275 | feeDescriptionKey 276 | } 277 | } 278 | fragment LoanDetailsData on LoanDetails { 279 | __typename 280 | ... on FullPrePayment { 281 | loanDescriptionKey 282 | } 283 | ... on Initiate { 284 | loanDescriptionKey 285 | } 286 | ... on MonthlyPayment { 287 | loanDescriptionKey 288 | loanPaymentNumber 289 | loanTotalPaymentsCount 290 | } 291 | ... on PartialPrePayment { 292 | loanDescriptionKey 293 | } 294 | } 295 | fragment MandateDetailsData on MandateDetails { 296 | __typename 297 | ... on MandatePayment { 298 | mandateDescriptionKey 299 | } 300 | ... on MandateReturnPayment { 301 | mandateDescriptionKey 302 | } 303 | } 304 | fragment SavingsDetailsData on SavingsDetails { 305 | __typename 306 | ... on FullSavingsWithdrawal { 307 | savingsDescriptionKey 308 | } 309 | ... on MonthlySavingsDeposit { 310 | savingsDepositNumber 311 | savingsDescriptionKey 312 | savingsTotalDepositCount 313 | } 314 | ... on PartialSavingsWithdrawal { 315 | savingsDescriptionKey 316 | } 317 | ... on SavingsClosing { 318 | savingsDescriptionKey 319 | } 320 | ... on SavingsDeposit { 321 | savingsDescriptionKey 322 | } 323 | ... on SavingsInterest { 324 | savingsDescriptionKey 325 | } 326 | ... on SavingsPenalty { 327 | savingsDescriptionKey 328 | } 329 | ... on SavingsTax { 330 | savingsDescriptionKey 331 | } 332 | } 333 | fragment SubscriptionDetailsData on SubscriptionDetails { 334 | __typename 335 | ... on SubscriptionPayment { 336 | subscriptionDescriptionKey 337 | } 338 | ... on SubscriptionReturnPayment { 339 | subscriptionDescriptionKey 340 | } 341 | } 342 | fragment TransactionsDetailsData on TransactionDetails { 343 | __typename 344 | ... on BankTransfer { 345 | bank_transfer_details { 346 | __typename 347 | ...BankTransferDetailsData 348 | } 349 | book_date 350 | categories { 351 | __typename 352 | ...CategoryData 353 | } 354 | recurrences { 355 | __typename 356 | ...RecurrenceData 357 | } 358 | value_date 359 | } 360 | ... on Card { 361 | card_details { 362 | __typename 363 | ...CardDetailsData 364 | } 365 | categories { 366 | __typename 367 | ...CategoryData 368 | } 369 | recurrences { 370 | __typename 371 | ...RecurrenceData 372 | } 373 | value_date 374 | } 375 | ... on Cash { 376 | cash_details { 377 | __typename 378 | ...CashDetailsData 379 | } 380 | categories { 381 | __typename 382 | ...CategoryData 383 | } 384 | recurrences { 385 | __typename 386 | ...RecurrenceData 387 | } 388 | value_date 389 | } 390 | ... on Cheques { 391 | categories { 392 | __typename 393 | ...CategoryData 394 | } 395 | chequesDetails { 396 | __typename 397 | ...ChequesDetailsData 398 | } 399 | recurrences { 400 | __typename 401 | ...RecurrenceData 402 | } 403 | valueDate 404 | referenceNumber 405 | frontImageUrl 406 | backImageUrl 407 | } 408 | ... on Default { 409 | default_details { 410 | __typename 411 | ...DefaultDetailsData 412 | } 413 | recurrences { 414 | __typename 415 | ...RecurrenceData 416 | } 417 | value_date 418 | } 419 | ... on Fee { 420 | categories { 421 | __typename 422 | ...CategoryData 423 | } 424 | fee_details { 425 | __typename 426 | ...FeeDetailsData 427 | } 428 | value_date 429 | } 430 | ... on Loans { 431 | categories { 432 | __typename 433 | ...CategoryData 434 | } 435 | loan_details { 436 | __typename 437 | ...LoanDetailsData 438 | } 439 | recurrences { 440 | __typename 441 | ...RecurrenceData 442 | } 443 | value_date 444 | } 445 | ... on Mandate { 446 | categories { 447 | __typename 448 | ...CategoryData 449 | } 450 | mandate_details { 451 | __typename 452 | ...MandateDetailsData 453 | } 454 | recurrences { 455 | __typename 456 | ...RecurrenceData 457 | } 458 | value_date 459 | } 460 | ... on Savings { 461 | categories { 462 | __typename 463 | ...CategoryData 464 | } 465 | recurrences { 466 | __typename 467 | ...RecurrenceData 468 | } 469 | savings_details { 470 | __typename 471 | ...SavingsDetailsData 472 | } 473 | value_date 474 | } 475 | ... on SubscriptionTransaction { 476 | categories { 477 | __typename 478 | ...CategoryData 479 | } 480 | recurrences { 481 | __typename 482 | ...RecurrenceData 483 | } 484 | subscription_details { 485 | __typename 486 | ...SubscriptionDetailsData 487 | } 488 | value_date 489 | } 490 | } 491 | fragment TransactionFragment on Transaction { 492 | __typename 493 | baseTransaction { 494 | __typename 495 | ...BaseTransactionFragment 496 | } 497 | enrichment { 498 | __typename 499 | ...TransactionEnrichmentFragment 500 | } 501 | metadata { 502 | __typename 503 | ...TransactionEventMetadataFragment 504 | } 505 | referenceNumber 506 | transactionDetails { 507 | __typename 508 | ...TransactionsDetailsData 509 | } 510 | } 511 | fragment MovementFragment on Movement { 512 | __typename 513 | accountId 514 | bankCurrencyAmount 515 | bookingDate 516 | conversionRate 517 | creditDebit 518 | description 519 | isReversed 520 | linkTransaction { 521 | __typename 522 | ...TransactionFragment 523 | } 524 | movementAmount 525 | movementCurrency 526 | movementId 527 | movementReversedId 528 | movementTimestamp 529 | movementType 530 | portfolioId 531 | runningBalance 532 | transaction { 533 | __typename 534 | ...TransactionFragment 535 | } 536 | valueDate 537 | } 538 | fragment PaginationFragment on Pagination { 539 | __typename 540 | cursor 541 | hasMore 542 | } 543 | fragment MovementsFragment on Movements { 544 | __typename 545 | isRunningBalanceInSync 546 | movements { 547 | __typename 548 | ...MovementFragment 549 | } 550 | pagination { 551 | __typename 552 | ...PaginationFragment 553 | } 554 | }`; 555 | --------------------------------------------------------------------------------