├── src ├── queify.js ├── pdf │ ├── templates │ │ └── zerodha-cn-old.js │ ├── create-file.js │ ├── enums.js │ ├── utils.js │ └── extract-tables.js ├── middleware │ ├── commonMiddleware.js │ ├── cookieSessionRefresh.js │ ├── cookieSession.js │ └── ensureAuth.js ├── sentry.js ├── queues │ ├── index.js │ ├── send-email.js │ ├── send-whatsapp.js │ ├── auto-unlock.js │ ├── notifications.js │ ├── task-status.js │ ├── mail-fetch.js │ ├── gsheet-sync.js │ ├── email-to-json.js │ └── pdf-to-json.js ├── hooks │ └── useLocalState.js ├── utils.js ├── firebase │ ├── initFirebase.js │ ├── logout.js │ ├── firebaseAdmin.js │ ├── firebaseSessionHandler.js │ ├── hooks.js │ └── user.js ├── isomorphic │ ├── ensureConfiguration.js │ └── applyConfigOnEmail.js ├── auth.js ├── integrations │ ├── utils │ │ └── index.js │ ├── email │ │ └── mailgun.js │ └── google-spreadsheet │ │ ├── reset-sheet.js │ │ └── sync.js ├── apps │ └── utils.js ├── redis-queue.js └── gmail.js ├── .dockerignore ├── .prettierrc.js ├── public ├── favicon.ico ├── static │ └── images │ │ ├── favicon.png │ │ └── progressive-disclosure-line@2x.png └── vercel.svg ├── .babelrc ├── alias-config.js ├── css ├── tailwind.css └── react-responsive-modal-override.css ├── pages ├── api │ ├── apps │ │ ├── auto-unlock │ │ │ ├── webhook.js │ │ │ └── index.js │ │ └── email-to-json │ │ │ ├── webhook.js │ │ │ ├── integrations │ │ │ ├── whatsapp.js │ │ │ ├── google-sheet.js │ │ │ └── sms.js │ │ │ └── index.js │ ├── firebase │ │ ├── logout.js │ │ ├── exchange-token.js │ │ ├── login.js │ │ ├── associate-user.js │ │ └── user.js │ ├── fetch │ │ ├── email.js │ │ ├── attachment.js │ │ ├── tables-from-attachment.js │ │ └── preview-attachment-rules.js │ ├── jsonbox │ │ ├── get-user.js │ │ └── put-user.js │ ├── dashboard │ │ └── get-services.js │ ├── webhooks │ │ ├── credit-cards │ │ │ ├── amex.js │ │ │ └── dcb.js │ │ └── zerodha-cn │ │ │ └── index.js │ ├── user │ │ └── delete.js │ ├── integrations │ │ ├── google-spreadsheet │ │ │ ├── preview.js │ │ │ └── index.js │ │ ├── whatsapp │ │ │ ├── subscriber.js │ │ │ ├── [path].js │ │ │ ├── checker.js │ │ │ └── index.js │ │ ├── twilio │ │ │ └── index.js │ │ └── airtable │ │ │ └── index.js │ ├── email-search │ │ ├── index.js │ │ └── attachment-unlock.js │ ├── cron │ │ ├── index.js │ │ ├── user-cron.js │ │ └── cleanup-accounts.js │ └── otp │ │ └── index.js ├── [uid] │ ├── ft │ │ ├── email-json │ │ │ ├── [id].js │ │ │ └── index.js │ │ └── attachment-unlocker │ │ │ ├── [id].js │ │ │ └── index.js │ └── home.js ├── _app.js ├── helpers │ └── iframe.js └── _document.js ├── webpack.config.js ├── components ├── admin │ └── email │ │ └── fns │ │ ├── generateKeyFromName.js │ │ ├── generateUniqueId.js │ │ └── fullPath.js ├── ft │ ├── email-json │ │ ├── rules-preview.js │ │ └── Grid.js │ └── attachment-unlocker │ │ ├── email-preview.js │ │ └── action-bar.js ├── common │ ├── common-header.js │ └── Atoms.js ├── service-creator │ ├── header.js │ ├── action-bar.js │ ├── email-results-nav.js │ ├── configuration-editor.js │ └── email-preview.js ├── pageWrappers │ ├── withAuthUserInfo.js │ └── withAuthUser.js └── FirebaseAuth.js ├── postcss.config.js ├── .vscode └── launch.json ├── styles ├── globals.css └── Home.module.css ├── tailwind.config.js ├── docker-compose-local.yml ├── next.config.js ├── .gitignore ├── .github └── FUNDING.yml ├── docker-compose.yml ├── docker-compose-dev.yml ├── LICENSE.md ├── .eslintrc.js ├── Dockerfile-local ├── Dockerfile ├── TODO.md └── package.json /src/queify.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .next 3 | .vscode -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | singleQuote: true, 3 | trailingComma: 'all', 4 | }; -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aakashlpin/emailapi/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["next/babel"], 3 | "plugins": [["styled-components", { "ssr": true }]] 4 | } 5 | -------------------------------------------------------------------------------- /alias-config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | '~': path.resolve(__dirname), 5 | }; 6 | -------------------------------------------------------------------------------- /css/tailwind.css: -------------------------------------------------------------------------------- 1 | @import "tailwindcss/base"; 2 | @import "tailwindcss/components"; 3 | @import "tailwindcss/utilities"; 4 | -------------------------------------------------------------------------------- /pages/api/apps/auto-unlock/webhook.js: -------------------------------------------------------------------------------- 1 | import webhook from '../email-to-json/webhook'; 2 | 3 | export default webhook; 4 | -------------------------------------------------------------------------------- /public/static/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aakashlpin/emailapi/HEAD/public/static/images/favicon.png -------------------------------------------------------------------------------- /pages/[uid]/ft/email-json/[id].js: -------------------------------------------------------------------------------- 1 | import EmailJson from '~/components/ft/email-json/EmailJson'; 2 | 3 | export default EmailJson; 4 | -------------------------------------------------------------------------------- /pages/[uid]/ft/email-json/index.js: -------------------------------------------------------------------------------- 1 | import EmailJson from '~/components/ft/email-json/EmailJson'; 2 | 3 | export default EmailJson; 4 | -------------------------------------------------------------------------------- /src/pdf/templates/zerodha-cn-old.js: -------------------------------------------------------------------------------- 1 | export default { 2 | camelotMethod: 'lattice', 3 | camelotScale: 50, 4 | rules: [], 5 | }; 6 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const alias = require('./alias-config'); 2 | 3 | module.exports = { 4 | resolve: { 5 | alias, 6 | }, 7 | }; 8 | -------------------------------------------------------------------------------- /public/static/images/progressive-disclosure-line@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aakashlpin/emailapi/HEAD/public/static/images/progressive-disclosure-line@2x.png -------------------------------------------------------------------------------- /components/admin/email/fns/generateKeyFromName.js: -------------------------------------------------------------------------------- 1 | export default function generateKeyFromName(name) { 2 | return name.trim().toLowerCase().replace(/\s/g, '_'); 3 | } 4 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: [ 3 | require('postcss-import'), 4 | require('tailwindcss'), 5 | require('autoprefixer'), 6 | ], 7 | }; 8 | -------------------------------------------------------------------------------- /pages/[uid]/ft/attachment-unlocker/[id].js: -------------------------------------------------------------------------------- 1 | import AttachmentUnlocker from '~/components/ft/attachment-unlocker/AttachmentUnlocker'; 2 | 3 | export default AttachmentUnlocker; 4 | -------------------------------------------------------------------------------- /pages/[uid]/ft/attachment-unlocker/index.js: -------------------------------------------------------------------------------- 1 | import AttachmentUnlocker from '~/components/ft/attachment-unlocker/AttachmentUnlocker'; 2 | 3 | export default AttachmentUnlocker; 4 | -------------------------------------------------------------------------------- /css/react-responsive-modal-override.css: -------------------------------------------------------------------------------- 1 | .react-responsive-modal-overlay { 2 | padding: 0; 3 | } 4 | 5 | .react-responsive-modal-modal { 6 | width: 100%; 7 | max-width: calc(100% - 4rem); 8 | padding: 0; 9 | } -------------------------------------------------------------------------------- /src/middleware/commonMiddleware.js: -------------------------------------------------------------------------------- 1 | import cookieSession from './cookieSession'; 2 | import cookieSessionRefresh from './cookieSessionRefresh'; 3 | 4 | export default (handler) => cookieSession(cookieSessionRefresh(handler)); 5 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "attach", 7 | "name": "Launch Program", 8 | "skipFiles": ["/**"], 9 | "port": 9229 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /src/sentry.js: -------------------------------------------------------------------------------- 1 | const Sentry = require('@sentry/node'); 2 | 3 | const Tracing = require('@sentry/tracing'); 4 | 5 | const { SENTRY_DSN } = process.env; 6 | 7 | Sentry.init({ 8 | dsn: SENTRY_DSN, 9 | tracesSampleRate: 1.0, 10 | }); 11 | 12 | export default Sentry; 13 | 14 | export { Tracing }; 15 | -------------------------------------------------------------------------------- /src/pdf/create-file.js: -------------------------------------------------------------------------------- 1 | const base64 = require('base64topdf'); 2 | const shortid = require('shortid'); 3 | 4 | export default function createFileFromApiResponse(b64Content) { 5 | const filePath = `/tmp/${shortid.generate()}.pdf`; 6 | base64.base64Decode(b64Content, filePath); 7 | 8 | return filePath; 9 | } 10 | -------------------------------------------------------------------------------- /styles/globals.css: -------------------------------------------------------------------------------- 1 | html, 2 | body { 3 | padding: 0; 4 | margin: 0; 5 | font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen, 6 | Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif; 7 | } 8 | 9 | a { 10 | color: inherit; 11 | text-decoration: none; 12 | } 13 | 14 | * { 15 | box-sizing: border-box; 16 | } 17 | -------------------------------------------------------------------------------- /pages/api/firebase/logout.js: -------------------------------------------------------------------------------- 1 | import commonMiddleware from '~/src/middleware/commonMiddleware'; 2 | 3 | const handler = (req, res) => { 4 | // Destroy the session. 5 | // https://github.com/expressjs/cookie-session#destroying-a-session 6 | req.session = null; 7 | res.status(200).json({ status: true }); 8 | }; 9 | 10 | export default commonMiddleware(handler); 11 | -------------------------------------------------------------------------------- /pages/_app.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import App from 'next/app'; 3 | import '../css/tailwind.css'; 4 | 5 | class EmailApiApp extends App { 6 | render() { 7 | const { Component, pageProps } = this.props; 8 | // eslint-disable-next-line react/jsx-props-no-spreading 9 | return ; 10 | } 11 | } 12 | 13 | export default EmailApiApp; 14 | -------------------------------------------------------------------------------- /src/queues/index.js: -------------------------------------------------------------------------------- 1 | require('~/src/queues/send-whatsapp'); 2 | require('~/src/queues/gsheet-sync'); 3 | require('~/src/queues/pdf-to-json'); 4 | require('~/src/queues/send-email'); 5 | require('~/src/queues/notifications'); 6 | require('~/src/queues/auto-unlock'); 7 | require('~/src/queues/mail-fetch'); 8 | require('~/src/queues/email-to-json'); 9 | require('~/src/queues/task-status'); 10 | -------------------------------------------------------------------------------- /src/pdf/enums.js: -------------------------------------------------------------------------------- 1 | export const RULE_TYPE = { 2 | INCLUDE_ROWS: 'INCLUDE_ROWS', 3 | INCLUDE_CELLS: 'INCLUDE_CELLS', 4 | }; 5 | 6 | export const CELL_MATCH_TYPE = { 7 | NOT_EMPTY: 'NOT_EMPTY', 8 | STARTS_WITH: 'STARTS_WITH', 9 | ENDS_WITH: 'ENDS_WITH', 10 | EQUALS: 'EQUALS', 11 | CONTAINS: 'CONTAINS', 12 | REGEX: 'REGEX', 13 | }; 14 | 15 | export const TEMPLATE_TYPE = { 16 | ZERODHA_CN: 'ZERODHA_CN', 17 | CUSTOM: 'CUSTOM', 18 | }; 19 | -------------------------------------------------------------------------------- /pages/api/fetch/email.js: -------------------------------------------------------------------------------- 1 | import ensureAuth from '~/src/middleware/ensureAuth'; 2 | import { fetchEmailByMessageId } from '~/src/gmail'; 3 | 4 | async function handle(req, res, resolve) { 5 | const { messageId } = req.body; 6 | 7 | const messageBody = await fetchEmailByMessageId({ 8 | messageId, 9 | refreshToken: req.refresh_token, 10 | }); 11 | 12 | res.json(messageBody); 13 | resolve(); 14 | } 15 | 16 | export default ensureAuth(handle); 17 | -------------------------------------------------------------------------------- /src/middleware/cookieSessionRefresh.js: -------------------------------------------------------------------------------- 1 | // Update a value in the cookie so that the set-cookie will be sent. 2 | // Only changes every minute so that it's not sent with every request. 3 | // https://github.com/expressjs/cookie-session#extending-the-session-expiration 4 | export default (handler) => (req, res, resolve) => { 5 | if (req.session) { 6 | req.session.nowInMinutes = Math.floor(Date.now() / 60e3); 7 | } 8 | resolve(handler(req, res)); 9 | }; 10 | -------------------------------------------------------------------------------- /src/hooks/useLocalState.js: -------------------------------------------------------------------------------- 1 | import { useState, useEffect } from 'react'; 2 | 3 | export default function useLocalState(key, defaultValue = '') { 4 | const [value, setValue] = useState(() => 5 | typeof window !== 'undefined' 6 | ? JSON.parse(window.localStorage.getItem(key)) 7 | : defaultValue, 8 | ); 9 | 10 | useEffect(() => { 11 | window.localStorage.setItem(key, JSON.stringify(value)); 12 | }, [value]); 13 | 14 | return [value, setValue]; 15 | } 16 | -------------------------------------------------------------------------------- /pages/api/fetch/attachment.js: -------------------------------------------------------------------------------- 1 | import ensureAuth from '~/src/middleware/ensureAuth'; 2 | import { fetchAttachment } from '~/src/gmail'; 3 | 4 | async function handle(req, res, resolve) { 5 | const { attachmentId, messageId } = req.body; 6 | const base64Encoded = await fetchAttachment({ 7 | attachmentId, 8 | messageId, 9 | refreshToken: req.refresh_token, 10 | }); 11 | res.json({ base64: base64Encoded }); 12 | resolve(); 13 | } 14 | 15 | export default ensureAuth(handle); 16 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | theme: { 3 | extend: { 4 | fontFamily: { 5 | body: ['"Space Mono"', 'monospace'], 6 | display: [ 7 | 'system-ui', 8 | '-apple-system', 9 | 'BlinkMacSystemFont', 10 | 'Segoe UI', 11 | 'Roboto', 12 | 'Ubuntu', 13 | 'Helvetica Neue', 14 | 'sans-serif', 15 | ], 16 | }, 17 | }, 18 | }, 19 | variants: {}, 20 | plugins: [], 21 | }; 22 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | import Sentry from '~/src/sentry'; 2 | 3 | // eslint-disable-next-line import/prefer-default-export 4 | export const genericErrorHandler = (e) => { 5 | if (e.isAxiosError) { 6 | const axiosError = e; 7 | const captureError = axiosError.toJSON(); 8 | console.log('genericErrorHandler Axios error', captureError); 9 | Sentry.captureException(captureError); 10 | } else { 11 | console.log('genericErrorHandler error', e); 12 | Sentry.captureException(e); 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /components/ft/email-json/rules-preview.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { getRuleDataFromTable } from '~/src/pdf/utils'; 3 | import Grid from './Grid'; 4 | 5 | export default function RulePreview({ data, rule }) { 6 | // function validateCell 7 | const ruleDataFromTable = getRuleDataFromTable({ data, rule }); 8 | return ( 9 |
10 |

Rows Extracted by cell rules:

11 | 12 |
13 | ); 14 | } 15 | -------------------------------------------------------------------------------- /pages/api/jsonbox/get-user.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | 4 | const { EMAILAPI_BASE_URL } = process.env; 5 | 6 | async function handle(req, res, resolve) { 7 | const { uid } = req.body; 8 | try { 9 | const { data } = await axios(`${EMAILAPI_BASE_URL}/users/${uid}`); 10 | res.json(data); 11 | } catch (e) { 12 | console.log(e); 13 | res.status(500).send(e); 14 | } 15 | return resolve(); 16 | } 17 | 18 | export default ensureAuth(handle); 19 | -------------------------------------------------------------------------------- /pages/api/jsonbox/put-user.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | 4 | const { EMAILAPI_BASE_URL } = process.env; 5 | 6 | async function handle(req, res, resolve) { 7 | const { uid, data } = req.body; 8 | try { 9 | await axios.put(`${EMAILAPI_BASE_URL}/users/${uid}`, data); 10 | res.json({}); 11 | } catch (e) { 12 | console.log(e); 13 | res.status(500).send(e); 14 | } 15 | return resolve(); 16 | } 17 | 18 | export default ensureAuth(handle); 19 | -------------------------------------------------------------------------------- /src/firebase/initFirebase.js: -------------------------------------------------------------------------------- 1 | import firebase from 'firebase/app'; 2 | import 'firebase/auth'; 3 | 4 | export default function initFirebase() { 5 | const config = { 6 | apiKey: process.env.NEXT_PUBLIC_FIREBASE_PUBLIC_API_KEY, 7 | authDomain: process.env.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN, 8 | databaseURL: process.env.NEXT_PUBLIC_FIREBASE_DATABASE_URL, 9 | projectId: process.env.NEXT_PUBLIC_FIREBASE_PROJECT_ID, 10 | }; 11 | 12 | if (!firebase.apps.length) { 13 | firebase.initializeApp(config); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /docker-compose-local.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | jsonbox: 5 | container_name: jsonbox 6 | build: ~/clones/jsonbox/. 7 | env_file: 8 | - ~/apps/jsonbox/.env 9 | ports: 10 | - '3001:3000' 11 | 12 | redis: 13 | image: 'bitnami/redis:latest' 14 | environment: 15 | - ALLOW_EMPTY_PASSWORD=yes 16 | 17 | emailapi: 18 | container_name: emailapi 19 | restart: always 20 | build: . 21 | ports: 22 | - '3000:3000' 23 | depends_on: 24 | - jsonbox 25 | - redis 26 | -------------------------------------------------------------------------------- /pages/api/dashboard/get-services.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | 4 | async function handle(req, res, resolve) { 5 | const { uid } = req.body; 6 | // [TODO] paginate this API 7 | const userServicesEndpoint = `${process.env.JSONBOX_NETWORK_URL}/${uid}/services?limit=100`; 8 | const userServicesResponse = await axios(userServicesEndpoint); 9 | const services = userServicesResponse.data; 10 | res.json(services); 11 | return resolve(); 12 | } 13 | 14 | export default ensureAuth(handle); 15 | -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | const nextEnv = require('next-env'); 2 | const withPlugins = require('next-compose-plugins'); 3 | const withCSS = require('@zeit/next-css'); 4 | const aliases = require('./alias-config'); 5 | 6 | const withNextEnv = nextEnv(); 7 | 8 | const nextConfig = { 9 | webpack: (config) => { 10 | const { alias } = config.resolve; 11 | config.resolve.alias = { // eslint-disable-line 12 | ...alias, 13 | ...aliases, 14 | }; 15 | return config; 16 | }, 17 | }; 18 | 19 | module.exports = withPlugins([withNextEnv, withCSS], nextConfig); 20 | -------------------------------------------------------------------------------- /src/isomorphic/ensureConfiguration.js: -------------------------------------------------------------------------------- 1 | const ensureConfiguration = (data, config) => { 2 | if (Object.keys(data).every((key) => !data[key])) { 3 | // ensure that if no key contains data, then you return false 4 | return false; 5 | } 6 | 7 | if (!config.fields.length) { 8 | return false; 9 | } 10 | 11 | return config.fields 12 | .filter((field) => field) 13 | .every((field) => { 14 | if (!data[field.fieldKey]) { 15 | return false; 16 | } 17 | return true; 18 | }); 19 | }; 20 | 21 | export default ensureConfiguration; 22 | -------------------------------------------------------------------------------- /pages/api/webhooks/credit-cards/amex.js: -------------------------------------------------------------------------------- 1 | import format from 'date-fns/format'; 2 | 3 | const inrToNumber = (str) => Number(str.replace(/[^0-9.-]+/g, '')); 4 | 5 | export default async function handle(req, res) { 6 | const { rows, header } = req.body; 7 | 8 | const updatedRowsData = rows.map((item) => ({ 9 | 'Last Statement Balance': inrToNumber( 10 | item['Last Statement Balance'].split('closing date')[0].replace('Rs.'), 11 | ), 12 | 'Email Date': format(new Date(item['Email Date']), 'yyyy/MM/dd'), 13 | })); 14 | 15 | res.json({ rows: updatedRowsData, header }); 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # local env files 28 | .env.local 29 | .env.development.local 30 | .env.test.local 31 | .env.production.local 32 | .env.docker.local 33 | 34 | # vercel 35 | .vercel 36 | google-service-account.json 37 | 38 | .vscode 39 | .do -------------------------------------------------------------------------------- /components/admin/email/fns/generateUniqueId.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-bitwise */ 2 | function createUniqueID() { 3 | let dt = new Date().getTime(); 4 | const uuid = 'xxyxxxxxxyxxxxxyxxxx'.replace(/[xy]/g, (c) => { 5 | const r = (dt + Math.random() * 16) % 16 | 0; 6 | dt = Math.floor(dt / 16); 7 | return (c === 'x' ? r : (r & 0x3) | 0x8).toString(16); 8 | }); 9 | return uuid; 10 | } 11 | 12 | function generateUniqueId(prefix = '') { 13 | if (prefix.length > 4) 14 | return new Error('prefix length cannot be more than 4'); 15 | return `${prefix}${createUniqueID()}`; 16 | } 17 | 18 | module.exports = generateUniqueId; 19 | -------------------------------------------------------------------------------- /src/auth.js: -------------------------------------------------------------------------------- 1 | const { google } = require('googleapis'); 2 | 3 | function authorize(refreshToken, callback) { 4 | // [TODO] add error handling 5 | const oAuth2Client = new google.auth.OAuth2( 6 | process.env.NEXT_PUBLIC_GOOGLE_CLIENT_ID, 7 | process.env.GOOGLE_CLIENT_SECRET, 8 | process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI, 9 | ); 10 | oAuth2Client.setCredentials({ 11 | refresh_token: refreshToken, 12 | }); 13 | callback(oAuth2Client); 14 | } 15 | 16 | module.exports = (refreshToken) => 17 | new Promise((resolve) => { 18 | return authorize(refreshToken, (auth) => { 19 | resolve(auth); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /pages/api/webhooks/credit-cards/dcb.js: -------------------------------------------------------------------------------- 1 | import format from 'date-fns/format'; 2 | 3 | const inrToNumber = (str) => Number(str.replace(/[^0-9.-]+/g, '')); 4 | 5 | export default async function handle(req, res) { 6 | const { rows, header } = req.body; 7 | 8 | const updatedRowsData = rows.map((item) => ({ 9 | 'Total Amount Due': inrToNumber(item['Total Amount Due']), 10 | 'Min Amount Due': inrToNumber(item['Min Amount Due']), 11 | 'Due Date': item['Due Date'].split('-').reverse().join('/'), 12 | 'Email Date': format(new Date(item['Email Date']), 'yyyy/MM/dd'), 13 | })); 14 | 15 | res.json({ rows: updatedRowsData, header }); 16 | } 17 | -------------------------------------------------------------------------------- /src/integrations/utils/index.js: -------------------------------------------------------------------------------- 1 | const log = (obj) => 2 | typeof obj === 'object' 3 | ? console.log(JSON.stringify(obj, null, 2)) 4 | : console.log(obj); 5 | 6 | const isLengthyArray = (arrayLike) => 7 | Array.isArray(arrayLike) && arrayLike.length; 8 | 9 | const getSimulationOptions = (query = {}) => { 10 | const { 11 | dry_run: dryRun = '0', 12 | emulate_first_run: emulateFirstRun = '0', 13 | } = query; 14 | 15 | const isDryRun = !!Number(dryRun); 16 | const isEmulateFirstRun = !!Number(emulateFirstRun); 17 | 18 | return { isDryRun, isEmulateFirstRun }; 19 | }; 20 | 21 | module.exports = { 22 | log, 23 | isLengthyArray, 24 | getSimulationOptions, 25 | }; 26 | -------------------------------------------------------------------------------- /src/queues/send-email.js: -------------------------------------------------------------------------------- 1 | import queues from '~/src/redis-queue'; 2 | import { sendEmail } from '~/src/integrations/email/mailgun'; 3 | import Sentry from '~/src/sentry'; 4 | 5 | async function processJob(job) { 6 | const { data: emailOptions } = job; 7 | 8 | try { 9 | await sendEmail({ 10 | from: `emailapi.io <${process.env.NEXT_PUBLIC_SENDING_EMAIL_ID}>`, 11 | ...emailOptions, 12 | }); 13 | } catch (e) { 14 | Sentry.captureException(e); 15 | Promise.reject(e); 16 | } 17 | } 18 | 19 | (() => { 20 | queues.sendEmailQueue.process(async (job) => { 21 | console.log('processing sendEmailQueue job#', job.id); 22 | await processJob(job); 23 | }); 24 | })(); 25 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: aakashgoel 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: ['buymeacoffee.com/aakashgoel', 'paypal.me/aakashlpin'] 13 | -------------------------------------------------------------------------------- /pages/api/firebase/exchange-token.js: -------------------------------------------------------------------------------- 1 | const { google } = require('googleapis'); 2 | 3 | export default async function exchangeToken(req, res) { 4 | const { token } = req.query; 5 | const oAuth2Client = new google.auth.OAuth2( 6 | process.env.NEXT_PUBLIC_GOOGLE_CLIENT_ID, 7 | process.env.GOOGLE_CLIENT_SECRET, 8 | process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI, 9 | ); 10 | 11 | return new Promise((resolve) => { 12 | oAuth2Client.getToken(token, (err, tokenResponse) => { 13 | if (err) res.status(500).json(err); 14 | 15 | // eslint-disable-next-line camelcase 16 | const { id_token, refresh_token } = tokenResponse; 17 | res.json({ id_token, refresh_token }); 18 | resolve(); 19 | }); 20 | }); 21 | } 22 | -------------------------------------------------------------------------------- /pages/api/user/delete.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | 4 | const { EMAILAPI_BASE_URL } = process.env; 5 | 6 | async function handle(req, res, resolve) { 7 | const { uid: id } = req.body; 8 | try { 9 | await axios.delete(`${EMAILAPI_BASE_URL}/users/${id}`); 10 | try { 11 | const attemptedData = await axios.get(`${EMAILAPI_BASE_URL}/users/${id}`); 12 | res.status(500).json({ couldNotDelete: true, yourData: attemptedData }); 13 | } catch (e) { 14 | res.json({}); 15 | } 16 | return resolve(); 17 | } catch (e) { 18 | console.log(e); 19 | res.status(500).send(e); 20 | return resolve(); 21 | } 22 | } 23 | 24 | export default ensureAuth(handle); 25 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | jsonbox: 5 | container_name: jsonbox 6 | image: 'docker.pkg.github.com/aakashlpin/jsonbox/jsonbox:latest' 7 | env_file: 8 | - ~/apps/jsonbox/.env 9 | ports: 10 | - '3001:3000' 11 | 12 | redis: 13 | image: 'bitnami/redis:latest' 14 | volumes: 15 | - /tmp/redis/data:/bitnami/redis/data 16 | environment: 17 | - ALLOW_EMPTY_PASSWORD=yes 18 | 19 | emailapi: 20 | container_name: emailapi 21 | restart: always 22 | image: 'docker.pkg.github.com/aakashlpin/emailapi/emailapi:latest' 23 | env_file: 24 | - ~/apps/emailapi-pipeline/docker.env 25 | volumes: 26 | - /tmp:/tmp 27 | ports: 28 | - '3000:3000' 29 | depends_on: 30 | - jsonbox 31 | - redis 32 | -------------------------------------------------------------------------------- /src/firebase/logout.js: -------------------------------------------------------------------------------- 1 | import firebase from 'firebase/app'; 2 | import 'firebase/auth'; 3 | 4 | export default async () => { 5 | return firebase 6 | .auth() 7 | .signOut() 8 | .then(() => { 9 | // Sign-out successful. 10 | if (typeof window !== 'undefined') { 11 | // Remove the server-side rendered user data element. See: 12 | // https://github.com/zeit/next.js/issues/2252#issuecomment-353992669 13 | try { 14 | const elem = window.document.getElementById('__MY_AUTH_USER_INFO'); 15 | elem.parentNode.removeChild(elem); 16 | } catch (e) { 17 | console.error(e); 18 | } 19 | } 20 | return true; 21 | }) 22 | .catch((e) => { 23 | console.error(e); 24 | return false; 25 | }); 26 | }; 27 | -------------------------------------------------------------------------------- /src/queues/send-whatsapp.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import queues from '~/src/redis-queue'; 3 | import Sentry from '~/src/sentry'; 4 | 5 | const { WA_API_URI, WA_API_KEY } = process.env; 6 | 7 | async function processJob(job) { 8 | const { 9 | data: { path, args }, 10 | } = job; 11 | 12 | try { 13 | axios.post( 14 | `${WA_API_URI}${path}`, 15 | { 16 | args, 17 | }, 18 | { 19 | headers: { 20 | key: WA_API_KEY, 21 | }, 22 | }, 23 | ); 24 | } catch (e) { 25 | Sentry.captureException(e); 26 | Promise.reject(e); 27 | } 28 | } 29 | 30 | (() => { 31 | queues.sendWhatsAppQueue.process(async (job) => { 32 | console.log('processing sendWhatsAppQueue job#', job.id); 33 | await processJob(job); 34 | }); 35 | })(); 36 | -------------------------------------------------------------------------------- /src/integrations/email/mailgun.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/prefer-default-export */ 2 | const mailgun = require('mailgun-js'); 3 | 4 | const { MAILGUN_API_KEY } = process.env; 5 | const { MAILGUN_DOMAIN } = process.env; 6 | 7 | const mg = mailgun({ apiKey: MAILGUN_API_KEY, domain: MAILGUN_DOMAIN }); 8 | 9 | export async function sendEmail(opts) { 10 | return new Promise((resolve, reject) => { 11 | mg.messages().send(opts, (error, res) => { 12 | if (error) { 13 | console.log('error from mailgun api', error); 14 | return reject(new Error(error)); 15 | } 16 | if (!error && !res) { 17 | const ERR = 'something went wrong sending email...'; 18 | console.log(ERR); 19 | return reject(new Error(ERR)); 20 | } 21 | return resolve(); 22 | }); 23 | }); 24 | } 25 | -------------------------------------------------------------------------------- /src/integrations/google-spreadsheet/reset-sheet.js: -------------------------------------------------------------------------------- 1 | import Sentry from '~/src/sentry'; 2 | 3 | const { GoogleSpreadsheet } = require('google-spreadsheet'); 4 | 5 | export default async function resetSheet({ googleSheetId }) { 6 | try { 7 | const doc = new GoogleSpreadsheet(googleSheetId); 8 | 9 | // use service account creds 10 | await doc.useServiceAccountAuth({ 11 | client_email: process.env.GOOGLE_SERVICE_ACCOUNT_EMAIL, 12 | private_key: process.env.GOOGLE_PRIVATE_KEY.replace(/\\n/g, '\n'), 13 | }); 14 | 15 | await doc.loadInfo(); 16 | 17 | // add another sheet 18 | await doc.addSheet(); 19 | // delete the first one 20 | const sheet = doc.sheetsByIndex[0]; 21 | await sheet.delete(); 22 | } catch (e) { 23 | Sentry.captureException(e); 24 | Promise.reject(e); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/firebase/firebaseAdmin.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/prefer-default-export */ 2 | import * as admin from 'firebase-admin'; 3 | 4 | export const verifyIdToken = (token) => { 5 | const firebasePrivateKey = process.env.FIREBASE_PRIVATE_KEY; 6 | 7 | if (!admin.apps.length) { 8 | admin.initializeApp({ 9 | credential: admin.credential.cert({ 10 | projectId: process.env.NEXT_PUBLIC_FIREBASE_PROJECT_ID, 11 | clientEmail: process.env.FIREBASE_CLIENT_EMAIL, 12 | // https://stackoverflow.com/a/41044630/1332513 13 | privateKey: firebasePrivateKey.replace(/\\n/g, '\n'), 14 | }), 15 | databaseURL: process.env.NEXT_PUBLIC_FIREBASE_DATABASE_URL, 16 | }); 17 | } 18 | 19 | return admin 20 | .auth() 21 | .verifyIdToken(token) 22 | .catch((error) => { 23 | throw error; 24 | }); 25 | }; 26 | -------------------------------------------------------------------------------- /components/common/common-header.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { withRouter } from 'next/router'; 3 | import styled from 'styled-components'; 4 | 5 | const Header = styled.header` 6 | display: grid; 7 | grid-template-columns: 400px 1fr 600px; 8 | border-bottom: 1px solid rgba(0, 0, 0, 0.1); 9 | align-items: center; 10 | `; 11 | 12 | const Logo = styled.img` 13 | max-height: 24px; 14 | `; 15 | 16 | function CommonHeader({ children, router }) { 17 | return ( 18 |
19 |
20 | 21 |
22 |
{children}
23 |
24 | Dashboard 25 |
26 |
27 | ); 28 | } 29 | 30 | export default withRouter(CommonHeader); 31 | -------------------------------------------------------------------------------- /docker-compose-dev.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | jsonbox_dev: 5 | container_name: jsonbox_dev 6 | image: 'docker.pkg.github.com/aakashlpin/jsonbox/jsonbox:latest' 7 | env_file: 8 | - ~/apps/jsonbox/dev.env 9 | ports: 10 | - '4001:3000' 11 | 12 | redis_dev: 13 | container_name: redis_dev 14 | image: 'bitnami/redis:latest' 15 | volumes: 16 | - /tmp/dev/redis/data:/bitnami/redis/data 17 | environment: 18 | - ALLOW_EMPTY_PASSWORD=yes 19 | 20 | emailapi_dev: 21 | container_name: emailapi_dev 22 | restart: always 23 | image: 'docker.pkg.github.com/aakashlpin/emailapi/emailapi:dev' 24 | env_file: 25 | - ~/apps/emailapi-pipeline/dev.env 26 | volumes: 27 | - /tmp:/tmp 28 | ports: 29 | - '4000:3000' 30 | depends_on: 31 | - jsonbox_dev 32 | - redis_dev 33 | -------------------------------------------------------------------------------- /src/firebase/firebaseSessionHandler.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/prefer-default-export */ 2 | // From: 3 | // https://github.com/zeit/next.js/blob/canary/examples/with-firebase-authentication/pages/index.js 4 | 5 | import fetch from 'isomorphic-unfetch'; 6 | 7 | export const setSession = (user) => { 8 | // Log in. 9 | if (user) { 10 | return user.getIdToken().then((token) => { 11 | return fetch('/api/firebase/login', { 12 | method: 'POST', 13 | // eslint-disable-next-line no-undef 14 | headers: new Headers({ 'Content-Type': 'application/json' }), 15 | credentials: 'same-origin', 16 | body: JSON.stringify({ token, uid: user.uid }), 17 | }); 18 | }); 19 | } 20 | 21 | // Log out. 22 | return fetch('/api/firebase/logout', { 23 | method: 'POST', 24 | credentials: 'same-origin', 25 | }); 26 | }; 27 | -------------------------------------------------------------------------------- /pages/api/integrations/google-spreadsheet/preview.js: -------------------------------------------------------------------------------- 1 | import ensureAuth from '~/src/middleware/ensureAuth'; 2 | import queues from '~/src/redis-queue'; 3 | 4 | require('~/src/queues'); 5 | 6 | async function handle(req, res, resolve) { 7 | const { 8 | data_endpoint: dataEndpoint, 9 | gsheet_id: googleSheetId, 10 | uid, 11 | token, 12 | } = req.body; 13 | 14 | if (!googleSheetId) { 15 | res.status(500).json({ code: 'gsheet_id is required prop!' }); 16 | return resolve(); 17 | } 18 | 19 | const { user } = req; 20 | const userProps = { 21 | uid, 22 | user, 23 | token, 24 | refreshToken: req.refresh_token, 25 | }; 26 | 27 | console.log({ dataEndpoint }); 28 | 29 | queues.gSheetSyncQueue.add({ 30 | userProps, 31 | dataEndpoint, 32 | googleSheetId, 33 | }); 34 | 35 | res.json({ status: 'ok' }); 36 | 37 | return resolve(); 38 | } 39 | 40 | export default ensureAuth(handle); 41 | -------------------------------------------------------------------------------- /components/admin/email/fns/fullPath.js: -------------------------------------------------------------------------------- 1 | export default function fullPath(el, stopProps = {}) { 2 | const names = []; 3 | while ( 4 | el.parentNode && 5 | (stopProps.className 6 | ? !el.parentNode.classList.contains(stopProps.className) 7 | : true) 8 | ) { 9 | if (el.id) { 10 | names.unshift(`#${el.id}`); 11 | break; 12 | } else { 13 | // eslint-disable-next-line 14 | if (el == el.ownerDocument.documentElement) 15 | names.unshift(el.tagName.toLowerCase()); 16 | else { 17 | for ( 18 | // eslint-disable-next-line 19 | var c = 1, e = el; 20 | e.previousElementSibling; 21 | e = e.previousElementSibling, c++ // eslint-disable-line 22 | ); 23 | names.unshift(`${el.tagName.toLowerCase()}:nth-child(${c})`); // eslint-disable-line 24 | } 25 | el = el.parentNode; // eslint-disable-line 26 | } 27 | } 28 | return names.join(' > '); 29 | } 30 | -------------------------------------------------------------------------------- /pages/api/integrations/whatsapp/subscriber.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | 3 | const { WA_DATABASE_URI } = process.env; 4 | 5 | export default async function handle(req, res) { 6 | const { event, data } = req.body; 7 | console.log(data); 8 | if (event !== 'message') { 9 | return res.json({}); 10 | } 11 | 12 | const { from, sender, body } = data; 13 | console.log({ from }); 14 | try { 15 | const { data: userFromDb } = await axios( 16 | `${WA_DATABASE_URI}/senders?q=from:${from}`, 17 | ); 18 | if (userFromDb.length) { 19 | console.log(`existing user whatsapp request from ${from} — "${body}"`); 20 | return res.json({}); 21 | } 22 | await axios.post(`${WA_DATABASE_URI}/senders`, { 23 | from, 24 | sender, 25 | }); 26 | console.log( 27 | `successfully synced new whatsapp request from ${from} — "${body}"`, 28 | ); 29 | return res.json({}); 30 | } catch (e) { 31 | console.log(e); 32 | return res.status(500).send(e); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/integrations/google-spreadsheet/sync.js: -------------------------------------------------------------------------------- 1 | import Sentry from '~/src/sentry'; 2 | 3 | const { GoogleSpreadsheet } = require('google-spreadsheet'); 4 | 5 | export default async function sync({ 6 | googleSheetId, 7 | sheetHeader, 8 | sheetRows, 9 | sheetName, 10 | }) { 11 | try { 12 | const doc = new GoogleSpreadsheet(googleSheetId); 13 | 14 | // use service account creds 15 | await doc.useServiceAccountAuth({ 16 | client_email: process.env.GOOGLE_SERVICE_ACCOUNT_EMAIL, 17 | private_key: process.env.GOOGLE_PRIVATE_KEY.replace(/\\n/g, '\n'), 18 | }); 19 | 20 | await doc.loadInfo(); 21 | 22 | const sheet = doc.sheetsByIndex[0]; // or use doc.sheetsById[id] 23 | if (sheetName) { 24 | sheet.updateProperties({ 25 | title: sheetName, 26 | }); 27 | } 28 | 29 | await sheet.setHeaderRow(sheetHeader); 30 | await sheet.addRows(sheetRows); 31 | } catch (e) { 32 | console.log(e); 33 | Sentry.captureException(e); 34 | Promise.reject(e); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pages/api/integrations/whatsapp/[path].js: -------------------------------------------------------------------------------- 1 | // import axios from 'axios'; 2 | // import { genericErrorHandler } from '~/src/utils'; 3 | import queues from '~/src/redis-queue'; 4 | 5 | require('~/src/queues'); 6 | 7 | const { WA_SELF_NUMBER, INLOOPWITH_API_KEY } = process.env; 8 | 9 | export default async function handle(req, res) { 10 | const API_KEY = req.headers['x-ilw-api-key']; 11 | if (INLOOPWITH_API_KEY !== API_KEY) { 12 | return res.status(401).send('Unauthorized'); 13 | } 14 | 15 | const { path } = req.query; 16 | 17 | const supportedPaths = ['sendText', 'sendLinkWithAutoPreview']; 18 | 19 | if (!supportedPaths.includes(path)) { 20 | return res.status(400).json({ error: `route not supported` }); 21 | } 22 | 23 | const { body } = req.body; 24 | if (!(typeof body === 'string' && body.length)) { 25 | return res.status(400).json({ error: `missing body` }); 26 | } 27 | 28 | queues.sendWhatsAppQueue.add({ 29 | path: `/${path}`, 30 | args: [WA_SELF_NUMBER, body], 31 | }); 32 | 33 | return res.json({}); 34 | } 35 | -------------------------------------------------------------------------------- /public/vercel.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 EmailAPI 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /pages/api/fetch/tables-from-attachment.js: -------------------------------------------------------------------------------- 1 | import ensureAuth from '~/src/middleware/ensureAuth'; 2 | import extractTableInJson from '~/src/pdf/extract-tables'; 3 | import savePdfAttachmentToDisk from '~/src/pdf/create-file'; 4 | import { fetchAttachment } from '~/src/gmail'; 5 | 6 | async function handle(req, res, resolve) { 7 | try { 8 | const { 9 | attachmentId, 10 | messageId, 11 | attachmentPassword, 12 | camelotMethod, 13 | camelotScale, 14 | } = req.body; 15 | 16 | const base64Encoded = await fetchAttachment({ 17 | attachmentId, 18 | messageId, 19 | refreshToken: req.refresh_token, 20 | }); 21 | 22 | const pdfDiskPath = await savePdfAttachmentToDisk(base64Encoded); 23 | const extractedTables = await extractTableInJson(pdfDiskPath, { 24 | stream: camelotMethod === 'stream', 25 | scale: camelotScale, 26 | password: attachmentPassword, 27 | }); 28 | 29 | res.json(extractedTables); 30 | } catch (e) { 31 | console.log(e); 32 | res.status(500).send(e); 33 | } 34 | resolve(); 35 | } 36 | 37 | export default ensureAuth(handle); 38 | -------------------------------------------------------------------------------- /pages/api/integrations/whatsapp/checker.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | 4 | const { WA_DATABASE_URI } = process.env; 5 | 6 | async function handle(req, res, resolve) { 7 | const { waPhoneNumber } = req.body; 8 | if (!waPhoneNumber) { 9 | res.status(401).json({ error: 'missing param waPhoneNumber' }); 10 | return resolve(); 11 | } 12 | 13 | try { 14 | const { data: userFromDb } = await axios( 15 | // q=name:*ya* 16 | `${WA_DATABASE_URI}/senders?q=from:*${waPhoneNumber}*`, 17 | ); 18 | if (userFromDb.length > 1) { 19 | res.status(500).json({ 20 | error: `DB BOTCHED! Mulitple entries for same phone number. endpoint — "${WA_DATABASE_URI}/senders?q=from:*${waPhoneNumber}"`, 21 | }); 22 | return resolve(); 23 | } 24 | if (userFromDb.length === 1) { 25 | res.json(userFromDb); 26 | return resolve(); 27 | } 28 | res.status(200).json([]); 29 | return resolve(); 30 | } catch (e) { 31 | console.log(e); 32 | res.status(500).send(e); 33 | return resolve(); 34 | } 35 | } 36 | 37 | export default ensureAuth(handle); 38 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | es6: true, 5 | node: true, 6 | }, 7 | parser: 'babel-eslint', 8 | extends: ['airbnb', 'prettier', 'prettier/react'], 9 | globals: { 10 | Atomics: 'readonly', 11 | SharedArrayBuffer: 'readonly', 12 | }, 13 | parserOptions: { 14 | ecmaFeatures: { 15 | jsx: true, 16 | modules: true, 17 | }, 18 | ecmaVersion: 2018, 19 | sourceType: 'module', 20 | }, 21 | plugins: ['react', 'prettier'], 22 | rules: { 23 | 'prettier/prettier': 'error', 24 | 'react/jsx-filename-extension': [1, { extensions: ['.js', '.jsx'] }], 25 | 'react/forbid-prop-types': [0, { forbid: ['any'] }], 26 | 'react/prop-types': 0, 27 | 'no-console': 0, 28 | 'no-debugger': 0, 29 | 'no-nested-ternary': 0, 30 | 'no-underscore-dangle': 0, 31 | 'global-require': 0, 32 | 'react/no-array-index-key': 0, 33 | 'jsx-a11y/no-static-element-interactions': 0, 34 | 'jsx-a11y/click-events-have-key-events': 0, 35 | }, 36 | settings: { 37 | 'import/resolver': { 38 | webpack: { 39 | config: './webpack.config.js', 40 | }, 41 | }, 42 | }, 43 | }; 44 | -------------------------------------------------------------------------------- /pages/api/email-search/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable consistent-return */ 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | import { genericErrorHandler } from '~/src/utils'; 4 | import { fetchEmails, processMessageBody } from '~/src/gmail'; 5 | 6 | async function handle(req, res, resolve) { 7 | const { 8 | query, 9 | nextPageToken = '', 10 | gmail_search_props: gmailSearchProps = {}, 11 | has_attachment: hasAttachment = false, 12 | } = req.body; 13 | 14 | try { 15 | const fetchEmailsRes = await fetchEmails( 16 | query, 17 | req.refresh_token, 18 | nextPageToken, 19 | gmailSearchProps, 20 | hasAttachment, 21 | ); 22 | const { emails, nextPageToken: resPageToken } = fetchEmailsRes || {}; 23 | if (Array.isArray(emails) && emails.length) { 24 | const items = emails 25 | .map((response) => processMessageBody(response.data)) 26 | .filter((item) => item) 27 | .filter((item) => item.date); 28 | 29 | res.json({ emails: items, nextPageToken: resPageToken }); 30 | } else { 31 | res.json({ emails: [] }); 32 | } 33 | } catch (e) { 34 | genericErrorHandler(e); 35 | res.status(500).json({}); 36 | } 37 | resolve(); 38 | } 39 | 40 | export default ensureAuth(handle); 41 | -------------------------------------------------------------------------------- /components/service-creator/header.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import styled from 'styled-components'; 3 | import CommonHeader from '~/components/common/common-header'; 4 | 5 | const InputSearch = styled.input.attrs({ 6 | className: 'w-full px-4 py-2 border border-gray-400 border-solid rounded', 7 | })` 8 | opacity: ${(props) => (props.disabled ? 0.4 : 1)}; 9 | `; 10 | 11 | const Header = ({ 12 | serviceId, 13 | searchInput, 14 | setTriggerSearch, 15 | isLoading, 16 | isServiceIdFetched, 17 | handleChangeSearchInput, 18 | }) => ( 19 | 20 | {serviceId ? ( 21 | searchInput ? ( 22 |
“{searchInput}”
23 | ) : null 24 | ) : ( 25 |
{ 28 | e.preventDefault(); 29 | setTriggerSearch(true); 30 | }} 31 | > 32 | handleChangeSearchInput(e.target.value)} 37 | placeholder="Search query from Google..." 38 | /> 39 | 40 | )} 41 |
42 | ); 43 | 44 | export default Header; 45 | -------------------------------------------------------------------------------- /src/firebase/hooks.js: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react'; 2 | import firebase from 'firebase/app'; 3 | import 'firebase/auth'; 4 | import initFirebase from './initFirebase'; 5 | import { setSession } from './firebaseSessionHandler'; 6 | import { createAuthUserInfo } from './user'; 7 | 8 | initFirebase(); 9 | 10 | // https://benmcmahen.com/using-firebase-with-react-hooks/ 11 | 12 | // Defaults to empty AuthUserInfo object. 13 | export const AuthUserInfoContext = React.createContext(createAuthUserInfo()); 14 | 15 | export const useAuthUserInfo = () => { 16 | return React.useContext(AuthUserInfoContext); 17 | }; 18 | 19 | // Returns a Firebase JS SDK user object. 20 | export const useFirebaseAuth = () => { 21 | const [state, setState] = useState(() => { 22 | const user = firebase.auth().currentUser; 23 | return { 24 | initializing: !user, 25 | user, 26 | }; 27 | }); 28 | 29 | function onChange(user) { 30 | setState({ initializing: false, user }); 31 | 32 | // Call server to update session. 33 | setSession(user); 34 | } 35 | 36 | useEffect(() => { 37 | // Listen for auth state changes. 38 | const unsubscribe = firebase.auth().onAuthStateChanged(onChange); 39 | 40 | // Unsubscribe to the listener when unmounting. 41 | return () => unsubscribe(); 42 | }, []); 43 | 44 | return state; 45 | }; 46 | -------------------------------------------------------------------------------- /components/ft/attachment-unlocker/email-preview.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import styled from 'styled-components'; 3 | 4 | const normalizeHtmlWhitespace = require('normalize-html-whitespace'); 5 | 6 | const EmailContainer = styled.div.attrs({ 7 | className: 'pt-4 p-8 pb-16', 8 | })` 9 | overflow-y: scroll; 10 | `; 11 | 12 | const EmailPreview = ({ showPreview, messageItem }) => { 13 | if (!showPreview) { 14 | return null; 15 | } 16 | 17 | if (!messageItem) { 18 | return ( 19 | Select an email on the left to preview 20 | ); 21 | } 22 | 23 | const { isHtmlContent = true, message } = messageItem; 24 | 25 | return ( 26 | 27 |
/g, // remove global styles sent in email. styles are always inlined anyways 34 | '', 35 | ) 36 | .replace(//g, '') 37 | .replace(//g, ''), 38 | ) 39 | : `
${message}
`, 40 | }} 41 | onClick={(e) => e.preventDefault()} 42 | /> 43 | 44 | ); 45 | }; 46 | 47 | export default EmailPreview; 48 | -------------------------------------------------------------------------------- /Dockerfile-local: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | # https://rtfm.co.ua/en/docker-configure-tzdata-and-timezone-during-build/ 3 | ENV TZ=Asia/Kolkata 4 | ENV LANG C.UTF-8 5 | ENV LC_ALL C.UTF-8 6 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 7 | RUN apt-get update -y 8 | 9 | # Install Node.js 10 | RUN apt-get install -y curl 11 | RUN apt-get install -y gnupg-agent 12 | RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - 13 | RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list 14 | RUN curl --silent --location https://deb.nodesource.com/setup_12.x | bash - 15 | RUN apt-get install -y nodejs 16 | RUN apt-get install -y build-essential 17 | RUN apt install -y yarn 18 | RUN node --version 19 | RUN yarn --version 20 | 21 | RUN apt-get install -y qpdf 22 | RUN apt-get install -y software-properties-common 23 | RUN add-apt-repository ppa:deadsnakes/ppa 24 | RUN apt-get update 25 | RUN apt-get install -y python3.8 26 | RUN python3 --version 27 | RUN apt-get install -y python3-tk ghostscript 28 | RUN gs -version 29 | RUN apt-get install -y python3-pip 30 | RUN apt-get install -y libsm6 libxext6 libxrender-dev 31 | RUN pip3 install 'opencv-python==4.2.0.34' 32 | RUN pip3 install "camelot-py[cv]" 33 | 34 | WORKDIR /codebase 35 | COPY package.json ./ 36 | COPY yarn.lock ./ 37 | COPY . ./ 38 | COPY .env.docker.local ./.env.local 39 | RUN yarn && yarn build 40 | EXPOSE 3000 41 | CMD ["yarn", "start"] -------------------------------------------------------------------------------- /src/firebase/user.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Take the user object from Firebase (from either the Firebase admin SDK or 3 | * or the client-side Firebase JS SDK) and return a consistent AuthUser object. 4 | * @param {Object} firebaseUser - A decoded Firebase user token or JS SDK 5 | * Firebase user object. 6 | * @return {Object|null} AuthUser - The user object. 7 | * @return {String} AuthUser.id - The user's ID 8 | * @return {String} AuthUser.email - The user's email 9 | * @return {Boolean} AuthUser.emailVerified - Whether the user has verified their email 10 | */ 11 | export const createAuthUser = (firebaseUser) => { 12 | if (!firebaseUser || !firebaseUser.uid) { 13 | return null; 14 | } 15 | return firebaseUser; 16 | }; 17 | 18 | /** 19 | * Create an object with an AuthUser object and AuthUserToken value. 20 | * @param {Object} firebaseUser - A decoded Firebase user token or JS SDK 21 | * Firebase user object. 22 | * @param {String} firebaseToken - A Firebase auth token string. 23 | * @return {Object|null} AuthUserInfo - The auth user info object. 24 | * @return {String} AuthUserInfo.AuthUser - An AuthUser object (see 25 | * `createAuthUser` above). 26 | * @return {String} AuthUser.token - The user's encoded Firebase token. 27 | */ 28 | export const createAuthUserInfo = ({ 29 | firebaseUser = null, 30 | token = null, 31 | uid = null, 32 | } = {}) => { 33 | return { 34 | AuthUser: createAuthUser(firebaseUser), 35 | token, 36 | uid, 37 | }; 38 | }; 39 | -------------------------------------------------------------------------------- /components/ft/attachment-unlocker/action-bar.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import styled from 'styled-components'; 3 | import { format } from 'date-fns'; 4 | 5 | import { Button, Label, FlexEnds } from '~/components/common/Atoms'; 6 | 7 | const Nudges = styled.div.attrs({ 8 | className: 'border-b', 9 | })` 10 | display: grid; 11 | grid-template-columns: 400px 1fr 600px; 12 | align-items: end; 13 | padding: 0.5rem; 14 | `; 15 | 16 | const ActionBar = ({ 17 | isLoading, 18 | searchResults, 19 | nextPageToken, 20 | handleFetchMoreMails, 21 | }) => ( 22 | 23 |
24 | 25 |
26 | 27 | {searchResults.length} /{' '} 28 | {searchResults.length 29 | ? format( 30 | new Date(searchResults[searchResults.length - 1].date), 31 | 'dd MMM yyyy', 32 | ) 33 | : 'Loading...'} 34 |
35 |
36 | 42 |
43 | 44 |
45 |
Original Email
46 |
47 | Attachment Unlocker Settings 48 |
49 | 50 | ); 51 | 52 | export default ActionBar; 53 | -------------------------------------------------------------------------------- /pages/api/apps/email-to-json/webhook.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | 4 | const GOOGLE_OAUTH_REDIRECT_URI = 5 | process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI; 6 | const EMAILAPI_DOMAIN = process.env.NEXT_PUBLIC_EMAILAPI_DOMAIN; 7 | 8 | async function handle(req, res) { 9 | const { apiId, serviceEndpoint, success, uid } = req.body; 10 | const { data: serviceData } = await axios(serviceEndpoint); 11 | const { data = [] } = serviceData; 12 | const { refresh_token: refreshToken } = req; 13 | 14 | const updatedServiceData = [ 15 | ...data, 16 | { 17 | id: apiId, 18 | is_successful: success, 19 | _isReadyOn: new Date().toISOString(), 20 | }, 21 | ]; 22 | 23 | await axios.put(serviceEndpoint, { 24 | ...serviceData, 25 | data: updatedServiceData, 26 | }); 27 | 28 | // this is where you can sync to other integrations 29 | // as data in our own store is now persisted 30 | 31 | if (serviceData.gsheet_id && success) { 32 | console.log('[bg] gsheet_id exists; syncing now'); 33 | try { 34 | await axios.post( 35 | `${GOOGLE_OAUTH_REDIRECT_URI}/api/integrations/google-spreadsheet`, 36 | { 37 | uid, 38 | refresh_token: refreshToken, 39 | service_id: serviceData._id, 40 | data_endpoint: `${EMAILAPI_DOMAIN}/${uid}/${apiId}`, 41 | }, 42 | ); 43 | } catch (e) { 44 | console.log(e); 45 | } 46 | } 47 | 48 | res.json({}); 49 | } 50 | 51 | export default ensureAuth(handle); 52 | -------------------------------------------------------------------------------- /src/apps/utils.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | 3 | const findLastIndex = require('lodash/findLastIndex'); 4 | 5 | export const getAfterTs = (ts) => parseInt(new Date(ts).getTime() / 1000, 10); 6 | 7 | // space is intentional 8 | const excludeFilter = ` -from:${process.env.NEXT_PUBLIC_SENDING_EMAIL_ID}`; 9 | 10 | /* eslint-disable import/prefer-default-export */ 11 | export async function getSearchQuery({ serviceEndpoint, newOnly = false }) { 12 | const { data: serviceData } = await axios(serviceEndpoint); 13 | const { data } = serviceData; 14 | let { search_query: searchQuery } = serviceData; 15 | if (!newOnly) { 16 | return `${searchQuery}${excludeFilter}`; 17 | } 18 | 19 | const hasData = Array.isArray(data) && data.length; 20 | let foundTsFromExistingData = false; 21 | 22 | if (hasData) { 23 | const lastSuccessfulDataEntry = findLastIndex( 24 | data, 25 | (item) => item.is_successful, 26 | ); 27 | 28 | if (lastSuccessfulDataEntry >= 0) { 29 | const lastProcessingTimestamp = getAfterTs( 30 | data[lastSuccessfulDataEntry]._isReadyOn, 31 | ); 32 | 33 | searchQuery = `${searchQuery} after:${lastProcessingTimestamp}`; 34 | foundTsFromExistingData = true; 35 | } 36 | } 37 | 38 | if (!foundTsFromExistingData) { 39 | // if service got created without past data 40 | // then use after timestamp from the _createdOn timestamp of the db record 41 | searchQuery = `${searchQuery} after:${getAfterTs(serviceData._createdOn)}`; 42 | } 43 | 44 | return `${searchQuery}${excludeFilter}`; 45 | } 46 | -------------------------------------------------------------------------------- /src/middleware/cookieSession.js: -------------------------------------------------------------------------------- 1 | import cookieSession from 'cookie-session'; 2 | 3 | const { SESSION_SECRET_CURRENT = 'current' } = process.env; 4 | const { SESSION_SECRET_PREVIOUS = 'previous' } = process.env; 5 | 6 | export const addSession = (req, res) => { 7 | // Ensure that session secrets are set. 8 | if (!(SESSION_SECRET_CURRENT && SESSION_SECRET_PREVIOUS)) { 9 | throw new Error( 10 | 'Session secrets must be set as env vars `SESSION_SECRET_CURRENT` and `SESSION_SECRET_PREVIOUS`.', 11 | ); 12 | } 13 | 14 | // An array is useful for rotating secrets without invalidating old sessions. 15 | // The first will be used to sign cookies, and the rest to validate them. 16 | // https://github.com/expressjs/cookie-session#keys 17 | const sessionSecrets = [SESSION_SECRET_CURRENT, SESSION_SECRET_PREVIOUS]; 18 | 19 | // Example: 20 | // https://github.com/billymoon/micro-cookie-session 21 | const includeSession = cookieSession({ 22 | keys: sessionSecrets, 23 | // TODO: set other options, such as "secure", "sameSite", etc. 24 | // https://github.com/expressjs/cookie-session#cookie-options 25 | maxAge: 604800000, // week 26 | httpOnly: true, 27 | overwrite: true, 28 | }); 29 | includeSession(req, res, () => {}); 30 | }; 31 | 32 | export default (handler) => (req, res) => { 33 | return new Promise((resolve) => { 34 | try { 35 | addSession(req, res); 36 | } catch (e) { 37 | res.status(500).json({ error: 'Could not get user session.' }); 38 | return resolve(); 39 | } 40 | return handler(req, res, resolve); 41 | }); 42 | }; 43 | -------------------------------------------------------------------------------- /src/queues/auto-unlock.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import Sentry from '~/src/sentry'; 3 | 4 | import queues from '../redis-queue'; 5 | 6 | const GOOGLE_OAUTH_REDIRECT_URI = 7 | process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI; 8 | 9 | async function processJob(jobData, done) { 10 | try { 11 | const { email, queueData } = jobData; 12 | const { 13 | unlockPassword, 14 | userProps: { uid, token }, 15 | } = queueData; 16 | 17 | /** 18 | * NB: 19 | * the code below sends 1 unique request per attachment per email 20 | * ideally assuming that there would be max 1-2 attachments/email 21 | */ 22 | const unlockRequests = [email] 23 | .reduce((accum, _email) => { 24 | return [ 25 | ...accum, 26 | ..._email.attachments.map((attachment) => ({ 27 | messageId: _email.messageId, 28 | attachmentId: attachment.id, 29 | filename: attachment.filename, 30 | })), 31 | ]; 32 | }, []) 33 | .map((apiProps) => 34 | axios.post( 35 | `${GOOGLE_OAUTH_REDIRECT_URI}/api/email-search/attachment-unlock`, 36 | { 37 | ...apiProps, 38 | uid, 39 | token, 40 | pdfPasswordInput: unlockPassword, 41 | }, 42 | ), 43 | ); 44 | 45 | await Promise.all(unlockRequests); 46 | } catch (e) { 47 | Sentry.captureException(e); 48 | } 49 | done(); 50 | } 51 | 52 | (() => { 53 | queues.autoUnlockQueue.process((job, done) => { 54 | console.log('processing autoUnlockQueue job#', job.id); 55 | processJob(job.data, done); 56 | }); 57 | })(); 58 | -------------------------------------------------------------------------------- /src/queues/notifications.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import queues from '~/src/redis-queue'; 3 | import Sentry from '~/src/sentry'; 4 | 5 | const normalizeHtmlWhitespace = require('normalize-html-whitespace'); 6 | 7 | async function sendEmail({ to, subject, body }) { 8 | queues.sendEmailQueue.add({ 9 | to, 10 | subject, 11 | html: normalizeHtmlWhitespace(body), 12 | }); 13 | } 14 | 15 | async function sendWebhook({ url, data, method }) { 16 | console.log('🔼 sendWebhook running:', { url, data, method }); 17 | try { 18 | const response = await axios[method.toLowerCase()](url, data); 19 | console.log('⬇️ sendWebhook success:', response.data); 20 | } catch (e) { 21 | console.log('🚨 sendWebhook error:', e); 22 | Sentry.captureException(e); 23 | } 24 | } 25 | 26 | async function processJob(job) { 27 | try { 28 | const { data: jobData } = job; 29 | const { type } = jobData; 30 | 31 | switch (type) { 32 | case 'email': { 33 | const { data: emailData } = jobData; 34 | await sendEmail(emailData); 35 | break; 36 | } 37 | 38 | case 'webhook': { 39 | const { data: webhookData } = jobData; 40 | await sendWebhook(webhookData); 41 | break; 42 | } 43 | 44 | default: { 45 | Promise.resolve(); 46 | break; 47 | } 48 | } 49 | } catch (e) { 50 | Sentry.captureException(e); 51 | console.log(e); 52 | } 53 | 54 | Promise.resolve(); 55 | } 56 | 57 | (() => { 58 | queues.notificationsQueue.process(async (job) => { 59 | console.log('processing notificationsQueue job#', job.id); 60 | await processJob(job); 61 | }); 62 | })(); 63 | -------------------------------------------------------------------------------- /pages/api/webhooks/zerodha-cn/index.js: -------------------------------------------------------------------------------- 1 | import { toArray, toObject } from '../../../../src/pdf/utils'; 2 | 3 | export default async function handle(req, res) { 4 | const inputData = req.body; 5 | const contractNote = inputData.find( 6 | (item) => item.rule.name === 'Contract note', 7 | ); 8 | /** 9 | * 10 | cn_no:'CNT-20/21-67361114' 11 | settlement_date:'' 12 | settlement_no:'' 13 | trade_date:'25/09/2020' 14 | */ 15 | 16 | const tradeDate = contractNote.data.contract_note.trade_date; 17 | 18 | const output = inputData.map((item) => { 19 | if (item.rule.name === 'Contract note') { 20 | return item; 21 | } 22 | 23 | const { data } = item; 24 | const processedData = Object.keys(data).reduce((accum, dataKey) => { 25 | const [headerRow, ...otherRows] = data[dataKey]; 26 | const headerRowArray = toArray(headerRow); 27 | const modifiedHeaderRow = toObject(['Trade Date', ...headerRowArray]); 28 | 29 | const modifiedOtherRows = otherRows.map((row) => { 30 | const rowArray = toArray(row); 31 | const modifiedRow = [tradeDate, ...rowArray]; 32 | const objectifiedRow = toObject(modifiedRow); 33 | if (item.rule.name !== 'Equity') { 34 | return objectifiedRow; 35 | } 36 | objectifiedRow[5] = objectifiedRow[5].split('/')[0].trim(); 37 | return objectifiedRow; 38 | }); 39 | 40 | return { 41 | ...accum, 42 | [dataKey]: [modifiedHeaderRow, ...modifiedOtherRows], 43 | }; 44 | }, {}); 45 | 46 | return { 47 | ...item, 48 | data: processedData, 49 | }; 50 | }); 51 | 52 | return res.json(output); 53 | } 54 | -------------------------------------------------------------------------------- /pages/api/cron/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * this endpoint receives recurring calls from an external source 3 | * with just 1 mandatory params - `uid` 4 | * 5 | * with uid we can create endpoints required to run cron jobs 6 | 7 | * 1. endpoint for user's db object 8 | * `${process.env.EMAILAPI_BASE_URL}/users/${uid}` 9 | * -> response contains `refreshToken` necessary to query gmail api 10 | * 11 | * 2. endpoint for user's services array 12 | * `${process.env.NEXT_PUBLIC_EMAILAPI_DOMAIN}/${uid}/services` 13 | * -> response contains array of objects with `{cron: true/false}` to identify services to run with cron 14 | */ 15 | import Sentry from '~/src/sentry'; 16 | 17 | const base64 = require('base-64'); 18 | const axios = require('axios'); 19 | 20 | export default async function handle(req, res) { 21 | const authHeader = req.headers.authorization; 22 | const encodedUsernamePassword = authHeader.replace('Basic', '').trim(); 23 | const decodedUsernamePassword = base64.decode(encodedUsernamePassword); 24 | const [uid] = decodedUsernamePassword.split(':'); 25 | 26 | try { 27 | // if valid user initiated request, then continue 28 | await axios(`${process.env.EMAILAPI_BASE_URL}/users/${uid}`); 29 | 30 | const { data: users } = await axios( 31 | `${process.env.EMAILAPI_BASE_URL}/users?limit=100`, 32 | ); 33 | 34 | const userCrons = users.map(({ refreshToken, _id }) => 35 | axios.post( 36 | `${process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI}/api/cron/user-cron`, 37 | { 38 | refresh_token: refreshToken, 39 | uid: _id, 40 | }, 41 | ), 42 | ); 43 | 44 | Promise.all(userCrons); 45 | } catch (e) { 46 | Sentry.captureException(e); 47 | console.log(e); 48 | } 49 | res.json({}); 50 | } 51 | -------------------------------------------------------------------------------- /pages/api/firebase/login.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-underscore-dangle */ 2 | import commonMiddleware from '~/src/middleware/commonMiddleware'; 3 | import { verifyIdToken } from '~/src/firebase/firebaseAdmin'; 4 | import { userExists } from './user'; 5 | 6 | const handler = async (req, res) => { 7 | if (!req.body) { 8 | return res.status(400); 9 | } 10 | 11 | const { token, uid } = req.body; 12 | if (!token) { 13 | return res.status(500).json({ error: 'token not found' }); 14 | } 15 | 16 | // Here, we decode the user's Firebase token and store it in a cookie. Use 17 | // express-session (or similar) to store the session data server-side. 18 | // An alternative approach is to use Firebase's `createSessionCookie`. See: 19 | // https://firebase.google.com/docs/auth/admin/manage-cookies 20 | // Firebase docs: 21 | // "This is a low overhead operation. The public certificates are initially 22 | // queried and cached until they expire. Session cookie verification can be 23 | // done with the cached public certificates without any additional network 24 | // requests." 25 | // However, in a serverless environment, we shouldn't rely on caching, so 26 | // it's possible Firebase's `verifySessionCookie` will make frequent network 27 | // requests in a serverless context. 28 | const decodedToken = await verifyIdToken(token); 29 | let _id; 30 | if (uid) { 31 | try { 32 | const dbUser = await userExists(uid); 33 | _id = dbUser._id; 34 | } catch (e) { 35 | return res.status(500).send(e); 36 | } 37 | } 38 | req.session.decodedToken = decodedToken; 39 | req.session.token = token; 40 | req.session.uid = _id; 41 | return res.json({ status: true, decodedToken, uid: _id }); 42 | }; 43 | 44 | export default commonMiddleware(handler); 45 | -------------------------------------------------------------------------------- /pages/api/cron/user-cron.js: -------------------------------------------------------------------------------- 1 | import Sentry from '~/src/sentry'; 2 | 3 | const axios = require('axios'); 4 | 5 | export default async function handle(req, res) { 6 | const { refresh_token: refreshToken, uid } = req.body; 7 | try { 8 | // NB: only processes 100 services 9 | const { data: userServices } = await axios( 10 | `${process.env.JSONBOX_NETWORK_URL}/${uid}/services?limit=100`, 11 | ); 12 | 13 | res.json({}); 14 | 15 | const prs = userServices 16 | .filter((service) => service.cron) 17 | .map(async (service) => { 18 | switch (service.app) { 19 | case 'EMAIL_TO_JSON': { 20 | return axios.post( 21 | `${process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI}/api/apps/email-to-json`, 22 | { 23 | uid, 24 | refresh_token: refreshToken, 25 | new_only: true, 26 | service_id: service._id, 27 | cron: true, 28 | }, 29 | ); 30 | } 31 | case 'AUTO_UNLOCK': { 32 | return axios.post( 33 | `${process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI}/api/apps/auto-unlock`, 34 | { 35 | uid, 36 | refresh_token: refreshToken, 37 | new_only: true, 38 | service_id: service._id, 39 | cron: true, 40 | }, 41 | ); 42 | } 43 | default: { 44 | return null; 45 | } 46 | } 47 | }); 48 | 49 | try { 50 | await Promise.all(prs.filter((pr) => pr)); 51 | } catch (e) { 52 | Sentry.captureException(e); 53 | console.log(e); 54 | } 55 | } catch (e) { 56 | Sentry.captureException(e); 57 | console.log(e); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pages/api/apps/email-to-json/integrations/whatsapp.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import Sentry from '~/src/sentry'; 3 | import ensureAuth from '~/src/middleware/ensureAuth'; 4 | 5 | const { JSONBOX_NETWORK_URL } = process.env; 6 | const GOOGLE_OAUTH_REDIRECT_URI = 7 | process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI; 8 | 9 | async function handle(req, res, resolve) { 10 | const { uid, service_id: serviceId, wa_phone_number: phoneNumber } = req.body; 11 | 12 | if (!phoneNumber) { 13 | res.status(400).json({ error: 'Missing required param `wa_phone_number`' }); 14 | return resolve(); 15 | } 16 | 17 | try { 18 | const serviceEndpoint = `${JSONBOX_NETWORK_URL}/${uid}/services/${serviceId}`; 19 | const { data: existingServiceData } = await axios(serviceEndpoint); 20 | 21 | await axios.put(`${JSONBOX_NETWORK_URL}/${uid}/services/${serviceId}`, { 22 | ...existingServiceData, 23 | wa_phone_number: phoneNumber, 24 | }); 25 | 26 | if (!Array.isArray(existingServiceData.data)) { 27 | res.json({ error: 'Please wait... data not synced yet!' }); 28 | return resolve(); 29 | } 30 | 31 | // setup a task per pre-existing data endpoint to sync to gSheet 32 | const dataEndpoints = existingServiceData.data 33 | .filter((item) => item.is_successful) 34 | .map(({ id }) => `${JSONBOX_NETWORK_URL}/${uid}/${id}`) 35 | .map((endpoint) => 36 | axios.post(`${GOOGLE_OAUTH_REDIRECT_URI}/api/integrations/whatsapp`, { 37 | uid, 38 | refresh_token: req.refresh_token, 39 | service_id: serviceId, 40 | data_endpoint: endpoint, 41 | }), 42 | ); 43 | 44 | await Promise.all(dataEndpoints); 45 | } catch (e) { 46 | console.log(e); 47 | Sentry.captureException(e); 48 | } 49 | 50 | res.json({}); 51 | return resolve(); 52 | } 53 | 54 | export default ensureAuth(handle); 55 | -------------------------------------------------------------------------------- /components/pageWrappers/withAuthUserInfo.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable react/jsx-props-no-spreading */ 2 | import React from 'react'; 3 | import PropTypes from 'prop-types'; 4 | import { get } from 'lodash/object'; 5 | import { AuthUserInfoContext } from '~/src/firebase/hooks'; 6 | 7 | // Provides an AuthUserInfo prop to the composed component. 8 | export default (ComposedComponent) => { 9 | const WithAuthUserInfoComp = (props) => { 10 | const { AuthUserInfo: AuthUserInfoFromSession, ...otherProps } = props; 11 | return ( 12 | 13 | {(AuthUserInfo) => ( 14 | 18 | )} 19 | 20 | ); 21 | }; 22 | 23 | WithAuthUserInfoComp.getInitialProps = async (ctx) => { 24 | const AuthUserInfo = get(ctx, 'myCustomData.AuthUserInfo', null); 25 | 26 | // Evaluate the composed component's getInitialProps(). 27 | let composedInitialProps = {}; 28 | if (ComposedComponent.getInitialProps) { 29 | composedInitialProps = await ComposedComponent.getInitialProps(ctx); 30 | } 31 | 32 | return { 33 | ...composedInitialProps, 34 | AuthUserInfo, 35 | }; 36 | }; 37 | 38 | WithAuthUserInfoComp.displayName = `WithAuthUserInfo(${ComposedComponent.displayName})`; 39 | 40 | WithAuthUserInfoComp.propTypes = { 41 | // eslint-disable-next-line react/require-default-props 42 | AuthUserInfo: PropTypes.shape({ 43 | AuthUser: PropTypes.shape({ 44 | id: PropTypes.string.isRequired, 45 | email: PropTypes.string.isRequired, 46 | emailVerified: PropTypes.bool.isRequired, 47 | }), 48 | token: PropTypes.string, 49 | }), 50 | }; 51 | 52 | WithAuthUserInfoComp.defaultProps = {}; 53 | 54 | return WithAuthUserInfoComp; 55 | }; 56 | -------------------------------------------------------------------------------- /pages/[uid]/home.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-underscore-dangle */ 2 | import React from 'react'; 3 | import { withRouter } from 'next/router'; 4 | import Head from 'next/head'; 5 | import Link from 'next/link'; 6 | import styled from 'styled-components'; 7 | import CommonHeader from '~/components/common/common-header'; 8 | import withAuthUser from '~/components/pageWrappers/withAuthUser'; 9 | import withAuthUserInfo from '~/components/pageWrappers/withAuthUserInfo'; 10 | import { Anchor } from '~/components/common/Atoms'; 11 | 12 | const Container = styled.div` 13 | display: grid; 14 | grid-template-columns: 1fr; 15 | grid-template-rows: 64px 1fr; 16 | `; 17 | 18 | const Body = styled.div` 19 | padding: 1rem 0.5rem; 20 | `; 21 | 22 | const Dashboard = ({ router }) => { 23 | const { 24 | query: { uid }, 25 | } = router; 26 | 27 | return ( 28 | <> 29 | 30 | emailapi.io | create new job 31 | 32 | 33 | 34 | 35 |
36 | Goto Dashboard 37 |
38 | 39 | 40 | 41 |

Select job type

42 |
43 |

44 | 45 | PDF attachment Unlocker 46 | 47 |

48 |
49 | 50 |
51 |

52 | 53 | Email to JSON 54 | 55 |

56 |
57 | 58 | 59 | 60 | ); 61 | }; 62 | 63 | export default withAuthUser(withAuthUserInfo(withRouter(Dashboard))); 64 | -------------------------------------------------------------------------------- /pages/api/cron/cleanup-accounts.js: -------------------------------------------------------------------------------- 1 | /** 2 | * this endpoint receives recurring calls from an external source 3 | * with just 1 mandatory params - `uid` 4 | * 5 | * with uid we can create endpoints required to run cron jobs 6 | 7 | * endpoint for user's db object 8 | * `${process.env.EMAILAPI_BASE_URL}/users/${uid}` 9 | * -> response contains `hostedOptin` boolean 10 | * 11 | * if hostedOptin is false or not set and more than 48 hours have elapsed since `_createdOn` 12 | * then delete record at `${process.env.EMAILAPI_BASE_URL}/users/${uid}` 13 | * 14 | */ 15 | import Sentry from '~/src/sentry'; 16 | 17 | const base64 = require('base-64'); 18 | const axios = require('axios'); 19 | const differenceInMinutes = require('date-fns/differenceInMinutes'); 20 | 21 | export default async function handle(req, res) { 22 | const authHeader = req.headers.authorization; 23 | const encodedUsernamePassword = authHeader.replace('Basic', '').trim(); 24 | const decodedUsernamePassword = base64.decode(encodedUsernamePassword); 25 | const [uid] = decodedUsernamePassword.split(':'); 26 | 27 | try { 28 | await axios(`${process.env.EMAILAPI_BASE_URL}/users/${uid}`); 29 | // if record is successfully found, then continue 30 | 31 | const response = await axios( 32 | `${process.env.EMAILAPI_BASE_URL}/users?limit=100`, 33 | ); 34 | 35 | const prs = response.data 36 | .filter( 37 | ({ hostedOptin = false, _createdOn }) => 38 | !hostedOptin && 39 | differenceInMinutes(new Date(), new Date(_createdOn)) >= 48 * 60, 40 | ) 41 | // eslint-disable-next-line no-shadow 42 | .map(({ _id }) => 43 | axios.delete(`${process.env.EMAILAPI_BASE_URL}/users/${_id}`), 44 | ); 45 | 46 | if (prs.length) { 47 | await Promise.all(prs); 48 | } 49 | } catch (e) { 50 | // invalid user 51 | Sentry.captureException(e); 52 | console.log(e); 53 | } 54 | 55 | res.json({}); 56 | } 57 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | # https://rtfm.co.ua/en/docker-configure-tzdata-and-timezone-during-build/ 3 | ENV TZ=Asia/Kolkata 4 | ENV LANG C.UTF-8 5 | ENV LC_ALL C.UTF-8 6 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 7 | RUN apt-get update -y 8 | 9 | # Install Node.js 10 | RUN apt-get install -y curl 11 | RUN apt-get install -y gnupg-agent 12 | RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - 13 | RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list 14 | RUN curl --silent --location https://deb.nodesource.com/setup_12.x | bash - 15 | RUN apt-get install -y nodejs 16 | RUN apt-get install -y build-essential 17 | RUN apt install -y yarn 18 | RUN node --version 19 | RUN yarn --version 20 | 21 | RUN apt-get install -y qpdf 22 | RUN apt-get install -y software-properties-common 23 | RUN add-apt-repository ppa:deadsnakes/ppa 24 | RUN apt-get update 25 | RUN apt-get install -y python3.8 26 | RUN python3 --version 27 | RUN apt-get install -y python3-tk ghostscript 28 | RUN gs -version 29 | RUN apt-get install -y python3-pip 30 | RUN apt-get install -y libsm6 libxext6 libxrender-dev 31 | RUN pip3 install 'opencv-python==4.2.0.34' 32 | RUN pip3 install "camelot-py[cv]" 33 | 34 | WORKDIR /codebase 35 | COPY package.json ./ 36 | COPY yarn.lock ./ 37 | RUN yarn 38 | COPY . ./ 39 | ARG NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN 40 | ARG NEXT_PUBLIC_FIREBASE_DATABASE_URL 41 | ARG FIREBASE_CLIENT_EMAIL 42 | ARG NEXT_PUBLIC_FIREBASE_PROJECT_ID 43 | ARG NEXT_PUBLIC_FIREBASE_PUBLIC_API_KEY 44 | ARG FIREBASE_PRIVATE_KEY 45 | ARG NEXT_PUBLIC_GOOGLE_CLIENT_ID 46 | ARG GOOGLE_CLIENT_SECRET 47 | ARG NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI 48 | ARG NEXT_PUBLIC_EMAILAPI_DOMAIN 49 | ARG EMAILAPI_BASE_URL 50 | ARG JSONBOX_NETWORK_URL 51 | ARG MAILGUN_API_KEY 52 | ARG MAILGUN_DOMAIN 53 | ARG NEXT_PUBLIC_SENDING_EMAIL_ID 54 | ARG MAILGUN_API_BASE_URL 55 | ARG OTP_EMAILAPI_USER_ID 56 | ARG NODE_DEBUG 57 | ARG NODE_ENV 58 | ARG SENTRY_DSN 59 | ARG GOOGLE_SERVICE_ACCOUNT_EMAIL 60 | ARG GOOGLE_PRIVATE_KEY 61 | ARG REDISCLOUD_URL 62 | RUN yarn build 63 | EXPOSE 3000 64 | CMD ["yarn", "start"] -------------------------------------------------------------------------------- /components/common/Atoms.js: -------------------------------------------------------------------------------- 1 | import styled from 'styled-components'; 2 | 3 | export const Shadow = styled.a.attrs({ 4 | className: 'inline-block uppercase text-gray-800 py-1 px-4 text-sm', 5 | })` 6 | background-color: hsl(228, 100%, 92%); 7 | box-shadow: 4px 4px hsla(228, 100%, 92%, 0.8); 8 | cursor: pointer; 9 | `; 10 | 11 | export const Separator = styled.div` 12 | background-image: url(/static/images/progressive-disclosure-line@2x.png); 13 | background-repeat: repeat-x; 14 | background-position: 0; 15 | background-size: 32px; 16 | height: 12px; 17 | `; 18 | 19 | export const VisibilityIcon = styled.span.attrs({ 20 | className: 'inline-block mr-1', 21 | })` 22 | vertical-align: bottom; 23 | `; 24 | 25 | export const StockItemContainer = styled.div` 26 | padding: 12px 24px 16px; 27 | background: ${(props) => 28 | props.isSelected ? 'hsla(219, 79%, 66%, 1)' : 'hsla(189, 50%, 95%, 1)'}; 29 | border: 1px solid hsla(214, 34%, 82%, 0.15); 30 | box-shadow: 0 4px 0px hsl(214, 34%, 82%); 31 | border-radius: 4px; 32 | cursor: ${(props) => (props.isActionable ? 'pointer' : 'initial')}; 33 | `; 34 | 35 | export const StockItemBadge = styled.span` 36 | background: hsl(0, 0%, 86%); 37 | border-radius: 4px; 38 | padding: 2px 4px; 39 | text-align: center; 40 | color: hsl(0, 0%, 17%); 41 | border: 1px dashed hsl(0, 0%, 60%); 42 | `; 43 | 44 | export const Anchor = styled.a.attrs({ 45 | className: 'cursor-pointer', 46 | })` 47 | border-bottom: 5px solid #ffc107; 48 | `; 49 | 50 | export const Button = styled.button.attrs((props) => ({ 51 | type: props.type || 'button', 52 | }))` 53 | border-bottom: 5px solid #ffc107; 54 | opacity: ${(props) => (props.disabled ? 0.25 : 1)}; 55 | cursor: ${(props) => (props.disabled ? 'not-allowed' : 'pointer')}; 56 | 57 | &:focus { 58 | outline: none; 59 | } 60 | `; 61 | 62 | export const Label = styled.div.attrs({ 63 | className: 'uppercase text-gray-800 tracking-wide text-sm', 64 | })``; 65 | 66 | export const Value = styled.span``; 67 | 68 | export const FlexEnds = styled.div.attrs({ 69 | className: 'flex flex-1 w-full items-center justify-between', 70 | })``; 71 | 72 | export const Row = styled.div.attrs({ 73 | className: 'flex items-center', 74 | })` 75 | > * { 76 | margin-right: 4px; 77 | } 78 | `; 79 | -------------------------------------------------------------------------------- /pages/api/otp/index.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | 3 | import flatten from 'lodash/flatten'; 4 | 5 | import applyConfigOnEmail from '~/src/isomorphic/applyConfigOnEmail'; 6 | import ensureConfiguration from '~/src/isomorphic/ensureConfiguration'; 7 | 8 | const { 9 | NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI: GOOGLE_OAUTH_REDIRECT_URI, 10 | } = process.env; 11 | const { OTP_EMAILAPI_USER_ID } = process.env; 12 | 13 | function isValidEmail(email) { 14 | // eslint-disable-next-line no-useless-escape 15 | const regex = /^([a-zA-Z0-9_.+-])+\@(([a-zA-Z0-9-])+\.)+([a-zA-Z0-9]{2,4})+$/; 16 | return regex.test(email); 17 | } 18 | 19 | export default async function handle(req, res) { 20 | const { 21 | email: reqEmail, 22 | search_query: reqSearchQuery = '', 23 | configurations = [], 24 | } = req.body; 25 | 26 | if (!isValidEmail(reqEmail)) { 27 | return res.status(400).json({ 28 | error_code: 'INVALID_EMAIL', 29 | }); 30 | } 31 | 32 | const searchQuery = `to: ${reqEmail} ${reqSearchQuery}`; 33 | 34 | const { data: emailSearchResults } = await axios.post( 35 | `${GOOGLE_OAUTH_REDIRECT_URI}/api/email-search`, 36 | { 37 | query: searchQuery, 38 | uid: OTP_EMAILAPI_USER_ID, 39 | api_only: true, 40 | gmail_search_props: { 41 | maxResults: 1, // only interested in the most recent email 42 | }, 43 | }, 44 | ); 45 | 46 | const { emails } = emailSearchResults; 47 | if (!emails.length) { 48 | // no emails found for this query 49 | return res.json({ 50 | status: 0, 51 | }); 52 | } 53 | 54 | if (!configurations.length) { 55 | return res.json({ 56 | status: 2, 57 | }); 58 | } 59 | 60 | const emailapi = flatten( 61 | configurations.map((config) => 62 | emails 63 | .map((email) => applyConfigOnEmail(email.message, config)) 64 | .filter((extactedData) => ensureConfiguration(extactedData, config)), 65 | ), 66 | ); 67 | 68 | if (!emailapi.length) { 69 | // [IMP!] emails were found but this configuration wasn't able to parse it. Email template has most likely changed. FIX IT! 70 | return res.json({ 71 | status: 1, 72 | }); 73 | } 74 | 75 | // all good! :) 76 | return res.json({ 77 | status: 2, 78 | results: emailapi, 79 | }); 80 | } 81 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODOs 2 | 3 | ## Feature: Auto PDF Unlock 4 | 5 | - After search query is entered and an email is selected, if the email contains a PDF, 6 | - Show an option which says "Enable automatic PDF unlocking?" 7 | - On click, show an input to accept PDF password 8 | - Do a test dry run for this specific messageId + attachmentId combination 9 | where we attempt to temporarily download and unlock PDF on server 10 | then send this user mail with same subject prefixed with [UNLOCKED] and attach unlocked PDF 11 | then delete the file on server 12 | - If unlocked pdf mail from m.emailapi.io arrives, show options to 13 | - unlock all previous emails and receive "unlocked" emails for all of them 14 | - or, unlock all emails going forward 15 | 16 | ## Tracking task completion between parent-child queue jobs 17 | 18 | - ### 1-many jobs 19 | mailFetch will always be the parent job 20 | mailFetch queue will spawn n jobs in child queue 21 | 22 | - ### Parent job completion user notifications 23 | 24 | 25 | ### TODOs 26 | - Remove `done` based callbacks from queue and move to Promise.resolve()s 27 | - Enable job success/failure notification system 28 | - Include all queues from filesystem instead of `require`ing them individually 29 | 30 | 31 | ## Extracting tables 32 | 33 | ### Scenarios 34 | 35 | > User knows this PDF template always contains X number of tables 36 | 1. User can reject tables as is 37 | 2. User can select tables as is 38 | 2.1. or/and add whitelist rules 39 | 2.1.1. grab a row only if it contains a cell where cell's value `==` or `contains` some value 40 | 2.2. or/and add blacklist rules 41 | 2.2.1. reject rows if it contains a cell where cell's value `==` or `contains` some value 42 | 43 | # Last Notes 11th Oct 2020 8.35PM 44 | - UI: Added feature to allow posting to different Google Sheets for each `extraction rule` 45 | - Backend: stopped accepting gSheetId as a global prop and made it part of each rule object 46 | - Next steps: 47 | 1. Data extraction seems funky as one sheet contains results from both `rules` 48 | 2. Google Sheets API is rate limiting 429 - culprit being /preview endpoint which creates a new spreadsheet instance per API call 49 | 3. Options to solve #2 - Create a Sheets Factory with references stored on a per sheet id level (in redis to make it serverless?) and reuse them 50 | 51 | BCCPG2423G -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "emailapi", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "NODE_OPTIONS='--inspect' next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "build_start": "next build && next start" 10 | }, 11 | "dependencies": { 12 | "@popperjs/core": "^2.4.4", 13 | "@sentry/node": "^5.22.3", 14 | "@sentry/tracing": "^5.22.3", 15 | "@zeit/next-css": "^1.0.1", 16 | "airtable": "^0.10.0", 17 | "autoprefixer": "^9.8.6", 18 | "axios": "^0.19.2", 19 | "base-64": "^0.1.0", 20 | "base64topdf": "^1.1.8", 21 | "bitly": "^7.1.0", 22 | "bluebird": "^3.7.2", 23 | "bull": "^3.18.0", 24 | "classnames": "^2.2.6", 25 | "cookie-session": "^1.4.0", 26 | "date-fns": "^2.15.0", 27 | "firebase": "^7.18.0", 28 | "firebase-admin": "^9.1.0", 29 | "follow-redirects": "^1.13.0", 30 | "get-urls": "^10.0.0", 31 | "google-spreadsheet": "^3.0.13", 32 | "googleapis": "^59.0.0", 33 | "ioredis": "^4.17.3", 34 | "isomorphic-unfetch": "^3.0.0", 35 | "jsdom": "^16.4.0", 36 | "json-formatter-js": "^2.3.4", 37 | "lodash": "^4.17.20", 38 | "mailgun-js": "^0.22.0", 39 | "next": "9.5.1", 40 | "next-compose-plugins": "^2.2.1", 41 | "next-env": "^1.1.1", 42 | "normalize-html-whitespace": "^1.0.0", 43 | "noty": "^3.2.0-beta", 44 | "postcss-import": "^12.0.1", 45 | "prop-types": "^15.7.2", 46 | "react": "16.13.1", 47 | "react-dom": "16.13.1", 48 | "react-feather": "^2.0.8", 49 | "react-google-login": "^5.1.21", 50 | "react-responsive-modal": "^5.0.3", 51 | "remove-pdf-password": "^0.1.0", 52 | "shelljs": "^0.8.4", 53 | "shortid": "^2.2.15", 54 | "styled-components": "^5.1.1", 55 | "tailwindcss": "^1.6.2", 56 | "twilio": "^3.50.0", 57 | "unzipper": "^0.10.11" 58 | }, 59 | "devDependencies": { 60 | "babel-eslint": "^10.1.0", 61 | "babel-plugin-styled-components": "^1.11.1", 62 | "eslint": "^7.7.0", 63 | "eslint-config-airbnb": "^18.2.0", 64 | "eslint-config-prettier": "^6.11.0", 65 | "eslint-import-resolver-webpack": "^0.12.2", 66 | "eslint-plugin-import": "^2.22.0", 67 | "eslint-plugin-jsx-a11y": "^6.3.1", 68 | "eslint-plugin-prettier": "^3.1.4", 69 | "eslint-plugin-react": "^7.20.6", 70 | "esm": "^3.2.25", 71 | "prettier": "^2.0.5" 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /pages/api/apps/email-to-json/integrations/google-sheet.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import Sentry from '~/src/sentry'; 3 | import ensureAuth from '~/src/middleware/ensureAuth'; 4 | 5 | import resetSheet from '~/src/integrations/google-spreadsheet/reset-sheet'; 6 | 7 | const EMAILAPI_DOMAIN = process.env.NEXT_PUBLIC_EMAILAPI_DOMAIN; 8 | const GOOGLE_OAUTH_REDIRECT_URI = 9 | process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI; 10 | 11 | async function handle(req, res, resolve) { 12 | const { 13 | uid, 14 | service_id: serviceId, 15 | gsheet_id: gSheetId, 16 | presync_webhook: preSyncWebhook, 17 | } = req.body; 18 | 19 | if (!gSheetId) { 20 | res.status(400).json({ error: 'Missing required param `gsheet_id`' }); 21 | resolve(); 22 | return; 23 | } 24 | 25 | try { 26 | const { data: existingServiceData } = await axios( 27 | `${EMAILAPI_DOMAIN}/${uid}/services/${serviceId}`, 28 | ); 29 | 30 | await axios.put(`${EMAILAPI_DOMAIN}/${uid}/services/${serviceId}`, { 31 | ...existingServiceData, 32 | gsheet_id: gSheetId, 33 | presync_webhook: preSyncWebhook, 34 | }); 35 | 36 | // acknowledge request 37 | res.json({}); 38 | 39 | if (existingServiceData.gsheet_id) { 40 | // reset existing sheet 41 | // [NB]: this has a side-effect of data existing on another sheet getting reset 42 | // if user is changing sheets 43 | try { 44 | await resetSheet({ 45 | googleSheetId: existingServiceData.gsheet_id, 46 | }); 47 | } catch (e) { 48 | console.log(e); 49 | Sentry.captureException(e); 50 | } 51 | } 52 | 53 | // setup a task per pre-existing data endpoint to sync to gSheet 54 | const dataEndpoints = existingServiceData.data 55 | .filter((item) => item.is_successful) 56 | .map(({ id }) => `${EMAILAPI_DOMAIN}/${uid}/${id}`) 57 | .map((endpoint) => 58 | axios.post( 59 | `${GOOGLE_OAUTH_REDIRECT_URI}/api/integrations/google-spreadsheet`, 60 | { 61 | uid, 62 | refresh_token: req.refresh_token, 63 | service_id: serviceId, 64 | data_endpoint: endpoint, 65 | }, 66 | ), 67 | ); 68 | 69 | await Promise.all(dataEndpoints); 70 | } catch (e) { 71 | console.log(e); 72 | Sentry.captureException(e); 73 | } 74 | 75 | resolve(); 76 | } 77 | 78 | export default ensureAuth(handle); 79 | -------------------------------------------------------------------------------- /pages/helpers/iframe.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable react/no-danger */ 2 | import React, { useState, useEffect } from 'react'; 3 | import { useRouter } from 'next/router'; 4 | import axios from 'axios'; 5 | import base64 from 'base-64'; 6 | 7 | const normalizeHtmlWhitespace = require('normalize-html-whitespace'); 8 | 9 | const isLengthyArray = (arr) => Array.isArray(arr) && arr.length; 10 | 11 | function findPartOfType(parts, type) { 12 | if (!isLengthyArray(parts)) { 13 | return null; 14 | } 15 | 16 | // eslint-disable-next-line no-plusplus 17 | for (let i = 0; i < parts.length; i++) { 18 | const childHtmlPart = parts.find((part) => part.mimeType === type); 19 | if (childHtmlPart) { 20 | return childHtmlPart; 21 | } 22 | if (isLengthyArray(parts[i].parts)) { 23 | const grandChildHtmlPart = findPartOfType(parts[i].parts, type); 24 | if (grandChildHtmlPart) { 25 | return grandChildHtmlPart; 26 | } 27 | } 28 | } 29 | 30 | return null; 31 | } 32 | 33 | export default function IframeComponent() { 34 | const router = useRouter(); 35 | const { 36 | query: { messageId, uid, isHtmlContent }, 37 | } = router; 38 | 39 | const [isEmailFetched, setIsEmailFetched] = useState(false); 40 | const [email, setEmail] = useState(''); 41 | 42 | useEffect(() => { 43 | async function fetchEmail() { 44 | const { data: emailData } = await axios.post( 45 | `${process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI}/api/fetch/email`, 46 | { 47 | uid, 48 | messageId, 49 | }, 50 | ); 51 | 52 | const part = 53 | emailData.payload.mimeType === 'text/html' 54 | ? emailData.payload.body.data 55 | : findPartOfType(emailData.payload.parts, 'text/html').body.data; 56 | 57 | setEmail( 58 | normalizeHtmlWhitespace( 59 | decodeURIComponent( 60 | escape(base64.decode(part.replace(/-/g, '+').replace(/_/g, '/'))), 61 | ), 62 | ), 63 | ); 64 | setIsEmailFetched(true); 65 | } 66 | if (uid) { 67 | fetchEmail(); 68 | } 69 | }, [uid]); 70 | 71 | if (!isEmailFetched) { 72 | return <>Loading...; 73 | } 74 | 75 | function child() { 76 | return { 77 | __html: JSON.parse(isHtmlContent) ? email : `
${email}
`, 78 | }; 79 | } 80 | 81 | const childDom = child().__html; 82 | return
; 83 | } 84 | -------------------------------------------------------------------------------- /styles/Home.module.css: -------------------------------------------------------------------------------- 1 | .container { 2 | min-height: 100vh; 3 | padding: 0 0.5rem; 4 | display: flex; 5 | flex-direction: column; 6 | justify-content: center; 7 | align-items: center; 8 | } 9 | 10 | .main { 11 | padding: 5rem 0; 12 | flex: 1; 13 | display: flex; 14 | flex-direction: column; 15 | justify-content: center; 16 | align-items: center; 17 | } 18 | 19 | .footer { 20 | width: 100%; 21 | height: 100px; 22 | border-top: 1px solid #eaeaea; 23 | display: flex; 24 | justify-content: center; 25 | align-items: center; 26 | } 27 | 28 | .footer img { 29 | margin-left: 0.5rem; 30 | } 31 | 32 | .footer a { 33 | display: flex; 34 | justify-content: center; 35 | align-items: center; 36 | } 37 | 38 | .title a { 39 | color: #0070f3; 40 | text-decoration: none; 41 | } 42 | 43 | .title a:hover, 44 | .title a:focus, 45 | .title a:active { 46 | text-decoration: underline; 47 | } 48 | 49 | .title { 50 | margin: 0; 51 | line-height: 1.15; 52 | font-size: 4rem; 53 | } 54 | 55 | .title, 56 | .description { 57 | text-align: center; 58 | } 59 | 60 | .description { 61 | line-height: 1.5; 62 | font-size: 1.5rem; 63 | } 64 | 65 | .code { 66 | background: #fafafa; 67 | border-radius: 5px; 68 | padding: 0.75rem; 69 | font-size: 1.1rem; 70 | font-family: Menlo, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono, 71 | Bitstream Vera Sans Mono, Courier New, monospace; 72 | } 73 | 74 | .grid { 75 | display: flex; 76 | align-items: center; 77 | justify-content: center; 78 | flex-wrap: wrap; 79 | 80 | max-width: 800px; 81 | margin-top: 3rem; 82 | } 83 | 84 | .card { 85 | margin: 1rem; 86 | flex-basis: 45%; 87 | padding: 1.5rem; 88 | text-align: left; 89 | color: inherit; 90 | text-decoration: none; 91 | border: 1px solid #eaeaea; 92 | border-radius: 10px; 93 | transition: color 0.15s ease, border-color 0.15s ease; 94 | } 95 | 96 | .card:hover, 97 | .card:focus, 98 | .card:active { 99 | color: #0070f3; 100 | border-color: #0070f3; 101 | } 102 | 103 | .card h3 { 104 | margin: 0 0 1rem 0; 105 | font-size: 1.5rem; 106 | } 107 | 108 | .card p { 109 | margin: 0; 110 | font-size: 1.25rem; 111 | line-height: 1.5; 112 | } 113 | 114 | .logo { 115 | height: 1em; 116 | } 117 | 118 | @media (max-width: 600px) { 119 | .grid { 120 | width: 100%; 121 | flex-direction: column; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /components/ft/email-json/Grid.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable react/jsx-props-no-spreading */ 2 | import React from 'react'; 3 | import styled from 'styled-components'; 4 | import { toArray } from '../../../src/pdf/utils'; 5 | 6 | const StyledGrid = styled.div` 7 | display: grid; 8 | grid-template-columns: ${(props) => `repeat(${props.cols}, 1fr)`}; 9 | border: 1px solid #ddd; 10 | overflow-x: scroll; 11 | font-size: 0.8rem; 12 | max-width: calc(50vw - 4rem - 1rem); 13 | `; 14 | 15 | const GridRowBaseCell = styled.div.attrs((props) => ({ 16 | className: props.selected ? 'bg-orange-100' : null, 17 | }))` 18 | border-bottom: 1px solid #ddd; 19 | border-right: 1px solid #ddd; 20 | padding: 2px 4px; 21 | `; 22 | 23 | const GridRowClickableCell = styled(GridRowBaseCell)` 24 | cursor: pointer; 25 | &:hover { 26 | background: orange; 27 | border: 1px solid orange; 28 | color: #281a01; 29 | } 30 | `; 31 | 32 | const GridCell = ({ isCellClickable, ...props }) => 33 | isCellClickable ? ( 34 | 35 | ) : ( 36 | 37 | ); 38 | 39 | export default function Grid({ 40 | data, 41 | isCellClickable = false, 42 | cellClickCb = () => {}, 43 | className = '', 44 | selectedRows = [], 45 | selectedCells = [], 46 | }) { 47 | if (!data) { 48 | return

Glitch!

; 49 | } 50 | console.log({ selectedRows }); 51 | return ( 52 | 53 | {data.map((row, rowIdx) => { 54 | const cells = toArray(row); 55 | return ( 56 | <> 57 | {cells.map((cell, colIdx) => ( 58 | 65 | selectedCell.rowIdx === rowIdx && 66 | selectedCell.colIdx === colIdx, 67 | ) 68 | } 69 | onClick={() => 70 | isCellClickable 71 | ? cellClickCb({ value: cell, rowIdx, colIdx }) 72 | : null 73 | } 74 | > 75 | {cell} 76 | 77 | ))} 78 | 79 | ); 80 | })} 81 | 82 | ); 83 | } 84 | -------------------------------------------------------------------------------- /pages/api/apps/auto-unlock/index.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import Sentry from '~/src/sentry'; 3 | import ensureAuth from '~/src/middleware/ensureAuth'; 4 | import queues from '~/src/redis-queue'; 5 | import { getSearchQuery } from '~/src/apps/utils'; 6 | 7 | const generateUniqueId = require('~/components/admin/email/fns/generateUniqueId'); 8 | 9 | const { NEXT_PUBLIC_EMAILAPI_DOMAIN, JSONBOX_NETWORK_URL } = process.env; 10 | const APP_HOST = process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI; 11 | 12 | require('~/src/queues'); 13 | 14 | async function handle(req, res, resolve) { 15 | const { refresh_token: refreshToken } = req; 16 | const { 17 | token, 18 | uid, 19 | service_id: serviceId, 20 | api_only: apiOnly, 21 | new_only: newOnly = false, 22 | } = req.body; 23 | 24 | const apiId = generateUniqueId(); 25 | const endpoint = `${JSONBOX_NETWORK_URL}/${uid}/${apiId}`; 26 | const publicEndpoint = `${NEXT_PUBLIC_EMAILAPI_DOMAIN}/${uid}/${apiId}`; 27 | 28 | res.json({ endpoint: publicEndpoint }); 29 | 30 | try { 31 | const serviceEndpoint = `${JSONBOX_NETWORK_URL}/${uid}/services/${serviceId}`; 32 | const searchQuery = await getSearchQuery({ 33 | serviceEndpoint, 34 | newOnly, 35 | }); 36 | 37 | const { 38 | data: { unlock_password: unlockPassword }, 39 | } = await axios(serviceEndpoint); 40 | 41 | const { user } = req; 42 | const userProps = { 43 | uid, 44 | user, 45 | token, 46 | refreshToken, 47 | }; 48 | 49 | queues.mailFetchQueue.add({ 50 | apiOnly, 51 | userProps, 52 | searchQuery, 53 | _nextQueue: 'autoUnlockQueue', 54 | _nextQueueData: { 55 | endpoint, 56 | userProps, 57 | unlockPassword, 58 | serviceEndpoint, 59 | }, 60 | completionNotifications: { 61 | success: { 62 | notifyConditions: { 63 | childJobsGotCreated: true, 64 | }, 65 | notifications: [ 66 | { 67 | type: 'webhook', 68 | data: { 69 | method: 'POST', 70 | url: `${APP_HOST}/api/apps/auto-unlock/webhook`, 71 | data: { 72 | uid: userProps.uid, 73 | refresh_token: userProps.refreshToken, 74 | apiId, 75 | serviceEndpoint, 76 | success: true, 77 | }, 78 | }, 79 | }, 80 | ], 81 | }, 82 | }, 83 | }); 84 | } catch (e) { 85 | console.log(e); 86 | Sentry.captureException(e); 87 | } 88 | 89 | return resolve(); 90 | } 91 | 92 | export default ensureAuth(handle); 93 | -------------------------------------------------------------------------------- /pages/api/email-search/attachment-unlock.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | import ensureAuth from '~/src/middleware/ensureAuth'; 3 | import queues from '~/src/redis-queue'; 4 | 5 | import { fetchEmailByMessageId, processMessageBody } from '~/src/gmail'; 6 | 7 | const base64 = require('base64topdf'); 8 | const removePdfPassword = require('remove-pdf-password'); 9 | const { getAfterTs } = require('~/src/apps/utils'); 10 | 11 | require('~/src/queues'); 12 | 13 | async function handle(req, res, resolve) { 14 | const { 15 | attachmentId, 16 | messageId, 17 | token, 18 | uid, 19 | filename, 20 | pdfPasswordInput, 21 | } = req.body; 22 | 23 | try { 24 | const { data } = await axios.post( 25 | `${process.env.NEXT_PUBLIC_GOOGLE_OAUTH_REDIRECT_URI}/api/fetch/attachment`, 26 | { 27 | messageId, 28 | attachmentId, 29 | token, 30 | uid, 31 | }, 32 | ); 33 | 34 | if (!data.base64) { 35 | throw new Error( 36 | 'base64 key not found in response object from /api/fetch/attachment', 37 | ); 38 | } 39 | 40 | try { 41 | const unixFriendlyFilename = filename.replace(/\s/g, '_'); 42 | const localFilePath = `/tmp/${unixFriendlyFilename}`; 43 | base64.base64Decode(data.base64, localFilePath); 44 | 45 | const params = { 46 | inputFilePath: localFilePath, 47 | password: pdfPasswordInput, 48 | outputFilePath: `/tmp/${unixFriendlyFilename 49 | .split('.') 50 | .map((v, idx) => (idx === 0 ? `${v}_unlocked` : v)) 51 | .join('.')}`, 52 | }; 53 | 54 | removePdfPassword(params); 55 | 56 | const messageBody = await fetchEmailByMessageId({ 57 | messageId, 58 | refreshToken: req.refresh_token, 59 | }); 60 | 61 | const messageProps = processMessageBody(messageBody); 62 | 63 | const emailOpts = { 64 | from: `${messageProps.from} <${process.env.NEXT_PUBLIC_SENDING_EMAIL_ID}>`, 65 | to: req.user.email, 66 | subject: `[UNLOCKED] ${messageProps.subject}`, 67 | 'h:Reply-To': 'aakash@emailapi.io', 68 | html: messageProps.message, 69 | attachment: params.outputFilePath, 70 | }; 71 | 72 | queues.sendEmailQueue.add(emailOpts); 73 | 74 | res.json({ 75 | pollQuery: `from:(${emailOpts.from}) subject:(${ 76 | emailOpts.subject 77 | }) after:${getAfterTs(new Date())}`, 78 | }); 79 | return resolve(); 80 | } catch (e) { 81 | res.status(500).send(e); 82 | console.log(e); 83 | return resolve(); 84 | } 85 | } catch (e) { 86 | res.status(500).send(e); 87 | console.log(e); 88 | return resolve(); 89 | } 90 | } 91 | 92 | export default ensureAuth(handle); 93 | -------------------------------------------------------------------------------- /pages/_document.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Document, { Html, Head, Main, NextScript } from 'next/document'; 3 | import { ServerStyleSheet } from 'styled-components'; 4 | 5 | class MyDocument extends Document { 6 | static async getInitialProps(ctx) { 7 | const sheet = new ServerStyleSheet(); 8 | const originalRenderPage = ctx.renderPage; 9 | 10 | try { 11 | ctx.renderPage = () => 12 | originalRenderPage({ 13 | enhanceApp: (App) => (props) => 14 | sheet.collectStyles(), 15 | }); 16 | 17 | const initialProps = await Document.getInitialProps(ctx); 18 | return { 19 | ...initialProps, 20 | styles: ( 21 | <> 22 | {initialProps.styles} 23 | {sheet.getStyleElement()} 24 | 25 | ), 26 | }; 27 | } finally { 28 | sheet.seal(); 29 | } 30 | } 31 | 32 | render() { 33 | return ( 34 | 35 | 36 | 37 | 38 | 39 | 43 | 44 | 48 | 54 | 55 | 56 | 60 |