├── .devcontainer ├── devcontainer.json └── postbuild ├── .gitattributes ├── .gitignore ├── .nvmrc ├── .prettierrc ├── .vscode ├── settings.json └── tasks.json ├── README.md ├── core ├── README.md ├── dev.js ├── package.json ├── src │ ├── BotSecrets.ts │ ├── CodeEvaluation.ts │ ├── Cron.ts │ ├── DataEncryption.ts │ ├── DeviceTracking.ts │ ├── ExpenseTracking.ts │ ├── HomeAutomation.ts │ ├── ImageMessageHandler.ts │ ├── LINEClient.ts │ ├── LINEMessageUtilities.ts │ ├── LanguageModelAssistant.ts │ ├── MessageHandler.ts │ ├── MessageHistory.ts │ ├── MongoDatabase.ts │ ├── NotificationProcessor.ts │ ├── PersistentState.ts │ ├── PhoneFinder.ts │ ├── PreludeCode.ts │ ├── RomanNumerals.ts │ ├── SMSHandler.ts │ ├── SlackMessageUtilities.ts │ ├── SpeedDial.ts │ ├── SpendingTracking.ts │ ├── TemporaryBlobStorage.ts │ ├── Tracing.ts │ ├── bot.ts │ ├── logger.ts │ ├── modules.d.ts │ ├── typedefs.d.ts │ └── types.ts └── tsconfig.json ├── images ├── .gitkeep ├── api.png ├── auto_expense.png ├── expense_tracking.png ├── home_automation.png ├── image_to_text.png ├── livescript.png ├── quick_replies.png └── transaction_aggregation.png ├── package.json ├── pnpm-lock.yaml ├── pnpm-workspace.yaml └── webui ├── .eslintrc.cjs ├── .gitignore ├── README.md ├── app ├── Clock.tsx ├── backend.ts ├── firebase.ts ├── index.css ├── requireAuth.ts ├── root.tsx └── routes │ ├── _index.tsx │ ├── automatron._index.tsx │ ├── automatron.knobs.tsx │ └── automatron.tsx ├── env.d.ts ├── package.json ├── playwright-report └── index.html ├── playwright.config.ts ├── postcss.config.cjs ├── public └── favicon.ico ├── tailwind.config.cjs ├── tests └── automatron.spec.ts ├── tsconfig.json ├── vercel.json └── vite.config.ts /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: 2 | // https://github.com/microsoft/vscode-dev-containers/tree/v0.192.0/containers/codespaces-linux 3 | { 4 | "image": "mcr.microsoft.com/devcontainers/universal:2", 5 | "containerEnv": { 6 | "GOOGLE_APPLICATION_CREDENTIALS": "/home/codespace/.google-cloud-service-account.json" 7 | }, 8 | "postCreateCommand": "pnpm install && ./.devcontainer/postbuild" 9 | } 10 | -------------------------------------------------------------------------------- /.devcontainer/postbuild: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | node core/dev set-up-codespaces -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Don't allow people to merge changes to these generated files, because the result 2 | # may be invalid. You need to run "rush update" again. 3 | pnpm-lock.yaml merge=binary 4 | shrinkwrap.yaml merge=binary 5 | npm-shrinkwrap.json merge=binary 6 | yarn.lock merge=binary 7 | 8 | # Rush's JSON config files use JavaScript-style code comments. The rule below prevents pedantic 9 | # syntax highlighters such as GitHub's from highlighting these comments as errors. Your text editor 10 | # may also require a special configuration to allow comments in JSON. 11 | # 12 | # For more information, see this issue: https://github.com/microsoft/rushstack/issues/1088 13 | # 14 | *.json linguist-language=JSON-with-Comments 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .cache 3 | node_modules 4 | /automatron.js 5 | /automatron.js.map 6 | /dist/ 7 | *.gz 8 | webpack.stats.json 9 | *.env 10 | 11 | # Rush temporary files 12 | common/deploy/ 13 | common/temp/ 14 | common/autoinstallers/*/.npmrc 15 | **/.rush/temp/ 16 | 17 | # Heft 18 | .heft 19 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v12 2 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "semi": false 4 | } 5 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": ["Automatron"] 3 | } 4 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "npm", 6 | "script": "dev", 7 | "path": "core/", 8 | "problemMatcher": [], 9 | "label": "npm: dev - core", 10 | "detail": "node dev.js", 11 | "group": { 12 | "kind": "build", 13 | "isDefault": true 14 | } 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # automatron 2 | 3 | This is my personal LINE bot that helps me automate various tasks of everyday life, such as 4 | **home control** (air conditioner, lights and plugs) and **expense tracking** (record how much I spend each day). 5 | [See below for a feature tour.](#features) 6 | 7 | I recommend every developer to try creating their own personal assistant chat bot. 8 | It’s a great way to practice coding and improve problem solving skills. 9 | And it helps make life more convenient! 10 | 11 | It is written in TypeScript and runs on [Google Cloud Run](https://cloud.google.com/run) on top of [evalaas](https://github.com/dtinth/evalaas) JavaScript-execution platform. 12 | 13 | ## features 14 | 15 | - [home automation](#home-automation) 16 | - [expense tracking](#expense-tracking) 17 | - [transaction aggregation](#transaction-aggregation) 18 | - [image-to-text](#image-to-text) 19 | - [livescript evaluation](#livescript-evaluation) 20 | 21 | ### home automation 22 | 23 | ![home automation](./images/home_automation.png) 24 | 25 | I have a Raspberry Pi set up which can [control lights](https://github.com/dtinth/hue.sh), [air conditioner](https://medium.com/@dtinth/remotely-turning-on-my-air-conditioner-through-google-assistant-1a1441471e9d), and [smart plugs](https://ifttt.com/services/kasa). It receives commands via [Google Cloud IoT Core](https://cloud.google.com/iot-core/), performs the action, and then reports back to automatron via [its API](#cli-api). 26 | 27 | ### expense tracking 28 | 29 | ![expense tracking](./images/expense_tracking.png) 30 | 31 | Simple expense tracking by typing in the amount + category. Example: 50f means ฿50 for food. Data is saved in [Airtable](https://airtable.com/). 32 | 33 | On mobile, tapping the bubble’s body (containing the amount) will take me to the created Airtable record. This allows me to easily edit or add remarks to the record. Tapping the bubble’s footer (containing the stats) will take me to Airtable view, which lets me see all the recorded data. 34 | 35 | ### transaction aggregation 36 | 37 | ![transaction_aggregation](./images/transaction_aggregation.png) 38 | 39 | I [set up IFTTT to read SMS messages](https://ifttt.com/services/android_messages) and send it to automatron. It then uses [transaction-parser-th](https://github.com/dtinth/transaction-parser-th) to parse SMS message and extract transaction information. It is then sent to me as a [flex message](https://developers.line.me/en/docs/messaging-api/using-flex-messages/). 40 | 41 | ![quick_replies](./images/quick_replies.png) 42 | 43 | In mobile phone, [quick reply buttons](https://developers.line.me/en/docs/messaging-api/using-quick-reply/) lets me quickly turn a transaction into an expense record by simply tapping on the category. 44 | 45 | ![auto_expense](./images/auto_expense.png) 46 | 47 | Certain kinds of transactions can be automatically be turned into an expense, for example, when I [take BTS Skytrain using Rabbit LINE Pay card](https://brandinside.asia/rabbit-line-pay-bts/). Having many features in one bot enabled this kind of tight integrations. 48 | 49 | ### image-to-text 50 | 51 | ![image_to_text](./images/image_to_text.png) 52 | 53 | automatron can also convert image to text using [Google Cloud Vision API](https://cloud.google.com/vision/). 54 | 55 | ### livescript evaluation 56 | 57 | ![livescript](./images/livescript.png) 58 | 59 | [LiveScript](https://livescript.net/) interpreter is included, which allows me to do some quick calculations. 60 | 61 | ### cli / api 62 | 63 | ![api](./images/api.png) 64 | 65 | `POST /text` sends a text command to automatron. This is equivalent to sending a text message through LINE. This allows me to create a CLI tool that lets me talk to automatron from my terminal. 66 | 67 | `POST /post` sends a message to my LINE account directly. This allows the [home automation](#home-automation) scripts to report back to me whenever the script is invoked. 68 | 69 | ## project structure 70 | 71 | This project is a monorepo managed by [Rush](https://rushjs.io/). It contains multiple subprojects: 72 | 73 | - [core](./core) — The core automatron service running on Google Cloud Run. 74 | - [webui](./webui) — The web-based UI running on Vercel. 75 | 76 | ### other projects 77 | 78 | - [automatron-prelude](https://github.dev/dtinth/automatron-prelude/blob/main/prelude.js) contains code experimentation. 79 | -------------------------------------------------------------------------------- /core/README.md: -------------------------------------------------------------------------------- 1 | # @dtinth/automatron-core 2 | 3 | The core service of automatron. Provides: 4 | 5 | - Chat bot interface (LINE, Slack) 6 | - REST API interface 7 | - Cron jobs 8 | 9 | ## secrets 10 | 11 | The secret data required to run the automation are defined in [BotSecrets.ts](./src/BotSecrets.ts). 12 | 13 | ## development workflow 14 | 15 | ### developing 16 | 17 | Watches for file changes and deploy the compiled code. Since it is my personal bot (I am the only one using it), I want a save-and-deploy workflow; there is no dev/staging environment at all. 18 | 19 | ```sh 20 | node dev 21 | ``` 22 | 23 | ### configuration 24 | 25 | ```sh 26 | # download 27 | gsutil cp gs://$GOOGLE_CLOUD_PROJECT-evalaas/evalaas/automatron.env automatron.env 28 | 29 | # upload 30 | gsutil cp automatron.env gs://$GOOGLE_CLOUD_PROJECT-evalaas/evalaas/automatron.env 31 | ``` 32 | -------------------------------------------------------------------------------- /core/dev.js: -------------------------------------------------------------------------------- 1 | require('dotenv').config() 2 | 3 | if (!process.env.GOOGLE_CLOUD_PROJECT) 4 | throw new Error('Missing GOOGLE_CLOUD_PROJECT environment variable.') 5 | 6 | const fs = require('fs') 7 | const ora = require('ora')() 8 | const FormData = require('form-data') 9 | const axios = require('axios').default 10 | const bucketName = `${process.env.GOOGLE_CLOUD_PROJECT}-evalaas` 11 | const { Storage } = require('@google-cloud/storage') 12 | const gcs = new Storage() 13 | let pushing = false 14 | let pending = false 15 | let latestResult 16 | 17 | const { GoogleAuth } = require('google-auth-library') 18 | const auth = new GoogleAuth() 19 | 20 | require('yargs') 21 | .command( 22 | '$0', 23 | 'Watches for file change and uploads automatron code.', 24 | {}, 25 | async (args) => { 26 | ora.info('Running bundler.') 27 | const ncc = require('@vercel/ncc')(require.resolve('./src/bot.ts'), { 28 | externals: [ 29 | '@google-cloud/firestore', 30 | '@google-cloud/storage', 31 | '@google-cloud/trace-agent', 32 | '@google-cloud/vision', 33 | 'mongodb', 34 | 'google-auth-library', 35 | ], 36 | sourceMap: true, 37 | sourceMapRegister: false, 38 | watch: true, 39 | }) 40 | ncc.handler((result) => { 41 | if (result.err) { 42 | console.error(result.err) 43 | return 44 | } 45 | let code = result.code 46 | const expectedFooter = '//# sourceMappingURL=index.js.map' 47 | const mapBase64 = Buffer.from(result.map).toString('base64') 48 | const mapComment = `//# sourceMappingURL=data:application/json;charset=utf-8;base64,${mapBase64}` 49 | if (code.endsWith(expectedFooter)) { 50 | code = code.slice(0, -expectedFooter.length) + mapComment 51 | } else { 52 | code += '\n' + mapComment 53 | } 54 | const codeBuffer = Buffer.from(code) 55 | const gzippedBuffer = require('zlib').gzipSync(codeBuffer) 56 | console.log( 57 | 'Compiled / Code %skb (%skb gzipped)', 58 | (codeBuffer.length / 1024).toFixed(1), 59 | (gzippedBuffer.length / 1024).toFixed(1) 60 | ) 61 | require('fs').writeFileSync('automatron.js.gz', gzippedBuffer) 62 | 63 | // https://github.com/zeit/ncc/pull/516#issuecomment-601708133 64 | require('fs').writeFileSync( 65 | 'webpack.stats.json', 66 | JSON.stringify(result.stats.toJson()) 67 | ) 68 | push() 69 | }) 70 | ncc.rebuild(() => { 71 | console.log('Rebuilding...') 72 | }) 73 | ora.info('Watching for file changes.') 74 | } 75 | ) 76 | .command('id-token', 'Prints ID token', {}, async () => { 77 | const jwt = await getJwt() 78 | console.log(jwt) 79 | }) 80 | .command('download-env', 'Downloads environment file', {}, async () => { 81 | await gcs 82 | .bucket(bucketName) 83 | .file('evalaas/automatron.env') 84 | .download({ destination: 'automatron.env' }) 85 | }) 86 | .command('upload-env', 'Uploads environment file', {}, async () => { 87 | await gcs 88 | .bucket(bucketName) 89 | .file('evalaas/automatron.env') 90 | .save(fs.readFileSync('automatron.env')) 91 | }) 92 | .command( 93 | 'set-up-codespaces', 94 | 'Downloads Google Cloud service account file for usage in GitHub Codespaces', 95 | {}, 96 | async () => { 97 | const encrypted = require('@dtinth/encrypted')( 98 | process.env.SERVICE_ACCOUNT_ENCRYPTION_KEY 99 | ) 100 | const encryptedServiceAccount = require('child_process') 101 | .execSync('curl $SERVICE_ACCOUNT_URL') 102 | .toString() 103 | .trim() 104 | const decryptedServiceAccount = encrypted(encryptedServiceAccount) 105 | 106 | const serviceAccountPath = 107 | process.env.HOME + '/.google-cloud-service-account.json' 108 | fs.writeFileSync( 109 | serviceAccountPath, 110 | JSON.stringify(decryptedServiceAccount, null, 2) 111 | ) 112 | console.log('Written service account file to', serviceAccountPath) 113 | } 114 | ) 115 | .strict() 116 | .help() 117 | .parse() 118 | 119 | async function getJwt() { 120 | const audience = 'https://github.com/dtinth/automatron' 121 | const client = await auth.getIdTokenClient(audience) 122 | const jwt = await client.idTokenProvider.fetchIdToken(audience) 123 | return jwt 124 | } 125 | 126 | async function push() { 127 | if (pushing) { 128 | pending = true 129 | return 130 | } 131 | pushing = true 132 | ora.start('Uploading code...') 133 | try { 134 | const jwt = await getJwt() 135 | const form = new FormData() 136 | const buffer = fs.readFileSync('automatron.js.gz') 137 | form.append('file', buffer, 'file') 138 | await axios.put( 139 | `${process.env.EVALAAS_URL}/admin/endpoints/automatron`, 140 | form.getBuffer(), 141 | { 142 | headers: Object.assign({}, form.getHeaders(), { 143 | Authorization: `Bearer ${jwt}`, 144 | }), 145 | } 146 | ) 147 | ora.succeed('Done! Code updated at ' + new Date().toString()) 148 | } catch (error) { 149 | ora.fail('Failed: ' + error) 150 | } finally { 151 | ora.stop() 152 | pushing = false 153 | if (pending) { 154 | pending = false 155 | push() 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@dtinth/automatron-core", 3 | "private": true, 4 | "description": "My LINE bot!", 5 | "version": "1.0.0", 6 | "license": "Apache-2.0", 7 | "author": "Google Inc.", 8 | "engines": { 9 | "node": ">=10" 10 | }, 11 | "repository": "https://github.com/dtinth/automatron", 12 | "main": "app.js", 13 | "scripts": { 14 | "start": "node app.js", 15 | "deploy": "gcloud app deploy --quiet", 16 | "dev": "node dev.js" 17 | }, 18 | "dependencies": { 19 | "@dtinth/encrypted": "^0.3.1", 20 | "@google-cloud/firestore": "^5.0.2", 21 | "@google-cloud/storage": "^5.20.5", 22 | "@google-cloud/trace-agent": "^5.1.6", 23 | "@google-cloud/vision": "^2.4.2", 24 | "@line/bot-sdk": "^9.2.2", 25 | "@slack/types": "^1.10.0", 26 | "@types/cors": "~2.8.13", 27 | "airtable": "0.5.9", 28 | "axios": "0.18.0", 29 | "body-parser": "1.18.3", 30 | "cors": "~2.8.5", 31 | "dotenv": "^7.0.0", 32 | "express": "4.16.4", 33 | "express-async-handler": "~1.2.0", 34 | "express-oauth2-jwt-bearer": "~1.1.0", 35 | "form-data": "~4.0.0", 36 | "google-auth-library": "~7.12.0", 37 | "jsonwebtoken": "^8.5.1", 38 | "lib": "~4.3.3", 39 | "livescript": "1.6.0", 40 | "lodash": "^4.17.21", 41 | "mongodb": "^4.13.0", 42 | "mqtt": "2.18.8", 43 | "nanoid": "~3.3.4", 44 | "pino": "^7.11.0", 45 | "prelude-ls": "1.1.2", 46 | "transaction-parser-th": "0.1.1", 47 | "tweetnacl": "^1.0.3", 48 | "tweetnacl-sealedbox-js": "^1.2.0", 49 | "verror": "^1.10.1" 50 | }, 51 | "devDependencies": { 52 | "@types/express": "^4.17.15", 53 | "@types/jsonwebtoken": "^8.5.9", 54 | "@types/node": "^20.11.0", 55 | "@vercel/ncc": "^0.36.0", 56 | "execa": "^1.0.0", 57 | "ora": "^3.4.0", 58 | "typescript": "^4.9.4", 59 | "yargs": "^13.3.2" 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /core/src/BotSecrets.ts: -------------------------------------------------------------------------------- 1 | export interface BotSecrets { 2 | /** The secret key that must be sent with the request to use the [API](../README.md#cli-api) */ 3 | API_KEY: string 4 | /** Self-explanatory */ 5 | LINE_CHANNEL_SECRET: string 6 | /** Self-explanatory */ 7 | LINE_CHANNEL_ACCESS_TOKEN: string 8 | /** My user ID, so that the bot receives commands from me only */ 9 | LINE_USER_ID: string 10 | /** Self-explanatory */ 11 | AIRTABLE_API_KEY: string 12 | /** The ID of the Airtable base used for tracking expenses (should start with ‘app’) */ 13 | AIRTABLE_EXPENSE_BASE: string 14 | /** The web URL to the Airtable base, used for deep linking */ 15 | AIRTABLE_EXPENSE_URI: string 16 | /** The ID of the Airtable base used for cron jobs (should start with ‘app’) */ 17 | AIRTABLE_CRON_BASE: string 18 | /** Two numbers in form of A/B, where A is usage budget per day (rolls over to the next day) and B is starting budget */ 19 | EXPENSE_PACEMAKER: string 20 | /** Slack webhook URL */ 21 | SLACK_WEBHOOK_URL: string 22 | /** My user ID, so that the bot receives commands from me only */ 23 | SLACK_USER_ID: string 24 | /** Google Cloud IoT Core device path to send */ 25 | CLOUD_IOT_CORE_DEVICE_PATH: string 26 | /** GitHub OAuth App credentials for making unauthenticated API calls with elevated rate limits. In form of ":" */ 27 | GITHUB_OAUTH_APP_CREDENTIALS: string 28 | /** Encryption secret for decrypting */ 29 | ENCRYPTION_SECRET: string 30 | /** Database storage */ 31 | MONGODB_URL: string 32 | /** Subject ID for Google Auth */ 33 | GOOGLE_AUTH_SUB: string 34 | } 35 | -------------------------------------------------------------------------------- /core/src/CodeEvaluation.ts: -------------------------------------------------------------------------------- 1 | import { getCodeExecutionContext } from './PreludeCode' 2 | import { AutomatronContext, TextMessageHandler } from './types' 3 | 4 | export async function evaluateCode(input: string, context: AutomatronContext) { 5 | const code = input.startsWith(';') 6 | ? input 7 | : require('livescript') 8 | .compile(input, { 9 | run: true, 10 | print: true, 11 | header: false, 12 | }) 13 | .replace(/^\(function/, '(async function') 14 | console.log('Code compilation result', code) 15 | const runner = new Function( 16 | ...['prelude', 'self', 'code', 'context'], 17 | 'with (prelude) { with (self) { return [ eval(code) ] } }' 18 | ) 19 | const self = await getCodeExecutionContext(context) 20 | const [value] = runner(require('prelude-ls'), self, code, context) 21 | const returnedValue = await Promise.resolve(value) 22 | let result = postProcessResult(returnedValue) 23 | const extraMessages = [...self.extraMessages] 24 | return { result, extraMessages } 25 | } 26 | 27 | function postProcessResult(returnedValue: any) { 28 | if (typeof returnedValue === 'string') { 29 | return returnedValue 30 | } 31 | return require('util').inspect(returnedValue) 32 | } 33 | 34 | export const CodeEvaluationMessageHandler: TextMessageHandler = ( 35 | text, 36 | context 37 | ) => { 38 | if (text.startsWith(';')) { 39 | return async () => { 40 | const input = text.slice(1) 41 | try { 42 | var { result, extraMessages } = await evaluateCode(input, context) 43 | return [{ type: 'text', text: result }, ...extraMessages] 44 | } catch (error: any) { 45 | const stack = String(error.stack).replace( 46 | /\/evalaas\/webpack:\/@dtinth\/automatron-core\//g, 47 | '' 48 | ) 49 | return [{ type: 'text', text: `❌ EVALUATION FAILED ❌\n${stack}` }] 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /core/src/Cron.ts: -------------------------------------------------------------------------------- 1 | import { ObjectId } from 'mongodb' 2 | import { getDb } from './MongoDatabase' 3 | import { AutomatronContext } from './types' 4 | 5 | interface CronEntry { 6 | name: string 7 | scheduledTime: string 8 | completed: boolean 9 | notes?: string 10 | } 11 | 12 | export async function addCronEntry( 13 | context: AutomatronContext, 14 | time: number, 15 | text: string 16 | ) { 17 | const targetTime = new Date(time) 18 | targetTime.setUTCSeconds(0) 19 | targetTime.setUTCMilliseconds(0) 20 | 21 | const collection = await getCronCollection(context) 22 | await collection.insertOne({ 23 | name: text, 24 | scheduledTime: targetTime.toISOString(), 25 | completed: false, 26 | }) 27 | 28 | return { 29 | localTime: new Date(targetTime.getTime() + 7 * 3600e3) 30 | .toJSON() 31 | .replace(/\.000Z/, ''), 32 | } 33 | } 34 | 35 | export async function getCronCollection(context: AutomatronContext) { 36 | const db = await getDb(context) 37 | return db.collection('cronJobs') 38 | } 39 | 40 | export async function getPendingCronJobs(context: AutomatronContext) { 41 | const collection = await getCronCollection(context) 42 | return collection.find({ completed: false }).toArray() 43 | } 44 | 45 | export async function updateCronJob( 46 | context: AutomatronContext, 47 | jobId: string, 48 | update: Partial 49 | ) { 50 | const collection = await getCronCollection(context) 51 | await collection.updateOne({ _id: new ObjectId(jobId) }, { $set: update }) 52 | } 53 | -------------------------------------------------------------------------------- /core/src/DataEncryption.ts: -------------------------------------------------------------------------------- 1 | import Encrypted from '@dtinth/encrypted' 2 | import { AutomatronContext } from './types' 3 | 4 | export function decrypt( 5 | context: AutomatronContext, 6 | encryptedPayload: string 7 | ): any { 8 | const encrypted = Encrypted(context.secrets.ENCRYPTION_SECRET) 9 | return encrypted(encryptedPayload) 10 | } 11 | 12 | export function encrypt(context: AutomatronContext, payload: any): string { 13 | const encrypted = Encrypted(context.secrets.ENCRYPTION_SECRET) 14 | return encrypted.encrypt(payload) 15 | } 16 | -------------------------------------------------------------------------------- /core/src/DeviceTracking.ts: -------------------------------------------------------------------------------- 1 | import { getDb } from './MongoDatabase' 2 | import { ref } from './PersistentState' 3 | import { AutomatronContext } from './types' 4 | import { logger } from './logger' 5 | 6 | async function getCollection(context: AutomatronContext) { 7 | const db = await getDb(context) 8 | return db.collection('deviceLog') 9 | } 10 | 11 | interface DeviceLogEntry { 12 | time: string 13 | deviceId: string 14 | key: string 15 | value: any 16 | } 17 | 18 | async function getDeviceStateUpdater( 19 | context: AutomatronContext, 20 | deviceId: string 21 | ) { 22 | const collection = await getCollection(context) 23 | const device = getDeviceRef(context, deviceId) 24 | const state = (await device.get()) || {} 25 | const time = new Date().toISOString() 26 | let changed = false 27 | return { 28 | time, 29 | state, 30 | update: async (key: string, value: any) => { 31 | if (JSON.stringify(state[key]) !== JSON.stringify(value)) { 32 | state[key] = value 33 | await collection.insertOne({ 34 | time, 35 | deviceId, 36 | key, 37 | value, 38 | }) 39 | logger.info( 40 | { deviceId, key, value }, 41 | `Device "${deviceId}" property "${key}" changed to "${value}"` 42 | ) 43 | changed = true 44 | } 45 | }, 46 | updateSilently: async (key: string, value: any) => { 47 | if (JSON.stringify(state[key]) !== JSON.stringify(value)) { 48 | state[key] = value 49 | changed = true 50 | } 51 | }, 52 | save: async () => { 53 | if (changed) { 54 | await device.set(state) 55 | } 56 | }, 57 | } 58 | } 59 | 60 | export async function trackDevice( 61 | context: AutomatronContext, 62 | deviceId: string, 63 | properties: Record 64 | ) { 65 | const updater = await getDeviceStateUpdater(context, deviceId) 66 | if (!('ip' in properties)) { 67 | properties.ip = context.requestInfo.ip 68 | } 69 | for (const [key, value] of Object.entries(properties)) { 70 | await updater.update(key, value) 71 | } 72 | await updater.updateSilently('lastSeen', updater.time) 73 | await updater.save() 74 | return updater.state 75 | } 76 | 77 | function getDeviceRef(context: AutomatronContext, deviceId: string) { 78 | return ref(context, 'devices.' + deviceId) 79 | } 80 | 81 | export async function checkDeviceOnlineStatus(context: AutomatronContext) { 82 | const deviceIds = await getDeviceIds(context) 83 | for (const deviceId of deviceIds) { 84 | const updater = await getDeviceStateUpdater(context, deviceId) 85 | if ( 86 | updater.state.online && 87 | Date.now() - new Date(updater.state.lastSeen).getTime() > 1000 * 60 * 3 88 | ) { 89 | await updater.update('online', false) 90 | } 91 | await updater.save() 92 | } 93 | } 94 | 95 | async function getDeviceIds(context: AutomatronContext) { 96 | const value = await ref(context, 'deviceIds').get() 97 | return (value || '').split(',').filter(Boolean) 98 | } 99 | -------------------------------------------------------------------------------- /core/src/ExpenseTracking.ts: -------------------------------------------------------------------------------- 1 | import { FlexBox, messagingApi } from '@line/bot-sdk' 2 | import Airtable, { AirtableRecord } from 'airtable' 3 | import { AutomatronContext } from './types' 4 | import { createBubble } from './LINEMessageUtilities' 5 | 6 | export async function recordExpense( 7 | context: AutomatronContext, 8 | amount: string, 9 | category: string, 10 | remarks = '' 11 | ) { 12 | const date = new Date().toJSON().split('T')[0] 13 | // Airtable 14 | const table = getExpensesTable(context) 15 | const record = await table.create( 16 | { 17 | Date: date, 18 | Category: category, 19 | Amount: amount, 20 | Remarks: remarks, 21 | }, 22 | { typecast: true } 23 | ) 24 | const body: messagingApi.FlexBox = { 25 | type: 'box', 26 | layout: 'vertical', 27 | contents: [ 28 | { 29 | type: 'text', 30 | text: '฿' + amount, 31 | size: 'xxl', 32 | weight: 'bold', 33 | }, 34 | { 35 | type: 'text', 36 | text: `${category}\nrecorded`, 37 | wrap: true, 38 | }, 39 | ], 40 | action: { 41 | type: 'uri', 42 | label: 'Open Airtable', 43 | uri: context.secrets.AIRTABLE_EXPENSE_URI + '/' + record.getId(), 44 | }, 45 | } 46 | const footer = await getExpensesSummaryData(context) 47 | const bubble = createBubble('expense tracking', body, { 48 | headerColor: '#ffffbb', 49 | footer: { 50 | type: 'box', 51 | layout: 'horizontal', 52 | spacing: 'sm', 53 | contents: footer.map(([label, text]) => ({ 54 | type: 'box', 55 | layout: 'vertical', 56 | contents: [ 57 | { 58 | type: 'text', 59 | text: label, 60 | color: '#8b8685', 61 | size: 'xs', 62 | align: 'end', 63 | }, 64 | { 65 | type: 'text', 66 | text: text, 67 | color: '#8b8685', 68 | size: 'sm', 69 | align: 'end', 70 | }, 71 | ], 72 | })), 73 | action: { 74 | type: 'uri', 75 | label: 'Open Airtable', 76 | uri: context.secrets.AIRTABLE_EXPENSE_URI, 77 | }, 78 | }, 79 | }) 80 | return bubble 81 | } 82 | function getExpensesTable(context: AutomatronContext) { 83 | return new Airtable({ apiKey: context.secrets.AIRTABLE_API_KEY }) 84 | .base(context.secrets.AIRTABLE_EXPENSE_BASE) 85 | .table('Expense records') 86 | } 87 | async function getExpensesSummaryData(context: AutomatronContext) { 88 | const date = new Date().toJSON().split('T')[0] 89 | const tableData = await getExpensesTable(context).select().all() 90 | const normalRecords = tableData.filter((r) => !r.get('Occasional')) 91 | const total = (records: AirtableRecord[]) => 92 | records.map((r) => +r.get('Amount') || 0).reduce((a, b) => a + b, 0) 93 | const firstDate = normalRecords 94 | .map((r) => r.get('Date')) 95 | .reduce((a, b) => (a < b ? a : b), date) 96 | const todayUsage = total(normalRecords.filter((r) => r.get('Date') === date)) 97 | const totalUsage = total(normalRecords) 98 | const dayNumber = 99 | Math.round((Date.parse(date) - Date.parse(firstDate)) / 86400e3) + 1 100 | const [pacemakerPerDay, pacemakerBase] = 101 | context.secrets.EXPENSE_PACEMAKER.split('/') 102 | const pacemaker = +pacemakerBase + +pacemakerPerDay * dayNumber - totalUsage 103 | const $ = (v: number) => `฿${v.toFixed(2)}` 104 | return [ 105 | ['today', $(todayUsage)], 106 | ['pace', $(pacemaker)], 107 | ['day', `${dayNumber}`], 108 | ] 109 | } 110 | -------------------------------------------------------------------------------- /core/src/HomeAutomation.ts: -------------------------------------------------------------------------------- 1 | import Encrypted from '@dtinth/encrypted' 2 | import { AutomatronContext } from './types' 3 | import axios from 'axios' 4 | 5 | export async function sendHomeCommand( 6 | context: AutomatronContext, 7 | cmd: string | string[] 8 | ): Promise { 9 | const cmds = Array.isArray(cmd) ? cmd : [cmd] 10 | const encrypted = Encrypted(context.secrets.ENCRYPTION_SECRET) 11 | const { url, key } = encrypted`mz8Dc0LiBPPI63I5lMJHdC3RC9VF//S2.QYg30oYhuEUS8b1u80vM7sO0cv4OLYNtxy52fTMnf2Vcg8QZHlLeXUV1qPnEjR/5jYTid5hG6of8mHDHjTE1A+luDzplgM4WJQBgDNM2pkRnKnbcmAUw8MXxBb4ZMqrrAyFELigKoELWwbDg51ErhFXrm+n3hzfpbRIcge1BdH6aEPtNipsUcMj7q7BfBqFLZA==` 12 | await Promise.all( 13 | cmds.map(async (command) => { 14 | const id = 15 | new Date().toJSON() + 16 | Math.floor(Math.random() * 10000) 17 | .toString() 18 | .padStart(2, '0') 19 | await axios.post(url, { id, topic: 'home', data: command }, { 20 | headers: { 21 | 'X-Api-Key': key 22 | } 23 | }) 24 | }) 25 | ) 26 | } 27 | -------------------------------------------------------------------------------- /core/src/ImageMessageHandler.ts: -------------------------------------------------------------------------------- 1 | import vision from '@google-cloud/vision' 2 | import { ref } from './PersistentState' 3 | import { getBlob, getBlobUrl } from './TemporaryBlobStorage' 4 | import { TextMessage, TextMessageHandler } from './types' 5 | 6 | export const ImageMessageHandler: TextMessageHandler = (text, context) => { 7 | if (text.startsWith('image:')) { 8 | return async () => { 9 | const blobName = text.slice(6) 10 | await ref(context, 'latestImage').set(blobName) 11 | return [{ type: 'text', text: blobName }] 12 | } 13 | } 14 | if (text === 'annotate') { 15 | return async () => { 16 | const blobName = await ref(context, 'latestImage').get() 17 | return await annotateImage(blobName) 18 | } 19 | } 20 | if (text === 'image url') { 21 | return async () => { 22 | const blobName = await ref(context, 'latestImage').get() 23 | return await getBlobUrl(blobName) 24 | } 25 | } 26 | } 27 | 28 | async function annotateImage(blobName: string) { 29 | const buffer = await getBlob(blobName) 30 | const imageAnnotator = new vision.ImageAnnotatorClient() 31 | const results = await imageAnnotator.documentTextDetection(buffer) 32 | const fullTextAnnotation = results[0].fullTextAnnotation 33 | const blocks: string[] = [] 34 | for (const page of fullTextAnnotation.pages) { 35 | blocks.push( 36 | ...page.blocks.map((block) => { 37 | return block.paragraphs 38 | .map((p) => 39 | p.words.map((w) => w.symbols.map((s) => s.text).join('')).join(' ') 40 | ) 41 | .join('\n\n') 42 | }) 43 | ) 44 | } 45 | const blocksToResponses = (blocks: string[]) => { 46 | if (blocks.length <= 4) return blocks 47 | let processedIndex = 0 48 | const outBlocks = [] 49 | for (let i = 0; i < 4; i++) { 50 | const targetIndex = Math.ceil(((i + 1) * blocks.length) / 4) 51 | outBlocks.push( 52 | blocks 53 | .slice(processedIndex, targetIndex) 54 | .map((x) => `・ ${x}`) 55 | .join('\n') 56 | ) 57 | processedIndex = targetIndex 58 | } 59 | return outBlocks 60 | } 61 | const responses = blocksToResponses(blocks) 62 | return [...responses.map((r): TextMessage => ({ type: 'text', text: r }))] 63 | } 64 | -------------------------------------------------------------------------------- /core/src/LINEClient.ts: -------------------------------------------------------------------------------- 1 | import { messagingApi } from '@line/bot-sdk' 2 | 3 | export class LINEClient { 4 | private readonly api: messagingApi.MessagingApiClient 5 | private readonly blobApi: messagingApi.MessagingApiBlobClient 6 | constructor(config: { channelAccessToken: string }) { 7 | this.api = new messagingApi.MessagingApiClient({ 8 | channelAccessToken: config.channelAccessToken, 9 | }) 10 | this.blobApi = new messagingApi.MessagingApiBlobClient({ 11 | channelAccessToken: config.channelAccessToken, 12 | }) 13 | } 14 | replyMessage(replyToken: string, messages: messagingApi.Message[]) { 15 | return this.api.replyMessage({ replyToken, messages }) 16 | } 17 | pushMessage(to: string, messages: messagingApi.Message[]) { 18 | return this.api.pushMessage({ to, messages }) 19 | } 20 | getMessageContent(messageId: string) { 21 | return this.blobApi.getMessageContent(messageId) 22 | } 23 | showLoadingAnimation(chatId: string) { 24 | return this.api.showLoadingAnimation({ chatId }) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/LINEMessageUtilities.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FlexBox, 3 | FlexText, 4 | FlexBubble, 5 | FlexMessage, 6 | messagingApi, 7 | } from '@line/bot-sdk' 8 | 9 | export function toMessages(data: any): messagingApi.Message[] { 10 | if (!data) data = '...' 11 | if (typeof data === 'string') data = [{ type: 'text', text: data }] 12 | return data 13 | } 14 | 15 | export function createBubble( 16 | title: string, 17 | text: string | messagingApi.FlexBox, 18 | { 19 | headerBackground = '#353433', 20 | headerColor = '#d7fc70', 21 | textSize = 'xl', 22 | altText = String(text), 23 | footer, 24 | }: { 25 | headerBackground?: string 26 | headerColor?: string 27 | textSize?: messagingApi.FlexText['size'] 28 | altText?: string 29 | footer?: string | messagingApi.FlexBox 30 | } = {} 31 | ): messagingApi.FlexMessage { 32 | const data: messagingApi.FlexContainer = { 33 | type: 'bubble', 34 | styles: { 35 | header: { backgroundColor: headerBackground }, 36 | }, 37 | header: { 38 | type: 'box', 39 | layout: 'vertical', 40 | contents: [ 41 | { type: 'text', text: title, color: headerColor, weight: 'bold' }, 42 | ], 43 | }, 44 | body: 45 | typeof text === 'string' 46 | ? { 47 | type: 'box', 48 | layout: 'vertical', 49 | contents: [ 50 | { type: 'text', text: text, wrap: true, size: textSize }, 51 | ], 52 | } 53 | : text, 54 | } 55 | if (footer) { 56 | data.styles!.footer = { backgroundColor: '#e9e8e7' } 57 | data.footer = 58 | typeof footer === 'string' 59 | ? { 60 | type: 'box', 61 | layout: 'vertical', 62 | contents: [ 63 | { 64 | type: 'text', 65 | text: footer, 66 | wrap: true, 67 | size: 'sm', 68 | color: '#8b8685', 69 | }, 70 | ], 71 | } 72 | : footer 73 | } 74 | return { 75 | type: 'flex', 76 | altText: truncate(`[${title}] ${altText}`, 400), 77 | contents: data, 78 | } 79 | } 80 | 81 | function truncate(text: string, maxLength: number) { 82 | return text.length + 5 > maxLength 83 | ? text.substr(0, maxLength - 5) + '…' 84 | : text 85 | } 86 | -------------------------------------------------------------------------------- /core/src/LanguageModelAssistant.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import { decrypt } from './DataEncryption' 3 | import { logger } from './logger' 4 | import { getDb } from './MongoDatabase' 5 | import { ref } from './PersistentState' 6 | import { 7 | AutomatronContext, 8 | AutomatronResponse, 9 | TextMessageHandler, 10 | } from './types' 11 | 12 | type Role = 'system' | 'user' | 'assistant' 13 | interface ChatMessage { 14 | role: Role 15 | content: string 16 | } 17 | interface LlmHistoryEntry { 18 | time: string 19 | contextMessages?: ChatMessage[] 20 | inText: string 21 | outText: string 22 | } 23 | 24 | async function getCollection(context: AutomatronContext) { 25 | const db = await getDb(context) 26 | return db.collection('llmHistory') 27 | } 28 | 29 | export const LanguageModelAssistantMessageHandler: TextMessageHandler = ( 30 | text, 31 | context 32 | ) => { 33 | const runLlm = async ( 34 | inText: string, 35 | continueFrom?: LlmHistoryEntry 36 | ): Promise => { 37 | const key = decrypt( 38 | context, 39 | 'c0wgjM3RJp/V40lLhcHbtjUDBvlT/NlI.yF9iWwImsHrUOiZkD6UMZdGyjLd3yCJm7WMkN6dxerzXlxCK4U6bSzNFHwyvUjPDop4+gLCs5Qa9Pcxwii3DvSVjjE+7' 40 | ) 41 | const nowIct = new Date(Date.now() + 7 * 3600e3) 42 | const day = [ 43 | 'Sunday', 44 | 'Monday', 45 | 'Tuesday', 46 | 'Wednesday', 47 | 'Thursday', 48 | 'Friday', 49 | 'Saturday', 50 | ][nowIct.getDay()] 51 | 52 | const prompt: string[] = [ 53 | // https://beta.openai.com/examples/default-chat 54 | 'Current date and time: ' + 55 | nowIct.toISOString().replace('Z', '+07:00') + 56 | ' (Asia/Bangkok).', 57 | 'Today is ' + day + '.', 58 | 'The following is a conversation with an AI assistant, automatron. ' + 59 | 'The assistant is helpful, creative, clever, funny, and very friendly. ' + 60 | (await ref(context, 'llmPrompt').get()), 61 | ] 62 | const newContext: ChatMessage[] = [ 63 | ...(continueFrom 64 | ? [ 65 | ...(continueFrom.contextMessages ?? []), 66 | { role: 'user' as Role, content: continueFrom.inText }, 67 | { role: 'assistant' as Role, content: continueFrom.outText }, 68 | ] 69 | : []), 70 | ] 71 | const messages: ChatMessage[] = [ 72 | { role: 'system', content: prompt.join('\n') }, 73 | ...newContext, 74 | { role: 'user', content: inText }, 75 | ] 76 | const payload = { 77 | model: 'chatgpt-4o-latest', 78 | messages, 79 | temperature: +(await ref(context, 'llmTemperature').get()) || 0.5, 80 | } 81 | const response = await axios.post( 82 | 'https://api.openai.com/v1/chat/completions', 83 | payload, 84 | { headers: { Authorization: `Bearer ${key}` } } 85 | ) 86 | const responseText = response.data.choices[0].message.content.trim() 87 | logger.info( 88 | { assistant: { prompt, response: response.data } }, 89 | 'Ran OpenAI assistant' 90 | ) 91 | const collection = await getCollection(context) 92 | await collection.insertOne({ 93 | time: new Date().toISOString(), 94 | contextMessages: newContext, 95 | inText, 96 | outText: responseText, 97 | }) 98 | return [{ type: 'text', text: responseText }] 99 | } 100 | 101 | if (text.match(/^hey\b/i)) { 102 | return async () => { 103 | return runLlm(text) 104 | } 105 | } 106 | if (text.match(/^hmm?\b/i)) { 107 | return async () => { 108 | const collection = await getCollection(context) 109 | const [lastEntry] = await collection 110 | .find({}) 111 | .sort({ _id: -1 }) 112 | .limit(1) 113 | .toArray() 114 | return runLlm(text, lastEntry) 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /core/src/MessageHandler.ts: -------------------------------------------------------------------------------- 1 | import { CodeEvaluationMessageHandler } from './CodeEvaluation' 2 | import { addCronEntry } from './Cron' 3 | import { recordExpense } from './ExpenseTracking' 4 | import { sendHomeCommand } from './HomeAutomation' 5 | import { ImageMessageHandler } from './ImageMessageHandler' 6 | import { LanguageModelAssistantMessageHandler } from './LanguageModelAssistant' 7 | import { 8 | MessageHistoryMessageHandler, 9 | saveMessageHistory, 10 | } from './MessageHistory' 11 | import { getDb } from './MongoDatabase' 12 | import { ref } from './PersistentState' 13 | import { PhoneFinderMessageHandler } from './PhoneFinder' 14 | import { getCodeExecutionContext } from './PreludeCode' 15 | import { decodeRomanNumerals } from './RomanNumerals' 16 | import { SpendingTrackingMessageHandler } from './SpendingTracking' 17 | import { putBlob } from './TemporaryBlobStorage' 18 | import { trace } from './Tracing' 19 | import { AutomatronContext, AutomatronResponse } from './types' 20 | 21 | const messageHandlers = [ 22 | CodeEvaluationMessageHandler, 23 | ImageMessageHandler, 24 | MessageHistoryMessageHandler, 25 | SpendingTrackingMessageHandler, 26 | PhoneFinderMessageHandler, 27 | LanguageModelAssistantMessageHandler, 28 | ] 29 | 30 | export async function handleTextMessage( 31 | context: AutomatronContext, 32 | message: string, 33 | options: { source: string } 34 | ): Promise { 35 | message = message.trim() 36 | let match: RegExpMatchArray | null 37 | 38 | context.addPromise( 39 | 'Save text history', 40 | getDb(context).then((db) => 41 | trace(context, 'Save history', () => 42 | saveMessageHistory(context, message, options.source) 43 | ) 44 | ) 45 | ) 46 | 47 | if (message === 'ac on' || message === 'sticker:2:27') { 48 | await sendHomeCommand(context, 'ac on') 49 | return 'ok, turning air-con on' 50 | } else if (message === 'ac off' || message === 'sticker:2:29') { 51 | await sendHomeCommand(context, 'ac off') 52 | return 'ok, turning air-con off' 53 | } else if (message === 'power on' || message === 'plugs on') { 54 | await sendHomeCommand(context, 'plugs on') 55 | return 'ok, turning smart plugs on' 56 | } else if (message === 'power off' || message === 'plugs off') { 57 | await sendHomeCommand(context, 'plugs off') 58 | return 'ok, turning smart plugs off' 59 | } else if ( 60 | message === 'home' || 61 | message === 'arriving' || 62 | message === 'sticker:2:503' 63 | ) { 64 | await sendHomeCommand(context, ['plugs on', 'lights normal', 'ac on']) 65 | return 'preparing home' 66 | } else if (message === 'leaving' || message === 'sticker:2:502') { 67 | await sendHomeCommand(context, ['plugs off', 'lights off', 'ac off']) 68 | return 'bye' 69 | } else if (message === 'lights' || message === 'sticker:4:275') { 70 | await sendHomeCommand(context, 'lights normal') 71 | return 'ok, lights normal' 72 | } else if ( 73 | message === 'bedtime' || 74 | message === 'gn' || 75 | message === 'gngn' || 76 | message === 'sticker:11539:52114128' 77 | ) { 78 | await sendHomeCommand(context, 'lights dimmed') 79 | await addCronEntry(context, Date.now() + 300e3, 'lights off') 80 | const prelude = await getCodeExecutionContext(context) 81 | await prelude.executeHandlers('bedtime') 82 | return 'ok, good night' 83 | } else if (message === 'ooo') { 84 | const prelude = await getCodeExecutionContext(context) 85 | await prelude.executeHandlers('ooo') 86 | return 'ok, out of office' 87 | } else if (message === 'work') { 88 | const prelude = await getCodeExecutionContext(context) 89 | await prelude.executeHandlers('work') 90 | return 'ok, set working status' 91 | } else if ((match = message.match(/^lights (\w+)$/))) { 92 | const cmd = match[1] 93 | await sendHomeCommand(context, 'lights ' + cmd) 94 | return 'ok, lights ' + cmd 95 | } else if ((match = message.match(/^in ([\d\.]+)([mh]),?\s+([^]+)$/))) { 96 | const targetTime = 97 | Date.now() + +match[1] * (match[2] === 'm' ? 60 : 3600) * 1e3 98 | const result = await addCronEntry(context, targetTime, match[3]) 99 | return `will run "${match[3]}" at ${result.localTime}` 100 | } else if ((match = message.match(/^([\d.]+|[ivxlcdm]+)(j?)([tfghmol])$/i))) { 101 | const m = match 102 | const enteredAmount = m[1].match(/[ivxlcdm]/) 103 | ? decodeRomanNumerals(m[1]) 104 | : +m[1] 105 | const conversionRate = m[2] ? 0.302909 : 1 106 | const amount = (enteredAmount * conversionRate).toFixed(2) 107 | const category = ( 108 | { 109 | t: 'transportation', 110 | f: 'food', 111 | g: 'game', 112 | h: 'health', 113 | m: 'miscellaneous', 114 | o: 'occasion', 115 | l: 'lodging', 116 | } as { 117 | [k: string]: string 118 | } 119 | )[m[3].toLowerCase()] 120 | const remarks = m[2] ? `${m[1]} JPY` : '' 121 | return await recordExpense(context, amount, category, remarks) 122 | } else if ((match = message.match(/^([ivxlcdm]+)$/i))) { 123 | return `${match[1]} = ${decodeRomanNumerals(match[1])}` 124 | } 125 | 126 | // Go through message handlers and see if any of them can handle the message 127 | for (const handler of messageHandlers) { 128 | const action = handler(message, context) 129 | if (action) { 130 | return action() 131 | } 132 | } 133 | 134 | // At this point, the message is not recognized. 135 | // Just save it to the stack. 136 | const size = await ref(context, 'stack').push(message) 137 | return '(unrecognized message, saved to stack (size=' + size + '))' 138 | } 139 | 140 | export async function handleImage( 141 | context: AutomatronContext, 142 | imageBuffer: Buffer, 143 | options: { source: string } 144 | ) { 145 | const blobName = await putBlob(imageBuffer, '.jpg') 146 | return await handleTextMessage(context, 'image:' + blobName, options) 147 | } 148 | -------------------------------------------------------------------------------- /core/src/MessageHistory.ts: -------------------------------------------------------------------------------- 1 | import { getDb } from './MongoDatabase' 2 | import { AutomatronContext, TextMessage, TextMessageHandler } from './types' 3 | 4 | export interface MessageHistoryEntry { 5 | time: string 6 | text: string 7 | source: string 8 | } 9 | 10 | export async function saveMessageHistory( 11 | context: AutomatronContext, 12 | text: string, 13 | source: string 14 | ) { 15 | const db = await getDb(context) 16 | await db.collection('history').insertOne({ 17 | time: new Date().toISOString(), 18 | text: text, 19 | source: source, 20 | }) 21 | } 22 | 23 | export async function getMessageHistory( 24 | context: AutomatronContext, 25 | options: { limit: number } 26 | ) { 27 | const db = await getDb(context) 28 | return await db 29 | .collection('history') 30 | .find({}) 31 | .sort({ _id: -1 }) 32 | .limit(options.limit) 33 | .toArray() 34 | } 35 | 36 | export const MessageHistoryMessageHandler: TextMessageHandler = ( 37 | text, 38 | context 39 | ) => { 40 | if (text === 'history') { 41 | return async () => { 42 | const history = await getMessageHistory(context, { limit: 5 }) 43 | return [...history] 44 | .reverse() 45 | .map((entry): TextMessage => ({ type: 'text', text: entry.text })) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /core/src/MongoDatabase.ts: -------------------------------------------------------------------------------- 1 | import { Db, MongoClient } from 'mongodb' 2 | import { trace } from './Tracing' 3 | import { AutomatronContext } from './types' 4 | 5 | export { Db } from 'mongodb' 6 | 7 | const globalCacheKey = Symbol.for('automatron/MongoDatabase') 8 | const cache: { dbPromise: Promise | null } = (() => { 9 | return ((global as any)[globalCacheKey] = (global as any)[globalCacheKey] || { 10 | dbPromise: null, 11 | }) 12 | })() 13 | 14 | export async function getDb(context: AutomatronContext): Promise { 15 | if (cache.dbPromise) { 16 | return cache.dbPromise 17 | } 18 | cache.dbPromise = trace(context, 'getDb', async () => { 19 | const client = await MongoClient.connect(context.secrets.MONGODB_URL) 20 | const db = client.db('automatron') 21 | return db 22 | }) 23 | cache.dbPromise.catch(() => { 24 | cache.dbPromise = null 25 | }) 26 | return cache.dbPromise 27 | } 28 | -------------------------------------------------------------------------------- /core/src/NotificationProcessor.ts: -------------------------------------------------------------------------------- 1 | import { getDb } from './MongoDatabase' 2 | import { AutomatronContext } from './types' 3 | import { logger } from './logger' 4 | 5 | export interface INotification { 6 | packageName: string 7 | text: string 8 | title: string 9 | time: string 10 | postTime: string 11 | when: string 12 | key: string 13 | } 14 | 15 | export async function handleNotification( 16 | context: AutomatronContext, 17 | notification: INotification 18 | ) { 19 | const text = notification.title + ' : ' + notification.text 20 | const time = notification.when || notification.postTime || notification.time 21 | const key = notification.key 22 | const promises: Promise[] = [] 23 | const process = (name: string, promise: Promise) => { 24 | promises.push( 25 | promise.catch((err) => { 26 | logger.error({ err, name }, `Unable to run processor "${name}": ${err}`) 27 | }) 28 | ) 29 | } 30 | 31 | process('save to DB', saveNotificationToDb(context, notification)) 32 | 33 | if (notification.packageName === 'com.kasikorn.retail.mbanking.wap') { 34 | process('process KBank', handleKbankNotification(context, key, text, time)) 35 | } 36 | 37 | return Promise.all(promises) 38 | } 39 | 40 | async function saveNotificationToDb( 41 | context: AutomatronContext, 42 | notification: INotification 43 | ) { 44 | const db = await getDb(context) 45 | await db 46 | .collection('notis') 47 | .insertOne({ received: new Date().toISOString(), ...notification }) 48 | } 49 | 50 | export async function handleKbankNotification( 51 | context: AutomatronContext, 52 | key: string, 53 | text: string, 54 | time: string = new Date().toISOString() 55 | ) { 56 | let m: RegExpMatchArray | null 57 | 58 | m = text.match( 59 | /^รายการใช้บัตร : หมายเลขบัตร (\S+) จำนวนเงิน (\S+) (\S+) ที่ ([^]*)$/ 60 | ) 61 | if (m) { 62 | const db = await getDb(context) 63 | db.collection('txs').insertOne({ 64 | notificationKey: key, 65 | time, 66 | type: 'charge', 67 | card: m[1], 68 | amount: parseAmount(m[2]), 69 | currency: m[3], 70 | merchant: m[4], 71 | }) 72 | return 73 | } 74 | 75 | m = text.match( 76 | /^รายการยกเลิก : หมายเลขบัตร (\S+) จำนวนเงิน (\S+) (\S+) ที่ ([^]*)$/ 77 | ) 78 | if (m) { 79 | const db = await getDb(context) 80 | db.collection('txs').insertOne({ 81 | notificationKey: key, 82 | time, 83 | type: 'refund', 84 | card: m[1], 85 | amount: parseAmount(m[2]), 86 | currency: m[3], 87 | merchant: m[4], 88 | }) 89 | return 90 | } 91 | } 92 | 93 | function parseAmount(text: string) { 94 | return +text.replace(/,/g, '') 95 | } 96 | -------------------------------------------------------------------------------- /core/src/PersistentState.ts: -------------------------------------------------------------------------------- 1 | import { getDb } from './MongoDatabase' 2 | import { trace } from './Tracing' 3 | import { AutomatronContext } from './types' 4 | 5 | interface StateDoc { 6 | _id: string 7 | value: any 8 | } 9 | 10 | interface StateDocStack extends StateDoc { 11 | value: any[] 12 | } 13 | 14 | async function push( 15 | context: AutomatronContext, 16 | key: string, 17 | value: any 18 | ): Promise { 19 | const db = await getDb(context) 20 | const result = await trace(context, `read(${key})`, () => 21 | db 22 | .collection('state') 23 | .findOneAndUpdate( 24 | { _id: key }, 25 | { $push: { value } }, 26 | { upsert: true, returnDocument: 'after' } 27 | ) 28 | ) 29 | return result.value!.value.length 30 | } 31 | 32 | async function pop(context: AutomatronContext, key: string): Promise { 33 | const db = await getDb(context) 34 | const result = await trace(context, `pop(${key})`, () => 35 | db 36 | .collection('state') 37 | .findOneAndUpdate({ _id: key }, { $pop: { value: 1 } }) 38 | ) 39 | return result.value!.value.pop() 40 | } 41 | 42 | async function get(context: AutomatronContext, key: string): Promise { 43 | const db = await getDb(context) 44 | const result = await trace(context, `get(${key})`, () => 45 | db.collection('state').findOne({ _id: key }) 46 | ) 47 | return result?.value 48 | } 49 | 50 | async function set( 51 | context: AutomatronContext, 52 | key: string, 53 | value: any 54 | ): Promise { 55 | const db = await getDb(context) 56 | const result = await trace(context, `set(${key})`, () => 57 | db 58 | .collection('state') 59 | .findOneAndUpdate( 60 | { _id: key }, 61 | { $set: { value } }, 62 | { upsert: true, returnDocument: 'after' } 63 | ) 64 | ) 65 | return !!result.ok 66 | } 67 | 68 | export function ref(context: AutomatronContext, key: string) { 69 | return { 70 | push: push.bind(null, context, key), 71 | pop: pop.bind(null, context, key), 72 | get: get.bind(null, context, key), 73 | set: set.bind(null, context, key), 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /core/src/PhoneFinder.ts: -------------------------------------------------------------------------------- 1 | import { TextMessageHandler } from './types' 2 | import { GoogleAuth } from 'google-auth-library' 3 | import axios from 'axios' 4 | import { decrypt } from './DataEncryption' 5 | 6 | const auth = new GoogleAuth() 7 | 8 | export const PhoneFinderMessageHandler: TextMessageHandler = ( 9 | text, 10 | context 11 | ) => { 12 | if (text === 'where is my phone') { 13 | return async () => { 14 | const url = decrypt( 15 | context, 16 | '09oTErU3ru/YqzCNmUBIH2ftVYz1jSfG.NODxXCib7BiuF0vAtnxpwbsvVQ5fxNVXoYDYGZHmJNxBRLPfAWj9KmAK89cOjCtFw3uxudSjqCV1F79Icmn3/SWwi5Z313xXkKyJFr9YWT/1dq8+EWcZfbPR' 17 | ) 18 | const client = await auth.getIdTokenClient(url) 19 | const jwt = await client.idTokenProvider.fetchIdToken(url) 20 | await axios.post(url, {}, { headers: { Authorization: `Bearer ${jwt}` } }) 21 | return 'calling you now...' 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/src/PreludeCode.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | import { Storage } from '@google-cloud/storage' 3 | import { AutomatronContext } from './types' 4 | import tweetnacl from 'tweetnacl' 5 | import jsonwebtoken from 'jsonwebtoken' 6 | import crypto from 'crypto' 7 | import util from 'util' 8 | import Encrypted from '@dtinth/encrypted' 9 | import { logger } from './logger' 10 | import * as mongodb from 'mongodb' 11 | import * as os from 'os' 12 | import { Db, getDb } from './MongoDatabase' 13 | import * as NotificationProcessor from './NotificationProcessor' 14 | import * as PersistentState from './PersistentState' 15 | import { getAllSpeedDials, getSpeedDialCode, saveSpeedDial } from './SpeedDial' 16 | 17 | const lib = require('lib') 18 | const storage = new Storage() 19 | const preludeFile = storage.bucket('dtinth-automatron-data').file('prelude.js') 20 | 21 | export async function deployPrelude(context: AutomatronContext) { 22 | const userPreludeResponse = await axios.get( 23 | 'https://api.github.com/repos/dtinth/automatron-prelude/contents/prelude.js', 24 | { 25 | auth: { 26 | username: context.secrets.GITHUB_OAUTH_APP_CREDENTIALS.split(':')[0], 27 | password: context.secrets.GITHUB_OAUTH_APP_CREDENTIALS.split(':')[1], 28 | }, 29 | } 30 | ) 31 | const buffer = Buffer.from(userPreludeResponse.data.content, 'base64') 32 | await preludeFile.save(buffer) 33 | await PersistentState.ref(context, 'preludeDeployedAt').set( 34 | new Date().toISOString() 35 | ) 36 | } 37 | 38 | let cache: { deployedAt: string; code: string } | undefined 39 | 40 | export async function getPreludeCode(context: AutomatronContext) { 41 | const latestDeployedAt = await PersistentState.ref( 42 | context, 43 | 'preludeDeployedAt' 44 | ).get() 45 | if (cache && cache.deployedAt === latestDeployedAt) { 46 | return cache.code 47 | } 48 | const [buffer] = await preludeFile.download() 49 | const code = buffer.toString('utf8') 50 | cache = { 51 | deployedAt: latestDeployedAt, 52 | code, 53 | } 54 | return code 55 | } 56 | 57 | export async function getCodeExecutionContext( 58 | context: AutomatronContext 59 | ): Promise { 60 | // Prepare "self" context 61 | const self: any = {} 62 | self.exec = (code: string) => { 63 | return new Function( 64 | ...['self', 'code'], 65 | 'with (self) { return eval(code) }' 66 | )(self, code) 67 | } 68 | self.encrypted = Encrypted(context.secrets.ENCRYPTION_SECRET) 69 | self.extraMessages = [] 70 | self.withDb = (f: (db: Db) => any) => getDb(context).then(f) 71 | self.ref = PersistentState.ref.bind(null, context) 72 | self.stack = self.ref('stack') 73 | self.require = (id: string) => { 74 | const availableModules: { [id: string]: any } = { 75 | axios, 76 | tweetnacl, 77 | jsonwebtoken, 78 | crypto, 79 | util, 80 | mongodb, 81 | os, 82 | lib, 83 | '@/NotificationProcessor': NotificationProcessor, 84 | } 85 | const available = {}.hasOwnProperty.call(availableModules, id) 86 | if (!available) { 87 | throw new Error( 88 | `Module ${id} not available; available modules: ${Object.keys( 89 | availableModules 90 | )}` 91 | ) 92 | } 93 | return availableModules[id] 94 | } 95 | self.json = (x: any) => JSON.stringify(x, null, 2) 96 | self.SD = async (name?: string, f?: () => any) => { 97 | if (name && f) { 98 | if (typeof f !== 'function') { 99 | throw new Error('f must be a function') 100 | } 101 | await saveSpeedDial(context, name, f.toString()) 102 | return 'Saved speed dial' 103 | } else if (name) { 104 | const code = await getSpeedDialCode(context, name) 105 | self.extraMessages.push({ 106 | type: 'text', 107 | text: `;;SD('${name}', ${code})`, 108 | }) 109 | return self.exec(code)() 110 | } else { 111 | const all = await getAllSpeedDials(context) 112 | return all.map((s) => s._id) 113 | } 114 | } 115 | 116 | // Plugin system 117 | type Handler = (...args: any[]) => Promise 118 | const registeredHandlers: Record> = {} 119 | self.registerHandler = (event: string, handler: Handler) => { 120 | if (!registeredHandlers[event]) { 121 | registeredHandlers[event] = new Set() 122 | } 123 | registeredHandlers[event].add(handler) 124 | } 125 | self.executeHandlers = async (event: string, ...args: any[]) => { 126 | await Promise.all( 127 | [...(registeredHandlers[event] || [])].map(async (handler, i) => { 128 | try { 129 | const start = Date.now() 130 | await handler(...args) 131 | const elapsed = Date.now() - start 132 | logger.info( 133 | `Done executing handler index ${i} for event ${event} in ${elapsed} ms` 134 | ) 135 | } catch (error) { 136 | logger.error( 137 | { err: error }, 138 | `Unable to execute handler index ${i} for event ${event}: ${error}` 139 | ) 140 | } 141 | }) 142 | ) 143 | } 144 | 145 | // Execute user prelude 146 | const userPrelude = await getPreludeCode(context) 147 | self.exec(userPrelude) 148 | 149 | return self 150 | } 151 | -------------------------------------------------------------------------------- /core/src/RomanNumerals.ts: -------------------------------------------------------------------------------- 1 | export function decodeRomanNumerals(romanNumerals: string) { 2 | const decode = (s: string): number => { 3 | if (s.startsWith('M')) return 1000 + decode(s.substr(1)) 4 | if (s.startsWith('CM')) return 900 + decode(s.substr(2)) 5 | if (s.startsWith('D')) return 500 + decode(s.substr(1)) 6 | if (s.startsWith('CD')) return 400 + decode(s.substr(2)) 7 | if (s.startsWith('C')) return 100 + decode(s.substr(1)) 8 | if (s.startsWith('XC')) return 90 + decode(s.substr(2)) 9 | if (s.startsWith('L')) return 50 + decode(s.substr(1)) 10 | if (s.startsWith('XL')) return 40 + decode(s.substr(2)) 11 | if (s.startsWith('X')) return 10 + decode(s.substr(1)) 12 | if (s.startsWith('IX')) return 9 + decode(s.substr(2)) 13 | if (s.startsWith('V')) return 5 + decode(s.substr(1)) 14 | if (s.startsWith('IV')) return 4 + decode(s.substr(2)) 15 | if (s.startsWith('I')) return 1 + decode(s.substr(1)) 16 | if (s === '') return 0 17 | throw new InvalidRomanNumeralError(s) 18 | } 19 | try { 20 | return decode(romanNumerals.toUpperCase()) 21 | } catch (e) { 22 | if (e instanceof InvalidRomanNumeralError) { 23 | throw new InvalidRomanNumeralError(romanNumerals) 24 | } 25 | throw e 26 | } 27 | } 28 | 29 | class InvalidRomanNumeralError extends Error { 30 | constructor(s: string) { 31 | super('Invalid roman numeral in input ' + s) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/SMSHandler.ts: -------------------------------------------------------------------------------- 1 | import { messagingApi, QuickReplyItem } from '@line/bot-sdk' 2 | import { AutomatronContext } from './types' 3 | import { recordExpense } from './ExpenseTracking' 4 | import { createBubble } from './LINEMessageUtilities' 5 | import { LINEClient } from './LINEClient' 6 | 7 | export async function handleSMS( 8 | context: AutomatronContext, 9 | client: LINEClient, 10 | text: string 11 | ) { 12 | const { parseSMS } = require('transaction-parser-th') 13 | const result = parseSMS(text) 14 | if (!result || !result.amount) return { match: false } 15 | console.log('SMS parsing result', result) 16 | const title = result.type 17 | const pay = result.type === 'pay' 18 | const moneyOut = ['pay', 'transfer', 'withdraw'].includes(result.type) 19 | const body: messagingApi.FlexBox = { 20 | type: 'box', 21 | layout: 'vertical', 22 | contents: [ 23 | { 24 | type: 'text', 25 | text: '฿' + result.amount, 26 | size: 'xxl', 27 | weight: 'bold', 28 | }, 29 | ], 30 | } 31 | const ordering = ['provider', 'from', 'to', 'via', 'date', 'time', 'balance'] 32 | const skip = ['type', 'amount'] 33 | const getOrder = (key: string) => ordering.indexOf(key) + 1 || 999 34 | for (const key of Object.keys(result) 35 | .filter((key) => !skip.includes(key)) 36 | .sort((a, b) => getOrder(a) - getOrder(b))) { 37 | body.contents.push({ 38 | type: 'box', 39 | layout: 'horizontal', 40 | spacing: 'md', 41 | contents: [ 42 | { 43 | type: 'text', 44 | text: key, 45 | align: 'end', 46 | color: '#888888', 47 | flex: 2, 48 | }, 49 | { 50 | type: 'text', 51 | text: String(result[key]), 52 | flex: 5, 53 | }, 54 | ], 55 | }) 56 | } 57 | const quickReply = (suffix: string, label: string): QuickReplyItem => ({ 58 | type: 'action', 59 | action: { 60 | type: 'message', 61 | label: label, 62 | text: result.amount + suffix, 63 | }, 64 | }) 65 | const messages: messagingApi.Message[] = [ 66 | { 67 | ...createBubble(title, body, { 68 | headerBackground: pay ? '#91918F' : moneyOut ? '#DA9E00' : '#9471FF', 69 | headerColor: '#FFFFFF', 70 | altText: require('util').inspect(result), 71 | }), 72 | quickReply: { 73 | items: [ 74 | quickReply('f', 'food'), 75 | quickReply('h', 'health'), 76 | quickReply('t', 'transport'), 77 | quickReply('m', 'misc'), 78 | quickReply('o', 'occasion'), 79 | ], 80 | }, 81 | }, 82 | ] 83 | if (result.type === 'pay' && result.to === 'LINEPAY*BTS01') { 84 | messages.push( 85 | await recordExpense(context, result.amount, 'transportation', 'BTS') 86 | ) 87 | } 88 | await client.pushMessage(context.secrets.LINE_USER_ID, messages) 89 | return { match: true } 90 | } 91 | -------------------------------------------------------------------------------- /core/src/SlackMessageUtilities.ts: -------------------------------------------------------------------------------- 1 | import { KnownBlock, MessageAttachment } from '@slack/types' 2 | 3 | export type SlackMessage = SlackMessageWithoutBlocks | SlackMessageWithBlocks 4 | export type SlackMessageWithoutBlocks = { 5 | text: string 6 | attachments?: MessageAttachment[] 7 | } 8 | export type SlackMessageWithBlocks = { text?: string; blocks: KnownBlock[] } 9 | 10 | export function createErrorMessage(error: Error): SlackMessage { 11 | const title = 12 | (error.name || 'Error') + (error.message ? `: ${error.message}` : '') 13 | return { 14 | text: title, 15 | attachments: [ 16 | { 17 | color: 'danger', 18 | blocks: [ 19 | { 20 | type: 'section', 21 | text: { 22 | type: 'mrkdwn', 23 | text: ['```', String(error.stack || error), '```'].join('') 24 | } 25 | } 26 | ] 27 | } 28 | ] 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/SpeedDial.ts: -------------------------------------------------------------------------------- 1 | import { getDb } from './MongoDatabase' 2 | import { AutomatronContext } from './types' 3 | 4 | async function getCollection(context: AutomatronContext) { 5 | const db = await getDb(context) 6 | return db.collection<{ _id: string; code: string }>('speedDial') 7 | } 8 | 9 | export async function saveSpeedDial( 10 | context: AutomatronContext, 11 | name: string, 12 | code: string 13 | ) { 14 | const collection = await getCollection(context) 15 | await collection.updateOne( 16 | { _id: name }, 17 | { $set: { code } }, 18 | { upsert: true } 19 | ) 20 | } 21 | 22 | export async function getSpeedDialCode( 23 | context: AutomatronContext, 24 | name: string 25 | ) { 26 | const collection = await getCollection(context) 27 | const result = await collection.findOne({ _id: name }) 28 | if (!result) { 29 | throw new Error(`Speed dial ${name} not found`) 30 | } 31 | return result.code 32 | } 33 | 34 | export async function getAllSpeedDials(context: AutomatronContext) { 35 | const collection = await getCollection(context) 36 | return collection.find({}).sort({ _id: 1 }).toArray() 37 | } 38 | -------------------------------------------------------------------------------- /core/src/SpendingTracking.ts: -------------------------------------------------------------------------------- 1 | import { getDb } from './MongoDatabase' 2 | import { ref } from './PersistentState' 3 | import { TextMessageHandler } from './types' 4 | 5 | export const SpendingTrackingMessageHandler: TextMessageHandler = ( 6 | text, 7 | context 8 | ) => { 9 | if (text === 'pace') { 10 | return async () => { 11 | const db = await getDb(context) 12 | const paceResetTimeRef = ref(context, 'SpendingTracking.paceResetTime') 13 | const paceResetTime = await paceResetTimeRef.get() 14 | const allowancePerMillis = 24000 / (32 * 86400e3) 15 | const elapsed = Date.now() - Date.parse(paceResetTime) 16 | const txs = await db 17 | .collection('txs') 18 | .find({ time: { $gt: paceResetTime } }) 19 | .sort({ _id: 1 }) 20 | .toArray() 21 | const warnings = [] 22 | let sum = 0 23 | for (const tx of txs) { 24 | let currencyMul 25 | if (tx.currency === 'บาท') { 26 | currencyMul = 1 27 | } else { 28 | warnings.push(`Unknown currency ${tx.currency}`) 29 | continue 30 | } 31 | const typeMul = tx.type === 'refund' ? -1 : 1 32 | sum += tx.amount * typeMul * currencyMul 33 | } 34 | const remaining = Math.round(allowancePerMillis * elapsed - sum) 35 | // return { sum, warnings } 36 | return `used ${sum} บาท\nremaining ${remaining} บาท` 37 | } 38 | } 39 | if (text === 'reset pace') { 40 | return async () => { 41 | const now = new Date().toISOString() 42 | await ref(context, 'SpendingTracking.paceResetTime').set(now) 43 | return `Pace reset to ${now}` 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /core/src/TemporaryBlobStorage.ts: -------------------------------------------------------------------------------- 1 | import { Storage } from '@google-cloud/storage' 2 | import { nanoid } from 'nanoid' 3 | 4 | const storage = new Storage() 5 | let latest: { blobName: string; buffer: Buffer } | undefined 6 | 7 | export async function putBlob(buffer: Buffer, extension: string) { 8 | const blobName = nanoid() + extension 9 | await storage.bucket('tmpblob').file(blobName).save(buffer) 10 | latest = { blobName, buffer } 11 | return blobName 12 | } 13 | 14 | export async function getBlob(blobName: string) { 15 | if (latest && latest.blobName === blobName) { 16 | return latest.buffer 17 | } 18 | const response = await storage.bucket('tmpblob').file(blobName).download() 19 | return response[0] 20 | } 21 | 22 | export async function getBlobUrl(blobName: string) { 23 | const result = await storage 24 | .bucket('tmpblob') 25 | .file(blobName) 26 | .getSignedUrl({ 27 | action: 'read', 28 | expires: new Date(Date.now() + 86400e3), 29 | version: 'v4', 30 | virtualHostedStyle: true, 31 | }) 32 | return result[0] 33 | } 34 | -------------------------------------------------------------------------------- /core/src/Tracing.ts: -------------------------------------------------------------------------------- 1 | import { AutomatronContext } from './types' 2 | 3 | export async function trace( 4 | context: AutomatronContext, 5 | name: string, 6 | f: () => Promise 7 | ) { 8 | const tracer = context.tracer 9 | if (!tracer) { 10 | return f() 11 | } 12 | const span = tracer.createChildSpan({ name }) 13 | try { 14 | return await f() 15 | } finally { 16 | span.endSpan() 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /core/src/bot.ts: -------------------------------------------------------------------------------- 1 | import Encrypted from '@dtinth/encrypted' 2 | import { MessageEvent, middleware, WebhookEvent } from '@line/bot-sdk' 3 | import axios from 'axios' 4 | import cors from 'cors' 5 | import express, { 6 | NextFunction, 7 | Request, 8 | RequestHandler, 9 | Response, 10 | } from 'express' 11 | import handler from 'express-async-handler' 12 | import { claimEquals, auth as jwtAuth } from 'express-oauth2-jwt-bearer' 13 | import { Stream } from 'stream' 14 | import sealedbox from 'tweetnacl-sealedbox-js' 15 | import { getPendingCronJobs, updateCronJob } from './Cron' 16 | import { encrypt } from './DataEncryption' 17 | import { checkDeviceOnlineStatus, trackDevice } from './DeviceTracking' 18 | import { LINEClient } from './LINEClient' 19 | import { toMessages } from './LINEMessageUtilities' 20 | import { logger } from './logger' 21 | import { handleImage, handleTextMessage } from './MessageHandler' 22 | import { getMessageHistory } from './MessageHistory' 23 | import { handleNotification } from './NotificationProcessor' 24 | import { ref } from './PersistentState' 25 | import { deployPrelude } from './PreludeCode' 26 | import { createErrorMessage, SlackMessage } from './SlackMessageUtilities' 27 | import { handleSMS } from './SMSHandler' 28 | import { getAllSpeedDials } from './SpeedDial' 29 | import { AutomatronContext } from './types' 30 | 31 | const app = express() 32 | app.set('trust proxy', true) 33 | 34 | function getAutomatronContext(req: Request, res: Response): AutomatronContext { 35 | return { 36 | secrets: req.env, 37 | tracer: req.tracer, 38 | requestInfo: { 39 | ip: req.ip || '', 40 | ips: req.ips, 41 | headers: req.headers, 42 | }, 43 | addPromise: (name, promise) => { 44 | if (!res.yields) res.yields = [] 45 | res.yields.push(promise) 46 | }, 47 | } 48 | } 49 | 50 | async function runMiddleware( 51 | req: Request, 52 | res: Response, 53 | middleware: RequestHandler 54 | ): Promise { 55 | return new Promise((resolve, reject) => { 56 | middleware(req, res, (error) => { 57 | if (error) { 58 | reject(error) 59 | } else { 60 | resolve() 61 | } 62 | }) 63 | }) 64 | } 65 | 66 | async function handleWebhook( 67 | context: AutomatronContext, 68 | events: WebhookEvent[], 69 | client: LINEClient 70 | ) { 71 | async function main() { 72 | for (const event of events) { 73 | if (event.type === 'message') { 74 | await handleMessageEvent(event) 75 | } 76 | } 77 | } 78 | 79 | async function handleMessageEvent(event: MessageEvent) { 80 | const { replyToken, message } = event 81 | if (event.source.userId !== context.secrets.LINE_USER_ID) { 82 | await client.replyMessage(replyToken, toMessages('unauthorized')) 83 | return 84 | } 85 | client.showLoadingAnimation(event.source.userId).catch((e) => { 86 | logger.error({ err: e }, 'Unable to show loading animation') 87 | }) 88 | if (message.type === 'text') { 89 | const reply = await handleTextMessage(context, message.text, { 90 | source: 'line', 91 | }) 92 | await client.replyMessage(replyToken, toMessages(reply)) 93 | } else if (message.type === 'sticker') { 94 | const reply = await handleTextMessage( 95 | context, 96 | 'sticker:' + message.packageId + ':' + message.stickerId, 97 | { source: 'line' } 98 | ) 99 | await client.replyMessage(replyToken, toMessages(reply)) 100 | } else if (message.type === 'image') { 101 | const content = await client.getMessageContent(message.id) 102 | const buffer = await readAsBuffer(content) 103 | const reply = await handleImage(context, buffer as Buffer, { 104 | source: 'line', 105 | }) 106 | await client.replyMessage(replyToken, toMessages(reply)) 107 | } else { 108 | await client.replyMessage(replyToken, [ 109 | { type: 'text', text: 'don’t know how to handle this yet!' }, 110 | ]) 111 | } 112 | } 113 | 114 | return main() 115 | } 116 | 117 | app.post( 118 | '/webhook', 119 | handler(async (req, res) => { 120 | const lineConfig = getLineConfig(req, res) 121 | await runMiddleware(req, res, middleware(lineConfig)) 122 | await handleRequest(req, res, async (context, services) => { 123 | const lineClient = services.line 124 | logger.info( 125 | { ingest: 'line', event: JSON.stringify(req.body) }, 126 | 'Received webhook from LINE' 127 | ) 128 | const data = await handleWebhook(context, req.body.events, lineClient) 129 | return data 130 | }) 131 | }) 132 | ) 133 | 134 | app.post( 135 | '/slack', 136 | require('body-parser').json(), 137 | (req, res, next) => { 138 | logger.info( 139 | { ingest: 'slack', event: JSON.stringify(req.body) }, 140 | 'Received an event from Slack' 141 | ) 142 | if (req.body.type === 'url_verification') { 143 | res.set('Content-Type', 'text/plain').send(req.body.challenge) 144 | return 145 | } 146 | next() 147 | }, 148 | endpoint(async (context, req, services) => { 149 | if (req.body.type === 'event_callback') { 150 | let globalScope = global as unknown as { 151 | automatronSlackEventCache: Set 152 | } 153 | let eventCache = globalScope.automatronSlackEventCache 154 | if (!eventCache) { 155 | eventCache = new Set() 156 | globalScope.automatronSlackEventCache = eventCache 157 | } 158 | const eventId = req.body.event_id 159 | if (eventCache.has(eventId)) { 160 | return 161 | } 162 | eventCache.add(eventId) 163 | if (req.body.event.user === req.env.SLACK_USER_ID) { 164 | const text = String(req.body.event.text) 165 | .replace(/>/g, '>') 166 | .replace(/</g, '>') 167 | .replace(/&/g, '&') 168 | const slackClient = services.slack 169 | const reply = await handleTextMessage(context, text, { 170 | source: 'slack', 171 | }) 172 | await slackClient.pushMessage({ 173 | text: `\`\`\`${JSON.stringify(reply, null, 2)}\`\`\``, 174 | }) 175 | } 176 | } 177 | 178 | return 1 179 | }) 180 | ) 181 | 182 | app.post( 183 | '/post', 184 | require('body-parser').json(), 185 | requireApiKey, 186 | endpoint(async (context, req, services) => { 187 | const lineClient = services.line 188 | const messages = toMessages(req.body.data) 189 | await lineClient.pushMessage(context.secrets.LINE_USER_ID, messages) 190 | }) 191 | ) 192 | 193 | app.post( 194 | '/text', 195 | require('body-parser').json(), 196 | requireApiKey, 197 | endpoint(async (context, req, services) => { 198 | logger.info( 199 | { ingest: 'text', event: JSON.stringify(req.body) }, 200 | 'Received a text API call' 201 | ) 202 | const text = String(req.body.text) 203 | logToSlack(context, services.auditSlack, text, req.body.source) 204 | const reply = await handleTextMessage(context, text, { 205 | source: 'text:' + req.body.source, 206 | }) 207 | return reply 208 | }) 209 | ) 210 | 211 | app.options('/webpost-firebase', cors() as any) 212 | app.post( 213 | '/webpost-firebase', 214 | require('body-parser').json(), 215 | requireFirebaseAuth, 216 | cors(), 217 | endpoint(async (context, req, services) => { 218 | logger.info( 219 | { ingest: 'webpost-firebase', event: JSON.stringify(req.body) }, 220 | 'Received a webpost API call with Firebase credentials' 221 | ) 222 | const text = String(req.body.text) 223 | logToSlack(context, services.auditSlack, text, req.body.source) 224 | const reply = await handleTextMessage(context, text, { 225 | source: 'webpost:' + req.body.source, 226 | }) 227 | return reply 228 | }) 229 | ) 230 | 231 | app.options('/history', cors() as any) 232 | app.get( 233 | '/history', 234 | requireFirebaseAuth, 235 | cors(), 236 | endpoint(async (context, req) => { 237 | logger.info( 238 | { ingest: 'history', event: JSON.stringify(req.body) }, 239 | 'Received a history API call' 240 | ) 241 | return { 242 | history: await getMessageHistory(context, { limit: 20 }), 243 | } 244 | }) 245 | ) 246 | 247 | app.options('/speed-dials', cors() as any) 248 | app.get( 249 | '/speed-dials', 250 | requireFirebaseAuth, 251 | cors(), 252 | endpoint(async (context, req) => { 253 | logger.info( 254 | { ingest: 'speed-dials', event: JSON.stringify(req.body) }, 255 | 'Received a speed dial API call' 256 | ) 257 | return { 258 | speedDials: await getAllSpeedDials(context), 259 | } 260 | }) 261 | ) 262 | 263 | app.options('/knobs', cors() as any) 264 | app.get( 265 | '/knobs', 266 | requireFirebaseAuth, 267 | cors(), 268 | endpoint(async (context, req) => { 269 | logger.info({ ingest: 'knobs' }, 'Received a knobs API call') 270 | const knobKeys = ((await ref(context, 'knobs').get()) as string).split(',') 271 | const knobs = Object.fromEntries( 272 | await Promise.all( 273 | knobKeys.map(async (key) => { 274 | const value = await ref(context, key).get() 275 | return [key, value] as [string, string] 276 | }) 277 | ) 278 | ) 279 | return { knobs } 280 | }) 281 | ) 282 | 283 | // http post $AUTOMATRON/encrypt data=meow 284 | app.post( 285 | '/encrypt', 286 | express.json(), 287 | endpoint(async (context, req) => { 288 | return encrypt(context, req.body.data) 289 | }) 290 | ) 291 | 292 | function logToSlack( 293 | context: AutomatronContext, 294 | slack: Slack, 295 | text: string, 296 | source: string 297 | ) { 298 | context.addPromise( 299 | 'Log to Slack', 300 | slack.pushMessage({ 301 | blocks: [ 302 | { 303 | type: 'context', 304 | elements: [{ type: 'plain_text', text: 'from ' + source }], 305 | }, 306 | { 307 | type: 'section', 308 | text: { type: 'plain_text', text: text }, 309 | }, 310 | ], 311 | }) 312 | ) 313 | } 314 | 315 | app.post( 316 | '/gh/prelude/push', 317 | require('body-parser').json(), 318 | endpoint(async (context, req, services) => { 319 | logger.info( 320 | { ingest: 'prelude-push', event: JSON.stringify(req.body) }, 321 | 'Received prelude push webhook from GitHub' 322 | ) 323 | await deployPrelude(context) 324 | return 'ok' 325 | }) 326 | ) 327 | 328 | app.post( 329 | '/sms', 330 | require('body-parser').json(), 331 | requireApiKey, 332 | endpoint(async (context, req, services) => { 333 | const text = String(req.body.text) 334 | return await handleSMS(context, services.line, text) 335 | }) 336 | ) 337 | 338 | app.post( 339 | '/notification', 340 | require('body-parser').text(), 341 | endpoint(async (context, req, services) => { 342 | try { 343 | const encrypted = Encrypted(context.secrets.ENCRYPTION_SECRET) 344 | const { publicKey, secretKey } = encrypted(` 345 | BpnjVPJcwkInfE/lPxxcl6E11BlDNh3v.KoCet77F7KC4pvuhRySH2wNP1AjYpUGVcHQSqhc 346 | rbFTDHUsXDaYjF/Jc584uh7Bd6yLl0a4scdEsX7EhxuHXUknD4bA8AXxkJe/OhI3EbmfleP5 347 | ByVNvvvxqScM9pHvCy/bURK33REznhvW0MsscwgRGsqMxvI7Km9RpxpglexWANMlrkuVBJbC 348 | G3CeOqs9QGI3QS0K+jse8PM7HvJ8vg43AAjQsx6o85xSzaGWVWE1wdNtWfkdusGf/NYbDyb6 349 | hgA9ddrCRJVMydqJ4g9A/LgpieO0v 350 | `) 351 | const result = sealedbox.open( 352 | Buffer.from(req.body, 'hex'), 353 | Buffer.from(publicKey, 'base64'), 354 | Buffer.from(secretKey, 'base64') 355 | ) 356 | const notification = JSON.parse(Buffer.from(result).toString('utf8')) 357 | logger.info( 358 | { ingest: 'notification', notification }, 359 | 'Received a notification from ' + notification.packageName 360 | ) 361 | const deviceId = String(req.query.deviceId ?? 'phone') 362 | await trackDevice(context, deviceId, {}) 363 | await handleNotification(context, notification) 364 | } catch (err) { 365 | logger.error({ err, data: req.body }, 'Unable to process notification') 366 | } 367 | }) 368 | ) 369 | 370 | app.get( 371 | '/cron', 372 | endpoint(async (context, req, services) => { 373 | const otherTasks: Promise[] = [] 374 | otherTasks.push( 375 | (async () => { 376 | await checkDeviceOnlineStatus(context) 377 | })() 378 | ) 379 | await Promise.all(otherTasks) 380 | 381 | const pendingJobs = await getPendingCronJobs(context) 382 | const jobsToRun = pendingJobs.filter( 383 | (j) => new Date().toISOString() >= j.scheduledTime 384 | ) 385 | logger.trace('Number of pending cron jobs found: %s', jobsToRun.length) 386 | try { 387 | for (const job of jobsToRun) { 388 | let result = 'No output' 389 | const logContext = { 390 | job: { id: job._id.toString(), name: job.name }, 391 | } 392 | try { 393 | const reply = await handleTextMessage(context, job.name, { 394 | source: 'cron:' + job._id.toString(), 395 | }) 396 | result = require('util').inspect(reply) 397 | logger.info( 398 | { ...logContext, result }, 399 | `Done processing cron job: ${job.name}` 400 | ) 401 | } catch (e) { 402 | logError('Unable to process cron job', e, logContext) 403 | result = `Error: ${e}` 404 | } 405 | await updateCronJob(context, job._id.toString(), { 406 | completed: true, 407 | notes: result, 408 | }) 409 | } 410 | return 'All OK' 411 | } catch (e) { 412 | logError('Unable to process cron jobs', e) 413 | return 'Error: ' + e 414 | } 415 | }) 416 | ) 417 | 418 | app.post( 419 | '/mac/ping', 420 | require('body-parser').json(), 421 | requireApiKey, 422 | endpoint(async (context, req, services) => { 423 | const deviceId = req.body.deviceId 424 | if (!deviceId) { 425 | throw new Error('Missing deviceId') 426 | } 427 | const newState = await trackDevice(context, deviceId, { 428 | locked: req.body.locked || null, 429 | powerSource: req.body.powerSource, 430 | online: true, 431 | }) 432 | return { newState } 433 | }) 434 | ) 435 | 436 | function requireApiKey(req: Request, res: Response, next: NextFunction) { 437 | const context = getAutomatronContext(req, res) 438 | if (req.body.key !== context.secrets.API_KEY) { 439 | return res.status(401).json({ error: 'Invalid API key' }) 440 | } 441 | next() 442 | } 443 | 444 | const firebaseAuthn = jwtAuth({ 445 | issuer: 'https://securetoken.google.com/dtinth-automatron', 446 | jwksUri: 447 | 'https://www.googleapis.com/service_accounts/v1/jwk/securetoken@system.gserviceaccount.com', 448 | audience: 'dtinth-automatron', 449 | }) 450 | function requireFirebaseAuth(req: Request, res: Response, next: NextFunction) { 451 | const encrypted = Encrypted(req.env.ENCRYPTION_SECRET) 452 | const sub = encrypted( 453 | `HnX5zHMDR/F7ZUy8sw2829Wi7cYbi/3c.6CBZ+I3sTbyYsXUg81qf+OvjRQExIfVkkhGvEs9E+w7LAO4y9KwU1DtKAtIutA==` 454 | ) 455 | return firebaseAuthn(req, res, (err) => { 456 | if (err) { 457 | return next(err) 458 | } 459 | claimEquals('sub', sub)(req, res, next) 460 | }) 461 | } 462 | 463 | class Slack { 464 | constructor(private webhookUrl: string) {} 465 | async pushMessage(message: SlackMessage) { 466 | await axios.post(this.webhookUrl, message) 467 | } 468 | } 469 | 470 | interface ThirdPartyServices { 471 | line: LINEClient 472 | slack: Slack 473 | auditSlack: Slack 474 | } 475 | 476 | function endpoint( 477 | f: ( 478 | context: AutomatronContext, 479 | req: Request, 480 | services: ThirdPartyServices 481 | ) => Promise 482 | ): RequestHandler { 483 | return handler(async (req, res) => { 484 | await handleRequest(req, res, (context, services) => 485 | f(context, req, services) 486 | ) 487 | }) 488 | } 489 | 490 | async function handleRequest( 491 | req: Request, 492 | res: Response, 493 | f: (context: AutomatronContext, services: ThirdPartyServices) => Promise 494 | ) { 495 | const context = getAutomatronContext(req, res) 496 | const encrypted = Encrypted(context.secrets.ENCRYPTION_SECRET) 497 | const lineConfig = getLineConfig(req, res) 498 | const lineClient = new LINEClient(lineConfig) 499 | const slackClient = new Slack(context.secrets.SLACK_WEBHOOK_URL) 500 | const auditSlackClient = new Slack( 501 | encrypted( 502 | 'j3o0uDUL3OuYfUsYxZUkI8ECdaUIGxW0.HX1CMjS27oaZHnQormJbPIoE9xdPB3GsITBVXW2oIFeuuAb4xyWVJZyywWMubR1I1ECkXtBJN+Fs+98MECYk+u9YnlnAgw6DlE9e8TezE88C5DeNtOO0DOnSO6ww39Cn/w==' 503 | ) 504 | ) 505 | try { 506 | const result = await f(context, { 507 | line: lineClient, 508 | slack: slackClient, 509 | auditSlack: auditSlackClient, 510 | }) 511 | await Promise.allSettled(res.yields || []) 512 | res.json({ ok: true, result }) 513 | } catch (e) { 514 | logError('Unable to execute endpoint ' + req.path, e) 515 | try { 516 | await slackClient.pushMessage(createErrorMessage(e as any)) 517 | } catch (ee) { 518 | console.error('Cannot send error message to LINE!') 519 | logError('Unable to send error message to Slack', ee) 520 | } 521 | await Promise.allSettled(res.yields || []) 522 | throw e 523 | } 524 | } 525 | 526 | function logError(title: string, e: any, extra: Record = {}) { 527 | var response = e.response || (e.originalError && e.originalError.response) 528 | var data = response && response.data 529 | const bindings: Record = { ...extra, err: e } 530 | if (data) { 531 | bindings.responseData = JSON.stringify(data) 532 | } 533 | logger.error(bindings, `${title}: ${e}`) 534 | } 535 | 536 | function getLineConfig(req: Request, res: Response) { 537 | const context = getAutomatronContext(req, res) 538 | return { 539 | channelAccessToken: context.secrets.LINE_CHANNEL_ACCESS_TOKEN, 540 | channelSecret: context.secrets.LINE_CHANNEL_SECRET, 541 | } 542 | } 543 | 544 | function readAsBuffer(stream: Stream) { 545 | return new Promise((resolve, reject) => { 546 | stream.on('error', (e: Error) => { 547 | reject(e) 548 | }) 549 | const bufs: Buffer[] = [] 550 | stream.on('end', () => { 551 | resolve(Buffer.concat(bufs)) 552 | }) 553 | stream.on('data', (buf: Buffer) => { 554 | bufs.push(buf) 555 | }) 556 | }) 557 | } 558 | 559 | module.exports = app 560 | -------------------------------------------------------------------------------- /core/src/logger.ts: -------------------------------------------------------------------------------- 1 | import pino from 'pino' 2 | 3 | export const logger = pino({ 4 | // https://github.com/pinojs/pino/issues/726#issuecomment-605814879 5 | messageKey: 'message', 6 | formatters: { 7 | level: (label) => { 8 | function getSeverity(label: string) { 9 | switch (label) { 10 | case 'trace': 11 | return 'DEBUG' 12 | case 'debug': 13 | return 'DEBUG' 14 | case 'info': 15 | return 'INFO' 16 | case 'warn': 17 | return 'WARNING' 18 | case 'error': 19 | return 'ERROR' 20 | case 'fatal': 21 | return 'CRITICAL' 22 | default: 23 | return 'DEFAULT' 24 | } 25 | } 26 | return { severity: getSeverity(label) } 27 | }, 28 | }, 29 | }).child({ name: 'automatron' }) 30 | -------------------------------------------------------------------------------- /core/src/modules.d.ts: -------------------------------------------------------------------------------- 1 | declare module '@google-cloud/vision' { 2 | export class ImageAnnotatorClient { 3 | constructor(options?: any) 4 | documentTextDetection( 5 | imageBuffer: Buffer 6 | ): Promise< 7 | { 8 | fullTextAnnotation: { 9 | pages: { 10 | blocks: { 11 | paragraphs: { 12 | words: { 13 | symbols: { 14 | text: string 15 | }[] 16 | }[] 17 | }[] 18 | }[] 19 | }[] 20 | } 21 | }[] 22 | > 23 | } 24 | } 25 | 26 | declare module 'airtable' { 27 | export interface AirtableRecord { 28 | get(field: string): any 29 | getId(): string 30 | } 31 | export default class Airtable { 32 | constructor(options: any) 33 | base( 34 | id: string 35 | ): { 36 | table( 37 | name: string 38 | ): { 39 | select( 40 | options?: any 41 | ): { 42 | all(): Promise 43 | } 44 | create(data: any, options?: any): Promise 45 | update(id: string, data: any): Promise 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /core/src/typedefs.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'tweetnacl-sealedbox-js' 2 | declare module 'lib' 3 | -------------------------------------------------------------------------------- /core/src/types.ts: -------------------------------------------------------------------------------- 1 | import { FlexMessage, messagingApi, TextMessage } from '@line/bot-sdk' 2 | import { BotSecrets } from './BotSecrets' 3 | import type { PluginTypes } from '@google-cloud/trace-agent' 4 | import { IncomingHttpHeaders } from 'http' 5 | 6 | export interface AutomatronContext { 7 | secrets: BotSecrets 8 | requestInfo: HttpRequestInfo 9 | tracer?: PluginTypes.Tracer 10 | addPromise: (name: string, promise: Promise) => void 11 | } 12 | 13 | export interface HttpRequestInfo { 14 | ip: string 15 | ips: string[] 16 | headers: IncomingHttpHeaders 17 | } 18 | 19 | declare global { 20 | module Express { 21 | interface Request { 22 | env: BotSecrets 23 | tracer?: PluginTypes.Tracer 24 | } 25 | interface Response { 26 | yields?: Promise[] 27 | } 28 | } 29 | module NodeJS { 30 | interface Global { 31 | automatronSlackEventCache?: Set 32 | } 33 | } 34 | } 35 | 36 | export { FlexMessage, TextMessage } from '@line/bot-sdk' 37 | 38 | export type AutomatronResponse = 39 | | string 40 | | messagingApi.TextMessage 41 | | messagingApi.TextMessage[] 42 | | messagingApi.FlexMessage 43 | | messagingApi.FlexMessage[] 44 | 45 | export type TextMessageHandler = ( 46 | text: string, 47 | context: AutomatronContext 48 | ) => (() => Promise) | undefined 49 | -------------------------------------------------------------------------------- /core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2020", 4 | "module": "commonjs", 5 | "noEmit": true, 6 | "lib": ["es2020"], 7 | "esModuleInterop": true, 8 | "strict": true, 9 | "rootDir": "src", 10 | "skipLibCheck": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /images/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /images/api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/api.png -------------------------------------------------------------------------------- /images/auto_expense.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/auto_expense.png -------------------------------------------------------------------------------- /images/expense_tracking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/expense_tracking.png -------------------------------------------------------------------------------- /images/home_automation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/home_automation.png -------------------------------------------------------------------------------- /images/image_to_text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/image_to_text.png -------------------------------------------------------------------------------- /images/livescript.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/livescript.png -------------------------------------------------------------------------------- /images/quick_replies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/quick_replies.png -------------------------------------------------------------------------------- /images/transaction_aggregation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtinth/automatron/544a6c6e552bcdca41d8acf57228452199bb7cec/images/transaction_aggregation.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "automatron-workspace", 3 | "private": true, 4 | "packageManager": "pnpm@9.7.1+sha512.faf344af2d6ca65c4c5c8c2224ea77a81a5e8859cbc4e06b1511ddce2f0151512431dd19e6aff31f2c6a8f5f2aced9bd2273e1fed7dd4de1868984059d2c4247" 5 | } 6 | -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | - core 3 | - webui 4 | -------------------------------------------------------------------------------- /webui/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | /** 2 | * This is intended to be a basic starting point for linting in your app. 3 | * It relies on recommended configs out of the box for simplicity, but you can 4 | * and should modify this configuration to best suit your team's needs. 5 | */ 6 | 7 | /** @type {import('eslint').Linter.Config} */ 8 | module.exports = { 9 | root: true, 10 | parserOptions: { 11 | ecmaVersion: 'latest', 12 | sourceType: 'module', 13 | ecmaFeatures: { 14 | jsx: true, 15 | }, 16 | }, 17 | env: { 18 | browser: true, 19 | commonjs: true, 20 | es6: true, 21 | }, 22 | 23 | // Base config 24 | extends: ['eslint:recommended'], 25 | 26 | overrides: [ 27 | // React 28 | { 29 | files: ['**/*.{js,jsx,ts,tsx}'], 30 | plugins: ['react', 'jsx-a11y'], 31 | extends: [ 32 | 'plugin:react/recommended', 33 | 'plugin:react/jsx-runtime', 34 | 'plugin:react-hooks/recommended', 35 | 'plugin:jsx-a11y/recommended', 36 | ], 37 | settings: { 38 | react: { 39 | version: 'detect', 40 | }, 41 | formComponents: ['Form'], 42 | linkComponents: [ 43 | { name: 'Link', linkAttribute: 'to' }, 44 | { name: 'NavLink', linkAttribute: 'to' }, 45 | ], 46 | 'import/resolver': { 47 | typescript: {}, 48 | }, 49 | }, 50 | rules: { 51 | 'jsx-a11y/no-autofocus': 'off', 52 | }, 53 | }, 54 | 55 | // Typescript 56 | { 57 | files: ['**/*.{ts,tsx}'], 58 | plugins: ['@typescript-eslint', 'import'], 59 | parser: '@typescript-eslint/parser', 60 | settings: { 61 | 'import/internal-regex': '^~/', 62 | 'import/resolver': { 63 | node: { 64 | extensions: ['.ts', '.tsx'], 65 | }, 66 | typescript: { 67 | alwaysTryTypes: true, 68 | }, 69 | }, 70 | }, 71 | extends: [ 72 | 'plugin:@typescript-eslint/recommended', 73 | 'plugin:import/recommended', 74 | 'plugin:import/typescript', 75 | ], 76 | rules: { 77 | '@typescript-eslint/no-unused-vars': 'off', 78 | 'import/no-unresolved': 'off', 79 | '@typescript-eslint/no-explicit-any': 'off', 80 | }, 81 | }, 82 | 83 | // Node 84 | { 85 | files: ['.eslintrc.js'], 86 | env: { 87 | node: true, 88 | }, 89 | }, 90 | ], 91 | } 92 | -------------------------------------------------------------------------------- /webui/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | 3 | /.cache 4 | /build 5 | .env 6 | /dist/ -------------------------------------------------------------------------------- /webui/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to Remix + Vite! 2 | 3 | 📖 See the [Remix docs](https://remix.run/docs) and the [Remix Vite docs](https://remix.run/docs/en/main/future/vite) for details on supported features. 4 | 5 | ## Development 6 | 7 | Run the Vite dev server: 8 | 9 | ```shellscript 10 | npm run dev 11 | ``` 12 | 13 | ## Deployment 14 | 15 | First, build your app for production: 16 | 17 | ```sh 18 | npm run build 19 | ``` 20 | 21 | Then run the app in production mode: 22 | 23 | ```sh 24 | npm start 25 | ``` 26 | 27 | Now you'll need to pick a host to deploy it to. 28 | 29 | ### DIY 30 | 31 | If you're familiar with deploying Node applications, the built-in Remix app server is production-ready. 32 | 33 | Make sure to deploy the output of `npm run build` 34 | 35 | - `build/server` 36 | - `build/client` 37 | -------------------------------------------------------------------------------- /webui/app/Clock.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from 'react' 2 | 3 | export function Clock() { 4 | const [time, setTime] = useState(getTime) 5 | useEffect(() => { 6 | const interval = setInterval(() => { 7 | setTime(getTime()) 8 | }, 1000) 9 | return () => clearInterval(interval) 10 | }, []) 11 | return ( 12 |
13 | {time} 14 |
15 | ) 16 | } 17 | const getTime = () => { 18 | return new Date().toString().split(' ')[4].split(':').slice(0, 2).join(':') 19 | } 20 | -------------------------------------------------------------------------------- /webui/app/backend.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getAuth, 3 | onAuthStateChanged, 4 | User, 5 | GoogleAuthProvider, 6 | signInWithPopup, 7 | signOut as signOutFirebase, 8 | } from 'firebase/auth' 9 | import { getFirestore, doc, getDoc } from 'firebase/firestore' 10 | import { app } from './firebase' 11 | import { SyncExternalStore } from 'sync-external-store' 12 | import axios from 'axios' 13 | 14 | const auth = getAuth(app) 15 | const firestore = getFirestore(app) 16 | 17 | class AutomatronBackend implements Backend { 18 | authReadyStatePromise: Promise 19 | authStore = new SyncExternalStore(undefined) 20 | private url?: string 21 | 22 | constructor() { 23 | this.authReadyStatePromise = new Promise((resolve) => { 24 | onAuthStateChanged(auth, (user) => { 25 | this.authStore.state = user 26 | resolve() 27 | if (user && !this.url) { 28 | this.getUrl() 29 | } 30 | }) 31 | }) 32 | } 33 | 34 | private async getUrl() { 35 | if (this.url) { 36 | return this.url 37 | } 38 | const s = await getDoc( 39 | doc(firestore, 'apps', 'automatron', 'config', 'url') 40 | ) 41 | this.url = s.data()?.service 42 | return this.url 43 | } 44 | 45 | async signIn() { 46 | try { 47 | const provider = new GoogleAuthProvider() 48 | await signInWithPopup(auth, provider) 49 | } catch (error) { 50 | console.error(error) 51 | alert(`Unable to sign in: ${error}`) 52 | } 53 | } 54 | 55 | async signOut() { 56 | await signOutFirebase(auth) 57 | } 58 | 59 | async send(text: string): Promise { 60 | const data = await this._post('/webpost-firebase', { text, source: 'web' }) 61 | if (typeof data.result === 'string') { 62 | data.result = [ 63 | { 64 | type: 'text', 65 | text: data.result, 66 | }, 67 | ] 68 | } 69 | return data 70 | } 71 | 72 | private async getHeaders() { 73 | return { 74 | Authorization: `Bearer ${await this.getIdToken()}`, 75 | } 76 | } 77 | 78 | async getHistory(): Promise { 79 | return this._get('/history') 80 | } 81 | 82 | async getSpeedDials(): Promise { 83 | return this._get('/speed-dials') 84 | } 85 | 86 | async getKnobs(): Promise }>> { 87 | return this._get('/knobs') 88 | } 89 | 90 | private async getIdToken() { 91 | return await auth.currentUser!.getIdToken() 92 | } 93 | 94 | async _get(url: string) { 95 | const response = await axios.get((await this.getUrl()) + url, { 96 | headers: await this.getHeaders(), 97 | }) 98 | return response.data 99 | } 100 | 101 | async _post(url: string, data: any) { 102 | const response = await axios.post((await this.getUrl()) + url, data, { 103 | headers: await this.getHeaders(), 104 | }) 105 | return response.data 106 | } 107 | } 108 | 109 | class FakeBackend implements Backend { 110 | authReadyStatePromise: Promise = Promise.resolve() 111 | authStore = new SyncExternalStore(null) 112 | 113 | async signIn() { 114 | this.authStore.state = {} as User 115 | } 116 | 117 | async signOut() { 118 | this.authStore.state = null 119 | } 120 | 121 | async getHistory(): Promise { 122 | return { ok: true, result: { history: [] } } 123 | } 124 | 125 | async getSpeedDials(): Promise { 126 | return { ok: true, result: { speedDials: [] } } 127 | } 128 | 129 | async getKnobs(): Promise }>> { 130 | return { ok: true, result: { knobs: {} } } 131 | } 132 | 133 | async send(text: string): Promise { 134 | return JSON.parse(text) 135 | } 136 | } 137 | 138 | type Ok = { ok: true; result: X } 139 | 140 | interface Backend { 141 | authReadyStatePromise: Promise 142 | authStore: SyncExternalStore 143 | signIn(): Promise 144 | signOut(): Promise 145 | send(text: string): Promise 146 | getHistory(): Promise 147 | getSpeedDials(): Promise 148 | getKnobs(): Promise }>> 149 | } 150 | 151 | export const backend = 152 | typeof location === 'undefined' || 153 | new URLSearchParams(location.search).get('backend') === 'fake' 154 | ? new FakeBackend() 155 | : new AutomatronBackend() 156 | 157 | if (typeof window !== 'undefined') { 158 | Object.assign(window, { backend }) 159 | } 160 | -------------------------------------------------------------------------------- /webui/app/firebase.ts: -------------------------------------------------------------------------------- 1 | import { initializeApp } from 'firebase/app' 2 | 3 | // Your web app's Firebase configuration 4 | const firebaseConfig = { 5 | apiKey: 'AIzaSyAmj0axCEWY5ojSMDs25h0hNNfMFCRIqys', 6 | authDomain: 'dtinth-automatron.firebaseapp.com', 7 | projectId: 'dtinth-automatron', 8 | storageBucket: 'dtinth-automatron.appspot.com', 9 | messagingSenderId: '347735770628', 10 | appId: '1:347735770628:web:8fce6a02b34c2d17c2d751', 11 | } 12 | 13 | // Initialize Firebase 14 | export const app = initializeApp(firebaseConfig) 15 | -------------------------------------------------------------------------------- /webui/app/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | body { 6 | font-family: Arimo, Helvetica, Arial, sans-serif; 7 | } 8 | 9 | .come-up { 10 | animation: 0.25s come-up; 11 | } 12 | 13 | @keyframes come-up { 14 | from { 15 | transform: scale(0.99); 16 | opacity: 0; 17 | } 18 | to { 19 | transform: scale(1); 20 | opacity: 1; 21 | } 22 | } 23 | 24 | body::-webkit-scrollbar { 25 | display: none; 26 | } 27 | 28 | .bg-bevel { 29 | background: #252423 linear-gradient(to bottom, #454443, #151413); 30 | } 31 | 32 | .bg-emboss { 33 | background: #252423 linear-gradient(to bottom, #151413, #292827); 34 | } 35 | 36 | .bg-glossy { 37 | background: #252423 38 | linear-gradient( 39 | to bottom, 40 | #353433 0%, 41 | #252423 50%, 42 | #151413 50%, 43 | #252423 100% 44 | ); 45 | } 46 | -------------------------------------------------------------------------------- /webui/app/requireAuth.ts: -------------------------------------------------------------------------------- 1 | import { backend } from './backend' 2 | import { redirect } from '@remix-run/react' 3 | 4 | export async function requireAuth() { 5 | await backend.authReadyStatePromise 6 | if (!backend.authStore.state) { 7 | throw redirect('/') 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /webui/app/root.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Links, 3 | LiveReload, 4 | Meta, 5 | Outlet, 6 | Scripts, 7 | ScrollRestoration, 8 | } from '@remix-run/react' 9 | 10 | import './index.css' 11 | 12 | export default function App() { 13 | return ( 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | ) 29 | } 30 | -------------------------------------------------------------------------------- /webui/app/routes/_index.tsx: -------------------------------------------------------------------------------- 1 | import { ReactNode, useSyncExternalStore } from 'react' 2 | import { Clock } from '~/Clock' 3 | import { backend } from '~/backend' 4 | import { Icon } from '@iconify-icon/react' 5 | import running from '@iconify-icons/cil/running' 6 | import menu from '@iconify-icons/cil/menu' 7 | import { useNavigate } from '@remix-run/react' 8 | 9 | export default function Index() { 10 | return ( 11 | <> 12 | 13 | 14 | 15 | ) 16 | } 17 | 18 | function AuthButton() { 19 | const authState = useSyncExternalStore( 20 | backend.authStore.subscribe, 21 | backend.authStore.getSnapshot 22 | ) 23 | const navigate = useNavigate() 24 | 25 | return ( 26 | <> 27 | {authState === undefined ? ( 28 | <> 29 | ) : authState === null ? ( 30 | backend.signIn()} title="Sign In"> 31 | 32 | 33 | ) : ( 34 | { 36 | navigate('/automatron') 37 | }} 38 | title="Automatron" 39 | > 40 | 41 | 42 | )} 43 | 44 | 45 | ) 46 | } 47 | 48 | export interface FloatingButton { 49 | title: string 50 | children: ReactNode 51 | onClick?: () => void 52 | } 53 | 54 | export function FloatingButton(props: FloatingButton) { 55 | return ( 56 |
57 | 64 |
65 | ) 66 | } 67 | -------------------------------------------------------------------------------- /webui/app/routes/automatron._index.tsx: -------------------------------------------------------------------------------- 1 | import { Icon } from '@iconify-icon/react' 2 | import chevronLeft from '@iconify-icons/cil/chevron-left' 3 | import chevronRight from '@iconify-icons/cil/chevron-right' 4 | import history from '@iconify-icons/cil/history' 5 | import send from '@iconify-icons/cil/send' 6 | import { 7 | Await, 8 | ClientActionFunctionArgs, 9 | Form, 10 | useActionData, 11 | useLoaderData, 12 | useNavigation, 13 | } from '@remix-run/react' 14 | import clsx from 'clsx' 15 | import { ReactNode, Suspense, useState } from 'react' 16 | import { z } from 'zod' 17 | import { backend } from '~/backend' 18 | import { requireAuth } from '~/requireAuth' 19 | 20 | export const clientLoader = async () => { 21 | await requireAuth() 22 | return { 23 | speedDialsPromise: backend.getSpeedDials(), 24 | historyPromise: backend.getHistory(), 25 | } 26 | } 27 | 28 | export interface ActionResult { 29 | result?: unknown 30 | error?: unknown 31 | } 32 | 33 | export const clientAction = async ( 34 | args: ClientActionFunctionArgs 35 | ): Promise => { 36 | await requireAuth() 37 | const form = await args.request.formData() 38 | const text = form.get('text') 39 | const result = await backend.send(String(text)) 40 | try { 41 | return { result } as ActionResult 42 | } catch (error) { 43 | return { error } 44 | } 45 | } 46 | 47 | const classes = { 48 | button: 49 | 'bg-bevel hover:border-#555453 block rounded border border-#454443 p-2 shadow-md shadow-black/50 active:border-#8b8685 flex flex-col items-center justify-center', 50 | } 51 | 52 | export default function AutomatronConsole() { 53 | const data = useLoaderData() 54 | const [speedDialEnabled, setSpeedDialEnabled] = useState(false) 55 | const [historyEnabled, setHistoryEnabled] = useState(false) 56 | const { error, result } = useActionData() ?? {} 57 | const navigation = useNavigation() 58 | const isSubmitting = navigation.state === 'submitting' 59 | 60 | return ( 61 | <> 62 |
63 |