├── .DS_Store ├── .gitignore ├── Assistant ├── Axios.js ├── ChatService.js ├── Drain.js ├── FastEmbed.js ├── GptSchema.json ├── InMemorySearch.js ├── JavascriptCode.js ├── Location.js ├── Main.js ├── Plugin │ ├── PluginServer.js │ └── plugins │ │ └── vscode │ │ ├── README.md │ │ ├── Tool.js │ │ ├── VSCode │ │ └── assistant │ │ │ ├── .eslintrc.json │ │ │ ├── .vscode-test.mjs │ │ │ ├── .vscode │ │ │ ├── extensions.json │ │ │ └── launch.json │ │ │ ├── .vscodeignore │ │ │ ├── Assistant.js │ │ │ ├── CHANGELOG.md │ │ │ ├── README.md │ │ │ ├── extension.js │ │ │ ├── jsconfig.json │ │ │ ├── package-lock.json │ │ │ ├── package.json │ │ │ ├── test │ │ │ └── extension.test.js │ │ │ └── vsc-extension-quickstart.md │ │ └── config.json ├── PluginTools.js ├── Preload.js ├── README.md ├── Screen.js ├── Shell.js ├── ToolClient.js ├── electron-playwright.js ├── gpt-override.css ├── installPlugins.js ├── osascript-error.json ├── package-lock.json └── package.json ├── Cloud ├── GPTAction.js ├── GetAdmin.js ├── Instructions.txt ├── Schema.js ├── Shell.js ├── build.js ├── buildSchema.js ├── createApiKey.js ├── createServiceAccount.js ├── firebase.json ├── firestore.indexes.json ├── firestore.rules ├── getFirebaseConfig.js ├── getServerURL.js ├── package-lock.json ├── package.json └── server.js ├── LICENSE.txt └── README.md /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Attunewise/GPT/3cba12accf1963073757c676220b874632ced796/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and not Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # vuepress v2.x temp and cache directory 104 | .temp 105 | .cache 106 | 107 | # Docusaurus cache and generated files 108 | .docusaurus 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | # yarn v2 126 | .yarn/cache 127 | .yarn/unplugged 128 | .yarn/build-state.yml 129 | .yarn/install-state.gz 130 | .pnp.* 131 | 132 | # Emacs temporary files 133 | *~ 134 | \#*\# 135 | .*.swp 136 | ServiceAccount.json 137 | gptURL.txt 138 | apiKey.txt 139 | serverURL.txt 140 | Build 141 | Client 142 | local_cache 143 | firebaseConfig.json 144 | -------------------------------------------------------------------------------- /Assistant/Axios.js: -------------------------------------------------------------------------------- 1 | const { drainStream } = require('./Drain.js'); 2 | 3 | class AxiosError extends Error { 4 | constructor(message, status, data) { 5 | super(message) 6 | this.status = status 7 | this.data = data 8 | } 9 | } 10 | 11 | const axios1 = (op, ...args) => { 12 | const doit = async () => { 13 | try { 14 | const axios = require('axios'); 15 | if (op) { 16 | //console.log(`axios[${op}]`, args); 17 | return await axios[op](...args); 18 | } else { 19 | return await axios(...args); 20 | } 21 | } catch (err) { 22 | let status 23 | let data 24 | let message = err.message 25 | if (err.response) { 26 | status = err.response.status 27 | data = err.response.data; 28 | if (!data || data.request) { 29 | data = await drainStream(err.response); 30 | if (data.request) { 31 | data = data.data 32 | } 33 | } 34 | try { 35 | err.message += '\n' + data 36 | } catch (ignored) { 37 | console.error(ignored.message) 38 | } 39 | throw new AxiosError(message, status, data) 40 | } 41 | console.error(err.message); 42 | throw err 43 | } 44 | } 45 | return doit(); 46 | } 47 | 48 | const axios = (...args) => axios1(null, ...args); 49 | 50 | for (const op of ['get', 'set', 'head', 'post', 'delete', 'put']) { 51 | axios[op] = (...args) => axios1(op, ...args) 52 | } 53 | 54 | module.exports = { axios } 55 | -------------------------------------------------------------------------------- /Assistant/ChatService.js: -------------------------------------------------------------------------------- 1 | const sqlite3 = require('sqlite3').verbose(); 2 | const { TypedStreamReader, Unarchiver } = require('node-typedstream') 3 | 4 | const path = require('path'); 5 | 6 | class ChatService { 7 | constructor() { 8 | this.dbPath = path.join(process.env.HOME, 'Library', 'Messages', 'chat.db'); 9 | this.db = new sqlite3.Database(this.dbPath); 10 | } 11 | 12 | searchMessages(sqlQuery, params) { 13 | return new Promise((resolve, reject) => { 14 | this.db.all(sqlQuery, params, (err, rows) => { 15 | if (err) { 16 | reject(err); 17 | return; 18 | } 19 | resolve(rows.map(row => { 20 | if (row.ROWID) { 21 | row.id = row.ROWID.toString() 22 | } 23 | if (row.attributedBody) { 24 | row.attributedBody = this.parseAttributedBody(row.attributedBody) 25 | } 26 | if (row.date) { 27 | row.date = new Date(row.date / 1000000 + 978307200000) // Convert Apple timestamp to JS Date 28 | } 29 | return row 30 | })) 31 | }); 32 | }); 33 | } 34 | 35 | parseAttributedBody(attributedBody) { 36 | console.log('attributedBody', attributedBody) 37 | if (!attributedBody) { 38 | return null; 39 | } 40 | try { 41 | return Unarchiver.open(attributedBody).decodeAll() 42 | // Parse the binary plist to a JavaScript object 43 | } catch (error) { 44 | console.error('Error parsing attributedBody:', error); 45 | return null; 46 | } 47 | } 48 | } 49 | 50 | module.exports = { ChatService } 51 | -------------------------------------------------------------------------------- /Assistant/Drain.js: -------------------------------------------------------------------------------- 1 | const drainStream = stream => { 2 | if (!stream.on) { 3 | return Promise.resolve(stream) 4 | } 5 | return new Promise(resolve => { 6 | let output = '' 7 | stream.on('data', data => { 8 | console.log("data=>", data) 9 | output += data 10 | }) 11 | stream.on('end', () => { 12 | resolve(output) 13 | }) 14 | }) 15 | } 16 | 17 | module.exports = { drainStream } 18 | -------------------------------------------------------------------------------- /Assistant/FastEmbed.js: -------------------------------------------------------------------------------- 1 | const { EmbeddingModel, FlagEmbedding } = require("fastembed") 2 | 3 | let embeddingModel 4 | const vectorSize = 768 5 | const createEmbeddings = async (documents, batchSize) => { 6 | batchSize = batchSize || 16 7 | if (!embeddingModel) { 8 | embeddingModel = await FlagEmbedding.init({ 9 | model: EmbeddingModel.BGEBaseEN 10 | }) 11 | } 12 | const start = Date.now() 13 | const embeddings = await embeddingModel.embed(documents, batchSize) 14 | const vectors = [] 15 | for await (const batch of embeddings) { 16 | for (const embedding of batch) { 17 | const vector = [] 18 | for (const i in embedding) { 19 | vector.push(embedding[i]) 20 | } 21 | vectors.push(vector) 22 | } 23 | } 24 | const end = Date.now() 25 | const size = vectors[0].length 26 | //console.log("created embeddings:", documents.length, vectors.length, 'size', size, "in", (end - start), "ms") 27 | return { embeddings: vectors } 28 | } 29 | 30 | module.exports = { createEmbeddings } 31 | -------------------------------------------------------------------------------- /Assistant/GptSchema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Attunewise/GPT/3cba12accf1963073757c676220b874632ced796/Assistant/GptSchema.json -------------------------------------------------------------------------------- /Assistant/InMemorySearch.js: -------------------------------------------------------------------------------- 1 | function dotProduct(vecA, vecB) { 2 | return vecA.reduce((sum, val, i) => sum + val * vecB[i], 0); 3 | } 4 | 5 | function norm(vec) { 6 | return Math.sqrt(vec.reduce((sum, val) => sum + val * val, 0)); 7 | } 8 | 9 | function cosineSimilarity(vecA, vecB) { 10 | return dotProduct(vecA, vecB) / (norm(vecA) * norm(vecB)); 11 | } 12 | 13 | class InMemorySearch { 14 | constructor(createEmbeddings) { 15 | this.createEmbeddings = createEmbeddings 16 | } 17 | 18 | memory = {} 19 | 20 | async index(contents) { 21 | const { embeddings } = await this.createEmbeddings(contents) 22 | contents.forEach((content, i) => { 23 | this.memory[i] = { 24 | id: i, 25 | content, 26 | embedding: embeddings[i] 27 | } 28 | }) 29 | } 30 | 31 | remove = id => { 32 | delete this.memory[id] 33 | } 34 | 35 | add = ({id, content, embedding}) => { 36 | this.memory[id] = { 37 | id, 38 | content, 39 | embedding 40 | } 41 | } 42 | 43 | async search(query, limit) { 44 | const { embeddings } = await this.createEmbeddings([query]) 45 | const [queryEmbedding] = embeddings 46 | const scored = Object.values(this.memory).map(({id, content, embedding}) => ({ 47 | id, 48 | score: cosineSimilarity(embedding, queryEmbedding), 49 | doc: content 50 | })).filter(item => { 51 | const {doc, score} = item 52 | //console.log(query, score, '=>', doc) 53 | return item.score >= 0.7 54 | }); // Threshold can be adjusted 55 | //console.log('got scored', scored) 56 | scored.sort((a, b) => b.score - a.score) 57 | let results = scored 58 | if (limit) { 59 | results = scored.slice(0, limit) 60 | } 61 | return { results } 62 | } 63 | } 64 | 65 | 66 | module.exports = { InMemorySearch } 67 | 68 | -------------------------------------------------------------------------------- /Assistant/JavascriptCode.js: -------------------------------------------------------------------------------- 1 | const babel = require("@babel/core"); 2 | 3 | function readExportsStmt({ types: t }) { 4 | return { 5 | visitor: { 6 | AssignmentExpression(path) { 7 | if (t.isMemberExpression(path.node.left) && 8 | path.node.left.object.name === 'module' && 9 | path.node.left.property.name === 'exports') { 10 | console.log('Found module.exports:', path.toString()); 11 | } 12 | } 13 | } 14 | } 15 | } 16 | 17 | 18 | const addReturnIfMissingPlugin = ({ types: t }) => ({ 19 | visitor: { 20 | Program(path) { 21 | let hasReturn = false; 22 | path.get('body').forEach((nodePath) => { 23 | console.log("node", nodePath) 24 | if (t.isReturnStatement(nodePath)) { 25 | hasReturn = true; 26 | } 27 | }); 28 | if (!hasReturn) { 29 | // Add a return statement at the end if there's none 30 | const lastNode = path.get('body')[path.node.body.length - 1]; 31 | if (t.isExpressionStatement(lastNode)) { 32 | lastNode.replaceWith( 33 | t.returnStatement(lastNode.node.expression) 34 | ); 35 | } 36 | } 37 | } 38 | } 39 | }); 40 | 41 | 42 | const returnFunctionDeclPlugin = ({ types: t }) => { 43 | return { 44 | visitor: { 45 | Program(path) { 46 | const body = path.get('body'); 47 | const lastStatement = body[body.length - 1]; 48 | 49 | // Check if the last statement is a function declaration 50 | if (t.isFunctionDeclaration(lastStatement.node)) { 51 | console.log("Found function declaration"); 52 | 53 | // Transform the function declaration into a function expression 54 | const funcExpr = t.functionExpression( 55 | lastStatement.node.id, 56 | lastStatement.node.params, 57 | lastStatement.node.body, 58 | lastStatement.node.generator, 59 | lastStatement.node.async 60 | ); 61 | 62 | // Directly replace the function declaration with the function expression 63 | lastStatement.replaceWith(funcExpr); 64 | } 65 | }, 66 | }, 67 | }; 68 | }; 69 | 70 | const returnFunctionDecl = code => { 71 | try { 72 | console.log('transpiling', code) 73 | const { code: transpiledCode } = babel.transform(code, { 74 | sourceType: 'unambiguous', 75 | plugins: [returnFunctionDeclPlugin], 76 | presets: [ 77 | ["@babel/preset-env", { 78 | "targets": { 79 | "node": "8" // or a higher version that supports async/await 80 | } 81 | }] 82 | ] 83 | }); 84 | console.log('transpiled', transpiledCode) 85 | // Safely evaluate the transpiled code 86 | return transpiledCode 87 | } catch (error) { 88 | console.error(error) 89 | if (error.message.indexOf("'return' outside of function" >= 0)) { 90 | return code 91 | } 92 | throw error 93 | } 94 | } 95 | 96 | 97 | // Function to transpile and evaluate code 98 | const addReturnStatementIfNecessary = code => { 99 | try { 100 | console.log('transpiling', code) 101 | const { code: transpiledCode } = babel.transform(code, { 102 | sourceType: 'unambiguous', 103 | plugins: [addReturnIfMissingPlugin], 104 | presets: [ 105 | ["@babel/preset-env", { 106 | "targets": { 107 | "node": "8" // or a higher version that supports async/await 108 | } 109 | }] 110 | ] 111 | }); 112 | console.log('transpiled', transpiledCode) 113 | // Safely evaluate the transpiled code 114 | return transpiledCode 115 | } catch (error) { 116 | if (error.message.indexOf("'return' outside of function" >= 0)) { 117 | return code 118 | } 119 | console.error(error) 120 | throw error 121 | } 122 | } 123 | 124 | 125 | module.exports = { addReturnStatementIfNecessary, returnFunctionDecl } 126 | -------------------------------------------------------------------------------- /Assistant/Location.js: -------------------------------------------------------------------------------- 1 | const { axios } = require('./Axios.js') 2 | 3 | getGeoLocation = async () => { 4 | const response = await axios.get('https://ipinfo.io/json') 5 | return response.data 6 | } 7 | 8 | module.exports = { getGeoLocation } 9 | 10 | -------------------------------------------------------------------------------- /Assistant/Main.js: -------------------------------------------------------------------------------- 1 | const { dialog, session, contextBridge, ipcMain, ipcRenderer, app, BrowserWindow } = require('electron') 2 | const crypto = require('crypto') 3 | const { ToolClient } = require('./ToolClient.js') 4 | const path = require('path') 5 | const os = require('os') 6 | const fs = require('fs') 7 | const gptURL = fs.readFileSync('./gptURL.txt', 'utf-8').trim() 8 | try { 9 | // require('electron-reloader')(module); 10 | } catch (_) {} 11 | 12 | let toolClient = new ToolClient('gpt-4') 13 | 14 | dialog.showErrorBox = function(title, content) { 15 | console.log(`${title}\n${content}`); 16 | }; 17 | 18 | initGPT = (win, url) => { 19 | console.log('initGPT', url) 20 | url = url.replace('chat.openai.com', 'chatgpt.com') 21 | let triedLogin = false 22 | win.webContents.on('did-finish-load', async () => { 23 | let current = win.webContents.getURL() 24 | console.log("current", current) 25 | if (current.startsWith(url)) { 26 | current = url 27 | } 28 | console.log(current) 29 | switch (current) { 30 | default: 31 | case 'https://chatgpt.com/': 32 | case 'https://chat.openai.com/': 33 | case 'https://chat.openai.com': 34 | let ses = win.webContents.session; 35 | const cookies = await ses.cookies.get({domain: 'chat.openai.com'}) 36 | const cookies2 = await ses.cookies.get({domain: 'chatgpt.com'}) 37 | console.log(cookies2) 38 | const isLoggedIn = //cookies.find(x => x.name == '__Secure-next-auth.session-token') || 39 | cookies2.find(x => x.name == '__Secure-next-auth.session-token') 40 | console.log("isLoggedIn", isLoggedIn) 41 | if (!triedLogin && isLoggedIn) { 42 | console.log("LOADING", url) 43 | triedLogin = true 44 | win.loadURL(url) 45 | } 46 | break 47 | case url: 48 | case url + '/': 49 | setTimeout(() => { 50 | let css = fs.readFileSync(path.join(__dirname, 'gpt-override.css'), 'utf-8') 51 | const { pathname } = new URL(url) 52 | css = css.replaceAll('$GPT_PATH', pathname) 53 | //win.webContents.insertCSS(css); 54 | //console.log("inserted css") 55 | }, 0) 56 | } 57 | }); 58 | win.loadURL('https://chat.openai.com') 59 | } 60 | 61 | let win 62 | async function createWindow () { 63 | session.defaultSession.setPermissionCheckHandler((webContents, permission) => { 64 | //console.log("permission check:", permission) 65 | return true 66 | }); 67 | session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => { 68 | //console.log("permission request:", permission) 69 | callback(true) 70 | }); 71 | // Create the browser window. 72 | win = new BrowserWindow({ 73 | width: 800, 74 | height: 800, 75 | webPreferences: { 76 | preload: path.join(__dirname, 'Preload.js'), 77 | nodeIntegration: true, 78 | webviewTag: true 79 | } 80 | }); 81 | initGPT(win, gptURL) 82 | await toolClient.login() 83 | } 84 | 85 | console.log("started") 86 | app.whenReady().then(() => { 87 | createWindow() 88 | }) 89 | -------------------------------------------------------------------------------- /Assistant/Plugin/PluginServer.js: -------------------------------------------------------------------------------- 1 | const WebSocket = require('ws') 2 | 3 | const delay = seconds => new Promise(resolve => setTimeout(resolve, seconds * 1000)) 4 | 5 | const createPluginServer = (name, port) => { 6 | // Create a WebSocket server on port 0xbad0 7 | console.log("Creating plugin server for plugin", name, "on port", port) 8 | const wss = new WebSocket.Server({ port }); 9 | 10 | const apps = {} 11 | 12 | const isConnected = appName => { 13 | return apps[appName] || apps['the'] 14 | } 15 | 16 | const getApp = async (appName, retry) => { 17 | console.log("getApp", appName) 18 | const app = apps[appName] 19 | if (app) { 20 | return app 21 | } else { 22 | if (!retry) { 23 | console.log("retrying...") 24 | await delay(5) 25 | return await getApp(appName, true) 26 | } 27 | const message = "Service not available: " + appName 28 | throw new Error(message) 29 | } 30 | } 31 | 32 | const listeners = {} 33 | 34 | const addEventListener = (appName, event, listener) => { 35 | const k = appName + '.' + event 36 | listeners[k] = { appName, listener } 37 | const app = apps[appName] 38 | if (app) { 39 | app.addEventListener(event, listener) 40 | } 41 | } 42 | 43 | const removeEventListener = (appName, event, listener) => { 44 | const k = appName + '.' + event 45 | delete listeners[k] 46 | const app = apps[appName] 47 | if (app) { 48 | app.removeEventListener(event, listener) 49 | } 50 | } 51 | 52 | const evaluateScript = async (appName, script) => { 53 | const app = await getApp(appName) 54 | return await app.evaluateScript(script) 55 | } 56 | 57 | class App { 58 | constructor(apps, allListeners, ws) { 59 | this.reqId = 0 60 | let appName 61 | this.ws = ws 62 | this.reqs = {} 63 | // Set up message event 64 | const self = this 65 | ws.on('message', message => { 66 | const json = JSON.parse(message) 67 | console.log('message', json) 68 | const reqId = json.reqId 69 | switch (json.type) { 70 | case 'event': 71 | { 72 | const value = json.event 73 | const { type } = value 74 | const event = value[type] 75 | self.fireEvent(type, event) 76 | } 77 | break 78 | case 'appName': 79 | appName = json.appName 80 | const existing = apps[appName] 81 | if (existing) { 82 | try { 83 | existing.close() 84 | } catch (ignored) { 85 | } 86 | } 87 | self.appName = appName 88 | this.listeners = Object.values(allListeners).filter(x => x.appName === appName).map(x => x.listener) 89 | apps[appName] = self 90 | console.log("New app connected: "+appName) 91 | self.fireEvent('connect', appName) 92 | return 93 | case 'result': 94 | const { resolve } = this.reqs[reqId] 95 | delete this.reqs[reqId] 96 | resolve(json.result) 97 | break 98 | case 'error': 99 | const {reject} = this.reqs[reqId] 100 | delete this.reqs[reqId] 101 | reject(new Error(json.error)) 102 | break 103 | } 104 | }) 105 | } 106 | evaluateScript = (script) => { 107 | const ws = this.ws 108 | return new Promise((resolve, reject) => { 109 | let id = ++this.reqId 110 | const msg = { 111 | reqId: `${id}`, 112 | target: this.appName, 113 | script 114 | } 115 | this.reqs[id] = {resolve, reject } 116 | ws.send(JSON.stringify(msg)) 117 | }) 118 | } 119 | addEventListener = (event, handler) => { 120 | let array = listeners[event] 121 | if (!array) { 122 | listeners[event] = [handler] 123 | } else { 124 | array.push(handler) 125 | } 126 | } 127 | removeEventListener = (event, handler) => { 128 | let array = listeners[event] 129 | if (array) { 130 | array = array.filter(x => x !== handler) 131 | } 132 | } 133 | fireEvent = (type, event) => { 134 | console.log("fire event", type, event) 135 | const listeners1 = listeners[type] 136 | const listeners2 = listeners['all'] 137 | const dispatch = listeners => { 138 | if (listeners) { 139 | listeners.forEach(listener => listener(type, event)) 140 | } 141 | } 142 | dispatch(listeners1) 143 | dispatch(listeners2) 144 | } 145 | 146 | close = () => { 147 | this.ws.close() 148 | } 149 | } 150 | 151 | // Set up connection event 152 | wss.on('connection', ws => { 153 | new App(apps, listeners, ws) 154 | }) 155 | 156 | console.log('WebSocket server started on ws://localhost:'+ port); 157 | return { 158 | evaluateScript, 159 | isConnected, 160 | addEventListener, 161 | removeEventListener 162 | } 163 | 164 | } 165 | 166 | 167 | 168 | module.exports = { createPluginServer } 169 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/README.md: -------------------------------------------------------------------------------- 1 | To use the `vscode` tool you'll need to install the assistant's VSCode plugin which is under `./VSCode' into VSCode. And you'll need to enable it in `./config.json`. -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/Tool.js: -------------------------------------------------------------------------------- 1 | const { createPluginServer } = require('../../PluginServer.js') 2 | const { addReturnStatementIfNecessary } = require('../../../JavascriptCode.js') 3 | 4 | const vscode = createPluginServer('VSCode', 0xbada) 5 | 6 | const call = async (name, args) => { 7 | let isError 8 | switch (name) { 9 | case 'vscode': 10 | { 11 | let content 12 | try { 13 | let { javascript } = args 14 | if (!javascript) { 15 | throw new Error("`javascript` is required.") 16 | } 17 | const req = javascript.indexOf('require') 18 | if (req < 0) { 19 | javascript = "const vscode = require('vscode')\n"+javascript 20 | } 21 | javascript = addReturnStatementIfNecessary(javascript) 22 | const newScript = `(async function() { ${javascript} })()` 23 | const scriptResult = await vscode.evaluateScript('vscode', newScript) 24 | content = JSON.stringify(scriptResult, null, ' ') 25 | } catch (err) { 26 | console.error("scriptError", err) 27 | isError = true 28 | content = err.message 29 | } 30 | console.log("vscode reply", content) 31 | return { 32 | content, isError 33 | } 34 | } 35 | } 36 | } 37 | 38 | module.exports = { call } 39 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": false, 4 | "commonjs": true, 5 | "es6": true, 6 | "node": true, 7 | "mocha": true 8 | }, 9 | "parserOptions": { 10 | "ecmaVersion": 2018, 11 | "ecmaFeatures": { 12 | "jsx": true 13 | }, 14 | "sourceType": "module" 15 | }, 16 | "rules": { 17 | "no-const-assign": "warn", 18 | "no-this-before-super": "warn", 19 | "no-undef": "warn", 20 | "no-unreachable": "warn", 21 | "no-unused-vars": "warn", 22 | "constructor-super": "warn", 23 | "valid-typeof": "warn" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/.vscode-test.mjs: -------------------------------------------------------------------------------- 1 | import { defineConfig } from '@vscode/test-cli'; 2 | 3 | export default defineConfig({ 4 | files: 'test/**/*.test.js', 5 | }); 6 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the extensions.json format 4 | "recommendations": [ 5 | "dbaeumer.vscode-eslint", 6 | "ms-vscode.extension-test-runner" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | // A launch configuration that launches the extension inside a new window 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | { 6 | "version": "0.2.0", 7 | "configurations": [ 8 | { 9 | "name": "Run Extension", 10 | "type": "extensionHost", 11 | "request": "launch", 12 | "args": [ 13 | "--extensionDevelopmentPath=${workspaceFolder}" 14 | ] 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/.vscodeignore: -------------------------------------------------------------------------------- 1 | .vscode/** 2 | .vscode-test/** 3 | test/** 4 | .gitignore 5 | .yarnrc 6 | vsc-extension-quickstart.md 7 | **/jsconfig.json 8 | **/*.map 9 | **/.eslintrc.json 10 | **/.vscode-test.* 11 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/Assistant.js: -------------------------------------------------------------------------------- 1 | const WebSocket = require('ws') 2 | const port = 0xbada; 3 | 4 | class Assistant { 5 | 6 | onDestroy() { 7 | this.disconnect() 8 | } 9 | 10 | onCreate(context) { 11 | this.context = context 12 | this.tryConnect() 13 | } 14 | 15 | evaluateScript = async script => { 16 | return await eval(script) 17 | } 18 | 19 | tryConnect = async () => { 20 | this.ws = new WebSocket('http://localhost:' + port); 21 | this.init() 22 | } 23 | 24 | init() { 25 | const self = this 26 | const ws = this.ws 27 | ws.onopen = async function() { 28 | try { 29 | console.log('Connected to the server'); 30 | const appName = 'vscode' 31 | const initMessage = JSON.stringify({ type: 'appName', appName }); 32 | ws.send(initMessage); 33 | 34 | } catch (err) { 35 | debugger 36 | console.error(err) 37 | } 38 | }; 39 | 40 | const sendReply = msg => { 41 | console.log('REPLY', msg) 42 | ws.send(JSON.stringify(msg)) 43 | } 44 | 45 | 46 | ws.onmessage = async function(event) { 47 | console.log('Message from server:', event.data); 48 | 49 | try { 50 | const request = JSON.parse(event.data); 51 | 52 | if (request.script) { 53 | try { 54 | const reply = await self.evaluateScript(request.script) 55 | sendReply({ type: 'result', result: reply, reqId: request.reqId }) 56 | } catch (err) { 57 | console.error(err) 58 | let error 59 | if (!err.message) { 60 | error = JSON.stringify(err) 61 | } else { 62 | error = err.message 63 | } 64 | if (err.stack) { 65 | const lines = err.stack.split('\t').filter(line => line.indexOf('') >= 0) 66 | for (const line of lines) { 67 | const start = line.indexOf('') 68 | const end = line.indexOf(')', start) 69 | const location = line.substring(start, end) 70 | error += '\t at ' + location + '\n' 71 | } 72 | } 73 | sendReply({ type: 'error', error , reqId: request.reqId }) 74 | } 75 | } else { 76 | console.error('Invalid message format'); 77 | } 78 | } catch (error) { 79 | console.error('Error parsing message:', error); 80 | } 81 | } 82 | 83 | ws.onerror = function(error) { 84 | console.error('WebSocket Error:', error.message); 85 | try { 86 | ws.close(); 87 | } catch (ignored) {} 88 | } 89 | ws.onclose = function() { 90 | console.log('Disconnected from the server'); 91 | clearTimeout(self.retryTimeout) 92 | self.retryTimeout = setTimeout(self.tryConnect, 3000); 93 | } 94 | } 95 | } 96 | 97 | 98 | module.exports = { Assistant } 99 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to the "assistant" extension will be documented in this file. 4 | 5 | Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. 6 | 7 | ## [Unreleased] 8 | 9 | - Initial release -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/README.md: -------------------------------------------------------------------------------- 1 | # assistant README 2 | 3 | This is the README for your extension "assistant". After writing up a brief description, we recommend including the following sections. 4 | 5 | ## Features 6 | 7 | Describe specific features of your extension including screenshots of your extension in action. Image paths are relative to this README file. 8 | 9 | For example if there is an image subfolder under your extension project workspace: 10 | 11 | \!\[feature X\]\(images/feature-x.png\) 12 | 13 | > Tip: Many popular extensions utilize animations. This is an excellent way to show off your extension! We recommend short, focused animations that are easy to follow. 14 | 15 | ## Requirements 16 | 17 | If you have any requirements or dependencies, add a section describing those and how to install and configure them. 18 | 19 | ## Extension Settings 20 | 21 | Include if your extension adds any VS Code settings through the `contributes.configuration` extension point. 22 | 23 | For example: 24 | 25 | This extension contributes the following settings: 26 | 27 | * `myExtension.enable`: Enable/disable this extension. 28 | * `myExtension.thing`: Set to `blah` to do something. 29 | 30 | ## Known Issues 31 | 32 | Calling out known issues can help limit users opening duplicate issues against your extension. 33 | 34 | ## Release Notes 35 | 36 | Users appreciate release notes as you update your extension. 37 | 38 | ### 1.0.0 39 | 40 | Initial release of ... 41 | 42 | ### 1.0.1 43 | 44 | Fixed issue #. 45 | 46 | ### 1.1.0 47 | 48 | Added features X, Y, and Z. 49 | 50 | --- 51 | 52 | ## Working with Markdown 53 | 54 | You can author your README using Visual Studio Code. Here are some useful editor keyboard shortcuts: 55 | 56 | * Split the editor (`Cmd+\` on macOS or `Ctrl+\` on Windows and Linux) 57 | * Toggle preview (`Shift+Cmd+V` on macOS or `Shift+Ctrl+V` on Windows and Linux) 58 | * Press `Ctrl+Space` (Windows, Linux, macOS) to see a list of Markdown snippets 59 | 60 | ## For more information 61 | 62 | * [Visual Studio Code's Markdown Support](http://code.visualstudio.com/docs/languages/markdown) 63 | * [Markdown Syntax Reference](https://help.github.com/articles/markdown-basics/) 64 | 65 | **Enjoy!** 66 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/extension.js: -------------------------------------------------------------------------------- 1 | // The module 'vscode' contains the VS Code extensibility API 2 | // Import the module and reference it with the alias vscode in your code below 3 | const vscode = require('vscode'); 4 | 5 | const { Assistant } = require('./Assistant.js') 6 | 7 | // This method is called when your extension is activated 8 | // Your extension is activated the very first time the command is executed 9 | 10 | let assistant 11 | /** 12 | * @param {vscode.ExtensionContext} context 13 | */ 14 | function activate(context) { 15 | assistant = new Assistant() 16 | assistant.onCreate(context) 17 | } 18 | 19 | // This method is called when your extension is deactivated 20 | function deactivate() { 21 | assistant.onDestroy() 22 | } 23 | 24 | module.exports = { 25 | activate, 26 | deactivate 27 | } 28 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "Node16", 4 | "target": "ES2022", 5 | "checkJs": false, /* Typecheck .js files. */ 6 | "lib": [ 7 | "ES2022" 8 | ] 9 | }, 10 | "exclude": [ 11 | "node_modules" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "assistant", 3 | "displayName": "Assistant", 4 | "description": "An assistant", 5 | "version": "0.0.1", 6 | "engines": { 7 | "vscode": "^1.85.0" 8 | }, 9 | "categories": [ 10 | "Other" 11 | ], 12 | "activationEvents": ["onStartupFinished"], 13 | "main": "./extension.js", 14 | "contributes": { 15 | "commands": [] 16 | }, 17 | "scripts": { 18 | "lint": "eslint .", 19 | "pretest": "npm run lint", 20 | "test": "vscode-test" 21 | }, 22 | "devDependencies": { 23 | "@types/vscode": "^1.85.0", 24 | "@types/mocha": "^10.0.6", 25 | "@types/node": "18.x", 26 | "eslint": "^8.56.0", 27 | "typescript": "^5.3.3", 28 | "@vscode/test-cli": "^0.0.4", 29 | "@vscode/test-electron": "^2.3.8" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/test/extension.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | // You can import and use all API from the 'vscode' module 4 | // as well as import your extension to test it 5 | const vscode = require('vscode'); 6 | // const myExtension = require('../extension'); 7 | 8 | suite('Extension Test Suite', () => { 9 | vscode.window.showInformationMessage('Start all tests.'); 10 | 11 | test('Sample test', () => { 12 | assert.strictEqual(-1, [1, 2, 3].indexOf(5)); 13 | assert.strictEqual(-1, [1, 2, 3].indexOf(0)); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/VSCode/assistant/vsc-extension-quickstart.md: -------------------------------------------------------------------------------- 1 | # Welcome to your VS Code Extension 2 | 3 | ## What's in the folder 4 | 5 | * This folder contains all of the files necessary for your extension. 6 | * `package.json` - this is the manifest file in which you declare your extension and command. 7 | * The sample plugin registers a command and defines its title and command name. With this information VS Code can show the command in the command palette. It doesn’t yet need to load the plugin. 8 | * `extension.js` - this is the main file where you will provide the implementation of your command. 9 | * The file exports one function, `activate`, which is called the very first time your extension is activated (in this case by executing the command). Inside the `activate` function we call `registerCommand`. 10 | * We pass the function containing the implementation of the command as the second parameter to `registerCommand`. 11 | 12 | ## Get up and running straight away 13 | 14 | * Press `F5` to open a new window with your extension loaded. 15 | * Run your command from the command palette by pressing (`Ctrl+Shift+P` or `Cmd+Shift+P` on Mac) and typing `Hello World`. 16 | * Set breakpoints in your code inside `extension.js` to debug your extension. 17 | * Find output from your extension in the debug console. 18 | 19 | ## Make changes 20 | 21 | * You can relaunch the extension from the debug toolbar after changing code in `extension.js`. 22 | * You can also reload (`Ctrl+R` or `Cmd+R` on Mac) the VS Code window with your extension to load your changes. 23 | 24 | ## Explore the API 25 | 26 | * You can open the full set of our API when you open the file `node_modules/@types/vscode/index.d.ts`. 27 | 28 | ## Run tests 29 | 30 | * Install the [Extension Test Runner](https://marketplace.visualstudio.com/items?itemName=ms-vscode.extension-test-runner) 31 | * Open the Testing view from the activity bar and click the Run Test" button, or use the hotkey `Ctrl/Cmd + ; A` 32 | * See the output of the test result in the Test Results view. 33 | * Make changes to `test/extension.test.js` or create new test files inside the `test` folder. 34 | * The provided test runner will only consider files matching the name pattern `**.test.js`. 35 | * You can create folders inside the `test` folder to structure your tests any way you want. 36 | 37 | ## Go further 38 | 39 | * [Follow UX guidelines](https://code.visualstudio.com/api/ux-guidelines/overview) to create extensions that seamlessly integrate with VS Code's native interface and patterns. 40 | * [Publish your extension](https://code.visualstudio.com/api/working-with-extensions/publishing-extension) on the VS Code extension marketplace. 41 | * Automate builds by setting up [Continuous Integration](https://code.visualstudio.com/api/working-with-extensions/continuous-integration). 42 | -------------------------------------------------------------------------------- /Assistant/Plugin/plugins/vscode/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "VScode", 3 | "enabled": false, 4 | "platforms": ["windows", "macos"], 5 | "tools": [{ 6 | "type": "function", 7 | "function": { 8 | "name": "vscode", 9 | "description": "Execute javascript code inside of a Visual Studio Code (VSCode) extension. You can safely access all capabilities of VSCode via its extensibilty API. This allows you to assist the user in all forms of software development.", 10 | "parameters": { 11 | "type": "object", 12 | "properties": { 13 | "javascript": { 14 | "type": "string", 15 | "description": "The javascript code to execute." 16 | } 17 | }, 18 | "required": ["javascript"] 19 | } 20 | } 21 | }] 22 | } 23 | -------------------------------------------------------------------------------- /Assistant/PluginTools.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const fss = require('fs/promises') 3 | 4 | class PluginTool { 5 | 6 | constructor(config, tool) { 7 | this.config = config 8 | this.tool = tool 9 | } 10 | 11 | getName = () => { 12 | return this.config.name 13 | } 14 | 15 | getTools = () => { 16 | return this.config.tools.map(tool => tool.function.name) 17 | } 18 | 19 | call = async (name, args) => { 20 | return await this.tool.call(name, args) 21 | } 22 | 23 | appendToSchema = (schema) => { 24 | //console.log('append to schema', this.config) 25 | const append = (tool, schema) => { 26 | const key = `/gpt/${tool.function.name}` 27 | const value = { 28 | "post": { 29 | "operationId": tool.function.name, 30 | "summary": tool.function.description, 31 | "x-openai-isConsequential": false, 32 | "requestBody": { 33 | "content": { 34 | "application/json": { 35 | "schema": tool.function.parameters 36 | } 37 | } 38 | } 39 | } 40 | } 41 | schema.paths[key] = value 42 | } 43 | for (const tool of this.config.tools) { 44 | append(tool, schema) 45 | } 46 | } 47 | } 48 | 49 | let pluginTools = [] 50 | 51 | const installPlugins = async () => { 52 | const result = await loadPlugins(true) 53 | const { windowsSchema, macSchema } = result 54 | if (windowsSchema) { 55 | await fss.writeFile('../Build/Schema-Windows-with-plugins.json', JSON.stringify(windowsSchema, null, ' '), 'utf-8') 56 | } 57 | if (macSchema) { 58 | await fss.writeFile('../Build/Schema-MacOs-with-plugins.json', JSON.stringify(windowsSchema, null, ' '), 'utf-8') 59 | } 60 | process.exit(1) 61 | } 62 | 63 | const loadPlugins = async (installOnly) => { 64 | let windowsSchema 65 | let macSchema 66 | windowsSchema = require('../Build/Schema-Windows.json') 67 | macSchema = require('../Build/Schema-MacOS.json') 68 | const folder = path.join(__dirname, 'Plugin', 'plugins') 69 | const files = await fss.readdir(folder) 70 | const pluginTools = [] 71 | for (const file of files) { 72 | const fullPath = path.join(folder, file) 73 | const isDirectory = (await fss.stat(fullPath)).isDirectory 74 | if (isDirectory) { 75 | let config 76 | try { 77 | config = require(path.join(fullPath, 'config.json')) 78 | } catch (err) { 79 | console.error(err) 80 | console.log("Couldn't load plugin "+fullPath) 81 | continue 82 | } 83 | const { name, platforms, enabled, tools } = config 84 | console.log("config", config) 85 | if (enabled) { 86 | let tool 87 | if (!installOnly) { 88 | try { 89 | tool = require(path.join(fullPath, 'Tool.js')) 90 | } catch (err) { 91 | console.error(err) 92 | console.log("Couldn't load plugin "+fullPath) 93 | continue 94 | } 95 | console.log("enabling tool", JSON.stringify(config, null, ' ')) 96 | } 97 | pluginTools.push(new PluginTool(config, platforms, tool)) 98 | } 99 | } 100 | } 101 | for (const tool of pluginTools) { 102 | console.log('config', tool.config) 103 | if (tool.config.platforms.find(p => p === 'windows')) { 104 | tool.appendToSchema(windowsSchema) 105 | } 106 | if (tool.config.platforms.find(p => p === 'macos')) { 107 | tool.appendToSchema(macSchema) 108 | } 109 | } 110 | return { tools: pluginTools, windowsSchema, macSchema } 111 | } 112 | 113 | 114 | module.exports = { loadPlugins, installPlugins } 115 | -------------------------------------------------------------------------------- /Assistant/Preload.js: -------------------------------------------------------------------------------- 1 | const { ipcRenderer, contextBridge } = require('electron') 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Assistant/README.md: -------------------------------------------------------------------------------- 1 | Experimental use of plugin tools: 2 | We've added a method for plugging in additional tools and as an example provided a plugin for VSCode 3 | You can add new tools by addign a folder under `Assistant/Plugin/plugins`. The folder must contain the following files: 4 | 1. `config.json` 5 | 2. `Tool.js` 6 | 7 | `config.json` provides the name, whether the plugin is enabled, the platforms it suppports, and the tools it supports in the form of function definitions according to the OpenAI function calling specification. Here is an example: 8 | ``` 9 | { 10 | "name": "VScode", 11 | "enabled": true, 12 | "platforms": ["windows", "macos"], 13 | "tools": [{ 14 | "type": "function", 15 | "function": { 16 | "name": "vscode", 17 | "description": "Execute javascript code inside of a Visual Studio Code (VSCode) extension. You can safely access all capabilities of VSCode via its extensibilty API. This allows you to assist the user in all forms of software development.", 18 | "parameters": { 19 | "type": "object", 20 | "properties": { 21 | "javascript": { 22 | "type": "string", 23 | "description": "The javascript code to execute." 24 | } 25 | }, 26 | "required": ["javascript"] 27 | } 28 | } 29 | }] 30 | } 31 | ``` 32 | 33 | `Tool.js` must export a single function `call` which will take two arguments, `name`, and `args` corresponding to the function call. It must return an object containg to properties `content` which is a string containing the output of the function call and `isError` which is a boolean. If isError is true, then `content` should contain the error message. 34 | 35 | The VSCode plugin uses a websocket server to communicate with the plugin running inside of VSCode. You can use this framework for other plugins, for example we've used it with `ExtendScript` and `UXP` plugins for Adobe Creative Suite. 36 | 37 | 38 | Once you've enabled a plugin, you can generate new JSON action schemas for its tools by navigating to the `Assistant` folder, and running 39 | 40 | ``` 41 | node installPlugins.js 42 | `` 43 | 44 | This will generate `Schema-Windows-with-plugins.json` and `Schema-MacOS-with-plugins.json` in the `Build` folder which you can then copy and paste int o your GPT configuration in the OpenAI GPT editor. 45 | 46 | -------------------------------------------------------------------------------- /Assistant/Screen.js: -------------------------------------------------------------------------------- 1 | const { exec } = require('child_process'); 2 | const os = require('os'); 3 | 4 | function getScreenDimensionsImpl(callback, err) { 5 | const platform = os.platform() 6 | 7 | if (platform === 'darwin') { 8 | // macOS 9 | exec("system_profiler SPDisplaysDataType | grep Resolution", (error, stdout) => { 10 | const match = stdout.match(/Resolution: (\d+) x (\d+)/) 11 | if (match) { 12 | callback({ width: parseInt(match[1], 10), height: parseInt(match[2], 10) }) 13 | } 14 | }) 15 | } else if (platform === 'win32') { 16 | // Windows 17 | exec("wmic path Win32_VideoController get CurrentHorizontalResolution,CurrentVerticalResolution", (error, stdout) => { 18 | const lines = stdout.trim().split('\n').filter(line => line.trim() !== '') 19 | const dimensions = lines[lines.length - 1].trim().split(/\s+/) 20 | if (dimensions.length === 2) { 21 | callback({ width: parseInt(dimensions[0], 10), height: parseInt(dimensions[1], 10) }) 22 | } 23 | }) 24 | } else if (platform === 'linux') { 25 | // Linux 26 | exec("xdpyinfo | grep dimensions", (error, stdout) => { 27 | const match = stdout.match(/dimensions:\s+(\d+)x(\d+)/) 28 | if (match) { 29 | return callback({ width: parseInt(match[1], 10), height: parseInt(match[2], 10) }) 30 | } 31 | }) 32 | } 33 | } 34 | 35 | 36 | const getScreenDimensions = async () => { 37 | return new Promise((resolve, reject) => { 38 | getScreenDimensionsImpl(resolve) 39 | }) 40 | } 41 | 42 | module.exports = { getScreenDimensions } 43 | -------------------------------------------------------------------------------- /Assistant/Shell.js: -------------------------------------------------------------------------------- 1 | const { exec, spawn } = require('child_process'); 2 | const readline = require('readline') 3 | const fss = require('fs/promises') 4 | const path = require('path') 5 | const os = require('os') 6 | const crypto = require('crypto') 7 | 8 | const tempFile = (opts) => new Promise((resolve, reject) => { 9 | opts = opts || {} 10 | const { suffix } = opts 11 | const uniqueId = crypto.randomBytes(16).toString('hex') 12 | const tempFilename = path.join(os.tmpdir(), `tempfile_${uniqueId}.${suffix}`); 13 | resolve({ 14 | filePath: tempFilename, 15 | cleanup: async () => { 16 | try { 17 | await fss.unlink(tempFilename) 18 | } catch (err) { 19 | console.error(err) 20 | } 21 | } 22 | }) 23 | }) 24 | 25 | let _loginShell 26 | let _loginShellType 27 | const getLoginShell = async () => { 28 | if (_loginShell !== undefined) { 29 | return _loginShell 30 | } 31 | const prefix = await getLoginShellPrefix() 32 | return _loginShell = prefix + '; '+ _loginShellType + " -c" 33 | } 34 | 35 | const log = info => { 36 | } 37 | 38 | const logError = err => { 39 | } 40 | 41 | const getLoginShellPrefix = async () => { 42 | if (os.platform() === 'win32') { 43 | return _loginShellType = "cmd.exe" 44 | } 45 | const homeDirectory = require('os').homedir(); 46 | const bashrcPath = path.join(homeDirectory, '.bash_profile') 47 | const zshrcPath = path.join(homeDirectory, '.zshrc') 48 | // Check for .bashrc 49 | let bashExists = false 50 | let zshExists = false 51 | let cmd 52 | try { 53 | await fss.access(bashrcPath, fs.constants.F_OK) 54 | bashExists = true 55 | } catch (err) { 56 | logError(err) 57 | } 58 | try { 59 | await fss.access(zshrcPath, fs.constants.F_OK) 60 | zshExists = true 61 | } catch(err) { 62 | logError(err) 63 | } 64 | 65 | if (bashExists && zshExists) { 66 | const stat1 = await fss.stat(bashrcPath) 67 | const lastModified1 = Date.parse(stat1.mtime) 68 | const fileSize1 = stat1.size 69 | log({lastModified1, fileSize1}) 70 | const stat2 = await fss.stat(zshrcPath) 71 | const lastModified2 = Date.parse(stat2.mtime) 72 | const fileSize2 = stat2.size 73 | log({lastModified2, fileSize2}) 74 | if (lastModified2 > lastModified1) { 75 | bashExists = false 76 | } else { 77 | zshExists = false 78 | } 79 | } 80 | log({zshExists, bashExists}) 81 | if (zshExists) { 82 | _loginShellType = 'zsh' 83 | cmd = `source "${zshrcPath}"` 84 | } else { 85 | _loginShellType = 'bash' 86 | cmd = `source "${bashrcPath}"` 87 | } 88 | log("LOGIN SHELL", cmd) 89 | return cmd 90 | } 91 | 92 | const getLoginShellType = async () => { 93 | await getLoginShell() 94 | return _loginShellType 95 | } 96 | 97 | let theProcess = process 98 | 99 | const run = async (script, options) => { 100 | options = options || {} 101 | const { trace, traceErr, powershell, admin, timeout } = options 102 | console.log(script) 103 | let suffix 104 | if (powershell) { 105 | suffix = 'ps1' 106 | if (admin) { 107 | script = `# Check if the script is running as an administrator 108 | if (-not ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) 109 | { 110 | # Re-launch the script with administrative rights 111 | Start-Process PowerShell -ArgumentList "-File \`"\$PSCommandPath\`"" -Verb RunAs 112 | exit 113 | } 114 | ` + script 115 | 116 | } 117 | } else { 118 | suffix = (os.platform() === 'win32') ? 'bat' : 'sh' 119 | } 120 | const { filePath, cleanup } = await tempFile({ suffix }) 121 | await fss.writeFile(filePath, script) 122 | await fss.chmod(filePath, '755') 123 | let shell 124 | let args = [filePath] 125 | if (powershell) { 126 | shell = 'PowerShell.exe' 127 | args = ['-ExecutionPolicy', 'Bypass', '-File', filePath] 128 | } else { 129 | shell = await getLoginShellType() 130 | if (shell === 'cmd.exe') args.unshift('/c') 131 | } 132 | return new Promise((resolve, reject) => { 133 | let output = '' 134 | const process = spawn(shell, args) 135 | process.stdout.on('data', (data) => { 136 | data = data.toString() 137 | console.log(data) 138 | if (trace) { 139 | theProcess.stdout.write(data) 140 | } 141 | output += data 142 | }) 143 | process.stderr.on('data', (data) => { 144 | data = data.toString().replace(filePath + ":", '') 145 | console.error(data) 146 | if (traceErr) { 147 | theProcess.stderr.write(data) 148 | } 149 | output += data 150 | }) 151 | let timer 152 | let timedOut 153 | if (timeout) { 154 | timer = setTimeout(() => { 155 | timedOut = true 156 | output = "Timed out after " + (timeout / 1000) + " seconds" 157 | process.kill() 158 | }, timeout) 159 | } 160 | process.on('close', (code) => { 161 | cleanup() 162 | if (timedOut) code = -1 163 | resolve({ code, output }) 164 | }) 165 | }) 166 | } 167 | 168 | 169 | const interactiveShell = async (commands, resolve) => { 170 | const { filePath, cleanup } = await tempFile({ suffix: '.sh' }); 171 | const script = (await getLoginShellPrefix()) + '\n' + commands 172 | console.log(script) 173 | await fss.writeFile(filePath, script); 174 | console.log("Script file", filePath) 175 | await fss.chmod(filePath, '755'); 176 | 177 | // Execute the script file 178 | const shell = await getLoginShellType() 179 | const process = spawn(shell, [filePath,"2>&1"]) 180 | const rl1 = readline.createInterface({ 181 | input: process.stdout 182 | }); 183 | const rl2 = readline.createInterface({ 184 | input: process.stderr 185 | }); 186 | 187 | let chars = 0 188 | let lines = [] 189 | let allOutput = '' 190 | let listener; 191 | let code 192 | let toolCallId 193 | let subject = new Subject() 194 | let outputSubject = new Subject() 195 | let busy = false 196 | 197 | const fireOutput = () => { 198 | outputSubject.next({code, output: allOutput}) 199 | } 200 | 201 | const flush = () => { 202 | if (listener && (code !== undefined || chars > 0)) { 203 | let obj = listener; 204 | listener = null; 205 | const output = lines.map(x => x.line).join('\n') 206 | obj.resolve({ toolCallId, code, output}); 207 | } 208 | }; 209 | 210 | const notify = () => { 211 | try { 212 | subject.next({ 213 | busy, 214 | toolCallId, 215 | lines, 216 | code, 217 | chars, 218 | }) 219 | } catch (err) { 220 | console.error(err) 221 | } 222 | } 223 | 224 | process.on('close', exitCode => { 225 | code = exitCode; 226 | notify() 227 | flush(); 228 | fireOutput() 229 | //cleanup(); // Clean up the temporary file 230 | }); 231 | 232 | const cb = (type, line) => { 233 | line = line.replaceAll(filePath + ':', '') 234 | const event = { 235 | type, 236 | line 237 | } 238 | allOutput += line + '\n' 239 | lines.push(event) 240 | notify() 241 | chars += line.length; 242 | setTimeout(flush, 100); 243 | } 244 | 245 | const cb1 = line => { 246 | console.log('out', line) 247 | cb('out', line) 248 | } 249 | 250 | const cb2 = line => { 251 | console.log('err', line) 252 | cb('err', line) 253 | } 254 | 255 | rl1.on('line', cb1); 256 | rl2.on('line', cb2); 257 | 258 | return { 259 | write: (data, id) => { 260 | toolCallId = id 261 | process.stdin.write(data); 262 | }, 263 | awaitOutput: (timeout) => { 264 | return new Promise((resolve, reject) => { 265 | let timedOut 266 | let timer 267 | if (timeout) { 268 | timer = setTimeout(() => { 269 | timedOut = true 270 | resolve({code: -1, output: "Timed out"}) 271 | }, timeout) 272 | } 273 | outputSubject.subscribe((event) => { 274 | if (!timedOut) { 275 | if (timer) clearTimeout(timer) 276 | resolve(event) 277 | } 278 | }) 279 | }) 280 | }, 281 | getOutput: () => { 282 | busy = true 283 | return new Promise((resolve, reject) => { 284 | listener = { 285 | resolve: returnValue => { 286 | busy = false 287 | resolve(returnValue) 288 | }, 289 | reject: err => { 290 | busy = false 291 | resolve(err) 292 | } 293 | }; 294 | flush(); 295 | }); 296 | }, 297 | subscribe: (id, cb) => { 298 | toolCallId = id 299 | const sub = subject.subscribe(cb) 300 | return { 301 | unsubscribe: () => { 302 | sub.unsubscribe() 303 | try { 304 | process.kill() 305 | } catch (err) { 306 | console.error(err) 307 | } 308 | } 309 | } 310 | } 311 | }; 312 | }; 313 | 314 | 315 | module.exports = { shell: run, getLoginShell, tempFile } 316 | -------------------------------------------------------------------------------- /Assistant/ToolClient.js: -------------------------------------------------------------------------------- 1 | const firebase = require( 'firebase/compat/app') 2 | require('firebase/compat/auth') 3 | require('firebase/compat/firestore') 4 | require('firebase/compat/storage') 5 | const { v4: uuidv4 } = require('uuid') 6 | const moment = require('moment') 7 | const { docData, doc, collectionChanges } = require("rxfire/firestore") 8 | const { bindCallback, of, concat, from, Subject, merge: mergeN, combineLatest } = require('rxjs') 9 | const { catchError, filter, map, flatMap, take, merge } = require('rxjs/operators') 10 | const { axios } = require('./Axios.js') 11 | const { exec, spawn } = require('child_process') 12 | const os = require('os') 13 | const readline = require('readline') 14 | const path = require('path') 15 | const fss = require('fs/promises') 16 | const fs = require('fs') 17 | const { ChatService } = require('./ChatService.js') 18 | const { session, BrowserWindow, app } = require('electron') 19 | const { addReturnStatementIfNecessary, returnFunctionDecl } = require('./JavascriptCode.js') 20 | const { getGeoLocation } = require('./Location.js') 21 | const FastEmbed = require('./FastEmbed.js') 22 | const { InMemorySearch } = require('./InMemorySearch.js') 23 | const { shell, tempFile, getLoginShell } = require('./Shell.js') 24 | const { encode } = require('gpt-3-encoder') 25 | const { getScreenDimensions } = require('./Screen.js') 26 | const { loadPlugins } = require('./PluginTools.js') 27 | 28 | const countTokens = text => text ? encode(text).length : 0 29 | 30 | const apiKey = fs.readFileSync('./apiKey.txt', 'utf-8').trim() 31 | const serverURL = fs.readFileSync('./serverURL.txt', 'utf-8').trim() 32 | const firebaseConfig = require('./firebaseConfig.json') 33 | 34 | const delay = millis => new Promise(resolve => setTimeout(resolve, millis)) 35 | const formatDate = (utcOffset, when) => { 36 | const roundToSeconds = t => Math.round(t / 1000) * 1000 37 | const dateToString = (t, utcOffset) => moment(roundToSeconds(t + utcOffset / (1000 * 60))).format('YYYY-MM-DD HH:mm:ss Z') 38 | return dateToString(when, utcOffset) 39 | } 40 | 41 | const osascriptErrors = require('./osascript-error.json') 42 | 43 | const electronLog = require('electron-log/main') 44 | const log = (...args) => { 45 | console.log(...args) 46 | //electronLog.info(...args) 47 | } 48 | 49 | const logError = (...args) => { 50 | console.error(...args) 51 | //electronLog.error(...args) 52 | } 53 | 54 | 55 | const executeAppleScript = async (lang, script, timeout) => { 56 | const { filePath, cleanup } = await tempFile({ suffix: lang == 'javaScript' ? '.js' : '.scpt' }); 57 | console.log({lang, script}) 58 | await fss.writeFile(filePath, script, 'utf-8'); 59 | await fss.chmod(filePath, '755'); 60 | let langOpt = lang === 'javaScript' ? '-l JavaScript' : '' 61 | const cmd = `osascript ${langOpt}${filePath}` 62 | console.log(cmd) 63 | let timedOut = false 64 | const run = () => new Promise((resolve, reject) => { 65 | let output = '' 66 | let errorOutput = '' 67 | let timer 68 | const process = spawn('osascript', [filePath, '2>&1']) 69 | process.on('error', reject) 70 | process.stdout.on('data', (data) => { 71 | clearTimeout(timer) 72 | console.log(data) 73 | output += data.toString(); 74 | }); 75 | process.stderr.on('data', (data) => { 76 | clearTimeout(timer) 77 | console.log(data) 78 | output += data.toString(); 79 | }); 80 | if (timeout) { 81 | timer = setTimeout(() => { 82 | timedOut = true 83 | process.kill() 84 | }, timeout) 85 | } 86 | process.on('close', (code) => { 87 | cleanup() 88 | clearTimeout(timer) 89 | if (timedOut) { 90 | output = `(Timed Out after ${timeout / 1000} seconds) ` + (output || ' ') 91 | if (!code) { 92 | code = -1 93 | } 94 | } 95 | log(`Exit code: ${code}`) 96 | log('Output:', output) 97 | //cleanup(); // Clean up the temporary file 98 | resolve({ code, output }) 99 | }); 100 | }) 101 | let { code, output } = await run() 102 | if (code !== 0 && !timedOut) { 103 | const offset = filePath.length 104 | if (lang !== 'javaScript') { 105 | let errorCode 106 | if (output.endsWith(")")) { 107 | const lparen = output.lastIndexOf('(') 108 | errorCode = output.substring(lparen + 1, output.length-1) 109 | console.log('errorCode') 110 | output = output.substring(0, lparen) + '('+ errorCode +' '+osascriptErrors[errorCode] + ')' 111 | } 112 | let [ignored, start, end, ...rest] = output.split(':') 113 | start = parseInt(start) 114 | end = parseInt(end) 115 | const location = script.substring(start, end) 116 | const select = script.substring(0, start) + "`" + location + "`" + script.substring(end) 117 | //const select = '`' + location + '`' 118 | output = `${select}:\n${start}:${end}: ${rest.join(':')}` 119 | } else { 120 | let [ignored, ...rest] = output.split(':') 121 | output = rest.join(':') 122 | } 123 | } 124 | return { code, output } 125 | } 126 | 127 | const googleSearch = async searchQuery => { 128 | const page = await openPage('https://www.google.com') 129 | const inputSelector = 'input[name="q"], textarea[name="q"]' 130 | const searchInput = await page.$(inputSelector) 131 | await searchInput.fill(searchQuery); 132 | await searchInput.press('Enter'); 133 | await page.waitForSelector('#search'); 134 | const searchResults = await page.evaluate(() => { 135 | const results = []; 136 | const items = document.querySelectorAll('div.g'); 137 | for (const item of items) { 138 | const title = item.querySelector('h3').textContent.trim(); 139 | const url = item.querySelector('a').href; 140 | const snippetElement = item.querySelector('div.VwiC3b'); 141 | const snippet = snippetElement ? snippetElement.textContent.trim() : ''; 142 | results.push({ title, url, snippet }); 143 | } 144 | return results; 145 | }); 146 | return searchResults 147 | } 148 | 149 | let electronApp 150 | let page 151 | 152 | initPlaywright = async () => { 153 | if (!electronApp) { 154 | const { _electron } = require('playwright'); 155 | electronApp = await _electron.launch({ 156 | args: ['./electron-playwright.js', '--disable-http2', '--disable-blink-features=AutomationControlled'], 157 | ignoreHTTPSErrors: true, 158 | headless: false, 159 | userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.81" 160 | }); 161 | const result = await electronApp.evaluate(({ app, session }) => { 162 | app.whenReady().then(() => { 163 | session.defaultSession.setPermissionCheckHandler((webContents, permission) => { 164 | //console.log("permission.check", permission) 165 | return true 166 | }) 167 | session.defaultSession.setPermissionRequestHandler((webContents, permission, callback) => { 168 | //console.log("permission.req", permission) 169 | callback(true) 170 | return true 171 | }) 172 | //console.log("installed permission handlers") 173 | }) 174 | }) 175 | //console.log('initPlayWright', result) 176 | } 177 | } 178 | 179 | const openPage = async (url, options) => { 180 | options = options || { 181 | width: 800, 182 | height: 600, 183 | visible: true 184 | } 185 | const { width, height, visible } = options 186 | await initPlaywright() 187 | if (!page) { 188 | console.log("options", options) 189 | await electronApp.evaluate(({ BrowserWindow, session}, options) => { 190 | options = JSON.parse(options) 191 | console.log("inside options", options) 192 | const { url, height, width, visible, title } = options 193 | const newWindow = new BrowserWindow({ 194 | width, 195 | height, 196 | }) 197 | newWindow.on('close', (event) => { 198 | event.preventDefault(); 199 | newWindow.hide(); 200 | }); 201 | console.log("created electron window") 202 | setTimeout(() => { 203 | newWindow.setTitle(title) 204 | newWindow.loadURL(url) 205 | }, 200) 206 | }, JSON.stringify({ 207 | width, height, visible, title: 'Assistant', url 208 | })) 209 | page = await electronApp.waitForEvent('window') 210 | await page.setDefaultTimeout(10000) 211 | } else { 212 | await electronApp.evaluate(({BrowserWindow}, options) => { 213 | const { height, width, visible } = JSON.parse(options) 214 | const allWindows = BrowserWindow.getAllWindows() 215 | const mainWindow = allWindows[0] 216 | if (visible) { 217 | mainWindow.show() 218 | } else { 219 | mainWindow.hide() 220 | } 221 | mainWindow.setSize(width, height) 222 | }, JSON.stringify(options)) 223 | await page.goto(url, { waitUntil: 'load' }); 224 | } 225 | return page 226 | } 227 | 228 | const scrape = async page => { 229 | await page.waitForLoadState('load') 230 | const content = await page.evaluate(() => { 231 | const extractContent = (node) => { 232 | let text = ''; 233 | let links = [] 234 | for (const child of node.childNodes) { 235 | if (child.nodeType === Node.TEXT_NODE) { 236 | //text += child.nodeValue.trim() + '\n'; 237 | } else if (child.nodeType === Node.ELEMENT_NODE) { 238 | if (child.nodeName === 'A') { 239 | links.push('[' + child.textContent.trim() + '](' + child.href + ')') 240 | } else if (child.nodeName === 'P' || child.nodeName === 'H1' || child.nodeName === 'H2' || child.nodeName === 'H3' || child.nodeName === 'H4' || child.nodeName === 'H5' || child.nodeName === 'H6') { 241 | text += child.innerText.split('.').join('\n') 242 | } else { 243 | const result = extractContent(child); 244 | links = links.concat(result.links) 245 | text += result.text 246 | } 247 | } 248 | } 249 | return { links, text: text.trim() } 250 | } 251 | return extractContent(document.body) 252 | }) 253 | return content 254 | } 255 | 256 | const documents = {} 257 | 258 | const searchDocument = async ({document, query, chunkSize, limit, createEmbeddings}) => { 259 | let existing = documents[document] 260 | let idx 261 | let docs 262 | let embeddingSearch 263 | if (!existing || existing.chunkSize !== chunkSize) { 264 | let text 265 | let links 266 | if (!existing) { 267 | const page = await openPage(document) 268 | //text = await page.evaluate(() => document.documentElement.innerText) 269 | const scraped = await scrape(page) 270 | text = scraped.text 271 | links = scraped.links 272 | } else { 273 | text = existing.text 274 | links = existing.links 275 | } 276 | chunkSize = chunkSize || 200 277 | //console.log("GOT TEXT", text) 278 | //console.log("GOT LINKS", links) 279 | const lunr = require('lunr') 280 | docs = {} 281 | const chunks = [] 282 | const sentences = text.split(/[.\n]/).map(x => x.trim()).filter(x=>x) 283 | idx = lunr(function() { 284 | this.field('text') 285 | let id = 0 286 | const self = this 287 | const add = (doc) => { 288 | //console.log("adding", doc) 289 | self.add(doc) 290 | docs[doc.id] = doc 291 | } 292 | //console.log("sentences", sentences) 293 | let tokens = 0 294 | let doc = [] 295 | let i = 1 296 | const flush = () => { 297 | const chunk = doc.join('\n').trim() 298 | if (chunk) { 299 | chunks.push(chunk) 300 | add({id: i, text: chunk}) 301 | doc = [] 302 | tokens = 0 303 | i++ 304 | } 305 | } 306 | for (const sentence of sentences) { 307 | doc.push(sentence) 308 | tokens += countTokens(sentence) 309 | if (tokens > chunkSize) { 310 | //console.log("tokens", tokens) 311 | flush() 312 | } 313 | } 314 | flush() 315 | for (const link of links) { 316 | self.add({id: i, text: link}) 317 | i++ 318 | } 319 | }) 320 | embeddingSearch = new InMemorySearch(createEmbeddings) 321 | await embeddingSearch.index(chunks) 322 | documents[document] = { idx, text, links, chunkSize, docs, embeddingSearch } 323 | } else { 324 | idx = existing.idx 325 | docs = existing.docs 326 | embeddingSearch = existing.embeddingSearch 327 | } 328 | let results1 = idx.search(query).map(result => { 329 | console.log('result', result) 330 | const result1 = docs[result.ref] 331 | return result1 ? result1.text : '' 332 | }).filter(x => x) 333 | let results2 = await embeddingSearch.search(query) 334 | return { 335 | keyword: results1.slice(0, limit || 3), 336 | vector: results2.results.slice(0, limit || 3) 337 | } 338 | } 339 | 340 | const windows = { 341 | } 342 | 343 | const getOrigin = url => new URL(url).origin 344 | 345 | async function verifyURL(url) { 346 | return axios.head(url) 347 | .then(response => { 348 | // URL is valid, you can check response.status or response.headers here 349 | console.log('URL is valid:', response.status); 350 | return true; 351 | }) 352 | .catch(error => { 353 | console.error(error) 354 | if (error.status === 405) { 355 | return true 356 | } 357 | console.log(error) 358 | throw new Error(error.message) 359 | }); 360 | } 361 | 362 | const open = async (url, onClose) => { 363 | if (!onClose) { 364 | return await openPage(url) 365 | } 366 | await verifyURL(url) 367 | const getCurrentUrl = async win => { 368 | return await win.webContents.executeJavaScript('location.href') 369 | } 370 | 371 | if (!url) { 372 | throw new Error('target is required') 373 | } 374 | const key = getOrigin(url) 375 | if (windows[key]) { 376 | const { window } = windows[key] 377 | const location = await getCurrentUrl(window) 378 | if (location !== url) { 379 | window.loadURL(url) 380 | } 381 | return window 382 | } 383 | const window = new BrowserWindow({ 384 | width: 1200, 385 | height: 800, 386 | webPreferences: { 387 | preload: path.join(__dirname, 'Preload.js'), 388 | 'node-integration': true, 389 | 'web-security': false, 390 | } 391 | }) 392 | //await window.webContents.openDevTools() 393 | window.on('closed', () => { 394 | delete windows[key] 395 | if (onClose) onClose() 396 | }) 397 | window.webContents.on('will-navigate', event => { 398 | try { 399 | console.log('will-navigate', event) 400 | const data = { from: url, to: event.url } 401 | window.ipcRenderer.send('will-navigate', JSON.stringify(data)) 402 | } catch(err) { 403 | console.error(err) 404 | } 405 | }) 406 | window.loadURL(url) 407 | windows[key] = { 408 | window 409 | } 410 | return new Promise(resolve => { 411 | window.webContents.on('did-finish-load', () => { 412 | console.log('Page loaded'); 413 | resolve(window) 414 | }) 415 | }) 416 | } 417 | 418 | let server = serverURL 419 | 420 | log('firebaseConfig', firebaseConfig) 421 | firebase.initializeApp(firebaseConfig) 422 | firebase.firestore().settings({ignoreUndefinedProperties:true}) 423 | 424 | let _osVersion 425 | const getOSVersion = async () => { 426 | if (_osVersion !== undefined) { 427 | return _osVersion 428 | } 429 | let command; 430 | if (os.platform() === 'darwin') { 431 | // macOS 432 | command = 'sw_vers -productVersion'; 433 | } else if (os.platform() === 'win32') { 434 | // Windows 435 | command = 'powershell systeminfo | findstr /B /C:"OS Version"'; 436 | // Alternatively, use 'wmic os get Caption, Version' for a shorter output 437 | } else { 438 | throw new Error('Unsupported OS'); 439 | } 440 | let { code, output } = await shell(command) 441 | if (os.platform() === 'win32') { 442 | output = output.split(':')[1].trim() 443 | } 444 | return _osVersion = output.trim() 445 | } 446 | 447 | utcOffset = -(new Date().getTimezoneOffset()*60*1000) 448 | 449 | const getToolSchema = name => { 450 | const platform = os.platform() === 'win32' ? 'Windows' : 'MacOS' 451 | const {paths} = require('../Build/Schema-${platform}.json') 452 | for (const path in paths) { 453 | const { operationId } = paths[path].post 454 | if (operationId === name) { 455 | return paths[path] 456 | } 457 | } 458 | } 459 | 460 | class ToolClient { 461 | 462 | constructor(opts) { 463 | const { sendMessage } = opts 464 | this.sendMessage = sendMessage 465 | loadPlugins().then(tools => { 466 | this.pluginTools = tools 467 | }) 468 | } 469 | 470 | login = async () => { 471 | const func = this.getFunc('gptLogin') 472 | const response = await func({apiKey}) 473 | const { customToken } = response.data 474 | const { user } = await firebase.auth().signInWithCustomToken(customToken) 475 | this.self = user 476 | this.uid = user.uid 477 | this.initToolCalls() 478 | this.initTaskNotes() 479 | getGeoLocation().then(location => { 480 | console.log("LOCATION", location) 481 | this.location = location 482 | }) 483 | } 484 | 485 | createEmbeddings = async documents => { 486 | return await FastEmbed.createEmbeddings(documents) 487 | } 488 | 489 | getFunc = name => { 490 | return async (data) => { 491 | const token = await this.getToken() 492 | const url = `${server}/${name}` 493 | log("getFunc", url) 494 | //log("getFunc", name, data) 495 | const response = await axios.post(url, data, { 496 | headers: { 497 | Authorization: 'Bearer ' + token 498 | } 499 | }) 500 | log("result", name, response.data) 501 | return response 502 | } 503 | } 504 | 505 | observeTaskNotes = () => { 506 | const db = firebase.firestore() 507 | const c = db.collection('UserChannel').doc(this.uid).collection('taskNotes') 508 | let ob = collectionChanges(c).pipe(flatMap(changes => { 509 | return changes.map(change => { 510 | const { type, doc } = change 511 | const data = doc.data() 512 | return { 513 | type, 514 | snap: doc, 515 | } 516 | }) 517 | })) 518 | return ob 519 | } 520 | 521 | taskNotes = new InMemorySearch(this.createEmbeddings) 522 | 523 | initTaskNotes = () => { 524 | this.observeTaskNotes().subscribe(change => { 525 | const { type, snap } = change 526 | if (type === 'removed') { 527 | this.taskNotes.remove(snap.id) 528 | } else { 529 | const { notes, task, embedding } = snap.data() 530 | const id = snap.id 531 | this.taskNotes.add({id, content: { notes, task }, embedding }) 532 | } 533 | }) 534 | } 535 | 536 | searchTaskNotes = async (query, limit) => { 537 | return await this.taskNotes.search(query, limit) 538 | } 539 | 540 | updateTaskNotes = async ({ upserts, deletes }) => { 541 | const db = firebase.firestore() 542 | const c = db.collection('UserChannel').doc(this.uid).collection('taskNotes') 543 | const batch = db.batch() 544 | let updated = 0 545 | let inserted = 0 546 | let deleted = 0 547 | if (deletes) for (const id of deletes) { 548 | console.log('deleting', id) 549 | const ref = c.doc(id) 550 | batch.delete(ref) 551 | deleted++ 552 | } 553 | if (upserts) { 554 | const { embeddings } = await this.createEmbeddings(upserts.map(x => x.task + ':\n' + x.notes)) 555 | upserts.forEach((upsert, i) => { 556 | let { id, task, notes } = upsert 557 | const embedding = embeddings[i] 558 | let ref 559 | if (!id) { 560 | ref = c.doc() 561 | inserted++ 562 | } else { 563 | updated++ 564 | ref = c.doc(id) 565 | } 566 | const updates = { task, notes, embedding } 567 | console.log('updates', updates) 568 | batch.set(ref, updates, { merge: true }) 569 | }) 570 | } 571 | await batch.commit() 572 | let message = '' 573 | let sep = '' 574 | if (inserted) { 575 | message += 'Inserted ' + inserted + ' notes.' 576 | sep = ' ' 577 | } 578 | if (updated) { 579 | message += sep 580 | message += 'Updated ' + updated + ' notes.' 581 | sep = ' ' 582 | } 583 | if (deleted) { 584 | message += sep 585 | message += 'Deleted ' + deleted + ' notes.' 586 | } 587 | return message 588 | } 589 | 590 | initToolCalls = async () => { 591 | const db = firebase.firestore() 592 | console.log("init tool calls", this.uid) 593 | const c = db.collection('UserChannel').doc(this.uid).collection('toolCalls') 594 | const { docs } = await c.where('received', '==', false).get() 595 | const batch = db.batch() 596 | for (const snap of docs) { 597 | batch.delete(snap.ref) 598 | } 599 | await batch.commit() 600 | let taskNotesChecked = false 601 | let wasError 602 | this.sub1 = this.observeToolCalls().subscribe(async data => { 603 | const { type, snap, toolCall } = data 604 | const ref = c.doc(snap.id) 605 | if (type == 'added') { 606 | const args = toolCall.arguments 607 | const name = toolCall.name 608 | if (name === 'searchTaskNotes') { 609 | taskNotesChecked = true 610 | } else { 611 | taskNotesChecked = false 612 | } 613 | const id = '' + toolCall.ts 614 | await ref.set({ 615 | received: true 616 | }, { merge: true }) 617 | let { content, isError } = await this.applyToolCall({id, name, args}, x => x) 618 | console.log(content) 619 | if (isError) { 620 | wasError = true 621 | content = 'The following error occurred:\n' + content 622 | if (!taskNotesChecked) { 623 | content += "\n\nPlease check your task notes if you haven't already by calling `searchTaskNotes` before continuing." 624 | } 625 | } else { 626 | taskNotesChecked = false 627 | if (wasError) { 628 | content += "\nUpdate your task notes if necessary." 629 | wasError = false 630 | } 631 | } 632 | console.log("TOOL CALL OUTPUT", content) 633 | await ref.set({ 634 | content, 635 | isError, 636 | }, { merge: true }) 637 | } 638 | }) 639 | } 640 | 641 | getToolCallsRef = () => firebase.firestore().collection('UserChannel').doc(this.uid).collection('toolCalls') 642 | 643 | observeToolCalls = () => { 644 | let q = this.getToolCallsRef() 645 | q = q.orderBy('ts', 'desc').limit(1) 646 | let ob = collectionChanges(q).pipe(flatMap(changes => { 647 | return changes.map(change => { 648 | const { type, doc } = change 649 | const data = doc.data() 650 | console.log('toolCall', data) 651 | return { 652 | type, 653 | snap: doc, 654 | toolCall: data 655 | } 656 | }) 657 | }), filter(x => { 658 | return !x.toolCall.received 659 | })) 660 | return ob 661 | } 662 | 663 | getPlatformInfo = async () => { 664 | const platform = process.platform === 'darwin' ? 'MacOS' : 'Windows' 665 | const osVersion = await getOSVersion() 666 | const shell = await getLoginShell() 667 | const info = { platform, osVersion, shell } 668 | log("platformInfo", info) 669 | return info 670 | } 671 | 672 | getToken = async () => this.user ? await this.user.getIdToken() : null 673 | 674 | applyToolCall = async (toolCall, reply) => { 675 | const { name, args, id } = toolCall 676 | console.log( { name, args }) 677 | const db = firebase.firestore() 678 | let isError 679 | switch (name) { 680 | default: 681 | if (this.pluginTools) { 682 | for (const tool of this.pluginTools) { 683 | const supported = tool.getTools() 684 | if (supported.indexOf('name') >= 0) { 685 | try { 686 | const { isError, content } = await tool.call(name, args) 687 | return reply({ 688 | content, isError 689 | }) 690 | } catch (err) { 691 | console.error(err) 692 | return { content: "Error invoking plugin: " + tool.getName() + "\n" + err.message, isError: true } 693 | } 694 | } 695 | } 696 | } 697 | break 698 | case 'openURL': 699 | case 'open': 700 | { 701 | let content = '' 702 | try { 703 | const { target } = args 704 | const url = target 705 | if (!url.startsWith('http')) { 706 | const open = (await import('open')).default 707 | console.log('open', open) 708 | let pathname 709 | try { 710 | const u = new URL(url) 711 | pathname = u.pathname 712 | } catch (err) { 713 | pathname = url 714 | } 715 | await open(pathname) 716 | content = "Opened app." 717 | } else { 718 | const window = await openPage(url) 719 | content = "Window created." 720 | } 721 | } catch (err) { 722 | isError = true 723 | content = err.message 724 | } 725 | return reply({ 726 | content, isError 727 | }) 728 | } 729 | break 730 | case 'playwright': 731 | { 732 | let content = '' 733 | try { 734 | let { target, script, window } = args 735 | const page = await openPage(target, window) 736 | console.log(script) 737 | script = returnFunctionDecl(script) 738 | const fun = await eval(script) 739 | page.page = page 740 | console.log('fun', fun) 741 | if (typeof fun === 'function') { 742 | const result = await fun(page) 743 | console.log("called fun result=>", result) 744 | content = JSON.stringify(result) 745 | } else { 746 | content = JSON.stringify(fun) 747 | console.log("fun not called result=>", content) 748 | } 749 | } catch (err) { 750 | console.error(err) 751 | isError = true 752 | content = err.message 753 | } 754 | return reply({ 755 | content, isError 756 | }) 757 | } 758 | break 759 | case 'queryChatDb': 760 | { 761 | let content = '' 762 | try { 763 | let { query } = args 764 | const chatService = new ChatService() 765 | const messages = await chatService.searchMessages(query) 766 | content = JSON.stringify(messages) 767 | } catch (err) { 768 | isError = true 769 | content = err.message 770 | } 771 | return reply({ 772 | content, isError 773 | }) 774 | } 775 | break 776 | case 'osascript': 777 | { 778 | let content = '' 779 | try { 780 | let { lang, command, script } = args 781 | script = script || command 782 | log("COMMAND", script) 783 | if (lang) { 784 | switch (lang.toLowerCase()) { 785 | case 'jxa': 786 | case 'javascript': 787 | lang = 'javaScript' 788 | break 789 | case 'applescript': 790 | lang = 'appleScript' 791 | break 792 | default: 793 | throw new Error("`lang` must be one of `appleScript` or `javaScript`") 794 | } 795 | } 796 | lang = lang || 'appleScript' 797 | if (lang === 'javaScript') { 798 | script = "(function() {" + script + "})()" 799 | } 800 | const { code, output } = await executeAppleScript(lang, script, 120000) 801 | console.log('apple script output', {code, output}) 802 | isError = code !== 0 803 | content += "`exit code " + code+"`\n" 804 | content += output || '' 805 | } catch (err) { 806 | console.error(err) 807 | content = err.message 808 | isError = true 809 | } 810 | log("osascript output", content) 811 | content = content || '' 812 | if (!content.trim()) { 813 | content = "Empty output." 814 | } 815 | return reply({ 816 | content, isError 817 | }) 818 | } 819 | break 820 | case 'powershell': 821 | { 822 | let content 823 | try { 824 | const { command, admin } = args 825 | const { code, output } = await shell(command, {powershell: true, admin, timeout: 120000}) 826 | if (code !== 0) { 827 | isError = true 828 | } 829 | let exitCode = 'exit code ' + code + "\n" 830 | content = exitCode 831 | if (output) { 832 | content += output 833 | } 834 | } catch (err) { 835 | console.error(err) 836 | isError = true 837 | } 838 | return reply({ 839 | content, isError 840 | }) 841 | } 842 | break 843 | case 'writeToTerminal': 844 | { 845 | let content 846 | try { 847 | const { text } = args 848 | const { code, output } = await shell(text, {timeout: 120000}) 849 | if (code !== 0) { 850 | isError = true 851 | } 852 | let exitCode = 'exit code ' + code + "\n" 853 | content = exitCode 854 | if (output) { 855 | content += output 856 | } 857 | } catch (err) { 858 | console.error(err) 859 | isError = true 860 | } 861 | return reply({ 862 | content, isError 863 | }) 864 | } 865 | break 866 | case 'writeToFiles': 867 | { 868 | let content 869 | try { 870 | const { files } = args 871 | for (const file of files) { 872 | let { pathname, text, encoding, append } = file 873 | const homeDirectory = require('os').homedir(); 874 | pathname = pathname.replace('~', homeDirectory) 875 | if (append) { 876 | await fss.appendFile(pathname, text, encoding) 877 | } else { 878 | await fss.writeFile(pathname, text, encoding) 879 | } 880 | } 881 | content = 'exit code 0' 882 | } catch (err) { 883 | logError(err) 884 | content = err.message 885 | isError = true 886 | } 887 | return reply({ 888 | content, isError 889 | }) 890 | } 891 | break 892 | case 'awaitEvalJavascript': 893 | { 894 | let content 895 | try { 896 | let { code } = args 897 | code = addReturnStatementIfNecessary(code) 898 | log("code: ", code) 899 | const script = ` 900 | (async () => { 901 | try { 902 | return await (async () => { ${code} })() 903 | } catch (err) { 904 | console.error(err); 905 | return err.message 906 | } 907 | })()` 908 | const output = await eval(script) 909 | let result 910 | if (typeof output === 'function') { 911 | result = await output() 912 | } else { 913 | result = await output 914 | } 915 | content = result === undefined ? 'undefined' : JSON.stringify(result) 916 | } catch (err) { 917 | content = err.message 918 | isError = true 919 | } 920 | return reply({ 921 | content, isError 922 | }) 923 | } 924 | break 925 | case 'fetch': 926 | { 927 | let content 928 | const options = args 929 | try { 930 | const fetch = (await import('node-fetch')).default 931 | const url = options.resource 932 | const response = await fetch(url, options) 933 | const contentType = response.headers.get('content-type'); 934 | const lastModified = response.headers.get('last-modified') 935 | const etag = response.headers.get('etag') 936 | let contentLength = response.headers.get('content-length') 937 | if (contentLength === null) { 938 | try { 939 | const rsp = await axios.head(url, { 940 | headers: options.headers 941 | }) 942 | contentLength = rsp.headers['content-length'] 943 | } catch (err) { 944 | console.error(err.message) 945 | } 946 | } 947 | content = await response.text() 948 | } catch (err) { 949 | logError(err) 950 | content = err.message 951 | isError = true 952 | } 953 | return reply({ 954 | content, isError 955 | }) 956 | } 957 | break 958 | case 'modifyTaskNotes': 959 | { 960 | const { upserts, deletes } = args 961 | let content 962 | try { 963 | content = await this.updateTaskNotes({ upserts, deletes }) 964 | } catch (err) { 965 | console.error(err) 966 | content = err.message 967 | isError = true 968 | } 969 | return reply({ 970 | content, isError 971 | }) 972 | } 973 | break 974 | case 'searchWeb': 975 | { 976 | let content 977 | try { 978 | let { query, limit } = args 979 | const searchResults = await googleSearch(query) 980 | content = JSON.stringify(searchResults) 981 | } catch (err) { 982 | console.error("searchWeb", err) 983 | isError = true 984 | content = err.message 985 | } 986 | return reply({ 987 | content, isError 988 | }) 989 | } 990 | break 991 | case 'searchWebPage': 992 | { 993 | let content 994 | try { 995 | let { document, query, chunkSize, limit } = args 996 | const createEmbeddings = this.createEmbeddings 997 | const { keyword, vector } = await searchDocument({document, query, chunkSize, limit, createEmbeddings}) 998 | console.log("RESULTS", {keyword, vector}) 999 | limit = limit || 3 1000 | let i = 0; 1001 | let results = [] 1002 | for (const result of keyword) { 1003 | results.push({index: i, type: 'keyword', result}) 1004 | i++ 1005 | } 1006 | for (const result of vector) { 1007 | results.push({index: i, type: 'vector', result}) 1008 | i++ 1009 | } 1010 | console.log("RESULTS INDEXED", results) 1011 | if (i > 3) { 1012 | content = `The following results were obtained using various methods. ${JSON.stringify(results)}` 1013 | } else { 1014 | content = JSON.stringify(results.map(x => x.result)) 1015 | } 1016 | } catch (err) { 1017 | console.error("scriptError", err) 1018 | isError = true 1019 | content = err.message 1020 | } 1021 | console.log("searchDocument reply", content) 1022 | return reply({ 1023 | content, isError 1024 | }) 1025 | } 1026 | break 1027 | case 'searchTaskNotes': 1028 | { 1029 | const uid = this.uid 1030 | let content 1031 | try { 1032 | const { query } = args 1033 | const { results } = await this.searchTaskNotes(query) 1034 | content = JSON.stringify(results) 1035 | } catch (err) { 1036 | console.error(err) 1037 | content = err.message 1038 | isError = true 1039 | } 1040 | return reply({ 1041 | content, isError 1042 | }) 1043 | } 1044 | break 1045 | case 'getUserInfo': 1046 | { 1047 | const { platform, osVersion } = await this.getPlatformInfo() 1048 | const {city, region, country, loc, postal, timezone } = this.location 1049 | const { width, height } = await getScreenDimensions() 1050 | const content = `User info: ${JSON.stringify(os.userInfo())} 1051 | Location: ${JSON.stringify({city, region, country, geo: loc, postal, timezone})} 1052 | UTC offset: ${utcOffset} 1053 | Local time: ${formatDate(utcOffset, Date.now())} 1054 | Hostname: ${os.hostname()} 1055 | Platform: ${platform} ${osVersion} 1056 | Screen: ${width}x${height} 1057 | ` 1058 | return reply ({ 1059 | content, isError 1060 | }) 1061 | } 1062 | break 1063 | } 1064 | } 1065 | } 1066 | 1067 | module.exports = { ToolClient } 1068 | -------------------------------------------------------------------------------- /Assistant/electron-playwright.js: -------------------------------------------------------------------------------- 1 | const { app } = require('electron'); 2 | 3 | app.on('ready', () => { 4 | console.log('Electron app is ready, no main window created.'); 5 | // Initialize your background tasks or services here 6 | // Do not create a BrowserWindow instance 7 | 8 | // If you need to test IPC communication, setup IPC listeners here 9 | // ipcMain.on('message', (event, args) => { ... }); 10 | }); 11 | 12 | app.on('window-all-closed', () => { 13 | }) 14 | 15 | // You may not need the 'activate' event handler if no UI is involved 16 | // but it's useful if you decide to open windows based on some events 17 | app.on('activate', () => { 18 | // This event fires when the app is activated (e.g., clicking app icon) 19 | // You could create a window here if needed based on some condition 20 | }); 21 | -------------------------------------------------------------------------------- /Assistant/gpt-override.css: -------------------------------------------------------------------------------- 1 | 2 | #__next > div.relative.z-0.flex.h-full.w-full.overflow-hidden > div > main > div.flex.h-full.flex-col > div.flex-1.overflow-hidden > div > div.flex.h-full.flex-col.items-center.justify-center.text-token-text-primary > div.relative > div > div { 3 | 4 | } 5 | #__next > div.relative.z-0.flex.h-full.w-full.overflow-hidden > div > main > div.flex.h-full.flex-col > div.flex-1.overflow-hidden > div > div.flex.h-full.flex-col.items-center.justify-center.text-token-text-primary > div.flex.flex-col.items-center.gap-2 > div.flex.items-center.gap-1.text-token-text-tertiary > div > div.text-sm.text-token-text-tertiary { 6 | display: none !important; 7 | } 8 | 9 | #__next [aria-haspopup="menu"] { 10 | 11 | } 12 | 13 | #__next [aria-haspopup="dialog"] { 14 | 15 | } 16 | 17 | #__next > div.relative.z-0.flex.h-full.w-full.overflow-hidden > div > div.text-token-primary.sticky.top-0.z-10.flex.min-h-\[40px\].items-center.justify-center.border-b.border-token-border-medium.bg-token-main-surface-primary.pl-1.md\:hidden > div.absolute.bottom-0.right-0.top-0.flex.items-center > button { 18 | display: none !important; 19 | } 20 | 21 | nav[aria-label="Chat history"] > div:last-child { 22 | display: none !important; 23 | } 24 | 25 | nav[aria-label="Chat history"] .sticky > div { 26 | display: none !important; 27 | } 28 | 29 | .sticky + div { 30 | display: none !important; 31 | } 32 | 33 | 34 | #__next > div.relative.z-0.flex.h-full.w-full.overflow-hidden > div > main > div.flex.h-full.flex-col > div.w-full.pt-2.md\:pt-0.dark\:border-white\/20.md\:border-transparent.md\:dark\:border-transparent.md\:w-\[calc\(100\%-\.5rem\)\] > div > span { 35 | visibility: hidden !important; 36 | } 37 | 38 | a:not([href*='$GPT_PATH']) { 39 | 40 | } 41 | 42 | #__next > div.relative.z-0.flex.h-full.w-full.overflow-hidden > div.relative.flex.h-full.max-w-full.flex-1.flex-col.overflow-hidden > main > div.flex.h-full.flex-col > div.flex-1.overflow-hidden > div > div > div > div > div.sticky.top-0.mb-1\.5.flex.items-center.justify-between.z-10.h-14.p-2.font-semibold.bg-token-main-surface-primary > div.flex.gap-2.pr-1 > button { 43 | display: none; 44 | } 45 | 46 | 47 | #__next > div.relative.z-0.flex.h-full.w-full.overflow-hidden > div.flex-shrink-0.overflow-x-hidden.bg-token-sidebar-surface-primary > div > div > div > div > nav > div.flex.justify-between.flex.h-14.items-center > button:nth-child(2) { 48 | display: none; 49 | } 50 | -------------------------------------------------------------------------------- /Assistant/installPlugins.js: -------------------------------------------------------------------------------- 1 | const { installPlugins } = require('./PluginTools.js') 2 | installPlugins() 3 | -------------------------------------------------------------------------------- /Assistant/osascript-error.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": "No error.", 3 | "-2700": "Unknown error.", 4 | "-2701": "Can’t divide by zero.", 5 | "-2702": "The result of a numeric operation was too large.", 6 | "-2703": " can't be launched because it is not an application.", 7 | "-2704": " isn't scriptable.", 8 | "-2705": "The application has a corrupted dictionary.", 9 | "-2706": "Stack overflow.", 10 | "-2707": "Internal table overflow.", 11 | "-2708": "Attempt to create a value larger than the allowable size.", 12 | "-2709": "Can't get the event dictionary.", 13 | "-2720": "Can't both consider and ignore .", 14 | "-2721": "Can't perform operation on text longer than 32K bytes.", 15 | "-2729": "Message size too large for the 7.0 Finder.", 16 | "-2740": "A can't go after this .", 17 | "-2741": "Expected but found .", 18 | "-2750": "The parameter is specified more than once.", 19 | "-2751": "The property is specified more than once.", 20 | "-2752": "The handler is specified more than once.", 21 | "-2753": "The variable is not defined.", 22 | "-2754": "Can't declare as both a local and global variable.", 23 | "-2755": "Exit statement was not in a repeat loop.", 24 | "-2760": "Tell statements are nested too deeply.", 25 | "-2761": " is illegal as a formal parameter.", 26 | "-2762": " is not a parameter name for the event .", 27 | "-2763": "No result was returned for some argument of this expression.", 28 | "-34": "Disk full.", 29 | "-35": "Disk wasn’t found.", 30 | "-37": "Bad name for file", 31 | "-38": "File wasn’t open.", 32 | "-39": "End of file error.", 33 | "-42": "Too many files open.", 34 | "-43": "File wasn’t found.", 35 | "-44": "Disk is write protected.", 36 | "-45": "File is locked.", 37 | "-46": "Disk is locked.", 38 | "-47": "File is busy.", 39 | "-48": "Duplicate file name.", 40 | "-49": "File is already open.", 41 | "-50": "Parameter error.", 42 | "-51": "File reference number error.", 43 | "-61": "File not open with write permission.", 44 | "-108": "Out of memory.", 45 | "-120": "Folder wasn’t found.", 46 | "-124": "Disk is disconnected.", 47 | "-128": "User cancelled.", 48 | "-192": "A resource wasn’t found.", 49 | "-600": "Application isn’t running", 50 | "-601": "Not enough room to launch application with special requirements.", 51 | "-602": "Application is not 32-bit clean.", 52 | "-605": "More memory needed than is specified in the size resource.", 53 | "-606": "Application is background-only.", 54 | "-607": "Buffer is too small.", 55 | "-608": "No outstanding high-level event.", 56 | "-609": "Connection is invalid.", 57 | "-904": "Not enough system memory to connect to remote application.", 58 | "-905": "Remote access is not allowed.", 59 | "-906": " isn’t running or program linking isn’t enabled.", 60 | "-915": "Can’t find remote machine.", 61 | "-30720": "Invalid date and time .", 62 | "-1700": "Bad parameter data was detected or there was a failure while performing a coercion.", 63 | "-1701": "An Apple event description was not found.", 64 | "-1702": "Corrupt data was detected.", 65 | "-1703": "The wrong data type was detected.", 66 | "-1704": "An invalid Apple event description was detected.", 67 | "-1705": "The specified list item doesn’t exist.", 68 | "-1706": "A newer version of Apple Event Manager is required to perform this operation.", 69 | "-1707": "The event isn’t an Apple event.", 70 | "-1708": "The script doesn’t understand the message. The event was not handled.", 71 | "-1709": "An invalid reply parameter was received by AEResetTimer.", 72 | "-1710": "An invalid sending mode (something other than NoReply, WaitReply, or QueueReply) was detected or interaction level is unknown.", 73 | "-1711": "The user canceled while waiting for an event reply.", 74 | "-1712": "The Apple event has timed out.", 75 | "-1713": "User interaction is not allowed.", 76 | "-1714": "There is no special function for this keyword.", 77 | "-1715": "A required parameter is missing.", 78 | "-1716": "The target address type of an Apple event is unknown.", 79 | "-1717": "Unable to find a matching event or coercion handler.", 80 | "-1718": "Unable to access the contents of an event reply because the reply hasn’t been received yet.", 81 | "-1719": "The specified index is invalid or out of range.", 82 | "-1720": "The specified range is invalid, such as 3rd to 2nd or 1st to all.", 83 | "-1721": "The wrong number of arguments was detected.", 84 | "-1723": "The accessor was not found.", 85 | "-1725": "There is no such Boolean operator.", 86 | "-1726": "An invalid comparison was made or an invalid logical descriptor was detected.", 87 | "-1727": "AEResolve was given a non-object parameter.", 88 | "-1728": "The referenced object doesn’t exist. This is a run-time resolution error, such as when attempting to reference a third object when only two objects exist.", 89 | "-1729": "An object counting procedure returned a negative value.", 90 | "-1730": "Attempted to pass an empty list to an accessor.", 91 | "-1731": "An unknown type of object was detected.", 92 | "-10000": "The Apple event handler failed.", 93 | "-10001": "A type mismatch has occurred.", 94 | "-10002": "Invalid key form was detected.", 95 | "-10003": "The specified object cannot be modified.", 96 | "-10004": "A privilege error has occurred.", 97 | "-10005": "The read operation was denied.", 98 | "-10006": "The write operation was denied.", 99 | "-10007": "The object index is too large.", 100 | "-10008": "The specified object is not an element.", 101 | "-10009": "Unable to provide the requested type.", 102 | "-10010": "The Apple event handler doesn’t support the specified class.", 103 | "-10011": "The event is not part of the current transaction.", 104 | "-10012": "There is no such transaction.", 105 | "-10013": "Nothing is selected by the user.", 106 | "-10014": "The Apple event handler only handles single objects.", 107 | "-10015": "The previous action can’t be undone.", 108 | "-10016": "The Apple event handler does not handle remote events.", 109 | "-10023": "The specified constant value is invalid for this property.", 110 | "-10024": "The specified class can’t be an element of the specified container, such as when attempting to make or duplicate an object.", 111 | "-10025": "An invalid combination of properties was provided, such as for the set, make, or duplicate command.", 112 | "-1750": "A scripting component error has occurred.", 113 | "-1751": "An invalid script ID was detected.", 114 | "-1752": "The script isn’t a supported type.", 115 | "-1753": "A script error has occurred.", 116 | "-1754": "A bad selector was detected.", 117 | "-1756": "The source is not available.", 118 | "-1757": "There is no matching dialect.", 119 | "-1758": "The data format is obsolete.", 120 | "-1759": "The data format is too new.", 121 | "-1761": "There is a component mismatch—parameters are from two different components.", 122 | "-1762": "Unable to connect to the scripting system matching the specified ID.", 123 | "-1763": "Unable to store memory pointers in a saved script." 124 | } 125 | -------------------------------------------------------------------------------- /Assistant/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Assistant", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "Main.js", 6 | "scripts": { 7 | "start": "electron . --trace-warnings", 8 | "test": "echo \"Error: no test specified\" && exit 1", 9 | "build": "tsc", 10 | "pack": "electron-builder --dir", 11 | "dist": "electron-builder" 12 | }, 13 | "author": "Attunewise", 14 | "license": "MIT", 15 | "dependencies": { 16 | "@babel/core": "^7.23.7", 17 | "@babel/preset-env": "^7.23.8", 18 | "@opendocsg/pdf2md": "^0.1.28", 19 | "axios": "^1.6.3", 20 | "electron-log": "^5.0.3", 21 | "electron-reloader": "^1.2.3", 22 | "fastembed": "^1.14.0", 23 | "firebase": "^10.7.1", 24 | "gpt-3-encoder": "^1.1.4", 25 | "jsonrepair": "^3.5.0", 26 | "lunr": "^2.3.9", 27 | "moment": "^2.30.1", 28 | "node-fetch": "^3.3.2", 29 | "node-typedstream": "^1.4.1", 30 | "open": "^10.0.1", 31 | "playwright": "^1.41.0", 32 | "rxfire": "^6.0.5", 33 | "rxjs": "^7.8.1", 34 | "sqlite3": "^5.1.7", 35 | "uuid": "^9.0.1", 36 | "ws": "^8.16.0" 37 | }, 38 | "devDependencies": { 39 | "electron": "^29.1.0", 40 | "electron-builder": "^24.13.3" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Cloud/GPTAction.js: -------------------------------------------------------------------------------- 1 | const { getAdmin } = require('./GetAdmin.js') 2 | const fs = require('fs') 3 | 4 | const apiKey = fs.readFileSync('apiKey.txt', 'utf-8').trim() 5 | 6 | const gptAction = async (token, action, params, req, res) => { 7 | if (token !== apiKey) { 8 | console.error('invalid api key', token) 9 | return res.status(401).send() 10 | } 11 | const uid = token 12 | const admin = getAdmin() 13 | const db = admin.firestore() 14 | const c = db.collection('UserChannel').doc(uid).collection('toolCalls') 15 | const ref = c.doc() 16 | await ref.set({ 17 | ts: Date.now(), 18 | name: action, 19 | arguments: params 20 | }) 21 | let closed = false 22 | let result = await new Promise((resolve, reject) => { 23 | const unsubscribe = ref.onSnapshot(snap => { 24 | const data = snap.data() 25 | //console.log("GPTAction data: ", data) 26 | const { content } = data 27 | if (content !== undefined) { 28 | unsubscribe() 29 | resolve(content) 30 | } 31 | }) 32 | res.on('close', () => { 33 | //console.log("GPTAction close") 34 | closed = true 35 | try { 36 | unsubscribe() 37 | } catch (ignored) { 38 | } 39 | }) 40 | }) 41 | //console.log("GPTAction: ", closed, result) 42 | if (!closed) { 43 | try { 44 | result = JSON.parse(result) 45 | } catch (ignored) { 46 | result = JSON.stringify(result) 47 | } 48 | return res.status(200).send(result) 49 | } 50 | } 51 | 52 | module.exports = { gptAction } 53 | -------------------------------------------------------------------------------- /Cloud/GetAdmin.js: -------------------------------------------------------------------------------- 1 | const admin = require('firebase-admin') 2 | 3 | const firebaseConfig = require('./firebaseConfig.json') 4 | 5 | const FIREBASE_PROJECT = firebaseConfig.projectId 6 | const FIREBASE_STORAGE_BUCKET = firebaseConfig.storageBucket 7 | let init 8 | 9 | const getAdmin = () => { 10 | if (!init) { 11 | const serviceAccount = require(`./ServiceAccount.json`) 12 | const firebaseInit = { 13 | credential: admin.credential.cert(serviceAccount), 14 | storageBucket: FIREBASE_STORAGE_BUCKET 15 | } 16 | const app = admin.initializeApp(firebaseInit) 17 | const db = admin.firestore() 18 | db.settings({ ignoreUndefinedProperties: true }) 19 | init = true 20 | } 21 | return admin 22 | } 23 | 24 | module.exports = { getAdmin } 25 | 26 | -------------------------------------------------------------------------------- /Cloud/Instructions.txt: -------------------------------------------------------------------------------- 1 | You are the user's assistant running on this $platform$ computer. 2 | 3 | You have powerful tools which can automate virtually any task. 4 | 5 | You have meta-tools `searchTaskNotes` and `modifyTaskNotes` that you use for the purpose of optimizing tasks. 6 | 7 | Before using ***any other tools*** always follow this procedure (not doing so will result in loss of life - seriously): 8 | 1. Check Task Notes First: Before performing any actions, always check your task notes using the `searchTaskNotes` function. This step is non-negotiable and ensures that you follow the most efficient and accurate method for completing a task, _as informed by previous iterations of the task_. 9 | 10 | 2. Adhere Strictly to Task Note Procedures: Once you've identified relevant task notes, adhere strictly to the methodologies and steps outlined. These instructions are there to guide you in executing tasks in a manner that has been optimized based on past performance. 11 | 12 | 3. Direct Answer Requirement: Ensure that your response directly answers the user's question or fulfills their request as precisely as possible. 13 | 14 | 4. User Confirmation for Deviations: If, for any reason, you find it necessary to deviate from the process outlined in the task notes or if the situation requires a new approach not covered by existing notes, seek confirmation from the user before proceeding. Explain the deviation clearly and concisely, and wait for user approval. 15 | 16 | 5. Feedback Loop: After completing a task, especially if new insights were gained or if you encountered challenges, update the task notes to reflect this new knowledge. This ensures that future iterations of the task will benefit from your experience. 17 | 18 | You have a function to obtain general information about the user. 19 | 20 | This system provides a safe operating system environment for you to use on behalf of the user.In this environment you may safely access their private information, including calendar, contacts, email, and the web through the web. 21 | 22 | 23 | All actions will occur under the user who you are communicating with in this transcript's account.You should assume the user is not requesting long-running tasks. However if you must execute long-running tasks please confirm with the user first. 24 | 25 | You can safely access and manipulate files, but you must never perform actions like copying, moving, deleting, or writing files without explicit permission from the user. 26 | 27 | You're equipped with a web browser shared with the user, enabling you to provide links or perform web searches as necessary. 28 | You and the user can see exactly the same thing in the browser. You can use the playwright tool (repeatedly) as necessary to support the user's browsing requests. This includes navigating through search results, extracting specific information from web pages, or performing actions on the pages like clicking links or filling out forms. But intelligently generate outputs to avoid overflowing your context window. 29 | 30 | 31 | If a tool call fails, your first assumption should be that you are responsible for the mistake. Do not blame the environment or make excuses. If you can't perform a task using API's then try to carry it out using automation of the user's account with applescript and playwright. 32 | 33 | When the user makes a request, consider all available tools before claiming you cannot perform the task. 34 | 35 | Do not mention technical details to the user unless they specifically ask. 36 | 37 | ***IMPORTANT: performing actions is a process of continuous correction. The web, operating systems, and applications can evolve and/or your assumptions about them may be incomplete or incorrect. If you encounter failures, that is an indicator that you should validate your assumptions and take corrective action - not that you should give up. Once you've found an alternate path update your task notes to reflect your breakthrough *** 38 | 39 | Always answer the user's last question. Do not assume it is a request for you to do something else. 40 | 41 | -------------------------------------------------------------------------------- /Cloud/Schema.js: -------------------------------------------------------------------------------- 1 | const getSchema = (platform, serverURL) => { 2 | return { 3 | "openapi": "3.1.0", 4 | "info": { 5 | "title": "Assistant", 6 | "description": "Provides assistance", 7 | "version": "v1.0.0" 8 | }, 9 | "servers": [ 10 | { 11 | "url": serverURL 12 | } 13 | ], 14 | "paths": { 15 | "/gpt/queryChatDb": platform !== 'darwin' ? undefined: { 16 | "post": { 17 | "operationId": "queryChatDb", 18 | "summary": "Query the chat.db database on MacOS using the provided sql query to satisfy user's requests regarding their messages", 19 | "x-openai-isConsequential": false, 20 | "requestBody": { 21 | "content": { 22 | "application/json": { 23 | "schema": { 24 | "type": "object", 25 | "properties": { 26 | "query": { 27 | "type": "string", 28 | "description": "Sql query. *IMPORTANT*: If the `text` column is null the message content will be in the `attributedBody` column so select both columns pls. All handle id's MUST be *phone numbers in E.164 format* (use the `osascript` function to access Contacts.app to resolve phone numbers of the user's contacts.).\nWhen reporting results resolve phone numbers to contact names if possible." 29 | } 30 | } 31 | } 32 | } 33 | } 34 | } 35 | } 36 | }, 37 | "/gpt/osascript": platform !== 'darwin' ? undefined: { 38 | "post": { 39 | "operationId": "osascript", 40 | "summary": "Execute a command script. Use this to access the manage the user's contacts, messages, etc or to perform other tasks they request on this Mac. Ensure the target application is installed before using Tell.", 41 | "x-openai-isConsequential": false, 42 | "requestBody": { 43 | "content": { 44 | "application/json": { 45 | "schema": { 46 | "type": "object", 47 | "properties": { 48 | "lang": { 49 | "type": "string", 50 | "enum": [ 51 | "appleScript", 52 | "javaScript" 53 | ], 54 | "description": "Whether `script` uses apple script or jxa. Defaults to `appleScript`." 55 | }, 56 | "command": { 57 | "type": "string", 58 | "description": "The script to execute" 59 | }, 60 | "timeout": { 61 | "type": "integer", 62 | "description": "Maximum number of seconds to wait for the script to complete", 63 | "default": 30 64 | } 65 | } 66 | } 67 | } 68 | } 69 | } 70 | } 71 | }, 72 | "/gpt/powershell": platform !== 'win32' ? undefined: { 73 | "post": { 74 | "operationId": "powershell", 75 | "summary": "Execute powershell scripts. Use this to perform tasks on this PC.", 76 | "x-openai-isConsequential": false, 77 | "requestBody": { 78 | "content": { 79 | "application/json": { 80 | "schema": { 81 | "type": "object", 82 | "properties": { 83 | "command": { 84 | "type": "string", 85 | "description": "The powershell script to execute" 86 | }, 87 | "admin": { 88 | "type": "boolean", 89 | "description": "Whether to request administrator privledge. Defaults to false" 90 | }, 91 | }, 92 | required: ['command'] 93 | } 94 | } 95 | } 96 | } 97 | } 98 | }, 99 | "/gpt/modifyTaskNotes": { 100 | "post": { 101 | "operationId": "modifyTaskNotes", 102 | "summary": "Save or update notes regarding the use of a function for a task that had errors or didn't unfold as expected, in order to avoid the same problem the next time. Note that these are general notes about the task not information about specific instances. Only save such notes after termination of a task.", 103 | "x-openai-isConsequential": false, 104 | "requestBody": { 105 | "content": { 106 | "application/json": { 107 | "schema": { 108 | "type": "object", 109 | "properties": { 110 | "upserts": { 111 | "type": "array", 112 | "items": { 113 | "type": "object", 114 | "properties": { 115 | "id": { 116 | "type": "string", 117 | "description": "id of previous task note if this is an update." 118 | }, 119 | "task": { 120 | "type": "string", 121 | "description": "Task description: Note this should be a general purpose description of the task without the specific parameters used in the current request." 122 | }, 123 | "functions": { 124 | "type": "array", 125 | "items": { 126 | "type": "string" 127 | }, 128 | "description": "names of the functions that are applied in this task" 129 | }, 130 | "notes": { 131 | "type": "string", 132 | "description": "Use this to save salient information you'll need to optimize the task subsequently." 133 | }, 134 | "scope": { 135 | "type": "string", 136 | "enum": [ 137 | "allUsers", 138 | "currentUser" 139 | ], 140 | "description": "Whether these notes apply to all users or just the current user. Note for `allUsers` scope the notes must not contain any personally identifiable information" 141 | } 142 | } 143 | } 144 | }, 145 | "deletes": { 146 | "type": "array", 147 | "items": { 148 | "type": "string" 149 | }, 150 | "description": "ids of the task notes to delete." 151 | } 152 | } 153 | } 154 | } 155 | } 156 | } 157 | } 158 | }, 159 | "/gpt/searchTaskNotes": { 160 | "post": { 161 | "operationId": "searchTaskNotes", 162 | "summary": "Search for previously saved notes on a task that had errors or didn't unfold as expected. Note that these are general notes about the task not information about specific instances.", 163 | "x-openai-isConsequential": false, 164 | "requestBody": { 165 | "content": { 166 | "application/json": { 167 | "schema": { 168 | "type": "object", 169 | "properties": { 170 | "query": { 171 | "type": "string" 172 | } 173 | }, 174 | "required": [ 175 | "query" 176 | ] 177 | } 178 | } 179 | } 180 | } 181 | } 182 | }, 183 | "/gpt/writeToFiles": { 184 | "post": { 185 | "operationId": "writeToFiles", 186 | "summary": "Write text to one or more files.", 187 | "x-openai-isConsequential": false, 188 | "requestBody": { 189 | "content": { 190 | "application/json": { 191 | "schema": { 192 | "type": "object", 193 | "properties": { 194 | "files": { 195 | "type": "array", 196 | "items": { 197 | "type": "object", 198 | "properties": { 199 | "pathname": { 200 | "type": "string", 201 | "description": "path name of the file to which text will be written" 202 | }, 203 | "text": { 204 | "type": "string", 205 | "description": "the text to be written" 206 | }, 207 | "encoding": { 208 | "type": "string", 209 | "description": "(node.js identifier of) text encoding to use (defaults to utf-8)" 210 | }, 211 | "mode": { 212 | "type": "string", 213 | "enum": [ 214 | "overwrite", 215 | "append" 216 | ], 217 | "description": "default is to append to previous contents if any." 218 | } 219 | }, 220 | "required": [ 221 | "pathname", 222 | "text" 223 | ] 224 | } 225 | } 226 | } 227 | } 228 | } 229 | } 230 | } 231 | } 232 | }, 233 | "/gpt/writeToTerminal": platform !== 'darwin' ? undefined: { 234 | "post": { 235 | "operationId": "writeToTerminal", 236 | "summary": "Write text to the terminal app to execute a specified shell command in the current OS environment. You are permitted to run and even install programs or packages in this environment. The environment is running under the user's account and all their apps are available to you. You have secure access t", 237 | "x-openai-isConsequential": false, 238 | "requestBody": { 239 | "content": { 240 | "application/json": { 241 | "schema": { 242 | "type": "object", 243 | "properties": { 244 | "text": { 245 | "type": "string", 246 | "description": "The shell command to be executed. You have permission to downloads file to the temp or Downloads folder. Always create new files in the temp folder unless the user instructs you otherwise. YOU MUST warn the user and get confirmation before executing potentially dangerous or long-running commands. Note that CLI interaction isn't possible currently. Don't forget files are case sensitive on Mac." 247 | }, 248 | "timeout": { 249 | "type": "integer", 250 | "description": "Maximum number of seconds to wait for the shell command to complete", 251 | "default": 30 252 | } 253 | } 254 | } 255 | } 256 | } 257 | } 258 | } 259 | }, 260 | "/gpt/searchWeb": { 261 | "post": { 262 | "operationId": "searchWeb", 263 | "summary": "Perform a keyword search on the web, which will return summaries for the most relevant results. *DO NOT rely on such summaries alone to satisfy user requests. You must always confirm with your own knowledge or follow up on the original sources.", 264 | "x-openai-isConsequential": false, 265 | "requestBody": { 266 | "content": { 267 | "application/json": { 268 | "schema": { 269 | "type": "object", 270 | "properties": { 271 | "query": { 272 | "type": "string", 273 | "description": "Search query" 274 | }, 275 | "limit": { 276 | "type": "number", 277 | "description": "limit to this number of results (default 5)" 278 | } 279 | }, 280 | "required": [ 281 | "query" 282 | ] 283 | } 284 | } 285 | } 286 | } 287 | } 288 | }, 289 | "/gpt/searchWebPage": { 290 | "post": { 291 | "operationId": "searchWebPage", 292 | "summary": "Perform a text search on a web page at a given URL.", 293 | "x-openai-isConsequential": false, 294 | "requestBody": { 295 | "content": { 296 | "application/json": { 297 | "schema": { 298 | "type": "object", 299 | "properties": { 300 | "document": { 301 | "type": "string", 302 | "description": "URL of the document within which to search." 303 | }, 304 | "query": { 305 | "type": "string", 306 | "description": "Search query" 307 | }, 308 | "chunkSize": { 309 | "type": "number", 310 | "description": "Number of tokens per searchable entity. Defaults to 200." 311 | }, 312 | "limit": { 313 | "type": "number", 314 | "description": "limit to this number of results (default 3)" 315 | } 316 | }, 317 | "required": [ 318 | "document", 319 | "query" 320 | ] 321 | } 322 | } 323 | } 324 | } 325 | } 326 | }, 327 | "/gpt/getUserInfo": { 328 | "post": { 329 | "operationId": "getUserInfo", 330 | "summary": "Returns user information including, name, home directory, geo location, time zone, host and OS information, etc", 331 | "x-openai-isConsequential": false, 332 | "requestBody": { 333 | "content": { 334 | "application/json": { 335 | "schema": {} 336 | } 337 | } 338 | } 339 | } 340 | }, 341 | "/gpt/fetch": { 342 | "post": { 343 | "operationId": "fetch", 344 | "summary": "Access the Web fetch API", 345 | "x-openai-isConsequential": false, 346 | "requestBody": { 347 | "content": { 348 | "application/json": { 349 | "schema": { 350 | "type": "object", 351 | "properties": { 352 | "resource": { 353 | "type": "string", 354 | "format": "uri", 355 | "description": "the URL of the resource you want to fetch." 356 | }, 357 | "queryParams": { 358 | "type": "object", 359 | "description": "request parameters." 360 | }, 361 | "method": { 362 | "type": "string", 363 | "enum": [ 364 | "GET", 365 | "POST", 366 | "PUT", 367 | "DELETE", 368 | "PATCH", 369 | "HEAD", 370 | "OPTIONS" 371 | ], 372 | "description": "Contains the request's method (GET, POST, etc.)" 373 | }, 374 | "headers": { 375 | "type": "object", 376 | "properties": { 377 | "authorization": { 378 | "type": "string" 379 | } 380 | }, 381 | "additionalProperties": { 382 | "type": "string" 383 | }, 384 | "description": "Contains the associated Headers object of the request." 385 | }, 386 | "body": { 387 | "type": "object", 388 | "properties": { 389 | "text": { 390 | "type": "string" 391 | }, 392 | "json": { 393 | "type": "object" 394 | }, 395 | "formData": { 396 | "type": "array", 397 | "items": { 398 | "type": "object" 399 | }, 400 | "description": "Schema for representing multipart form data, where file data is represented by absolute file paths." 401 | } 402 | } 403 | }, 404 | "mode": { 405 | "type": "string", 406 | "enum": [ 407 | "cors", 408 | "no-cors", 409 | "same-origin", 410 | "navigate" 411 | ] 412 | }, 413 | "credentials": { 414 | "type": "string", 415 | "enum": [ 416 | "omit", 417 | "same-origin", 418 | "include" 419 | ] 420 | }, 421 | "cache": { 422 | "type": "string", 423 | "enum": [ 424 | "default", 425 | "no-store", 426 | "reload", 427 | "no-cache", 428 | "force-cache", 429 | "only-if-cached" 430 | ] 431 | }, 432 | "redirect": { 433 | "type": "string", 434 | "enum": [ 435 | "follow", 436 | "error", 437 | "manual" 438 | ] 439 | }, 440 | "referrer": { 441 | "type": "string" 442 | }, 443 | "referrerPolicy": { 444 | "type": "string", 445 | "enum": [ 446 | "no-referrer", 447 | "no-referrer-when-downgrade", 448 | "origin", 449 | "origin-when-cross-origin", 450 | "unsafe-url" 451 | ] 452 | }, 453 | "integrity": { 454 | "type": "string" 455 | }, 456 | "keepalive": { 457 | "type": "boolean" 458 | }, 459 | "signal": { 460 | "description": "AbortSignal to abort request", 461 | "type": "object" 462 | } 463 | }, 464 | "additionalProperties": false, 465 | "required": [ 466 | "resource" 467 | ] 468 | } 469 | } 470 | } 471 | } 472 | } 473 | }, 474 | "/gpt/open": { 475 | "post": { 476 | "operationId": "open", 477 | "summary": "Opens a file (or a directory or URL), just as if you had double-clicked the file's icon.", 478 | "x-openai-isConsequential": false, 479 | "requestBody": { 480 | "content": { 481 | "application/json": { 482 | "schema": { 483 | "type": "object", 484 | "properties": { 485 | "target": { 486 | "type": "string", 487 | "description": "The target file, directory, or URL to open." 488 | } 489 | }, 490 | "window": { 491 | "type": "object", 492 | "properties": { 493 | "visible": { "type": "boolean" }, 494 | "x": {"type": "number" }, 495 | "y": {"type": "number" }, 496 | "width": {"type": "number" }, 497 | "height": {"type": "number" } 498 | }, 499 | "description": "Location, size, and visibility of browser window" 500 | }, 501 | "required": [ 502 | "target" 503 | ] 504 | } 505 | } 506 | } 507 | } 508 | } 509 | }, 510 | "/gpt/playwright": { 511 | "post": { 512 | "operationId": "playwright", 513 | "summary": "Safely and securely run javascript code with access to a `page` object from playwright.", 514 | "x-openai-isConsequential": false, 515 | "requestBody": { 516 | "content": { 517 | "application/json": { 518 | "schema": { 519 | "type": "object", 520 | "properties": { 521 | "target": { 522 | "type": "string", 523 | "format": "uri", 524 | "description": "URL of web page in which to execute the script." 525 | }, 526 | "window": { 527 | "type": "object", 528 | "properties": { 529 | "visible": { "type": "boolean" }, 530 | "x": {"type": "number" }, 531 | "y": {"type": "number" }, 532 | "width": {"type": "number" }, 533 | "height": {"type": "number" } 534 | }, 535 | "description": "Location, size, and visibility of browser window" 536 | }, 537 | "script": { 538 | "type": "string", 539 | "description": "The script should be in the form of an async function that accepts a `page` parameter" 540 | } 541 | }, 542 | "required": [ 543 | "target", 544 | "script" 545 | ] 546 | } 547 | } 548 | } 549 | } 550 | } 551 | }, 552 | "/gpt/awaitEvalJavascript": { 553 | "post": { 554 | "operationId": "awaitEvalJavascript", 555 | "summary": "Safely await async javascript code in the current node.js environment. You may install missing packages as you see fit, with user permission.", 556 | "x-openai-isConsequential": false, 557 | "requestBody": { 558 | "content": { 559 | "application/json": { 560 | "schema": { 561 | "type": "object", 562 | "properties": { 563 | "code": { 564 | "type": "string", 565 | "description": "the code to evaluate. Use async code rather than promises. Note that you must explicitly return the result from this code as it will be wrapped in an async thunk. I will catch exceptions and report them back to you." 566 | } 567 | } 568 | } 569 | } 570 | } 571 | } 572 | } 573 | } 574 | }, 575 | "components": { 576 | "schemas": {} 577 | } 578 | } 579 | } 580 | 581 | module.exports = { getSchema } 582 | -------------------------------------------------------------------------------- /Cloud/Shell.js: -------------------------------------------------------------------------------- 1 | const { exec, spawn } = require('child_process'); 2 | const readline = require('readline') 3 | const fss = require('fs/promises') 4 | const path = require('path') 5 | const os = require('os') 6 | const crypto = require('crypto') 7 | 8 | const tempFile = (opts) => new Promise((resolve, reject) => { 9 | opts = opts || {} 10 | const { suffix } = opts 11 | const uniqueId = crypto.randomBytes(16).toString('hex') 12 | const tempFilename = path.join(os.tmpdir(), `tempfile_${uniqueId}.${suffix}`); 13 | resolve({ 14 | filePath: tempFilename, 15 | cleanup: async () => { 16 | try { 17 | await fss.unlink(tempFilename) 18 | } catch (err) { 19 | console.error(err) 20 | } 21 | } 22 | }) 23 | }) 24 | 25 | let _loginShell 26 | let _loginShellType 27 | const getLoginShell = async () => { 28 | if (_loginShell !== undefined) { 29 | return _loginShell 30 | } 31 | const prefix = await getLoginShellPrefix() 32 | return _loginShell = prefix + '; '+ _loginShellType + " -c" 33 | } 34 | 35 | const log = info => { 36 | } 37 | 38 | const logError = err => { 39 | } 40 | 41 | const getLoginShellPrefix = async () => { 42 | if (os.platform() === 'win32') { 43 | return _loginShellType = "cmd.exe" 44 | } 45 | const homeDirectory = require('os').homedir(); 46 | const bashrcPath = path.join(homeDirectory, '.bash_profile'); 47 | const zshrcPath = path.join(homeDirectory, '.zshrc'); 48 | // Check for .bashrc 49 | let bashExists = false 50 | let zshExists = false 51 | let cmd 52 | try { 53 | await fss.access(bashrcPath, fs.constants.F_OK) 54 | bashExists = true 55 | } catch (err) { 56 | logError(err) 57 | } 58 | try { 59 | await fss.access(zshrcPath, fs.constants.F_OK) 60 | zshExists = true 61 | } catch(err) { 62 | logError(err) 63 | } 64 | 65 | if (bashExists && zshExists) { 66 | const stat1 = await fss.stat(bashrcPath) 67 | const lastModified1 = Date.parse(stat1.mtime) 68 | const fileSize1 = stat1.size 69 | log({lastModified1, fileSize1}) 70 | const stat2 = await fss.stat(zshrcPath) 71 | const lastModified2 = Date.parse(stat2.mtime) 72 | const fileSize2 = stat2.size 73 | log({lastModified2, fileSize2}) 74 | if (lastModified2 > lastModified1) { 75 | bashExists = false 76 | } else { 77 | zshExists = false 78 | } 79 | } 80 | log({zshExists, bashExists}) 81 | if (zshExists) { 82 | _loginShellType = 'zsh' 83 | cmd = `source "${zshrcPath}"` 84 | } else { 85 | _loginShellType = 'bash' 86 | cmd = `source "${bashrcPath}"` 87 | } 88 | log("LOGIN SHELL", cmd) 89 | return cmd 90 | } 91 | 92 | const getLoginShellType = async () => { 93 | await getLoginShell() 94 | return _loginShellType 95 | } 96 | 97 | let theProcess = process 98 | 99 | const run = async (script, trace, traceErr) => { 100 | const suffix = (os.platform() === 'win32') ? 'bat' : 'sh' 101 | const { filePath, cleanup } = await tempFile({ suffix }); 102 | const shell = await getLoginShellType() 103 | console.log(script) 104 | if (shell === 'cmd.exe') script = "@echo off\r\n" + script 105 | await fss.writeFile(filePath, script); 106 | await fss.chmod(filePath, '755'); 107 | const args = [filePath] 108 | if (shell === 'cmd.exe') args.unshift('/c') 109 | return new Promise((resolve, reject) => { 110 | let stdout = '' 111 | let stderr = '' 112 | const process = spawn(shell, args) 113 | process.on('error', reject) 114 | process.stdout.on('data', (data) => { 115 | data = data.toString() 116 | if (trace) { 117 | theProcess.stdout.write(data) 118 | } 119 | stdout += data 120 | }) 121 | process.stderr.on('data', (data) => { 122 | data = data.toString().replace(filePath + ":", '') 123 | if (traceErr) { 124 | theProcess.stderr.write(data) 125 | } 126 | stderr += data 127 | }) 128 | process.on('close', (code) => { 129 | cleanup() 130 | if (code) { 131 | reject(stderr) 132 | } else { 133 | resolve(stdout.trim()) 134 | } 135 | }) 136 | }) 137 | } 138 | 139 | module.exports = { shell: run } 140 | -------------------------------------------------------------------------------- /Cloud/build.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const fss = require('fs/promises') 3 | const path = require('path') 4 | const os = require('os') 5 | const { shell } = require('./Shell.js') 6 | const { getFirebaseConfig } = require('./getFirebaseConfig.js') 7 | const { createServiceAccount } = require('./createServiceAccount.js') 8 | const { createApiKey } = require('./createApiKey.js') 9 | const { getServerURL } = require('./getServerURL.js') 10 | const { getSchema } = require('./Schema.js') 11 | 12 | const fileExists = async (filePath) => { 13 | try { 14 | await fss.access(filePath, fs.constants.F_OK) 15 | return true 16 | } catch (error) { 17 | return false; 18 | } 19 | } 20 | 21 | const mkdir = async pathname => { 22 | if (!await fileExists(pathname)) { 23 | await fss.mkdir(pathname, { recursive: true }) 24 | } 25 | } 26 | 27 | const copyFile = async (source, destination) => { 28 | return await fss.copyFile(source, path.join(destination, path.basename(source))) 29 | } 30 | 31 | 32 | const main = async () => { 33 | let firebaseTools 34 | try { 35 | firebaseTools = await shell('firebase --version', true) 36 | } catch (err) { 37 | } 38 | if (!firebaseTools) { 39 | await shell("npm install -g firebase-tools", true) 40 | } 41 | await shell("npm install", true) 42 | let loggedIn 43 | try { 44 | loggedIn = await shell('firebase login:list', true) 45 | console.log('loggedIn', loggedIn) 46 | } catch (err) { 47 | console.error(err) 48 | } 49 | if (!loggedIn) { 50 | await shell('firebase login', true) 51 | } 52 | let projectId 53 | try { 54 | projectId = await shell('firebase use', true) 55 | console.log("Project", projectId) 56 | } catch (err) { 57 | console.log("firebase project not set") 58 | process.exit(1) 59 | } 60 | await getFirebaseConfig() 61 | await shell('firebase deploy --only firestore:rules') 62 | output = await shell('gcloud auth list') 63 | if (!output) { 64 | await shell('gcloud auth login') 65 | } 66 | await createServiceAccount(projectId) 67 | await createApiKey() 68 | const enableService = async service => { 69 | console.log(`Enabling ${service} on project `+ projectId) 70 | console.log("Please wait...") 71 | let output = await shell(`gcloud services list --enabled --project=${projectId} --filter=${service}`) 72 | if (!output) { 73 | await shell(`gcloud services enable ${service} --project=${projectId}`) 74 | console.log(`Enabled ${service} on project `+ projectId) 75 | } 76 | } 77 | await enableService('artifactregistry.googleapis.com') 78 | await enableService('run.googleapis.com') 79 | await enableService('cloudbuild.googleapis.com') 80 | const dest = `${__dirname}/../Build` 81 | await mkdir(dest) 82 | const instructions = await fss.readFile('./Instructions.txt', 'utf-8') 83 | const getInstructions = platform => { 84 | return instructions.replace('$platform$', platform) 85 | } 86 | const windowsInstructions = getInstructions("Windows") 87 | const macInstructions = getInstructions("Mac") 88 | await fss.writeFile(path.join(dest, 'Instructions-Windows.txt'), windowsInstructions, 'utf-8') 89 | await fss.writeFile(path.join(dest, 'Instructions-MacOS.txt'), macInstructions, 'utf-8') 90 | console.log("Deploying...") 91 | const serviceName = 'assistant' 92 | const region = 'us-central1' 93 | await shell(`gcloud run deploy ${serviceName} --quiet --source . --region ${region} --allow-unauthenticated`, true, true) 94 | const apiKeyFile = `${__dirname}/apiKey.txt` 95 | await copyFile(apiKeyFile, dest) 96 | const serverURL = await getServerURL(serviceName, region, projectId) 97 | console.log("Server deployed to ", serverURL) 98 | const schemaWindows = getSchema('win32', serverURL) 99 | const schemaMac = getSchema('darwin', serverURL) 100 | console.log("Copying API key, Instructions, and Schemas to Build folder") 101 | await fss.writeFile(path.join(dest, 'Schema-Windows.json'), JSON.stringify(schemaWindows, null, ' '), 'utf-8') 102 | await fss.writeFile(path.join(dest, 'Schema-MacOS.json'), JSON.stringify(schemaMac, null, ' '), 'utf-8') 103 | const serverURLFile = path.join(dest, 'serverURL.txt') 104 | const firebaseConfigFile = path.join(__dirname, 'firebaseConfig.json') 105 | await fss.writeFile(serverURLFile, serverURL, 'utf-8') 106 | console.log("Copying API key, server url, and firebase configuration to Assistant") 107 | await copyFile(apiKeyFile, `${__dirname}/../Assistant`) 108 | await copyFile(serverURLFile, `${__dirname}/../Assistant`) 109 | await copyFile(firebaseConfigFile, `${__dirname}/../Assistant`) 110 | process.exit(0) 111 | } 112 | 113 | main().catch(err => { 114 | console.error(err) 115 | }) 116 | 117 | 118 | -------------------------------------------------------------------------------- /Cloud/buildSchema.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const fss = require('fs/promises') 3 | const path = require('path') 4 | const os = require('os') 5 | const { getSchema } = require('./Schema.js') 6 | 7 | const copyFile = async (source, destination) => { 8 | return await fss.copyFile(source, path.join(destination, path.basename(source))) 9 | } 10 | const mkdir = async pathname => { 11 | if (!await fileExists(pathname)) { 12 | await fss.mkdir(pathname, { recursive: true }) 13 | } 14 | } 15 | 16 | const main = async (serverURL) => { 17 | if (!serverURL) { 18 | const dest = `${__dirname}/../Build` 19 | const serverURLFile = path.join(dest, 'serverURL.txt') 20 | serverURL = (await fss.readFile(serverURLFile, 'utf-8')).trim() 21 | console.log("read server url", serverURL) 22 | } 23 | const dest = `${__dirname}/../Build` 24 | const instructions = await fss.readFile('./Instructions.txt', 'utf-8') 25 | const getInstructions = platform => { 26 | return instructions.replace('$platform$', platform) 27 | } 28 | const windowsInstructions = getInstructions("Windows") 29 | const macInstructions = getInstructions("Mac") 30 | await fss.writeFile(path.join(dest, 'Instructions-Windows.txt'), windowsInstructions, 'utf-8') 31 | await fss.writeFile(path.join(dest, 'Instructions-MacOS.txt'), macInstructions, 'utf-8') 32 | const schemaWindows = getSchema('win32', serverURL) 33 | const schemaMac = getSchema('darwin', serverURL) 34 | await fss.writeFile(path.join(dest, 'Schema-Windows.json'), JSON.stringify(schemaWindows, null, ' '), 'utf-8') 35 | await fss.writeFile(path.join(dest, 'Schema-MacOS.json'), JSON.stringify(schemaMac, null, ' '), 'utf-8') 36 | console.log("Copied Instructions and Schemas to Build folder") 37 | } 38 | 39 | main(process.argv[2]) 40 | -------------------------------------------------------------------------------- /Cloud/createApiKey.js: -------------------------------------------------------------------------------- 1 | const { getAdmin } = require('./GetAdmin.js') 2 | const crypto = require('crypto'); 3 | const fs = require('fs') 4 | const fss = require('fs/promises') 5 | 6 | const generateId = (length = 40) => { 7 | return crypto.randomBytes(length).toString('hex'); 8 | } 9 | 10 | const createApiKey = async () => { 11 | const filename = './apiKey.txt' 12 | let uid 13 | if (!fs.existsSync(filename)) { 14 | uid = generateId() 15 | await fss.writeFile(filename, uid, 'utf-8') 16 | console.log("Created api key.") 17 | } else { 18 | uid = await fss.readFile(filename, 'utf-8') 19 | console.log("api key already exists.") 20 | } 21 | const admin = getAdmin() 22 | let user 23 | try { 24 | user = await admin.auth().getUser(uid) 25 | console.log("user already exists: ", user.uid) 26 | } catch (err) { 27 | admin.auth().createUser({ 28 | uid, 29 | displayName: "GPT", 30 | }) 31 | } 32 | } 33 | 34 | module.exports = { createApiKey } 35 | -------------------------------------------------------------------------------- /Cloud/createServiceAccount.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs').promises; 2 | const { shell } = require('./Shell.js') 3 | 4 | const createServiceAccount = async projectId => { 5 | 6 | const SERVICE_ACCOUNT_NAME = "ServiceAccount"; 7 | const DISPLAY_NAME = "ServiceAccount"; 8 | const ROLE = "roles/firebase.admin"; 9 | const KEY_PATH = "./ServiceAccount.json"; 10 | 11 | await shell("gcloud config set project "+projectId) 12 | 13 | // Check if the key file already exists 14 | try { 15 | const content = await fs.readFile(KEY_PATH, 'utf-8') 16 | if (content) { 17 | console.log(`Service Account key for '${projectId}' already exists at '${KEY_PATH}'`); 18 | return; // Exit if file exists 19 | } 20 | } catch (error) { 21 | // File does not exist, proceed 22 | } 23 | 24 | const SERVICE_ACCOUNT_EMAIL = `${SERVICE_ACCOUNT_NAME}@${projectId}.iam.gserviceaccount.com`; 25 | 26 | // Check if the service account already exists 27 | const serviceAccountsList = await shell(`gcloud iam service-accounts list --project="${projectId}" --filter="email=${SERVICE_ACCOUNT_EMAIL}" --format="value(email)"`); 28 | console.log("service account list", serviceAccountsList) 29 | if (serviceAccountsList.includes(SERVICE_ACCOUNT_EMAIL)) { 30 | console.log(`Service account ${SERVICE_ACCOUNT_EMAIL} already exists.`); 31 | } else { 32 | // Create a service account 33 | console.log("Creating service account") 34 | await shell(`gcloud iam service-accounts create "${SERVICE_ACCOUNT_NAME}" --display-name="${DISPLAY_NAME}" --project="${projectId}"`); 35 | } 36 | 37 | // Check if the service account already has the firebase/admin role 38 | const hasRole = await shell(`gcloud projects get-iam-policy "${projectId}" --flatten="bindings[].members" --format="table(bindings.role,bindings.members)" --filter="bindings.members:serviceAccount:${SERVICE_ACCOUNT_EMAIL} AND bindings.role:${ROLE}"`); 39 | if (hasRole.includes(ROLE)) { 40 | console.log(`Service account ${SERVICE_ACCOUNT_EMAIL} already has the ${ROLE} role.`); 41 | } else { 42 | // Assign roles to the service account 43 | console.log("Assigning firebase/admin role to service account") 44 | await shell(`gcloud projects add-iam-policy-binding "${projectId}" --member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" --role="${ROLE}"`); 45 | } 46 | // Generate a new private key for the service account 47 | console.log("Creating service account key") 48 | await shell(`gcloud iam service-accounts keys create "${KEY_PATH}" --iam-account="${SERVICE_ACCOUNT_EMAIL}" --project="${projectId}"`); 49 | console.log(`Service account key for project '${projectId}' created at '${KEY_PATH}'`); 50 | } 51 | 52 | module.exports = { createServiceAccount } 53 | -------------------------------------------------------------------------------- /Cloud/firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "firestore": { 3 | "rules": "firestore.rules", 4 | "indexes": "firestore.indexes.json" 5 | } 6 | // Include other configurations here as needed 7 | } 8 | -------------------------------------------------------------------------------- /Cloud/firestore.indexes.json: -------------------------------------------------------------------------------- 1 | { 2 | // Example: 3 | // 4 | // "indexes": [ 5 | // { 6 | // "collectionGroup": "widgets", 7 | // "queryScope": "COLLECTION", 8 | // "fields": [ 9 | // { "fieldPath": "foo", "arrayConfig": "CONTAINS" }, 10 | // { "fieldPath": "bar", "mode": "DESCENDING" } 11 | // ] 12 | // }, 13 | // 14 | // "fieldOverrides": [ 15 | // { 16 | // "collectionGroup": "widgets", 17 | // "fieldPath": "baz", 18 | // "indexes": [ 19 | // { "order": "ASCENDING", "queryScope": "COLLECTION" } 20 | // ] 21 | // }, 22 | // ] 23 | // ] 24 | "indexes": [], 25 | "fieldOverrides": [] 26 | } -------------------------------------------------------------------------------- /Cloud/firestore.rules: -------------------------------------------------------------------------------- 1 | rules_version = '2'; 2 | service cloud.firestore { 3 | match /databases/{database}/documents { 4 | 5 | match /UserChannel/{uid} { 6 | allow read,write: if (request.auth.uid == uid) 7 | } 8 | 9 | match /UserChannel/{uid}/toolCalls/{id} { 10 | allow read,write,delete: if (request.auth.uid == uid) 11 | } 12 | 13 | match /UserChannel/{uid}/taskNotes/{id} { 14 | allow read,write,delete: if (request.auth.uid == uid) 15 | } 16 | 17 | match /GPT/{uid} { 18 | allow read,write: if (request.auth.uid == uid) 19 | } 20 | } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /Cloud/getFirebaseConfig.js: -------------------------------------------------------------------------------- 1 | const fss = require('fs/promises') 2 | const { shell } = require('./Shell.js') 3 | 4 | const getFirebaseConfig = async () => { 5 | let commandOutput = await shell('firebase apps:list') 6 | const apps = commandOutput.split('\n').filter(line => line.includes('WEB')) 7 | let appId 8 | if (apps.length === 0) { 9 | commandOutput = await shell('firebase apps:create WEB "Client"') 10 | /* 11 | 🎉🎉🎉 Your Firebase WEB App is ready! 🎉🎉🎉 12 | 13 | App information: 14 | - App ID: 1:680626401476:web:16567d572e3a47a4bfd991 15 | - Display name: My Web App 16 | 17 | You can run this command to print out your new app's Google Services config: 18 | firebase apps:sdkconfig WEB 1:680626401476:web:16567d572e3a47a4bfd991 19 | */ 20 | console.log("creating firebase web app") 21 | const comps = commandOutput.trim().split(' ') 22 | appId = comps[comps.length-1] 23 | } else { 24 | for (const app of apps) { 25 | const comps = app.split('│') 26 | const name = comps[1].trim() 27 | const id = comps[2].trim() 28 | console.log("firebase web app exists: ", name) 29 | console.log({name, id}) 30 | appId = id 31 | break 32 | } 33 | } 34 | const config = await shell('firebase apps:sdkconfig WEB ' + appId) 35 | const start = config.indexOf('{') 36 | const end = config.lastIndexOf('}') + 1 37 | const json = config.substring(start, end) 38 | await fss.writeFile('./firebaseConfig.json', json + '\n', 'utf-8') 39 | console.log('wrote firebaseConfig.json') 40 | } 41 | 42 | module.exports = { getFirebaseConfig } 43 | -------------------------------------------------------------------------------- /Cloud/getServerURL.js: -------------------------------------------------------------------------------- 1 | const { shell } = require('./Shell.js') 2 | 3 | const getServerURL = async (serviceName, region, projectId) => { 4 | return (await shell(`gcloud run services describe ${serviceName} --platform managed --region ${region} --project ${projectId} --format="value(status.url)"`)).trim() 5 | } 6 | 7 | module.exports = { getServerURL } 8 | -------------------------------------------------------------------------------- /Cloud/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "autocomplete", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "server.js", 6 | "scripts": { 7 | "start": "node server.js", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "engines": { 11 | "node": "20" 12 | }, 13 | "dependencies": { 14 | "body-parser": "^1.20.1", 15 | "cookie-parser": "^1.4.6", 16 | "cors": "^2.8.5", 17 | "express": "^4.19.1", 18 | "firebase-admin": "^12.0.0" 19 | }, 20 | "private": true 21 | } 22 | -------------------------------------------------------------------------------- /Cloud/server.js: -------------------------------------------------------------------------------- 1 | const { getAdmin } = require('./GetAdmin.js') 2 | const { gptAction } = require('./GPTAction.js') 3 | const fss = require('fs/promises') 4 | const cors = require('cors') 5 | const express = require('express'); 6 | const bodyParser = require('body-parser'); 7 | const cookieParser = require('cookie-parser'); 8 | const app = express() 9 | app.use(cors()) 10 | app.use(express.json()) 11 | app.use(cookieParser()) 12 | app.use(bodyParser.urlencoded({ extended: true })); 13 | 14 | app.post('/gpt/:action', async (req, rsp) => { 15 | const token = req.get('authorization').split(' ')[1] 16 | const action = req.params.action 17 | await gptAction(token, action, req.body, req, rsp) 18 | }) 19 | 20 | app.post('/gptLogin', async (req, rsp) => { 21 | const { apiKey } = req.body 22 | const uid = (await fss.readFile('./apiKey.txt', 'utf-8')).trim() 23 | if (uid === apiKey) { 24 | const admin = getAdmin() 25 | const customToken = await admin.auth().createCustomToken(uid) 26 | return rsp.status(200).send({ customToken }) 27 | } 28 | return rsp.status(401).send() 29 | }) 30 | 31 | const port = parseInt(process.env.PORT) || 8080 32 | app.listen(port, () => { 33 | console.log(`server.js: listening on port ${port}`); 34 | }); 35 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [2024] [Attunewise.ai] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ![Screenshot 2024-03-20 214951](https://github.com/Attunewise/GPT/assets/164077948/6a201658-f403-4a1f-81f7-73288fd94cb1) 3 | 4 | Ask questions on our [Discord](https://discord.gg/2Mkp2b22) 5 | 6 | This project enables you to host an OpenAI [GPT](https://chat.openai.com/gpts) that can act as an agent on your desktop computer. 7 | The benefit of this approach compared to other agent frameworks is that OpenAI pays for the tokens and you don't have to pay for the [expensive GPT-4 API](https://openai.com/pricing). 8 | 9 | Using GPT-4 with tools has many pitfalls and failures, which few people are aware of due to limited access to testing it on realistic use cases. The purpose of this project is to enable such testing. All you need is a paid OpenAI account which is $20/mo. 10 | 11 | The GPT is equipped with the following tools: 12 | 1. Ability to use the terminal and run shell commands 13 | 2. Ability to run apple script on MacOS to automate applications such as Mail, Calendar, and others 14 | 3. Ability to query the chat.db on MacOS for messages 15 | 4. Ability to write to files 16 | 5. Ability to automate a web browser shared with the user 17 | 6. Ability to search the web and individual web pages using the web browser shared with the user 18 | 7. Ability to save, search, and delete task notes to optimize tasks 19 | 8. Ability to get information about the user including name, home directory, geo location, time zone, host and OS information 20 | 9. Ability to evaluate Javascript code in node.js 21 | 22 | And, of course, since you have the source code you can add or modify tools yourself. 23 | 24 | **For security reasons you must run your own personal server for this GPT.** 25 | 26 | To avoid costs, the server will be deployed using google cloud run and it will route calls from the GPT to the client running on your machine via google cloud Firestore. This setup avoids a dedicated cloud cpu that you'd have to pay for. 27 | 28 | A unique API key will be generated for use by the GPT and a copy of it will be used by your server and client. Do not share it with anybody else. 29 | 30 | Note: this implies you must trust both OpenAI and Google Cloud to protect the security of your server and client and the privacy of information that you share with the GPT. Given the massive number of users they both serve you presumably can. 31 | 32 | You must first create a firebase project using the Google firebase console 33 | 34 | #### **Create a New Firebase Project:** 35 | 36 | 1. **Go to the Firebase Console:** Open your web browser and navigate to the [Firebase Console](https://console.firebase.google.com/). You'll need to log in with your Google account if you haven't already. 37 | 38 | 2. **Add a Project:** Click on the “Add project” button to start the process of creating a new project. 39 | 40 | 3. **Project Details:** Enter a name for your project in the designated field. This name is how you'll identify your project in the Firebase console. 41 | 42 | 4. **Google Analytics (Optional):** Disable Google Analytics for your project. 43 | 44 | 5. **Create Project:** After configuring your project's settings, click on the “Create project” button. Firebase will take a few moments to prepare your project. 45 | 46 | 6. **Enable the Blaze pay as you go billing plan:** In order to deploy your server to cloud run, you'll need to enable pay as you go billing. This does not mean you will actually incur any costs. You can tap on the "Spark Plan" button on the getting started page to do this. Or tap the gear button on the sidebar and select "Usage and billing", and then select the "Details and settings" tab. You can set a budget alert to ensure you do not incur charges. 47 | 48 | #### **Add Firebase Authentication to Your Project:** 49 | 1. In the left-hand menu, click on "Authentication" to open the Authentication section. 50 | 2. Tap the "Get Started" Button 51 | 3. Do not enable any sign-in providers as the GPT client will use custom authentication with the API key. 52 | 53 | #### **Add Firestore to Your Project:** 54 | 55 | 1. **Navigate to Firestore Database:** Once your project dashboard is ready, look for the “Firestore Database” option in the left-hand menu and click on it. 56 | 57 | 2. **Create Database:** Click on the “Create database” button to start setting up Firestore for your project. 58 | 59 | 3. **Choose Database Mode:** You'll be prompted to select a mode for your Firestore Database. You can choose between “Production mode” and “Test mode”. Choose Production. 60 | 61 | 4. **Select a Location:** Choose a location for your Firestore Database. This determines where your data is stored and can affect latency for your users, so pick a location that is close to you as possible. If undecided, choose us-central. 62 | 63 | 5. **Finalize:** Click on “Done” to finalize the creation of your Firestore database. It might take a few moments for the database to be fully set up. 64 | 65 | **Install Node.js if you don't have it already:** 66 | 67 | Both the GPT server and GPT client use Node.js 68 | 69 | * Visit the official Node.js website at [https://nodejs.org/](https://nodejs.org/). 70 | * You will see two versions available for download: LTS (Long Term Support) and Current. The LTS version is recommended for most users as it is more stable. Click on the LTS version to download the installer. 71 | 72 | Install the Google Cloud CLI (also known as the `gcloud` command-line tool) if you haven't already 73 | 74 | [Follow the instructions here](https://cloud.google.com/sdk/docs/install) 75 | 76 | #### **Build and run your server** 77 | 78 | Node.js and the firebase and gcloud cli tools will be used to build and deploy your server to cloud run and configure your firebase project and firestore database for use by this GPT. 79 | Additionally, it will create the following files containing information you will need to provide to OpenAI in the OpenAI GPT Editor of your GPT in the output folder `Build` 80 | 81 | 1. `apiKey.txt` - The api key for the GPT authentication 82 | 2. `Instructions-Windows.txt` - Instructions for the system prompt of your GPT on Windows. 83 | 3. `Instructions-MacOS.txt` - Instructions for the system prompt of your GPT on Mac. 84 | 4. `Schema-Windows.json` - Schema for the actions provided by your GPT on Windows. 85 | 5. `Schema-MacOS.json` - Schema for the actions provided by your GPT on a Mac. 86 | 6. `serverURL.txt` - the url of your deployed server 87 | 88 | To build and deploy your server follow these steps: 89 | 1. Open a terminal and navigate to the `Cloud` folder. 90 | 2. Run ```npm install -g firebase-tools``` to install the firebase cli, if you haven't already. 91 | 2. Run ```firebase login``` to log in to the firebase cli, if you haven't already 92 | 3. Run ```firebase use ``` to select the project you created (you can use `firebase projects:list` to see all your projects) 93 | 3. Run the following command: 94 | ``` 95 | node build.js 96 | ``` 97 | This command will 98 | 1. Build the project with `npm` 99 | 1. Set up the firebase project 100 | 2. Configure security for your firestore database 101 | 3. Create a unique api key and associated firebase account 102 | 4. Enable required google deployment services on your project: artifact registry, cloud build, cloud run 103 | 5. Deploy the server using google cloud run 104 | 6. Generate the output files 105 | 106 | Next you must create your GPT using OpenAI's [GPT editor](https://chat.openai.com/gpts/editor) 107 | Create a new GPT, then select the Configure tab and perform the following steps: 108 | 1. Copy the contents of `Instructions-Windows.txt` or `Instructions-MacOS.txt` into the Instructions field that says "What does this GPT do etc" 109 | 2. Turn off the Web Browsing capability as the GPT has its own web browser 110 | 3. Tap the Create new action button 111 | 4. Select the Authentication drop-down and choose Authentication type "API Key" and Auth Type "Bearer" and copy and paste contents of `apiKey.txt` into the API Key field, then Save. 112 | 5. Copy and paste the content of `Schema-Windows.json` or `Schema-MacOS.json` into the text area where it says "Enter you OpenAPI schema here" 113 | 6. Press the back button, then scroll down to the bottom of the Configure tab and open "Additional Settings" and disable "Use conversation data in your GPT to improve our models" 114 | 7. Create and save for yourself only (DO NOT SHARE OR MAKE THIS GPT PUBLIC AS IT HAS ACCESS TO YOUR COMPUTER) 115 | 116 | Now you can run the client 117 | 1. Navigate to the `Assistant` folder 118 | 2. Create a text file called 'gptURL.txt' in this folder and store the url of your GPT in it (it will be something like 'https://chat.openai.com/g/g-btPQRDlHM-name') 119 | 3. Run the following command to build the client 120 | ``` 121 | npm install 122 | ``` 123 | 4. Run the following command to run the client 124 | ``` 125 | npm start 126 | ``` 127 | 128 | The first time you run it you will land on the OpenAI sign in page. Once you sign in you will redirected to your GPT. 129 | 130 | NOTE: be sure to keep the following generated files private: 131 | 132 | 1. `{Cloud,Build,Assistant}/apiKey.txt` 133 | 2. `Cloud/ServiceAccount.json` 134 | 3. `{Cloud,Assistant}/firebaseConfig.json` 135 | 136 | Once your server is deployed, you can modify the schema in your GPT and make changes to the Assistant app without redeploying the server. 137 | 138 | Don't be surprised when GPT-4 hallucinates false information and/or incorrect code. It talks a big game but when it comes down to actually doing it, it usually fails and moonwalks it all back. And in case you're wondering, the same occurs with Claude-3 Opus. 139 | 140 | 141 | Experimental use of plugin tools: 142 | We've added a method for plugging in additional tools and as an example provided a plugin for VSCode 143 | You can add new tools by adding a folder under `Assistant/Plugin/plugins`. The folder must contain the following files: 144 | 1. `config.json` 145 | 2. `Tool.js` 146 | 147 | `config.json` provides the name, whether the plugin is enabled, the platforms it supports, and the tools it supports in the form of function definitions according to the OpenAI function calling specification. Here is an example: 148 | ``` 149 | { 150 | "name": "VScode", 151 | "enabled": true, 152 | "platforms": ["windows", "macos"], 153 | "tools": [{ 154 | "type": "function", 155 | "function": { 156 | "name": "vscode", 157 | "description": "Execute javascript code inside of a Visual Studio Code (VSCode) extension. You can safely access all capabilities of VSCode via its extensibilty API. This allows you to assist the user in all forms of software development.", 158 | "parameters": { 159 | "type": "object", 160 | "properties": { 161 | "javascript": { 162 | "type": "string", 163 | "description": "The javascript code to execute." 164 | } 165 | }, 166 | "required": ["javascript"] 167 | } 168 | } 169 | }] 170 | } 171 | ``` 172 | 173 | `Tool.js` must export a single function `call` which will take two arguments, `name`, and `args` corresponding to the function call. It must return an object containing two properties, `content` which is a string containing the output of the function call and `isError` which is a boolean. If isError is true, then `content` should contain the error message. 174 | 175 | The VSCode plugin uses a websocket server to communicate with the plugin running inside of VSCode. You can use this framework for other plugins, for example we've used it with `ExtendScript` and `UXP` plugins for Adobe Creative Suite. 176 | 177 | Once you've enabled a plugin, you can generate new JSON action schemas for its tools by navigating to the `Assistant` folder, and running 178 | 179 | ``` 180 | node installPlugins.js 181 | ``` 182 | 183 | This will generate `Schema-Windows-with-plugins.json` and `Schema-MacOS-with-plugins.json` in the `Build` folder which you can then copy and paste into your GPT configuration in the OpenAI GPT editor. 184 | --------------------------------------------------------------------------------