├── doc ├── html.png └── accessibility_tree.png ├── package.json ├── .gitignore ├── LICENSE ├── gpt.ts ├── README.md ├── gpt-aria.ts ├── browser.ts └── prompt.ts /doc/html.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thegpvc/gpt-aria/HEAD/doc/html.png -------------------------------------------------------------------------------- /doc/accessibility_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thegpvc/gpt-aria/HEAD/doc/accessibility_tree.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gpt-aria", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "dependencies": { 7 | "exponential-backoff": "^3.1.0", 8 | "openai-api": "1.3.1", 9 | "puppeteer": "^19.6.1", 10 | "tsx": "^3.12.2", 11 | "yargs": "^17.7.1" 12 | }, 13 | "devDependencies": { 14 | "@types/yargs": "^17.0.22" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | log.txt 3 | # Created by https://www.toptal.com/developers/gitignore/api/macos 4 | # Edit at https://www.toptal.com/developers/gitignore?templates=macos 5 | 6 | ### macOS ### 7 | # General 8 | .DS_Store 9 | .AppleDouble 10 | .LSOverride 11 | 12 | # Icon must end with two \r 13 | Icon 14 | 15 | 16 | # Thumbnails 17 | ._* 18 | 19 | # Files that might appear in the root of a volume 20 | .DocumentRevisions-V100 21 | .fseventsd 22 | .Spotlight-V100 23 | .TemporaryItems 24 | .Trashes 25 | .VolumeIcon.icns 26 | .com.apple.timemachine.donotpresent 27 | 28 | # Directories potentially created on remote AFP share 29 | .AppleDB 30 | .AppleDesktop 31 | Network Trash Folder 32 | Temporary Items 33 | .apdisk 34 | 35 | ### macOS Patch ### 36 | # iCloud generated files 37 | *.icloud 38 | 39 | # End of https://www.toptal.com/developers/gitignore/api/macosnode_modules/ 40 | node_modules/** 41 | google-chrome/** 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Nat Friedman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /gpt.ts: -------------------------------------------------------------------------------- 1 | import { promises as fs } from "fs"; 2 | import OpenAI, { Completion } from 'openai-api' 3 | import { ObjectiveState } from "./prompt"; 4 | import { backOff, BackoffOptions } from "exponential-backoff"; 5 | 6 | export class GPTDriver { 7 | private lastAttemptCount = 0 8 | private OPENAI_API_KEY: string 9 | 10 | constructor(OPENAI_API_KEY: string) { 11 | this.OPENAI_API_KEY = OPENAI_API_KEY 12 | } 13 | 14 | async prompt(state: ObjectiveState): Promise<[string, string]> { 15 | let promptTemplate = await fs.readFile("prompt.ts", "utf8") 16 | let prefix = '{"progressAssessment":' 17 | let prompt = promptTemplate.trim() 18 | .replace("$objective", (state.objective)) 19 | .replace("$url", (state.url)) 20 | .replace('"$output"}})', '') 21 | .replace('$ariaTreeJSON', state.ariaTree) 22 | // .replace('"$browserError"', state.browserError ? JSON.stringify(state.browserError) : 'undefined') 23 | .replace('["$objectiveProgress"]', JSON.stringify(state.progress)) 24 | ; 25 | return [prompt, prefix] 26 | } 27 | 28 | async askCommand(prompt:string): Promise<[Completion, string]> { 29 | const openai = new OpenAI(this.OPENAI_API_KEY); 30 | 31 | const suffix = '})' 32 | let self = this 33 | const backOffOptions: BackoffOptions = { 34 | // try to delay the first attempt if we had fails in previous runs 35 | delayFirstAttempt: !!this.lastAttemptCount, 36 | startingDelay: (Math.max(0, this.lastAttemptCount - 1) + 1) * 100, 37 | retry: (e: any, attemptNumber: number) => { 38 | self.lastAttemptCount = attemptNumber 39 | console.warn(`Retry #${attemptNumber} after openai.complete error ${e}`) 40 | return true; 41 | } 42 | } 43 | const completion = await backOff(() => { 44 | return openai.complete({ 45 | engine: "text-davinci-003", 46 | prompt: prompt, 47 | maxTokens: 256, 48 | temperature: 0.5, 49 | bestOf: 10, 50 | n: 3, 51 | suffix: suffix, 52 | stop: suffix, 53 | }) 54 | }, backOffOptions); 55 | 56 | return [completion,suffix]; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gpt-aria 2 | 3 | Experiment to teach gpt to make use of the chrome accessibility tree to turn the web into a textual interface and access it like a user of a screen-reader. This avoids html parsing, supports dynamic content, etc. 4 | 5 | https://taras.glek.net/post/gpt-aria-experiment/ 6 | 7 | Running: 8 | 9 | * Install node 10 | * run `npm install` 11 | * run `export OPENAI_API_KEY=` 12 | * Run gpt-aria: `./gpt-aria.ts --objective "Whats the price of iphone 13 pro"` 13 | * Note first run will take a while as puppeteer has to download chrome 14 | * Run it starting custom start page: `./gpt-aria.ts --objective "Whats the price of iphone 14 pro" --start-url https://duckduckgo.com` 15 | 16 | Prompt lives in `prompt.ts`, log of execution is in `log.txt` 17 | 18 | Questions? @tarasglek on twitter or file github issues 19 | 20 | Sample queries: 21 | * `./gpt-aria.ts --objective "What is the cultural capital of western ukraine" --start-url https://bing.com --headless` 22 | * `./gpt-aria.ts --objective "Who was king of england when lviv was founded" --headless` 23 | * who was president when first starwars was released? 24 | 25 | # Design 26 | ```mermaid 27 | graph TD; 28 | subgraph GPT 29 | gpt["decide if enough info\nto return an ObjectiveComplete\nor if a BrowserAction is needed"] 30 | end 31 | subgraph gpt-aria 32 | BrowserAction 33 | ObjectiveComplete 34 | BrowserResponse 35 | end 36 | gpt-->ObjectiveComplete 37 | gpt--command-->BrowserAction 38 | BrowserAction--"command"-->Browser 39 | Browser --"url,ariaTree"--> BrowserResponse 40 | UserInput --"{objective, start-url}"--> BrowserAction 41 | BrowserResponse --"objective,progress[],url,ariaTree"--> gpt 42 | ObjectiveComplete--"result"--> UserOutput 43 | ``` 44 | 45 | # Prior art: 46 | * https://github.com/nat/natbot 47 | * https://yihui.dev/actgpt 48 | 49 | # Why ARIA is superior to raw html 50 | 51 | ### html: 52 | ![html](doc/html.png?raw=true "HTML is only good for renders") 53 | 54 | ### html with ARIA accessibility tree: 55 | ![accessibility_tree](doc/accessibility_tree.png?raw=true "HTML is only good for renders") 56 | 57 | ## Follow-up ideas 58 | ### Scrolling 59 | 60 | Would be nice to have a command so gpt could scroll up/down the page to summarize content in it 61 | 62 | ### Have an index of gpt prompts that explain in natural language how to navigate a particular website. 63 | * Eg for twitter it was say "In order to 'tweet', one goes to twitter.com and posts a tweet. in order to scan the latest news on twitter, one can pick use default timeline or pick a twitter list for a particular category". 64 | * For buying a house "redfin provides search functionality and ability to narrow down location and prices" 65 | * for shopping "amazon.com is a shopping site" 66 | * likewise for google, wikipedia, etc 67 | * eventually we'd want langchain-style website modules so you could specify "Summarize my inbox and news" which would be a composition of gmail and news modules 68 | -------------------------------------------------------------------------------- /gpt-aria.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node --loader tsx 2 | import { Browser } from "./browser"; 3 | import { GPTDriver } from "./gpt" 4 | import { promises as fs } from "fs"; 5 | import { ActionStep } from "./prompt"; 6 | import yargs from 'yargs/yargs'; 7 | 8 | (async () => { 9 | const argv = yargs(process.argv.slice(2)).options({ 10 | objective: { type: 'string', demandOption: true, description: 'Natural language objective for gpt-aria to complete' }, 11 | "start-url": { default: 'https://google.com/?hl=en' }, 12 | "log-output": { default: 'log.txt' }, 13 | "openai-api-key": { type: 'string', default: process.env.OPENAI_API_KEY , demandOption: true, description: 'OpenAI.com API key. Can also be set via OPENAI_API_KEY environment variable'}, 14 | "headless": { type: 'boolean', default: false, description: 'Run in headless mode (no browser window)' }, 15 | }). 16 | usage('Usage: $0 --objective [--start-url ]'). 17 | parseSync(); 18 | 19 | const browser = await Browser.create(argv.headless); 20 | const gpt = new GPTDriver(argv["openai-api-key"]!); 21 | const logFile = "log.txt" 22 | let fd = await fs.open(logFile, "w") 23 | console.log(`logging to ${logFile}`) 24 | async function log(info) { 25 | await fs.appendFile(fd, info) 26 | } 27 | 28 | const startUrl = argv["start-url"] 29 | await browser.goTo(startUrl); 30 | let objectiveProgress = [] as string[]; 31 | do { 32 | const state = await browser.state(argv.objective, objectiveProgress); 33 | const [prompt, prefix] = await gpt.prompt(state) 34 | let trimmed_prompt = prompt.split('// prompt //', 2)[1].trim() 35 | let interaction = trimmed_prompt + "\n////////////////////////////\n" 36 | await log(interaction) 37 | const [completions, _suffix] = await gpt.askCommand(prompt) 38 | log(JSON.stringify(completions.data.choices[0])) 39 | // filter debug a bit 40 | let debugChoices = [] as string[] 41 | for (let choice of completions.data.choices) { 42 | delete (choice as any)['index'] 43 | delete choice['logprobs'] 44 | let json = JSON.stringify(choice) 45 | let json_debug = "DEBUG:" + json + "\n" 46 | if (debugChoices.length && debugChoices[debugChoices.length - 1] == json_debug) { 47 | continue 48 | } 49 | debugChoices.push(json_debug) 50 | } 51 | log(debugChoices.join("")) 52 | let responseObj: ActionStep | undefined = undefined 53 | for (const choice of completions.data.choices) { 54 | let response = prefix + choice.text //+ suffix 55 | try { 56 | responseObj = JSON.parse(response) 57 | break 58 | } catch (e) { 59 | console.error("invalid JSON:" + response) 60 | continue 61 | } 62 | } 63 | if (!responseObj) { 64 | console.error("Did not receive a valid response") 65 | process.exit(1) 66 | } 67 | objectiveProgress.push(responseObj.description); 68 | interaction = JSON.stringify(responseObj) 69 | await log(interaction) 70 | if (responseObj.command.kind === "ObjectiveComplete") { 71 | console.log("Objective:" + argv.objective) 72 | console.log("Objective Progress:") 73 | console.log(objectiveProgress.join("\n")) 74 | console.log("Progress Assessment:") 75 | console.log(responseObj.progressAssessment) 76 | console.log("Result:") 77 | console.log(responseObj.command.result) 78 | process.exit(0) 79 | } else { 80 | console.log(responseObj) 81 | await browser.performAction(responseObj.command) 82 | } 83 | } while (true); 84 | })(); 85 | -------------------------------------------------------------------------------- /browser.ts: -------------------------------------------------------------------------------- 1 | import puppeteer, { Browser as PuppeteerBrowser, ElementHandle, Page, SerializedAXNode } from "puppeteer"; 2 | import { AccessibilityTree, BrowserAction, ObjectiveState } from "./prompt"; 3 | import { MAIN_WORLD } from "puppeteer"; 4 | 5 | export class Browser { 6 | private browser: PuppeteerBrowser; 7 | private page: Page; 8 | private idMapping = new Map() 9 | private error?: string 10 | 11 | constructor() { 12 | } 13 | 14 | private async init(headless: boolean) { 15 | this.browser = await puppeteer.launch({ 16 | headless: headless, 17 | userDataDir: "google-chrome", 18 | }); 19 | this.page = await this.browser.newPage(); 20 | let self = this 21 | // this helps us work when links are opened in new tab 22 | this.browser.on('targetcreated', async function(target){ 23 | let page = await target.page() 24 | if (page) { 25 | self.page = page 26 | } 27 | }) 28 | } 29 | 30 | async state(objective: string, objectiveProgress: string[], limit=4000): Promise { 31 | let contentJSON = await this.parseContent() 32 | let content: ObjectiveState = { 33 | url: this.url().replace(/[?].*/g, ""), 34 | ariaTree: contentJSON.substring(0, limit), 35 | progress: objectiveProgress, 36 | // error: this.error, 37 | objective: objective 38 | } 39 | return content 40 | } 41 | 42 | async performAction(command: BrowserAction) { 43 | this.error = undefined 44 | try { 45 | if (command.index !== undefined) { 46 | let e = await this.findElement(command.index) 47 | // cause text to get selected prior to replacing it(instead of appending) 48 | if (command.params) { 49 | await e.click({ clickCount: 3 }) 50 | await new Promise(resolve => setTimeout(resolve, 100)); 51 | await e.type(command.params[0] as string + "\n") 52 | } else { 53 | await e.click() 54 | await new Promise(resolve => setTimeout(resolve, 100)); 55 | } 56 | } else { 57 | throw new Error("Unknown command:"+ JSON.stringify(command)); 58 | } 59 | await new Promise(resolve => setTimeout(resolve, 1000)); 60 | } catch (e) { 61 | this.error = e.toString() 62 | console.error(this.error) 63 | } 64 | } 65 | 66 | async goTo(url: string) { 67 | await this.page.goto(url); 68 | await new Promise(resolve => setTimeout(resolve, 1000)); 69 | } 70 | 71 | url(): string { 72 | return this.page.url(); 73 | } 74 | 75 | async parseContent(): Promise { 76 | const tree = await this.getAccessibilityTree(this.page); 77 | this.idMapping = new Map() 78 | let tree_ret = this.simplifyTree(tree) 79 | let ret= JSON.stringify(tree_ret) 80 | return ret 81 | } 82 | 83 | private async getAccessibilityTree(page: Page): Promise { 84 | return await page.accessibility.snapshot({ interestingOnly: true }); 85 | } 86 | 87 | private simplifyTree(node: SerializedAXNode): AccessibilityTree { 88 | switch (node.role) { 89 | case "StaticText": 90 | case "generic": 91 | return node.name! 92 | case "img": 93 | return ["img", node.name!] 94 | default: 95 | break; 96 | } 97 | let index = this.idMapping.size 98 | let e: AccessibilityTree = [index, node.role, node.name!] 99 | this.idMapping.set(index, e) 100 | let children = [] as AccessibilityTree[] 101 | if (node.children) { 102 | const self = this; 103 | children = node.children.map(child => self.simplifyTree(child)) 104 | } else if (node.value) { 105 | children = [node.value] 106 | } 107 | if (children.length) { 108 | e.push(children) 109 | } 110 | return e 111 | } 112 | 113 | private async queryAXTree( 114 | client: CDPSession, 115 | element: ElementHandle, 116 | accessibleName?: string, 117 | role?: string 118 | ): Promise { 119 | const {nodes} = await client.send('Accessibility.queryAXTree', { 120 | objectId: element.remoteObject().objectId, 121 | accessibleName, 122 | role, 123 | }); 124 | const filteredNodes: Protocol.Accessibility.AXNode[] = nodes.filter( 125 | (node: Protocol.Accessibility.AXNode) => { 126 | return !node.role || node.role.value !== 'StaticText'; 127 | } 128 | ); 129 | return filteredNodes; 130 | } 131 | 132 | private async findElement(index:number): Promise> { 133 | let e = this.idMapping.get(index) 134 | let role = e[1] 135 | let name = e[2] 136 | 137 | // console.log(index + " " + role + " " + name) 138 | 139 | let client = (this.page as any)._client(); 140 | const body = await this.page.$("body"); 141 | const res = await this.queryAXTree(client, body, name, role); 142 | if (!res[0] || !res[0].backendDOMNodeId) { 143 | throw new Error(`Could not find element with role ${node.role} and name ${node.name}`); 144 | } 145 | const backendNodeId = res[0].backendDOMNodeId; 146 | 147 | const ret = (await this.page.mainFrame().worlds[MAIN_WORLD].adoptBackendNode(backendNodeId)) as ElementHandle 148 | 149 | if (!ret) { 150 | throw new Error(`Could not find element by backendNodeId with role ${role} and name ${name}`); 151 | } 152 | return ret 153 | } 154 | 155 | static async create(headless: boolean): Promise { 156 | const crawler = new Browser(); 157 | await crawler.init(headless); 158 | return crawler; 159 | } 160 | } -------------------------------------------------------------------------------- /prompt.ts: -------------------------------------------------------------------------------- 1 | type Generic = [number, string, string, AccessibilityTree[]?] 2 | type Content = string | number 3 | type Image = ["img", string] 4 | export type AccessibilityTree = Generic | Content | Image 5 | export type ObjectiveState = { 6 | objective: string, // objective set by user 7 | progress: string[], // summary of previous actions taken towards objective 8 | url: string, // current page url 9 | ariaTree: string //JSON of ariaTree of AccessibilityTree type 10 | } 11 | export type BrowserAction = { 12 | kind: "BrowserAction", 13 | index: number, // index for ariaTree element 14 | params?: string[] // input for combobox, textbox, or searchbox elements 15 | } 16 | export type ObjectiveComplete = { 17 | kind: "ObjectiveComplete", 18 | result: string // objective result in conversational tone 19 | } 20 | export type GptResponse = BrowserAction | ObjectiveComplete // either the next browser action or a final response to the objectivePrompt 21 | export type ActionStep = { 22 | progressAssessment: string, // decide if enough info to return an ObjectiveComplete or if another BrowserAction is needed 23 | command: GptResponse, // action 24 | description: string // brief description of actionCommand 25 | } 26 | /** Function that controls the browser 27 | @returns the next ActionStep 28 | */ 29 | declare function assertNextActionStep(input_output:{objectivestate:ObjectiveState, actionstep:ActionStep}) 30 | /* 31 | For each assertActionNextStep function below, use information from the ObjectiveState within that function only to complete the ActionStep within that function.Only write valid code. 32 | */ 33 | assertNextActionStep({ 34 | objectivestate: { 35 | objective: "how much is an gadget 11 pro", 36 | progress: [], 37 | url: "https://www.google.com/", 38 | ariaTree: `[0,"RootWebArea","Google",[[1,"link","Gmail"],[2,"link","Images"],[3,"button","Google apps"],[4,"link","Sign in"],["img","Google"],[5,"combobox","Search"]` 39 | }, 40 | actionstep: { 41 | "progressAssessment": "Do not have enough information in ariaTree to return an Objective Result.", 42 | "command": {"kind": "BrowserAction", "index": 5, "params": ["gadget 11 pro price"]}, 43 | "description": "Searched `gadget 11 pro price`" 44 | }}) 45 | 46 | assertNextActionStep({ 47 | objectivestate: { 48 | objective: "Who was president when Early Voting won Preakness Stakes", 49 | progress: ["Searched `early voting Preakness Stakes win`"], 50 | url: "https://www.google.com/search", 51 | ariaTree: `[0,"RootWebArea","early voting Preakness Stakes win - Google Search",[[1,"heading","Accessibility Links"],[2,"link","Skip to main content"],[3,"link","Switch to page by page results"],[4,"link","Accessibility help"],[5,"link","Accessibility feedback"],[6,"link","Google"],[7,"combobox","Search",["early voting Preakness Stakes win"]],[8,"button"," Clear"],[9,"button","Search by voice"],[10,"button","Search by image"],[11,"button","Search"],[12,"button","Settings"],[13,"button","Google apps"],[14,"link","Sign in"],[15,"heading","Search Modes"],"All",[16,"link","News"],[17,"link","Images"],[18,"link","Shopping"],[19,"link","Videos"],[20,"button","More"],[21,"button","Tools"],"About 166,000 results"," (0.39 seconds) ",[22,"heading","Search Results"],[23,"heading","Featured snippet from the web"],[24,"button","Image result for early voting Preakness Stakes win"],[25,"heading","Early Voting, a colt owned by the billionaire hedge fund investor Seth Klarman, repelled the challenge of the heavily favored Epicenter to capture the 147th running of the Preakness Stakes.May 21, 2022"],[26,"link"," Early Voting Wins Preakness Stakes - The New York Times https://www.nytimes.com › Sports › Horse Racing"],[27,"button","About this result"]` 52 | }, 53 | actionstep: { 54 | "progressAssessment": "Per search results in ariaTree: Early Voting won Preakness Stakes in 2022. Do not have enough information to return objective result. Now need to find out who was president in 2022", 55 | "command": {"kind": "BrowserAction", "index": 7, "params": ["2022 president"]}, 56 | "description": "Early Voting won Preakness Stakes on `May 21, 2022`. This is a partial answer to `early voting Preakness Stakes win` so searched `2022 president`" 57 | }}) 58 | 59 | assertNextActionStep({ 60 | objectivestate: { 61 | objective: "When was Ted Kennedy Born", 62 | progress: ["Searched `Ted Kennedy born`"], 63 | url: "https://www.google.com/", 64 | ariaTree: `[0,"RootWebArea","Ted Kennedy born - Google Search",[[1,"heading","Accessibility Links"],[2,"link","Skip to main content"],[3,"link","Switch to page by page results"],[4,"link","Accessibility help"],[5,"link","Accessibility feedback"],[6,"link","Google"],[7,"combobox","Search",["Ted Kennedy born"]],[8,"button"," Clear"],[9,"button","Search by voice"],[10,"button","Search by image"],[11,"button","Search"],[12,"button","Settings"],[13,"button","Google apps"],[14,"link","Sign in"],[15,"heading","Search Modes"],"All",[16,"link","Images"],[17,"link","News"],[18,"link","Videos"],[19,"link","Shopping"],[20,"button","More"],[21,"button","Tools"],"About 13,200,000 results"," (0.50 seconds) ",[22,"heading","Search Results"],[23,"heading","Ted Kennedy/Born"],[24,"link","February 22, 1932, Dorchester, Boston, MA"],[25,"button","Image result for Ted Kennedy born"],[26,"button","Feedback"],[27,"link"," Ted Kennedy - Wikipedia https://en.wikipedia.org › wiki › Ted_Kennedy"],[28,"button","About this result"],"Edward Moore Kennedy (","February 22, 1932 – August 25, 2009",") was an American lawyer` 65 | }, 66 | actionstep: { 67 | "progressAssessment": "Per search results in ariaTree: Ted Kennedy was born on February 22, 1932, returning Objective result.", 68 | "command": {"kind": "ObjectiveComplete", "result": "Ted Kennedy was born on February 22, 1932."}, 69 | "description": "Ted Kennedy was born on `February 22, 1932` according to search results. This is a reasonable answer to `When was Ted Kennedy Born` in objectivePrompt." 70 | }}) 71 | 72 | assertNextActionStep({ 73 | objectivestate: { 74 | objective: "", 75 | progress: [], 76 | url: "https://www.google.com/", 77 | ariaTree: `[0,"RootWebArea","Google",[[1,"dialog","Before you continue to Google Search",[["img","Google"],[2,"button","Choose language, en"],[3,"link","Sign in"],[4,"heading","Before you continue to Google"],"We use ",[5,"link","cookies"]," and data to","Deliver and maintain Google services","Track outages can also include more relevant results, recommendations and tailored ads based on past activity from this browser, like previous Google searches. We also use cookies and data to tailor the experience to be age-appropriate, if relevant.","Select 'More options' to see additional information, including details about managing your privacy settings. You can also visit ","g.co/privacytools"," at any time.",[6,"button","Reject all"],[7,"button","Accept all"],[8,"link","More options",["More options"]],[9,"link","Privacy"],[10,"link","Terms"]]]]]`, 78 | }, 79 | actionstep: { 80 | "progressAssessment": "Content in ariaTree wants me to accept terms. I will click accept when nothing more relevant to click.", 81 | "command": {"kind": "BrowserAction", "index" : 7}, 82 | "description": "Clicked Accept" 83 | }}) 84 | 85 | // prompt // 86 | assertNextActionStep({ 87 | objectivestate: { 88 | objective: "$objective", 89 | progress: ["$objectiveProgress"], 90 | url: "$url", 91 | ariaTree: `$ariaTreeJSON`, 92 | }, 93 | actionstep: { 94 | "progressAssessment":"$output"}}) --------------------------------------------------------------------------------