├── .gitignore ├── LICENSE ├── package.json ├── promptAugmentors.ts ├── README.md └── index.ts /.gitignore: -------------------------------------------------------------------------------- 1 | askcli 2 | builds/ 3 | test scripts -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License Copyright (c) 2022 swyx 2 | 3 | Permission is hereby granted, free of 4 | charge, to any person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, copy, modify, merge, 7 | publish, distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to the 9 | following conditions: 10 | 11 | The above copyright notice and this permission notice 12 | (including the next paragraph) shall be included in all copies or substantial 13 | portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO 18 | EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "askcli", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "build": "deno compile --unstable --allow-read --allow-write --allow-net --allow-env --allow-run --output builds/ask index.ts", 8 | "build-all": "npm run build-linux && npm run build-macx86 && npm run build-macm1 && npm run build-windows", 9 | "build-linux": "deno compile --unstable --allow-read --allow-write --allow-net --allow-env --allow-run --target x86_64-unknown-linux-gnu --output builds/ask-linux index.ts", 10 | "build-windows": "deno compile --unstable --allow-read --allow-write --allow-net --allow-env --allow-run --target x86_64-pc-windows-msvc --output builds/ask-windows.exe index.ts", 11 | "build-macx86": "deno compile --unstable --allow-read --allow-write --allow-net --allow-env --allow-run --target x86_64-apple-darwin --output builds/ask-macx86 index.ts", 12 | "build-macm1": "deno compile --unstable --allow-read --allow-write --allow-net --allow-env --allow-run --target aarch64-apple-darwin --output builds/ask-m1 index.ts" 13 | }, 14 | "keywords": [], 15 | "author": "", 16 | "license": "MIT" 17 | } -------------------------------------------------------------------------------- /promptAugmentors.ts: -------------------------------------------------------------------------------- 1 | // ---- stuff to change in future 2 | 3 | export type Augmentor = Partial<{ 4 | type: "basic"; 5 | prefix: string; 6 | postfix: string; 7 | stop: string; 8 | }>; 9 | 10 | export const promptAugmentors = { 11 | // https://twitter.com/arankomatsuzaki/status/1529285884817707008 12 | "Step By Step": { type: "basic", postfix: "Let's think step by step.\n\n" }, 13 | "Before we dive into the answer": { 14 | type: "basic", 15 | postfix: "Before we dive into the answer, ", 16 | }, 17 | "First, ": { type: "basic", postfix: "First, " }, 18 | // https://twitter.com/goodside/status/1568448128495534081 19 | "You Cant Do Math": { 20 | prefix: 21 | "You are GPT-3, and you can't do math.\n\n You can do basic math, and your memorization abilities are impressive, but you can't do any complex calculations that a human could not do in their head. You also have an annoying tendency to just make up highly specific, but wrong, answers.\n\n So we hooked you up to a JavaScript kernel, and now you can execute code. If anyone gives you a hard math problem, just use this format and we’ll take care of the rest:\n\n Question: ${Question with hard calculation.}\n```javascript\n${Code that prints what you need to know}\n```\n```output\n${Output of your code}\n```\nAnswer: ${Answer}\n\n Otherwise, use this simpler format:\n\n Question: ${Question without hard calculation}\nAnswer: ${Answer}\n\n Begin.\n\n Question: What is 37593 * 67?\n\n ```javascript\nconsole.log(37593 * 67)\n```\n```output\n2518731\n```\nAnswer: 2518731\n\n", 22 | postfix: "Question: ", 23 | stop: "```output", 24 | }, 25 | } as Record; 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ask CLI 2 | 3 | Ask is a Deno CLI for pinging GPT-3 and iterating with chain of thought prompting and [other prompt engineering tricks](https://github.com/sw-yx/prompt-eng/blob/main/GPT.md). 4 | 5 | > Inspired by [Linus' prototype](https://twitter.com/thesephist/status/1587593832002072576) (his impl [here](https://gist.github.com/thesephist/28786aa80ac6e26241116c5ed2be97ca)) 6 | 7 | ![image](https://user-images.githubusercontent.com/6764957/199749592-fd252e21-0da3-4c31-8497-ee17d37e803f.png) 8 | 9 | But it can also iterate on prompts: 10 | 11 | ![image](https://user-images.githubusercontent.com/6764957/199750459-83968ede-539b-4508-9535-98cfdc40b245.png) 12 | 13 | And then it runs them in parallel for you to choose: 14 | 15 | ![image](https://user-images.githubusercontent.com/6764957/199750666-0a1489b6-8bea-4657-8b16-329c06e1e03e.png) 16 | 17 | ## Install instructions 18 | 19 | We use [`deno compile`](https://deno.land/manual@v1.27.0/tools/compiler) to ship a dedicated executable for each system architecture. 20 | 21 | See [the Releases page](https://github.com/sw-yx/ask-cli/releases) to download the appropriate binaries. 22 | 23 | > If you know what you're doing, you may also just run from source locally, see Local Dev below 24 | 25 | Put the binary in your $PATH or just navigate your terminal to the folder where it is. 26 | 27 | ## Usage instructions 28 | 29 | These instructions are for Mac, but you should be able to adjust for Linux/Windows accordingly. 30 | 31 | First get your OpenAI API Key from: https://beta.openai.com/account/api-keys 32 | 33 | Then: 34 | 35 | ```bash 36 | export OPENAI_API_KEY=sk_your_api_key_here 37 | ask "How much wood would a woodchuck chuck if a woodchuck could chuck wood?" 38 | ``` 39 | 40 | By default you get a panel of options, but you can always exit with Ctrl+C. 41 | 42 | The real power of this CLI comes from chaining prompts. You can run candidate prompts in parallel and choose prompts based on the responses. 43 | 44 | ![image](https://user-images.githubusercontent.com/6764957/199750459-83968ede-539b-4508-9535-98cfdc40b245.png) 45 | 46 | And then it runs them in parallel for you to choose: 47 | 48 | ![image](https://user-images.githubusercontent.com/6764957/199750666-0a1489b6-8bea-4657-8b16-329c06e1e03e.png) 49 | 50 | ## Local Dev 51 | 52 | Make sure to have Deno installed. 53 | 54 | Then, you can either: 55 | 56 | - `deno run -A index.ts "how do i sample from a normal distribution from scratch in python?"` for local dev 57 | - `npm run build && ./builds/askcli "Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?"` to do a production build 58 | 59 | Debugging: 60 | 61 | ```bash 62 | deno run --inspect-brk -A index.ts -d "How many of the integers between 0 and 99 inclusive are divisible by 8?" 63 | ``` 64 | 65 | and then open `chrome://inspect`, open the directory in devtools and put in breakpoints. 66 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Command, 3 | EnumType, 4 | } from "https://deno.land/x/cliffy@v0.25.4/command/mod.ts"; 5 | import { promptAugmentors, Augmentor } from "./promptAugmentors.ts"; 6 | import { Input } from "https://deno.land/x/cliffy@v0.25.4/prompt/input.ts"; 7 | import { Checkbox } from "https://deno.land/x/cliffy@v0.25.4/prompt/checkbox.ts"; 8 | import { Select } from "https://deno.land/x/cliffy@v0.25.4/prompt/select.ts"; 9 | import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/mod.ts"; 10 | // import { Secret } from "https://deno.land/x/cliffy@v0.25.4/prompt/secret.ts"; 11 | import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/mod.ts"; 12 | // import { Aes } from "https://deno.land/x/crypto/aes.ts"; 13 | // import { Cbc, Padding } from "https://deno.land/x/crypto/block-modes.ts"; 14 | 15 | let basePrompt = ""; 16 | const currentPrompt = colors.bold.blue; 17 | const showNext = colors.bold.yellow; 18 | const system = colors.bold.cyan; 19 | const logLevelType = new EnumType(["debug", "info", "warn", "error"]); 20 | const te = new TextEncoder(); 21 | // const de = new TextDecoder(); 22 | // const key = te.encode("SuperDuperSecret"); 23 | // const iv = new Uint8Array(16); 24 | 25 | await new Command() 26 | .name("ask") 27 | .version("0.0.1") 28 | .description( 29 | "Ask CLI is a command line tool for pinging GPT3 in the command line." 30 | ) 31 | .type("log-level", logLevelType) 32 | // .env("DEBUG=", "Enable debug output.") 33 | .globalEnv("OPENAI_API_KEY=", "OpenAI API key...", { 34 | required: true, 35 | }) 36 | .option("-d, --debug", "Enable debug output.") 37 | // .option("-k, --key ", "Set OpenAI API key.") 38 | // .option("-l, --log-level ", "Set log level.", { 39 | // default: "info" as const, 40 | // }) 41 | .arguments("") 42 | // .arguments(" [output:string]") 43 | .action(async (options, ...args) => { 44 | if (options.debug) { 45 | console.log(system("Debug mode enabled.")); 46 | console.log({ options }); 47 | console.log(system("End debug mode.")); 48 | } 49 | async function pingGPT3( 50 | prompt: string, 51 | openaiApiKey: string, 52 | gpt3options: Object 53 | ) { 54 | if (options.debug) console.log("log pingGPT3", { prompt }); 55 | const res = await fetch("https://api.openai.com/v1/completions", { 56 | method: "POST", 57 | headers: { 58 | "Content-Type": "application/json", 59 | Authorization: `Bearer ${openaiApiKey}`, 60 | }, 61 | body: JSON.stringify({ 62 | prompt: prompt, 63 | // defaults 64 | model: "text-davinci-002", 65 | max_tokens: 256, 66 | temperature: 0.3, 67 | top_p: 0.9, 68 | n: 1, 69 | frequency_penalty: 0, 70 | presence_penalty: 0, 71 | stop: ["```output"], 72 | // override defaults 73 | ...gpt3options, 74 | }), 75 | }); 76 | let x = await res.json(); 77 | x = x.choices[0].text.trim(); 78 | // https://replit.com/@amasad/gptpy?v=1#main.py 79 | if (options.debug) { 80 | console.log("------GPT3 response------------"); 81 | console.log(x); 82 | console.log("------GPT3 response------------"); 83 | } 84 | if (x.startsWith("```javascript")) { 85 | const code = x.slice(13, -3); 86 | console.log("------Executing...------------"); 87 | console.log(code); 88 | console.log("------Done!-------------------"); 89 | // https://stackoverflow.com/questions/46417440/how-to-get-console-log-output-from-eval 90 | const str = ` 91 | (function() { 92 | console.oldLog = console.log; 93 | let output = ""; 94 | console.log = function(value){ 95 | output += value 96 | }; 97 | ${code} 98 | console.log = console.oldLog; 99 | return output.trim(); 100 | }()) 101 | `; 102 | const result = eval(str); 103 | x = `Eval Answer: ${result}`; 104 | // let response = await exec(code, {output: OutputMode.Capture}); 105 | } 106 | return x; 107 | } 108 | basePrompt = args[0]; 109 | const promptStack = [] as Augmentor[]; 110 | let nextPrompt = null as Augmentor | null; 111 | function compilePrompt(pStack: Augmentor[], candidate: Augmentor) { 112 | let result = ""; 113 | result += candidate.prefix ?? ""; 114 | for (const augmentor of pStack) { 115 | result += augmentor.prefix ?? ""; 116 | } 117 | result += basePrompt; 118 | for (const augmentor of pStack) { 119 | result += augmentor.postfix ?? ""; 120 | } 121 | result += candidate.postfix ?? ""; 122 | return result; 123 | } 124 | 125 | console.log(currentPrompt(basePrompt)); 126 | while (true) { 127 | if (options.debug) { 128 | console.log("start cycle", { promptStack, nextPrompt }); 129 | } 130 | nextPrompt = nextPrompt ?? { postfix: "" }; 131 | const answer = await pingGPT3( 132 | compilePrompt(promptStack, nextPrompt), 133 | options.openaiApiKey, 134 | {} 135 | ); 136 | if (options.debug) console.log("log1", { answer }); 137 | if (answer === "") { 138 | console.log( 139 | system( 140 | "GPT3 returned an empty string. We reached a stop sequence. Pop the stack or Ctrl+C exit." 141 | ) 142 | ); 143 | } else { 144 | console.log(showNext("GPT3 response: ")); 145 | console.log(answer); 146 | } 147 | console.log("\n ---------- \n"); 148 | const choices: string[] = await Checkbox.prompt({ 149 | message: "What do you want to do with this response?", 150 | options: [ 151 | Checkbox.separator("---Run Prompt candidates in Parallel!-----"), 152 | { name: "Display progress so far", value: "$display" }, 153 | { name: "Rewrite current prompt", value: "$rewrite" }, 154 | ...Object.entries(promptAugmentors).map(([augName, augData]) => { 155 | return { name: `Add "${augName.trim()}"`, value: augName }; 156 | }), 157 | Checkbox.separator( 158 | "---Interrupts (choosing one of these overrides all others). Use Ctrl+C to exit-----" 159 | ), 160 | { name: "Pop previous candidate off prompt stack", value: "$back" }, 161 | { 162 | name: "Add to prompt stack, get GPT3 to continue", 163 | value: "$continue", 164 | }, 165 | ], 166 | }); 167 | if (choices.length === 0) { 168 | continue; 169 | } 170 | if (choices.includes("$display")) { 171 | console.log(system("---- Prompt Progress: ----")); 172 | console.log(currentPrompt(basePrompt)); 173 | console.log(showNext("prompt Stack:")); 174 | console.log(promptStack.join("\n")); 175 | console.log(showNext("current candidate:")); 176 | console.log(JSON.stringify(nextPrompt)); 177 | console.log(system("---- End of Progress Print ----")); 178 | console.log(" "); 179 | } 180 | if (choices.includes("$back")) { 181 | if (promptStack.length > 1) { 182 | nextPrompt = promptStack.pop() || {}; 183 | continue; // next iteration 184 | } else { 185 | console.log(system("Can't go back any further.")); 186 | continue; // next iteration 187 | } 188 | } 189 | if (choices.includes("$continue")) { 190 | promptStack.push(nextPrompt); 191 | continue; 192 | } 193 | 194 | // run searches in parallel from here 195 | const values = await Promise.all( 196 | choices.map(async (choice) => { 197 | let value; 198 | let rewriteMsg = ""; 199 | let augData = {} as Augmentor; 200 | if (choice === "$rewrite") { 201 | console.log(system(`[extra info needed for rewriting prompt]`)); 202 | rewriteMsg = await Input.prompt({ 203 | message: 204 | "What do you want to rewrite the prompt to? (press up for suggestion)", 205 | minLength: 1, 206 | // info: true, 207 | suggestions: [answer], 208 | }); 209 | augData = { postfix: rewriteMsg } as Augmentor; 210 | value = await pingGPT3( 211 | compilePrompt(promptStack, augData), 212 | options.openaiApiKey, 213 | {} 214 | ); 215 | } else if (choice in promptAugmentors) { 216 | augData = promptAugmentors[choice as keyof typeof promptAugmentors]; 217 | value = await pingGPT3( 218 | compilePrompt(promptStack, augData), 219 | options.openaiApiKey, 220 | {} 221 | ); 222 | } 223 | return { 224 | name: choice === "$rewrite" ? rewriteMsg : choice, 225 | value: value, 226 | augData, 227 | }; 228 | }) 229 | ); 230 | if (values.length === 1) { 231 | // just immediately print out the value and carry on 232 | if (options.debug) console.log("log2", { values }); 233 | if (values[0].value === "") { 234 | console.log( 235 | showNext( 236 | "Null response from GPT3. We may have reached a natural stop sequence. Pop the stack or Ctrl+C to exit." 237 | ) 238 | ); 239 | } 240 | console.log(currentPrompt(values[0].value)); 241 | nextPrompt = values[0].augData; 242 | } else { 243 | values.push({ 244 | name: "accept current candidate without rewrite", 245 | value: answer, 246 | augData: { postfix: answer }, 247 | }); 248 | if (options.debug) console.log("table", { values }); 249 | // const table: Table = 250 | new Table() 251 | .header(["Candidate", "GPT3 Response"]) 252 | .body(values.map(({ value, name }) => [name, value])) 253 | .padding(1) 254 | .indent(2) 255 | .maxColWidth(60) 256 | .border(true) 257 | .render(); 258 | 259 | const selectedCandidate = await Select.prompt({ 260 | message: "Pick a candidate you like to add onto the prompt stack.", 261 | options: values, 262 | }); 263 | console.log(showNext(selectedCandidate)); 264 | if (options.debug) console.log("log3", { selectedCandidate }); 265 | nextPrompt = values.find((x) => x.value === selectedCandidate)!.augData; 266 | } 267 | } 268 | }) 269 | // // Child command 1. 270 | // .command("clear", "Foo sub-command.") 271 | // .option("-f, --foo", "Foo option.") 272 | // // .arguments("") 273 | // .action((options, ...args) => localStorage.clear()) 274 | .parse(Deno.args); 275 | 276 | 277 | // // cant do these yet https://github.com/denoland/deno/issues/10693 278 | // function encryptAndSaveKey(apikey: string) { 279 | // // const data = te.encode(apikey); 280 | // // const cipher = new Cbc(Aes, key, iv, Padding.PKCS7); 281 | 282 | // // const encrypted = cipher.encrypt(data); 283 | // // localStorage.setItem("myDemo", de.decode(encrypted)); 284 | // localStorage.setItem("myDemo", apikey); 285 | // } 286 | // async function getAndDecryptKey() { 287 | // const encrypted = localStorage.getItem("myDemo"); 288 | // // console.log({ encrypted: te.encode(encrypted) }); 289 | // if (encrypted) { 290 | // // const decipher = new Cbc(Aes, key, iv, Padding.PKCS7); 291 | // // return decipher.decrypt(te.encode(encrypted)); 292 | // return encrypted; 293 | // } else { 294 | // const password = await Secret.prompt( 295 | // "Enter your OpenAI API key (will be encrypted):" 296 | // ); 297 | // await encryptAndSaveKey(password); 298 | // return password; 299 | // } 300 | // } 301 | --------------------------------------------------------------------------------