├── .changeset ├── README.md └── config.json ├── .devcontainer └── devcontainer.json ├── .editorconfig ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── apps └── colors │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── package.json │ ├── public │ ├── example.html │ └── index.html │ ├── src │ ├── api.ts │ ├── entityColors.ts │ ├── entityTagline.ts │ └── index.ts │ └── tsconfig.json ├── examples ├── 00_hello.js ├── 01_hello_entity.js ├── 02_colors.js ├── 03_colors_raw_json.js ├── 04_tagline.js ├── 05_postcodes.js ├── 06_prompt_schema.js ├── 07_tools.js ├── README.md └── package.json ├── package-lock.json ├── package.json ├── packages └── openai-partial-stream │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── jest.config.js │ ├── package.json │ ├── src │ ├── entity.ts │ ├── index.ts │ ├── jsonCloser.ts │ ├── openAiHandler.ts │ ├── streamParser.ts │ └── utils.ts │ ├── tests │ ├── jsonCloser.test.ts │ ├── streamParser.test.ts │ └── utils.test.ts │ └── tsconfig.json ├── spec ├── CHALLENGES.md └── THE_SPEC.md └── turbo.json /.changeset/README.md: -------------------------------------------------------------------------------- 1 | # Changesets 2 | 3 | Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works 4 | with multi-package repos, or single-package repos to help you version and publish your code. You can 5 | find the full documentation for it [in our repository](https://github.com/changesets/changesets) 6 | 7 | We have a quick list of common questions to get you started engaging with this project in 8 | [our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md) 9 | -------------------------------------------------------------------------------- /.changeset/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://unpkg.com/@changesets/config@2.3.1/schema.json", 3 | "changelog": "@changesets/cli/changelog", 4 | "commit": false, 5 | "fixed": [], 6 | "linked": [], 7 | "access": "restricted", 8 | "baseBranch": "main", 9 | "updateInternalDependencies": "patch", 10 | "ignore": [] 11 | } 12 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node 3 | { 4 | "name": "Node.js & TypeScript", 5 | // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile 6 | "image": "mcr.microsoft.com/devcontainers/typescript-node:1-20-bullseye", 7 | "customizations": { 8 | "vscode": { 9 | "extensions": [ 10 | "esbenp.prettier-vscode" 11 | ] 12 | } 13 | } 14 | 15 | // Features to add to the dev container. More info: https://containers.dev/features. 16 | // "features": {}, 17 | 18 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 19 | // "forwardPorts": [], 20 | 21 | // Use 'postCreateCommand' to run commands after the container is created. 22 | // "postCreateCommand": "yarn install", 23 | 24 | // Configure tool-specific properties. 25 | // "customizations": {}, 26 | 27 | // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. 28 | // "remoteUser": "root" 29 | } 30 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.{ts,js}] 4 | indent_style = space 5 | indent_size = 4 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | end_of_line = lf 9 | charset = utf-8 10 | 11 | 12 | [*.md] 13 | indent_style = space 14 | indent_size = 2 15 | insert_final_newline = true 16 | trim_trailing_whitespace = true 17 | end_of_line = lf 18 | charset = utf-8 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore 2 | 3 | # Logs 4 | 5 | logs 6 | _.log 7 | npm-debug.log_ 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | .pnpm-debug.log* 12 | 13 | # Diagnostic reports (https://nodejs.org/api/report.html) 14 | 15 | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json 16 | 17 | # Runtime data 18 | 19 | pids 20 | _.pid 21 | _.seed 22 | \*.pid.lock 23 | 24 | # Directory for instrumented libs generated by jscoverage/JSCover 25 | 26 | lib-cov 27 | 28 | # Coverage directory used by tools like istanbul 29 | 30 | coverage 31 | \*.lcov 32 | 33 | # nyc test coverage 34 | 35 | .nyc_output 36 | 37 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 38 | 39 | .grunt 40 | 41 | # Bower dependency directory (https://bower.io/) 42 | 43 | bower_components 44 | 45 | # node-waf configuration 46 | 47 | .lock-wscript 48 | 49 | # Compiled binary addons (https://nodejs.org/api/addons.html) 50 | 51 | build/Release 52 | 53 | # Dependency directories 54 | 55 | node_modules/ 56 | jspm_packages/ 57 | 58 | # Snowpack dependency directory (https://snowpack.dev/) 59 | 60 | web_modules/ 61 | 62 | # TypeScript cache 63 | 64 | \*.tsbuildinfo 65 | 66 | # Optional npm cache directory 67 | 68 | .npm 69 | 70 | # Optional eslint cache 71 | 72 | .eslintcache 73 | 74 | # Optional stylelint cache 75 | 76 | .stylelintcache 77 | 78 | # Microbundle cache 79 | 80 | .rpt2_cache/ 81 | .rts2_cache_cjs/ 82 | .rts2_cache_es/ 83 | .rts2_cache_umd/ 84 | 85 | # Optional REPL history 86 | 87 | .node_repl_history 88 | 89 | # Output of 'npm pack' 90 | 91 | \*.tgz 92 | 93 | # Yarn Integrity file 94 | 95 | .yarn-integrity 96 | 97 | # dotenv environment variable files 98 | 99 | .env 100 | .env.development.local 101 | .env.test.local 102 | .env.production.local 103 | .env.local 104 | 105 | # parcel-bundler cache (https://parceljs.org/) 106 | 107 | .cache 108 | .parcel-cache 109 | 110 | # Next.js build output 111 | 112 | .next 113 | out 114 | 115 | # Nuxt.js build / generate output 116 | 117 | .nuxt 118 | dist 119 | 120 | # Gatsby files 121 | 122 | .cache/ 123 | 124 | # Comment in the public line in if your project uses Gatsby and not Next.js 125 | 126 | # https://nextjs.org/blog/next-9-1#public-directory-support 127 | 128 | # public 129 | 130 | # vuepress build output 131 | 132 | .vuepress/dist 133 | 134 | # vuepress v2.x temp and cache directory 135 | 136 | .temp 137 | .cache 138 | 139 | # Docusaurus cache and generated files 140 | 141 | .docusaurus 142 | 143 | # Serverless directories 144 | 145 | .serverless/ 146 | 147 | # FuseBox cache 148 | 149 | .fusebox/ 150 | 151 | # DynamoDB Local files 152 | 153 | .dynamodb/ 154 | 155 | # TernJS port file 156 | 157 | .tern-port 158 | 159 | # Stores VSCode versions used for testing VSCode extensions 160 | 161 | .vscode-test 162 | 163 | # yarn v2 164 | 165 | .yarn/cache 166 | .yarn/unplugged 167 | .yarn/build-state.yml 168 | .yarn/install-state.gz 169 | .pnp.\* 170 | 171 | 172 | buid/** 173 | dist/** 174 | 175 | .turbo 176 | 177 | .DS_Store 178 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Yanael Barbier 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: install build 2 | 3 | install: 4 | npm install turbo --global 5 | npm install 6 | 7 | build: 8 | turbo build 9 | 10 | lib: 11 | turbo build --filter=openai-partial-stream 12 | 13 | web: 14 | turbo build --filter=partial-ai-stream-website 15 | 16 | dev: 17 | turbo dev 18 | 19 | test: 20 | turbo test 21 | 22 | test-watch: 23 | turbo test:watch 24 | 25 | server: build 26 | node apps/colors/dist/server.js 27 | 28 | pack: lib 29 | cd packages/openai-partial-stream && npm pack 30 | 31 | version: 32 | npx changeset 33 | 34 | publish: 35 | npx changeset version 36 | npx changeset publish 37 | 38 | 39 | format: 40 | turbo format 41 | 42 | .PHONY: all install build lib web test test-watch server pack version publish format dev 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Parse Partial JSON Stream - Turn your slow AI app into an engaging real-time app 2 | 3 | - Convert a **stream of token** into a **parsable JSON** object before the stream ends. 4 | - Implement **Streaming UI** in **LLM**-based AI application. 5 | - Leverage **OpenAI Function Calling** for early stream processing. 6 | - Parse **JSON streams** into distinct **entities**. 7 | - Engage your users with a **real-time** experience. 8 | 9 | ![json_stream_color](https://github.com/st3w4r/openai-partial-stream/assets/4228332/04c4bdfc-d991-4ad0-85fc-04eb35b106f3) 10 | 11 | ## Follow the Work 12 | 13 | - [✖️ Twitter](https://twitter.com/YanaelBarbier) 14 | - [🧵 Threads](https://www.threads.net/@yanaelbarbier) 15 | - [📰 Blog](https://yanael.io/subscribe/) 16 | 17 | ## Install 18 | 19 | To install dependencies: 20 | 21 | ```bash 22 | npm install --save openai-partial-stream 23 | ``` 24 | 25 | ## Usage with simple stream 26 | 27 | Turn a stream of token into a parsable JSON object as soon as possible. 28 | 29 | ```javascript 30 | import OpenAi from "openai"; 31 | import { OpenAiHandler, StreamMode } from "openai-partial-stream"; 32 | 33 | // Set your OpenAI API key as an environment variable: OPENAI_API_KEY 34 | const openai = new OpenAi({ apiKey: process.env.OPENAI_API_KEY }); 35 | 36 | const stream = await openai.chat.completions.create({ 37 | messages: [{ role: "system", content: "Say hello to the world." }], 38 | model: "gpt-3.5-turbo", // OR "gpt-4" 39 | stream: true, // ENABLE STREAMING 40 | temperature: 1, 41 | functions: [ 42 | { 43 | name: "say_hello", 44 | description: "say hello", 45 | parameters: { 46 | type: "object", 47 | properties: { 48 | sentence: { 49 | type: "string", 50 | description: "The sentence generated", 51 | }, 52 | }, 53 | }, 54 | }, 55 | ], 56 | function_call: { name: "say_hello" }, 57 | }); 58 | 59 | const openAiHandler = new OpenAiHandler(StreamMode.StreamObjectKeyValueTokens); 60 | const entityStream = openAiHandler.process(stream); 61 | 62 | for await (const item of entityStream) { 63 | console.log(item); 64 | } 65 | ``` 66 | 67 | Output: 68 | 69 | ```js 70 | { index: 0, status: 'PARTIAL', data: {} } 71 | { index: 0, status: 'PARTIAL', data: { sentence: '' } } 72 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hello' } } 73 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hello,' } } 74 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hello, world' } } 75 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hello, world!' } } 76 | { index: 0, status: 'COMPLETED', data: { sentence: 'Hello, world!' } } 77 | ``` 78 | 79 | ## Usage with stream and entity parsing 80 | 81 | Validate the data against a schema and only return the data when it is valid. 82 | 83 | ```javascript 84 | import { z } from "zod"; 85 | import OpenAi from "openai"; 86 | import { OpenAiHandler, StreamMode, Entity } from "openai-partial-stream"; 87 | 88 | // Set your OpenAI API key as an environment variable: OPENAI_API_KEY 89 | const openai = new OpenAi({ apiKey: process.env.OPENAI_API_KEY }); 90 | 91 | const stream = await openai.chat.completions.create({ 92 | messages: [{ role: "system", content: "Say hello to the world." }], 93 | model: "gpt-3.5-turbo", // OR "gpt-4" 94 | stream: true, // ENABLE STREAMING 95 | temperature: 1, 96 | functions: [ 97 | { 98 | name: "say_hello", 99 | description: "say hello", 100 | parameters: { 101 | type: "object", 102 | properties: { 103 | sentence: { 104 | type: "string", 105 | description: "The sentence generated", 106 | }, 107 | }, 108 | }, 109 | }, 110 | ], 111 | function_call: { name: "say_hello" }, 112 | }); 113 | 114 | const openAiHandler = new OpenAiHandler(StreamMode.StreamObjectKeyValueTokens); 115 | const entityStream = openAiHandler.process(stream); 116 | 117 | // Entity Parsing to validate the data 118 | const HelloSchema = z.object({ 119 | sentence: z.string().optional(), 120 | }); 121 | 122 | const entityHello = new Entity("sentence", HelloSchema); 123 | const helloEntityStream = entityHello.genParse(entityStream); 124 | 125 | for await (const item of helloEntityStream) { 126 | console.log(item); 127 | } 128 | ``` 129 | 130 | Output: 131 | 132 | ```js 133 | { index: 0, status: 'PARTIAL', data: {}, entity: 'sentence' } 134 | { index: 0, status: 'PARTIAL', data: { sentence: '' }, entity: 'sentence' } 135 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hi' }, entity: 'sentence' } 136 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hi,' }, entity: 'sentence' } 137 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hi, world' }, entity: 'sentence' } 138 | { index: 0, status: 'PARTIAL', data: { sentence: 'Hi, world!' }, entity: 'sentence' } 139 | { index: 0, status: 'COMPLETED', data: { sentence: 'Hi, world!' }, entity: 'sentence'} 140 | ``` 141 | 142 | ## Usage with stream and entity parsing with multiple entities 143 | 144 | ```javascript 145 | import { z } from "zod"; 146 | import OpenAi from "openai"; 147 | import { OpenAiHandler, StreamMode, Entity } from "openai-partial-stream"; 148 | 149 | // Intanciate OpenAI client with your API key 150 | const openai = new OpenAi({ 151 | apiKey: process.env.OPENAI_API_KEY, 152 | }); 153 | 154 | const PostcodeSchema = z.object({ 155 | name: z.string().optional(), 156 | postcode: z.string().optional(), 157 | population: z.number().optional(), 158 | }); 159 | 160 | // Call the API with stream enabled and a function 161 | const stream = await openai.chat.completions.create({ 162 | messages: [ 163 | { 164 | role: "system", 165 | content: "Give me 3 cities and their postcodes in California.", 166 | }, 167 | ], 168 | model: "gpt-3.5-turbo", // OR "gpt-4" 169 | stream: true, // ENABLE STREAMING 170 | temperature: 1.1, 171 | functions: [ 172 | { 173 | name: "set_postcode", 174 | description: "Set a postcode and a city", 175 | parameters: { 176 | type: "object", 177 | properties: { 178 | // The name of the entity 179 | postcodes: { 180 | type: "array", 181 | items: { 182 | type: "object", 183 | properties: { 184 | name: { 185 | type: "string", 186 | description: "Name of the city", 187 | }, 188 | postcode: { 189 | type: "string", 190 | description: "The postcode of the city", 191 | }, 192 | population: { 193 | type: "number", 194 | description: "The population of the city", 195 | }, 196 | }, 197 | }, 198 | }, 199 | }, 200 | }, 201 | }, 202 | ], 203 | function_call: { name: "set_postcode" }, 204 | }); 205 | 206 | // Select the mode of the stream parser 207 | // - StreamObjectKeyValueTokens: (REALTIME) Stream of JSON objects, key value pairs and tokens 208 | // - StreamObjectKeyValue: (PROGRESSIVE) Stream of JSON objects and key value pairs 209 | // - StreamObject: (ONE-BY-ONE) Stream of JSON objects 210 | // - NoStream: (ALL-TOGETHER) All the data is returned at the end of the process 211 | const mode = StreamMode.StreamObject; 212 | 213 | // Create an instance of the handler 214 | const openAiHandler = new OpenAiHandler(mode); 215 | // Process the stream 216 | const entityStream = openAiHandler.process(stream); 217 | // Create an entity with the schema to validate the data 218 | const entityPostcode = new Entity("postcodes", PostcodeSchema); 219 | // Parse the stream to an entity, using the schema to validate the data 220 | const postcodeEntityStream = entityPostcode.genParseArray(entityStream); 221 | 222 | // Iterate over the stream of entities 223 | for await (const item of postcodeEntityStream) { 224 | if (item) { 225 | // Display the entity 226 | console.log(item); 227 | } 228 | } 229 | ``` 230 | 231 | Output: 232 | 233 | ```js 234 | { index: 0, status: 'COMPLETED', data: { name: 'Los Angeles', postcode: '90001', population: 3971883 }, entity: 'postcodes' } 235 | { index: 1, status: 'COMPLETED', data: { name: 'San Francisco', postcode: '94102', population: 883305 }, entity: 'postcodes' } 236 | { index: 2, status: 'COMPLETED', data: { name: 'San Diego', postcode: '92101', population: 1425976 }, entity: 'postcodes'} 237 | ``` 238 | 239 | # Modes 240 | 241 | Select a mode from the list below that best suits your requirements: 242 | 243 | 1. **NoStream** 244 | 2. **StreamObject** 245 | 3. **StreamObjectKeyValue** 246 | 4. **StreamObjectKeyValueTokens** 247 | 248 | --- 249 | 250 | ### NoStream 251 | 252 | Results are returned only after the entire query completes. 253 | 254 | | **NoStream Details** | 255 | | ---------------------------------------------------------------- | 256 | | ✅ Single query retrieves all data | 257 | | ✅ Reduces network traffic | 258 | | ⚠️ User experience may be compromised due to extended wait times | 259 | 260 | --- 261 | 262 | ### StreamObject 263 | 264 | An event is generated for each item in the list. Items appear as they become ready. 265 | 266 | | **StreamObject Details** | 267 | | ------------------------------------------------------------------------------- | 268 | | ✅ Each message corresponds to a fully-formed item | 269 | | ✅ Fewer messages | 270 | | ✅ All essential fields are received at once | 271 | | ⚠️ Some delay: users need to wait until an item is fully ready to update the UI | 272 | 273 | --- 274 | 275 | ### StreamObjectKeyValue 276 | 277 | Objects are received in fragments: both a key and its corresponding value are sent together. 278 | 279 | | **StreamObjectKeyValue Details** | 280 | | --------------------------------------------------------- | 281 | | ✅ Users can engage with portions of the UI | 282 | | ✅ Supports more regular UI updates | 283 | | ⚠️ Higher network traffic | 284 | | ⚠️ Challenges in enforcing keys due to incomplete objects | 285 | 286 | --- 287 | 288 | ### StreamObjectKeyValueTokens 289 | 290 | Keys are received in full, while values are delivered piecemeal until they're complete. This method offers token-by-token UI updating. 291 | 292 | | **StreamObjectKeyValueToken Details** | 293 | | ------------------------------------------------------------------- | 294 | | ✅ Offers a dynamic user experience | 295 | | ✅ Enables step-by-step content consumption | 296 | | ✅ Decreases user waiting times | 297 | | ⚠️ Possible UI inconsistencies due to values arriving incrementally | 298 | | ⚠️ Augmented network traffic | 299 | 300 | ## Demo 301 | 302 | Stream of JSON object progressively by key value pairs: 303 | 304 | https://github.com/st3w4r/openai-partial-stream/assets/4228332/55643614-b92b-4b1f-9cf9-e60d6d783a0c 305 | 306 | Stream of JSON objects in realtime: 307 | 308 | https://github.com/st3w4r/openai-partial-stream/assets/4228332/73289d38-8526-46cf-a68c-ac80019092ab 309 | 310 | ## References 311 | 312 | [npm pakcage](https://www.npmjs.com/package/openai-partial-stream) 313 | -------------------------------------------------------------------------------- /apps/colors/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .wrangler 4 | .dev.vars 5 | 6 | # Change them to your taste: 7 | wrangler.toml 8 | package-lock.json 9 | yarn.lock 10 | pnpm-lock.yaml -------------------------------------------------------------------------------- /apps/colors/Makefile: -------------------------------------------------------------------------------- 1 | dev: 2 | npm run dev 3 | 4 | dev-remote: 5 | npx wrangler dev --minify ./src/index.ts --remote 6 | 7 | deploy: 8 | npm run deploy 9 | 10 | 11 | .PHONY: dev dev-remote deploy 12 | -------------------------------------------------------------------------------- /apps/colors/README.md: -------------------------------------------------------------------------------- 1 | # Deployment 2 | 3 | ```bash 4 | npm install 5 | make dev 6 | make deploy 7 | ``` 8 | 9 | # Sercrets 10 | 11 | Set `.dot.vars` file for local development 12 | 13 | On production, set the following environment variables: 14 | 15 | ```bash 16 | wrangler secret put 17 | ``` 18 | 19 | # Cloudflare Workers 20 | 21 | Use the wrangler CLI to deploy the worker. 22 | 23 | ```bash 24 | 25 | npm install -g wrangler 26 | 27 | wrangler login 28 | 29 | wrangler init 30 | 31 | wrangler deploy 32 | 33 | wrangler secret put OPENAI_API_KEY 34 | 35 | ``` 36 | -------------------------------------------------------------------------------- /apps/colors/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web-app-colors", 3 | "private": true, 4 | "scripts": { 5 | "dev": "wrangler dev src/index.ts --port 8888", 6 | "deploy": "wrangler deploy --minify src/index.ts" 7 | }, 8 | "dependencies": { 9 | "@hono/zod-validator": "^0.1.9", 10 | "hono": "^3.7.2", 11 | "openai-partial-stream": "*", 12 | "zod": "^3.22.4" 13 | }, 14 | "devDependencies": { 15 | "@cloudflare/workers-types": "^4.20230914.0", 16 | "wrangler": "^3.12.0" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /apps/colors/public/example.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | SSE Example 7 | 8 | 9 | 10 |

SSE Example

11 |
12 | 13 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /apps/colors/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Dynamic Color Blocks 8 | 674 | 675 | 676 | 677 | 678 | 679 | 680 | 681 | 682 | 683 |
684 | GitHub 687 |
688 | 689 |
690 |
691 | 692 |
693 | 694 | 696 |
697 |
698 | 699 | 700 |
701 | 702 | 703 | 704 | 705 | 706 | 707 |
708 |
709 | 710 | 712 | 713 | 714 | 715 | 716 | 717 | 718 | 719 | 720 | 722 | 723 | 725 | 727 | 728 | 729 |
730 | 731 | 732 | 733 |
734 | 735 | 736 |
Voila!
737 | 738 | 739 |
740 |
741 |
742 | How AI color generation works

743 | 744 | We start by sending a prompt to ChatGPT, asking for colors using OpenAI Function Calling. 745 | In 746 | response, the API 747 | sends 748 | back 749 | a JSON text stream that isn't fully formatted yet. But here's where the efficiency kicks 750 | in: 751 | even 752 | before we've 753 | received all of the data, we're already parsing it on-the-fly and turning it into 754 | structured 755 | payload entities. And send each of these entities straight to the browser using Server Sent 756 | Events, 757 | ensuring a real-time experience. 758 |
759 |
760 | Modes
761 | All-Together the UI waits for complete data.
762 | One-By-One goes object by object.
763 | Progressive operates via key-value pairs.
764 | Real-time feeds data by individual tokens.
765 | 766 |
767 | 768 |
769 | The Prompt
770 | 771 | Give me a palette of 5 gorgeous colors with their hex codes, names, and descriptions. 772 | 773 |
774 |
775 | The Function 776 | 
name: give_colors
 777 | description: Give a list of colors
 778 | parameters:
 779 |   type: object
 780 |   properties:
 781 |     colors:
 782 |       type: array
 783 |       items:
 784 |         type: object
 785 |         properties:
 786 |           hex:
 787 |             type: string
 788 |             description: The hexadecimal code of the color
 789 |           name:
 790 |             type: string
 791 |             description: The color name
 792 |           description:
 793 |             type: string
 794 |             description: The description of the color
795 | 796 | 797 | The raw stream of tokens 798 | 
data: {"he
 799 | data: x":"#FF
 800 | data: 69B4"
 801 | data: ,"na
 802 | data: me":"Ho
 803 | data: t Pink"
804 | The parsable entity stream 805 | 
{ "index": 0, "status": "PARTIAL",
 806 | "data": { "hex": "#FF69B4", "name": "Hot Pink" },
 807 | "entity": "colors" }
808 | 809 |
810 |
811 | 812 |
813 | 814 |
815 |
816 | Installation

817 | To get started with JavaScript / TypeScript: 818 |

819 | npm i openai-partial-stream 820 |

821 | Import, and you'll want the manual, check out the documentation on GitHub 823 | 824 |
825 |
826 | 827 | 828 | 829 | 830 |
831 | Stepping into the arena with Partial-Stream 833 | and ChatGPT API function calling. 834 | 835 |
836 | 837 | X/Twitter 839 | 840 | 841 | Threads 843 | 844 | 845 | 846 | 848 | 849 | 850 |
851 |
852 | 853 | 854 | 855 | 1158 | 1159 | 1160 | 1161 | 1162 | 1163 | 1164 | -------------------------------------------------------------------------------- /apps/colors/src/api.ts: -------------------------------------------------------------------------------- 1 | import { Hono } from "hono"; 2 | import { OpenAI } from "openai"; 3 | import { z } from "zod"; 4 | 5 | import { callGenerateTagline } from "./entityTagline"; 6 | import { callGenerateColors } from "./entityColors"; 7 | import { StreamMode } from "openai-partial-stream"; 8 | import { zValidator } from "@hono/zod-validator"; 9 | 10 | type Bindings = { 11 | OPENAI_API_KEY: string; 12 | }; 13 | 14 | type Variables = { 15 | openai: OpenAI; 16 | }; 17 | 18 | let openai: OpenAI; 19 | 20 | const api = new Hono<{ Bindings: Bindings; Variables: Variables }>(); 21 | 22 | const queryMode = z.object({ 23 | mode: z 24 | .enum([ 25 | StreamMode.StreamObjectKeyValueTokens, 26 | StreamMode.StreamObjectKeyValue, 27 | StreamMode.StreamObject, 28 | StreamMode.NoStream, 29 | ]) 30 | .optional(), 31 | number: z 32 | .string() 33 | .optional() 34 | .default("5") 35 | .transform((v) => parseInt(v)) 36 | .refine((v) => !isNaN(v) && v >= 1 && v <= 10, { 37 | message: "not a valid number", 38 | }), 39 | 40 | prompt: z.string().max(100).optional(), 41 | }); 42 | 43 | api.use("*", async (c, next) => { 44 | if (!openai) { 45 | openai = new OpenAI({ apiKey: c.env.OPENAI_API_KEY }); 46 | } 47 | c.set("openai", openai); 48 | await next(); 49 | }); 50 | 51 | api.get("/", (c) => { 52 | return c.json({ 53 | message: "Welcome to Partial Stream API!", 54 | }); 55 | }); 56 | 57 | api.use("/sse/*", async (c, next) => { 58 | // Set SSE headers 59 | c.header("Content-Type", "text/event-stream"); 60 | c.header("Cache-Control", "no-cache"); 61 | c.header("Connection", "keep-alive"); 62 | await next(); 63 | }); 64 | 65 | api.get("/sse/tagline", (c) => { 66 | const openai = c.get("openai"); 67 | 68 | return c.stream(async (stream) => { 69 | const gen = await callGenerateTagline( 70 | openai, 71 | StreamMode.StreamObjectKeyValueTokens, 72 | ); 73 | 74 | for await (const data of gen) { 75 | const jsonStr = JSON.stringify(data); 76 | stream.write(`data: ${jsonStr}\n\n`); 77 | } 78 | // Stream is done 79 | stream.write(`event: CLOSE\n`); 80 | stream.write(`data: [DONE]\n\n`); 81 | }); 82 | }); 83 | 84 | api.get("/sse/colors", zValidator("query", queryMode), (c) => { 85 | const { mode, number, prompt } = c.req.valid("query"); 86 | const openai = c.get("openai"); 87 | 88 | return c.stream(async (stream) => { 89 | const gen = await callGenerateColors(openai, mode, number, prompt); 90 | 91 | for await (const data of gen) { 92 | const jsonStr = JSON.stringify(data); 93 | 94 | // Return the json as the message 95 | stream.write(`data: ${jsonStr}\n\n`); 96 | } 97 | // Stream is done 98 | stream.write(`event: CLOSE\n`); 99 | stream.write(`data: [DONE]\n\n`); 100 | }); 101 | }); 102 | 103 | export { api }; 104 | -------------------------------------------------------------------------------- /apps/colors/src/entityColors.ts: -------------------------------------------------------------------------------- 1 | import OpenAI from "openai"; 2 | import { z } from "zod"; 3 | 4 | import { StreamMode, OpenAiHandler, Entity } from "openai-partial-stream"; 5 | 6 | const ColorSchema = z.object({ 7 | hex: z.string().optional(), 8 | name: z.string().optional(), 9 | description: z.string().optional(), 10 | }); 11 | 12 | function getColorMessages(number: string, prompt: string): any[] { 13 | return [ 14 | { 15 | role: "system", 16 | content: "Write JSON only", 17 | }, 18 | { 19 | role: "system", 20 | content: 21 | "Give me a palette of " + 22 | number + 23 | " gorgeous color with the hex code, name and a description.", 24 | }, 25 | { 26 | role: "user", 27 | content: "The palette will have the ton and theme of:" + prompt, 28 | }, 29 | ]; 30 | } 31 | 32 | function getColorListFunction() { 33 | return { 34 | name: "give_colors", 35 | description: "Give a list of color", 36 | parameters: { 37 | type: "object", 38 | properties: { 39 | colors: { 40 | type: "array", 41 | items: { 42 | type: "object", 43 | properties: { 44 | hex: { 45 | type: "string", 46 | description: 47 | "The hexadecimal code of the color", 48 | }, 49 | name: { 50 | type: "string", 51 | description: "The color name", 52 | }, 53 | description: { 54 | type: "string", 55 | description: "The description of the color", 56 | }, 57 | }, 58 | }, 59 | }, 60 | }, 61 | }, 62 | }; 63 | } 64 | 65 | export async function callGenerateColors( 66 | openai: OpenAI, 67 | mode: StreamMode = StreamMode.StreamObjectKeyValueTokens, 68 | number: number = 5, 69 | prompt: string = "", 70 | ) { 71 | // Call OpenAI API, with function calling 72 | // Function calling: https://openai.com/blog/function-calling-and-other-api-updates 73 | const stream = await openai.chat.completions.create({ 74 | messages: getColorMessages(number.toString(), prompt), 75 | model: "gpt-4-turbo-preview", // OR "gpt-3.5-turbo" "gpt-4" 76 | stream: true, // ENABLE STREAMING - Server Sent Event (SSE) 77 | temperature: 1.1, 78 | functions: [getColorListFunction()], 79 | function_call: { name: "give_colors" }, 80 | }); 81 | 82 | // Handle the stream from OpenAI client 83 | const openAiHandler = new OpenAiHandler(mode); 84 | // Parse the stream to valid JSON 85 | const entityStream = openAiHandler.process(stream); 86 | // Handle the JSON to specific entity, return null if the JSON does not match the schema 87 | const entityColors = new Entity("colors", ColorSchema); 88 | // Transfrom each item of an array to a unique entity 89 | const colorEntityStream = entityColors.genParseArray(entityStream); 90 | // Return the stream of entity 91 | return colorEntityStream; 92 | } 93 | -------------------------------------------------------------------------------- /apps/colors/src/entityTagline.ts: -------------------------------------------------------------------------------- 1 | import OpenAI from "openai"; 2 | import { z } from "zod"; 3 | 4 | import { StreamMode, OpenAiHandler, Entity } from "openai-partial-stream"; 5 | 6 | const TaglineSchema = z.object({ 7 | tagline: z.string().optional(), 8 | }); 9 | 10 | function getTaglineMessages(): any[] { 11 | return [ 12 | { 13 | role: "system", 14 | content: ` 15 | Write for developers who are skilled into frontend, backend, API, who build AI app based on Large Language Model. 16 | The project: 17 | Partial Stream Spec is a specification for a stream of raw text or structured JSON that can be partially parsed and return early results for an early consumption. 18 | Use cases are: 19 | - Parse Partial JSON Stream 20 | - Turn your slow AI app into an engaging real-time app 21 | - Convert a stream of token into a parsable JSON object before the stream ends. 22 | - Implement Streaming UI in LLM-based AI application. 23 | - Leverage OpenAI Function Calling for early stream processing. 24 | - Parse JSON streams into distinct entities. 25 | - Engage your users with a real-time experience. 26 | Keywords: 27 | - Parse 28 | - Json 29 | - Stream 30 | - UI 31 | - LLM 32 | - APP 33 | - Fast 34 | - Realtime 35 | - User experience 36 | - Blocking UI 37 | Ban words: 38 | - Elevate 39 | - Unleash 40 | 41 | Generate a technical tagline with the keywords without the banned words. (MAXIUM 60 CHARACTERS)" 42 | `, 43 | }, 44 | ]; 45 | } 46 | 47 | function getTaglineFunction() { 48 | return { 49 | name: "tagline", 50 | description: "Generate a tagline", 51 | parameters: { 52 | type: "object", 53 | properties: { 54 | tagline: { 55 | type: "string", 56 | description: "The tagline generated", 57 | }, 58 | }, 59 | }, 60 | }; 61 | } 62 | 63 | export async function callGenerateTagline( 64 | openai: OpenAI, 65 | mode: StreamMode = StreamMode.StreamObjectKeyValueTokens, 66 | ) { 67 | const stream = await openai.chat.completions.create({ 68 | messages: getTaglineMessages(), 69 | model: "gpt-3.5-turbo", // OR "gpt-4" 70 | stream: true, // ENABLE STREAMING - Server Sent Event (SSE) 71 | temperature: 0.8, 72 | functions: [getTaglineFunction()], 73 | function_call: { name: "tagline" }, 74 | }); 75 | 76 | const openAiHandler = new OpenAiHandler(mode); 77 | const entityStream = openAiHandler.process(stream); 78 | const entityTagline = new Entity("tagline", TaglineSchema); 79 | const taglineEntityStream = entityTagline.genParse(entityStream); 80 | 81 | return taglineEntityStream; 82 | } 83 | -------------------------------------------------------------------------------- /apps/colors/src/index.ts: -------------------------------------------------------------------------------- 1 | import { Hono } from "hono"; 2 | import { cors } from "hono/cors"; 3 | 4 | import { api } from "./api"; 5 | 6 | const app = new Hono(); 7 | 8 | app.use( 9 | "*", 10 | cors({ 11 | origin: "*", 12 | allowMethods: ["GET", "POST", "PUT", "DELETE"], 13 | allowHeaders: ["Content-Type"], 14 | }), 15 | ); 16 | 17 | app.route("/api", api); 18 | 19 | app.get("/", (c) => c.text("Welcome to Partial Stream!")); 20 | 21 | export default app; 22 | -------------------------------------------------------------------------------- /apps/colors/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "ESNext", 5 | "moduleResolution": "node", 6 | "esModuleInterop": true, 7 | "strict": true, 8 | "lib": [ 9 | "esnext" 10 | ], 11 | "types": [ 12 | "@cloudflare/workers-types" 13 | ], 14 | "jsx": "react-jsx", 15 | "jsxImportSource": "hono/jsx" 16 | }, 17 | } 18 | -------------------------------------------------------------------------------- /examples/00_hello.js: -------------------------------------------------------------------------------- 1 | import { OpenAiHandler, StreamMode } from "openai-partial-stream"; 2 | import OpenAi from "openai"; 3 | 4 | // Intanciate OpenAI client with your API key 5 | const openai = new OpenAi({ 6 | apiKey: process.env.OPENAI_API_KEY, 7 | }); 8 | 9 | async function main() { 10 | // Call the API with stream enabled and a function 11 | const stream = await openai.chat.completions.create({ 12 | messages: [ 13 | { 14 | role: "system", 15 | content: "Say hi to the world.", 16 | }, 17 | ], 18 | model: "gpt-3.5-turbo", // OR "gpt-4" 19 | stream: true, // ENABLE STREAMING 20 | temperature: 1, 21 | functions: [ 22 | { 23 | name: "say_hello", 24 | description: "say hello", 25 | parameters: { 26 | type: "object", 27 | properties: { 28 | sentence: { 29 | type: "string", 30 | description: "The sentence generated", 31 | }, 32 | }, 33 | }, 34 | }, 35 | ], 36 | function_call: { name: "say_hello" }, 37 | }); 38 | 39 | // Select the mode of the stream parser 40 | // - StreamObjectKeyValueTokens: (REALTIME) Stream of JSON objects, key value pairs and tokens 41 | // - StreamObjectKeyValue: (PROGRESSIVE) Stream of JSON objects and key value pairs 42 | // - StreamObject: (ONE-BY-ONE) Stream of JSON objects 43 | // - NoStream: (ALL-TOGETHER) All the data is returned at the end of the process 44 | const mode = StreamMode.StreamObjectKeyValueTokens; 45 | 46 | // Create an instance of the handler 47 | const openAiHandler = new OpenAiHandler(mode); 48 | // Process the stream 49 | const entityStream = openAiHandler.process(stream); 50 | 51 | // Iterate over the stream of entities 52 | for await (const item of entityStream) { 53 | if (item) { 54 | // Display the entity 55 | console.log(item); 56 | } 57 | } 58 | } 59 | 60 | main(); 61 | -------------------------------------------------------------------------------- /examples/01_hello_entity.js: -------------------------------------------------------------------------------- 1 | import { OpenAiHandler, StreamMode, Entity } from "openai-partial-stream"; 2 | import OpenAi from "openai"; 3 | import { z } from "zod"; 4 | 5 | const HelloSchema = z.object({ 6 | sentence: z.string().optional(), 7 | }); 8 | 9 | // Intanciate OpenAI client with your API key 10 | const openai = new OpenAi({ 11 | apiKey: process.env.OPENAI_API_KEY, 12 | }); 13 | 14 | async function main() { 15 | // Call the API with stream enabled and a function 16 | const stream = await openai.chat.completions.create({ 17 | messages: [ 18 | { 19 | role: "system", 20 | content: "Say hi to the world.", 21 | }, 22 | ], 23 | model: "gpt-3.5-turbo", // OR "gpt-4" 24 | stream: true, // ENABLE STREAMING 25 | temperature: 1, 26 | functions: [ 27 | { 28 | name: "say_hello", 29 | description: "say hello", 30 | parameters: { 31 | type: "object", 32 | properties: { 33 | sentence: { 34 | type: "string", 35 | description: "The sentence generated", 36 | }, 37 | }, 38 | }, 39 | }, 40 | ], 41 | function_call: { name: "say_hello" }, 42 | }); 43 | 44 | // Select the mode of the stream parser 45 | // - StreamObjectKeyValueTokens: (REALTIME) Stream of JSON objects, key value pairs and tokens 46 | // - StreamObjectKeyValue: (PROGRESSIVE) Stream of JSON objects and key value pairs 47 | // - StreamObject: (ONE-BY-ONE) Stream of JSON objects 48 | // - NoStream: (ALL-TOGETHER) All the data is returned at the end of the process 49 | const mode = StreamMode.StreamObjectKeyValueTokens; 50 | 51 | // Create an instance of the handler 52 | const openAiHandler = new OpenAiHandler(mode); 53 | // Process the stream 54 | const entityStream = openAiHandler.process(stream); 55 | // Create an entity with the schema to validate the data 56 | const entityHello = new Entity("sentence", HelloSchema); 57 | // Parse the stream to an entity, using the schema to validate the data 58 | const helloEntityStream = entityHello.genParse(entityStream); 59 | 60 | // Iterate over the stream of entities 61 | for await (const item of helloEntityStream) { 62 | if (item) { 63 | // Display the entity 64 | console.log(item); 65 | } 66 | } 67 | } 68 | 69 | main(); 70 | -------------------------------------------------------------------------------- /examples/02_colors.js: -------------------------------------------------------------------------------- 1 | import OpenAI from "openai"; 2 | import { z } from "zod"; 3 | 4 | import { StreamMode, OpenAiHandler, Entity } from "openai-partial-stream"; 5 | 6 | // OPENAI INSTANCE 7 | if (!process.env.OPENAI_API_KEY) { 8 | console.error("OPENAI_API_KEY environment variable not found"); 9 | process.exit(1); 10 | } 11 | 12 | const openai = new OpenAI({ 13 | apiKey: process.env.OPENAI_API_KEY, 14 | }); 15 | 16 | // Schema of the entity 17 | const ColorSchema = z.object({ 18 | hex: z.string().optional(), 19 | name: z.string().optional(), 20 | description: z.string().optional(), 21 | }); 22 | 23 | async function callGenerateColors( 24 | mode = StreamMode.StreamObjectKeyValueTokens, 25 | ) { 26 | // Call OpenAI API, with function calling 27 | // Function calling: https://openai.com/blog/function-calling-and-other-api-updates 28 | const stream = await openai.chat.completions.create({ 29 | messages: [ 30 | { 31 | role: "user", 32 | content: 33 | "Give me a palette of 5 gorgeous color with the hex code, name and a description.", 34 | }, 35 | ], 36 | model: "gpt-4-turbo-preview", // OR "gpt-4" 37 | stream: true, // ENABLE STREAMING 38 | temperature: 1.3, 39 | functions: [ 40 | { 41 | name: "give_colors", 42 | description: "Give a list of color", 43 | parameters: { 44 | type: "object", 45 | properties: { 46 | colors: { 47 | type: "array", 48 | items: { 49 | type: "object", 50 | properties: { 51 | hex: { 52 | type: "string", 53 | description: 54 | "The hexadecimal code of the color", 55 | }, 56 | name: { 57 | type: "string", 58 | description: "The color name", 59 | }, 60 | description: { 61 | type: "string", 62 | description: 63 | "The description of the color", 64 | }, 65 | }, 66 | }, 67 | }, 68 | }, 69 | }, 70 | }, 71 | ], 72 | function_call: { name: "give_colors" }, 73 | }); 74 | 75 | // Handle the stream from OpenAI client 76 | const openAiHandler = new OpenAiHandler(mode); 77 | // Parse the stream to valid JSON 78 | const entityStream = openAiHandler.process(stream); 79 | // Handle the JSON to specific entity, return null if the JSON does not match the schema 80 | const entityColors = new Entity("colors", ColorSchema); 81 | // Transfrom each item of an array to a unique entity 82 | const colorEntityStream = entityColors.genParseArray(entityStream); 83 | // Return the stream of entity 84 | 85 | return colorEntityStream; 86 | } 87 | 88 | async function main() { 89 | // Select the mode of the stream parser 90 | const mode = StreamMode.StreamObject; // ONE-BY-ONE 91 | const colorEntityStream = await callGenerateColors(mode); 92 | 93 | for await (const item of colorEntityStream) { 94 | if (item) { 95 | console.log(item); 96 | } 97 | } 98 | } 99 | 100 | main(); 101 | -------------------------------------------------------------------------------- /examples/03_colors_raw_json.js: -------------------------------------------------------------------------------- 1 | import OpenAI from "openai"; 2 | 3 | import { StreamMode, OpenAiHandler } from "openai-partial-stream"; 4 | 5 | // OPENAI INSTANCE 6 | if (!process.env.OPENAI_API_KEY) { 7 | console.error("OPENAI_API_KEY environment variable not found"); 8 | process.exit(1); 9 | } 10 | 11 | // Intanciate OpenAI client with your API key 12 | const openai = new OpenAI({ 13 | apiKey: process.env.OPENAI_API_KEY, 14 | }); 15 | 16 | async function callGenerateColors( 17 | mode = StreamMode.StreamObjectKeyValueTokens, 18 | ) { 19 | // Call OpenAI API, with function calling 20 | // Function calling: https://openai.com/blog/function-calling-and-other-api-updates 21 | const stream = await openai.chat.completions.create({ 22 | messages: [ 23 | { 24 | role: "user", 25 | content: 26 | "Give me a palette of 5 gorgeous color with the hex code, name and a description.", 27 | }, 28 | ], 29 | model: "gpt-4-turbo-preview", // OR "gpt-4" 30 | stream: true, // ENABLE STREAMING 31 | temperature: 1.3, 32 | functions: [ 33 | { 34 | name: "give_colors", 35 | description: "Give a list of color", 36 | parameters: { 37 | type: "object", 38 | properties: { 39 | colors: { 40 | type: "array", 41 | items: { 42 | type: "object", 43 | properties: { 44 | hex: { 45 | type: "string", 46 | description: 47 | "The hexadecimal code of the color", 48 | }, 49 | name: { 50 | type: "string", 51 | description: "The color name", 52 | }, 53 | description: { 54 | type: "string", 55 | description: 56 | "The description of the color", 57 | }, 58 | }, 59 | }, 60 | }, 61 | }, 62 | }, 63 | }, 64 | ], 65 | function_call: { name: "give_colors" }, 66 | }); 67 | 68 | // Handle the stream from OpenAI client 69 | const openAiHandler = new OpenAiHandler(mode); 70 | // Parse the stream to valid JSON 71 | const entityStream = openAiHandler.process(stream); 72 | 73 | return entityStream; 74 | } 75 | 76 | async function main() { 77 | // Select the mode of the stream parser 78 | const mode = StreamMode.StreamObject; // ONE-BY-ONE 79 | const colorEntityStream = await callGenerateColors(mode); 80 | 81 | for await (const item of colorEntityStream) { 82 | if (item) { 83 | console.log(item.data); 84 | } 85 | } 86 | } 87 | 88 | main(); 89 | -------------------------------------------------------------------------------- /examples/04_tagline.js: -------------------------------------------------------------------------------- 1 | import { OpenAiHandler, StreamMode, Entity } from "openai-partial-stream"; 2 | import OpenAi from "openai"; 3 | import { z } from "zod"; 4 | 5 | // Intanciate OpenAI client with your API key 6 | const openai = new OpenAi({ 7 | apiKey: process.env.OPENAI_API_KEY, 8 | }); 9 | 10 | // Schema of the entity 11 | const TaglineSchema = z.object({ 12 | tagline: z.string().optional(), // Optional because the model can return a partial result 13 | }); 14 | 15 | async function main() { 16 | // Call the API with stream enabled and a function 17 | const stream = await openai.chat.completions.create({ 18 | messages: [ 19 | { 20 | role: "system", 21 | content: ` 22 | Generate a tagline related to the following text:(MAXIUM 60 CHARACTERS) 23 | Partial Stream Spec is a specification for a stream of raw text or structured JSON that can be partially parsed and return early results for an early consumption. 24 | Use cases are: 25 | - LLM stream of token as JSON format. 26 | - OpenAI Function calling, handling stream of data. 27 | - Improve UI/UX by showing partial results to the end user. 28 | 29 | What is the goal of this project?: 30 | - Make AI apps more interactive and responsive. 31 | `, 32 | }, 33 | ], 34 | model: "gpt-3.5-turbo", // OR "gpt-4" 35 | stream: true, // ENABLE STREAMING 36 | temperature: 1.1, 37 | functions: [ 38 | { 39 | name: "tagline", 40 | description: "Generate a tagline", 41 | parameters: { 42 | type: "object", 43 | properties: { 44 | tagline: { 45 | type: "string", 46 | description: "The tagline generated", 47 | }, 48 | }, 49 | }, 50 | }, 51 | ], 52 | function_call: { name: "tagline" }, 53 | }); 54 | 55 | // Select the mode of the stream parser 56 | // - StreamObjectKeyValueTokens: (REALTIME) Stream of JSON objects, key value pairs and tokens 57 | // - StreamObjectKeyValue: (PROGRESSIVE) Stream of JSON objects and key value pairs 58 | // - StreamObject: (ONE-BY-ONE) Stream of JSON objects 59 | // - NoStream: (ALL-TOGETHER) All the data is returned at the end of the process 60 | const mode = StreamMode.StreamObjectKeyValueTokens; 61 | 62 | // Create an instance of the handler 63 | const openAiHandler = new OpenAiHandler(mode); 64 | // Process the stream 65 | const entityStream = openAiHandler.process(stream); 66 | // Create an entity with the schema to validate the data 67 | const entityTagline = new Entity("tagline", TaglineSchema); 68 | // Parse the stream to an entity, using the schema to validate the data 69 | const taglineEntityStream = entityTagline.genParse(entityStream); 70 | 71 | // Iterate over the stream of entities 72 | for await (const item of taglineEntityStream) { 73 | if (item) { 74 | // Display the entity 75 | console.log(item); 76 | } 77 | } 78 | } 79 | 80 | main(); 81 | -------------------------------------------------------------------------------- /examples/05_postcodes.js: -------------------------------------------------------------------------------- 1 | import { OpenAiHandler, StreamMode, Entity } from "openai-partial-stream"; 2 | import OpenAi from "openai"; 3 | import { z } from "zod"; 4 | 5 | // Intanciate OpenAI client with your API key 6 | const openai = new OpenAi({ 7 | apiKey: process.env.OPENAI_API_KEY, 8 | }); 9 | 10 | // Schema of the entity 11 | const PostcodeSchema = z.object({ 12 | name: z.string().optional(), 13 | postcode: z.string().optional(), 14 | population: z.number().optional(), 15 | }); 16 | 17 | async function main() { 18 | // Call the API with stream enabled and a function 19 | const stream = await openai.chat.completions.create({ 20 | messages: [ 21 | { 22 | role: "system", 23 | content: "Give me 3 cities and their postcodes in California.", 24 | }, 25 | ], 26 | model: "gpt-3.5-turbo", // OR "gpt-4" OR "gpt-4-turbo-preview" 27 | stream: true, // ENABLE STREAMING 28 | temperature: 1.1, 29 | functions: [ 30 | { 31 | name: "set_postcode", 32 | description: "Set a postcode and a city", 33 | parameters: { 34 | type: "object", 35 | properties: { 36 | postcodes: { 37 | // <--The name of the entity 38 | type: "array", 39 | items: { 40 | type: "object", 41 | properties: { 42 | name: { 43 | type: "string", 44 | description: "Name of the city", 45 | }, 46 | postcode: { 47 | type: "string", 48 | description: "The postcode of the city", 49 | }, 50 | population: { 51 | type: "number", 52 | description: 53 | "The population of the city", 54 | }, 55 | }, 56 | }, 57 | }, 58 | }, 59 | }, 60 | }, 61 | ], 62 | function_call: { name: "set_postcode" }, 63 | }); 64 | 65 | // Select the mode of the stream parser 66 | // - StreamObjectKeyValueTokens: (REALTIME) Stream of JSON objects, key value pairs and tokens 67 | // - StreamObjectKeyValue: (PROGRESSIVE) Stream of JSON objects and key value pairs 68 | // - StreamObject: (ONE-BY-ONE) Stream of JSON objects 69 | // - NoStream: (ALL-TOGETHER) All the data is returned at the end of the process 70 | const mode = StreamMode.StreamObject; 71 | 72 | // Create an instance of the handler 73 | const openAiHandler = new OpenAiHandler(mode); 74 | // Process the stream 75 | const entityStream = openAiHandler.process(stream); 76 | // Create an entity with the schema to validate the data 77 | const entityPostcode = new Entity("postcodes", PostcodeSchema); 78 | // Parse the stream to an entity, using the schema to validate the data 79 | const postcodeEntityStream = entityPostcode.genParseArray(entityStream); 80 | 81 | // Iterate over the stream of entities 82 | for await (const item of postcodeEntityStream) { 83 | if (item) { 84 | // Display the entity 85 | console.log(item); 86 | } 87 | } 88 | } 89 | 90 | main(); 91 | -------------------------------------------------------------------------------- /examples/06_prompt_schema.js: -------------------------------------------------------------------------------- 1 | import { OpenAiHandler, StreamMode, Entity } from "openai-partial-stream"; 2 | import OpenAi from "openai"; 3 | import { z } from "zod"; 4 | 5 | // Intanciate OpenAI client with your API key 6 | const openai = new OpenAi({ 7 | apiKey: process.env.OPENAI_API_KEY, 8 | }); 9 | 10 | // Schema of the entity 11 | const PostcodeSchema = z.object({ 12 | name: z.string().optional(), 13 | postcode: z.string().optional(), 14 | }); 15 | 16 | async function main() { 17 | // Create an entity with the schema to validate the data 18 | const entityPostcode = new Entity("postcodes", PostcodeSchema); 19 | // Parse the stream to an entity, using the schema to validate the data 20 | const promptSchema = entityPostcode.generatePromptSchema(); 21 | // Call the API with stream enabled and a function 22 | const stream = await openai.chat.completions.create({ 23 | messages: [ 24 | { 25 | role: "system", 26 | content: "Give me 3 cities and their postcodes in California.", 27 | }, 28 | { 29 | role: "system", 30 | content: promptSchema, 31 | }, 32 | ], 33 | model: "gpt-3.5-turbo", // OR "gpt-4" 34 | stream: true, // ENABLE STREAMING 35 | temperature: 1.1, 36 | }); 37 | 38 | // Select the mode of the stream parser 39 | // - StreamObjectKeyValueTokens: (REALTIME) Stream of JSON objects, key value pairs and tokens 40 | // - StreamObjectKeyValue: (PROGRESSIVE) Stream of JSON objects and key value pairs 41 | // - StreamObject: (ONE-BY-ONE) Stream of JSON objects 42 | // - NoStream: (ALL-TOGETHER) All the data is returned at the end of the process 43 | const mode = StreamMode.StreamObjectKeyValueTokens; 44 | 45 | // Create an instance of the handler 46 | const openAiHandler = new OpenAiHandler(mode); 47 | // Process the stream 48 | const entityStream = openAiHandler.process(stream); 49 | // Parse the stream to an entity, using the schema to validate the data 50 | const postcodeEntityStream = entityPostcode.genParseArray(entityStream); 51 | 52 | // Iterate over the stream of entities 53 | for await (const item of postcodeEntityStream) { 54 | if (item) { 55 | // Display the entity 56 | console.log(item); 57 | } 58 | } 59 | } 60 | 61 | main(); 62 | -------------------------------------------------------------------------------- /examples/07_tools.js: -------------------------------------------------------------------------------- 1 | import { OpenAiHandler, StreamMode, Entity } from "openai-partial-stream"; 2 | import OpenAi from "openai"; 3 | import { z } from "zod"; 4 | 5 | // Intanciate OpenAI client with your API key 6 | const openai = new OpenAi({ 7 | apiKey: process.env.OPENAI_API_KEY, 8 | }); 9 | 10 | // Schema of the entity 11 | const TaglineSchema = z.object({ 12 | tagline: z.string().optional(), // Optional because the model can return a partial result 13 | }); 14 | 15 | async function main() { 16 | // Call the API with stream enabled and a function 17 | const stream = await openai.chat.completions.create({ 18 | messages: [ 19 | { 20 | role: "system", 21 | content: ` 22 | Generate a tagline related to the following text:(MAXIUM 60 CHARACTERS) 23 | Partial Stream Spec is a specification for a stream of raw text or structured JSON that can be partially parsed and return early results for an early consumption. 24 | Use cases are: 25 | - LLM stream of token as JSON format. 26 | - OpenAI Function calling, handling stream of data. 27 | - Improve UI/UX by showing partial results to the end user. 28 | 29 | What is the goal of this project?: 30 | - Make AI apps more interactive and responsive. 31 | `, 32 | }, 33 | ], 34 | model: "gpt-3.5-turbo", // OR "gpt-4" 35 | stream: true, // ENABLE STREAMING 36 | temperature: 1.1, 37 | tools: [ 38 | { 39 | type: "function", 40 | function: { 41 | name: "tagline", 42 | description: "Generate a tagline", 43 | parameters: { 44 | type: "object", 45 | properties: { 46 | tagline: { 47 | type: "string", 48 | description: "The tagline generated", 49 | }, 50 | }, 51 | }, 52 | }, 53 | }, 54 | ], 55 | tool_choice: { type: "function", function: { name: "tagline" } }, 56 | }); 57 | 58 | // Select the mode of the stream parser 59 | // - StreamObjectKeyValueTokens: (REALTIME) Stream of JSON objects, key value pairs and tokens 60 | // - StreamObjectKeyValue: (PROGRESSIVE) Stream of JSON objects and key value pairs 61 | // - StreamObject: (ONE-BY-ONE) Stream of JSON objects 62 | // - NoStream: (ALL-TOGETHER) All the data is returned at the end of the process 63 | const mode = StreamMode.StreamObjectKeyValueTokens; 64 | 65 | // Create an instance of the handler 66 | const openAiHandler = new OpenAiHandler(mode); 67 | // Process the stream 68 | const entityStream = openAiHandler.process(stream); 69 | 70 | for await (const item of entityStream) { 71 | if (item) { 72 | // Display the entity 73 | console.log(item); 74 | } 75 | } 76 | 77 | // Create an entity with the schema to validate the data 78 | const entityTagline = new Entity("tagline", TaglineSchema); 79 | // Parse the stream to an entity, using the schema to validate the data 80 | const taglineEntityStream = entityTagline.genParse(entityStream); 81 | 82 | // Iterate over the stream of entities 83 | for await (const item of taglineEntityStream) { 84 | if (item) { 85 | // Display the entity 86 | console.log(item); 87 | } 88 | } 89 | } 90 | 91 | main(); 92 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | Examples of how to use the library. 4 | 5 | ```bash 6 | npm install --save openai-partial-stream 7 | 8 | # REPLACE WITH YOUR OPENAI API KEY 9 | export OPENAI_API_KEY=sk-xxxxx 10 | 11 | node 00_hello.js 12 | ``` 13 | -------------------------------------------------------------------------------- /examples/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "module", 3 | "dependencies": { 4 | "openai-partial-stream": "^0.3.9" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "partial-ai", 3 | "version": "0.1.0", 4 | "type": "commonjs", 5 | "workspaces": [ 6 | "packages/*", 7 | "apps/*" 8 | ], 9 | "author": "Yanael Barbier", 10 | "license": "MIT", 11 | "devDependencies": { 12 | "@changesets/cli": "^2.26.2", 13 | "changeset": "^0.2.6", 14 | "prettier": "3.0.3", 15 | "tsup": "^7.2.0" 16 | }, 17 | "packageManager": "npm@9.8.1", 18 | "dependencies": { 19 | "@changesets/cli": "^2.26.2", 20 | "openai": "^4.26.0", 21 | "turbo": "^1.10.14" 22 | }, 23 | "scripts": { 24 | "publish-packages": "turbo run build && changeset version && changeset publish", 25 | "test": "turbo run test", 26 | "test:watch": "turbo run test:watch", 27 | "format": "turbo run format" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # partial-ai-stream-lib 2 | 3 | ## 0.3.9 4 | 5 | ### Patch Changes 6 | 7 | - support tools call for function 8 | 9 | ## 0.3.8 10 | 11 | ### Patch Changes 12 | 13 | - Fix long tokens with multi braces 14 | 15 | ## 0.3.7 16 | 17 | ### Patch Changes 18 | 19 | - Change zod-to-ts to zod-to-json-schema 20 | 21 | ## 0.3.6 22 | 23 | ### Patch Changes 24 | 25 | - Rename repo to openai-partial-stream 26 | 27 | ## 0.3.5 28 | 29 | ### Patch Changes 30 | 31 | - Update readme 32 | 33 | ## 0.3.4 34 | 35 | ### Patch Changes 36 | 37 | - Improve completion on object closure 38 | 39 | ## 0.3.3 40 | 41 | ### Patch Changes 42 | 43 | - Add types over entity 44 | 45 | ## 0.3.2 46 | 47 | ### Patch Changes 48 | 49 | - small fixes 50 | 51 | ## 0.3.1 52 | 53 | ### Patch Changes 54 | 55 | - Remove console logs 56 | 57 | ## 0.3.0 58 | 59 | ### Minor Changes 60 | 61 | - Rename package 62 | 63 | ## 0.2.1 64 | 65 | ### Patch Changes 66 | 67 | - Add readme 68 | 69 | ## 0.2.0 70 | 71 | ### Minor Changes 72 | 73 | - Add turbo for the build system 74 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Yanael Barbier 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/README.md: -------------------------------------------------------------------------------- 1 | # OpenaAI Partial Stream Library 2 | 3 | - Convert a **stream of token** into a **parsable JSON** object before the stream ends. 4 | - Implement **Streaming UI** in **LLM**-based AI application. 5 | - Leverage **OpenAI Function Calling** for early stream processing. 6 | - Parse **JSON stream** into distinct **entities**. 7 | - Engage your users with a **real-time** experience. 8 | 9 | ## Installation 10 | 11 | ```bash 12 | npm install openai-partial-stream 13 | ``` 14 | 15 | # Usage Example on GitHub 16 | 17 | - [GitHub repo](https://github.com/st3w4r/openai-partial-stream) 18 | 19 | ## Follow the Work 20 | 21 | - [X/Twitter](https://twitter.com/YanaelBarbier) 22 | - [Threads](https://www.threads.net/@yanaelbarbier) 23 | - [Blog](https://yanael.io/subscribe/) 24 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | module.exports = { 3 | preset: "ts-jest", 4 | testEnvironment: "node", 5 | }; 6 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "openai-partial-stream", 3 | "author": "Yanael Barbier", 4 | "license": "MIT", 5 | "private": false, 6 | "version": "0.3.9", 7 | "repository": { 8 | "type": "git", 9 | "url": "https://github.com/st3w4r/openai-partial-stream.git", 10 | "directory": "packages/openai-partial-stream" 11 | }, 12 | "keywords": [ 13 | "stream", 14 | "tokens", 15 | "openai", 16 | "json-parser", 17 | "json-parsing", 18 | "token-passing", 19 | "ai-app", 20 | "gpt-3", 21 | "openai-api", 22 | "gpt-4", 23 | "llm", 24 | "chatgpt", 25 | "chatgpt-api", 26 | "openai-function-calling", 27 | "llm-apps", 28 | "streaming-ui", 29 | "blocking-ui" 30 | ], 31 | "devDependencies": { 32 | "@jest/globals": "^29.7.0", 33 | "@types/jest": "^29.5.5", 34 | "@types/node": "^20.6.0", 35 | "jest": "^29.7.0", 36 | "ts-jest": "^29.1.1", 37 | "typescript": "^5.2.2" 38 | }, 39 | "peerDependencies": { 40 | "typescript": "^5.0.0" 41 | }, 42 | "dependencies": { 43 | "openai": "^4.26.0", 44 | "zod": "^3.22.3", 45 | "zod-to-json-schema": "^3.21.4" 46 | }, 47 | "main": "./dist/index.js", 48 | "module": "./dist/index.mjs", 49 | "types": "./dist/index.d.ts", 50 | "exports": { 51 | ".": { 52 | "require": "./dist/index.js", 53 | "import": "./dist/index.mjs", 54 | "types": "./dist/index.d.ts" 55 | } 56 | }, 57 | "scripts": { 58 | "build": "tsup src/**/*.ts --format esm,cjs --dts --clean", 59 | "dev": "npm run build -- --watch", 60 | "lint": "tsc", 61 | "test": "jest", 62 | "test:watch": "jest --watchAll", 63 | "format": "npx prettier --write src/*.ts" 64 | }, 65 | "publishConfig": { 66 | "access": "public" 67 | }, 68 | "files": [ 69 | "README.md", 70 | "LICENSE", 71 | "dist" 72 | ] 73 | } 74 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/src/entity.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { zodToJsonSchema } from "zod-to-json-schema"; 3 | 4 | import { ParsedResponse, StreamResponseWrapper } from "./utils"; 5 | 6 | export class Entity { 7 | // The name of the entity 8 | private name: K; 9 | // Zod schema 10 | private schema: z.ZodType; 11 | 12 | constructor(name: K, schema: z.ZodType) { 13 | this.name = name; 14 | this.schema = schema; 15 | } 16 | 17 | generatePromptSchema() { 18 | const jsonSchema = 19 | zodToJsonSchema(this.schema, this.name)?.definitions?.[this.name] ?? 20 | ""; 21 | const strJsonSchema = JSON.stringify(jsonSchema); 22 | 23 | const prompt = ` 24 | Format an array of json object to respect this json schema definition: 25 | ${strJsonSchema} 26 | 27 | Output as a json array: 28 | example: [{"name": "value"}, {"name": "value"}] 29 | 30 | Now convert to the JSON format, write directly to JSON. No explanation needed. 31 | `; 32 | return prompt; 33 | } 34 | 35 | parse(entityObject: unknown): T | null { 36 | const parserRes = this.schema.safeParse(entityObject); 37 | return parserRes.success ? parserRes.data : null; 38 | } 39 | 40 | async *genParse( 41 | entityObject: AsyncGenerator< 42 | StreamResponseWrapper | null, 43 | void, 44 | unknown 45 | >, 46 | ): AsyncIterable | null> { 47 | for await (const item of entityObject) { 48 | const data = item && this.parse(item.data); 49 | yield item && data ? { ...item, data, entity: this.name } : null; 50 | } 51 | } 52 | 53 | async *genParseArray( 54 | entityObject: AsyncGenerator< 55 | StreamResponseWrapper | null, 56 | void, 57 | unknown 58 | >, 59 | ): AsyncIterable | null> { 60 | for await (const item of entityObject) { 61 | if (item) { 62 | let childrens = item.data?.[this.name] ?? item.data; 63 | if (Array.isArray(childrens) && childrens.length > 0) { 64 | let index = childrens.length - 1; 65 | let data = this.parse(childrens[index]); 66 | yield data && { 67 | ...item, 68 | entity: this.name, 69 | index, 70 | data, 71 | }; 72 | } 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/src/index.ts: -------------------------------------------------------------------------------- 1 | export { StreamMode } from "./utils"; 2 | export { StreamParser } from "./streamParser"; 3 | export { OpenAiHandler } from "./openAiHandler"; 4 | export { Entity } from "./entity"; 5 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/src/jsonCloser.ts: -------------------------------------------------------------------------------- 1 | import { StreamMode } from "./utils"; 2 | 3 | export class JsonCloser { 4 | private mode: StreamMode; 5 | 6 | private buffer = ""; 7 | private stack: any[] = []; 8 | 9 | private prevSize = 0; 10 | 11 | private closedObject = false; 12 | private closedArray = false; 13 | 14 | constructor(mode: StreamMode = StreamMode.StreamObject) { 15 | this.mode = mode; 16 | } 17 | 18 | append(chunk: string) { 19 | for (const char of chunk) { 20 | this.buffer += char; 21 | 22 | switch (char) { 23 | case "{": 24 | this.stack.push(char); 25 | this.closedObject = false; 26 | break; 27 | case "}": 28 | if (this.stack[this.stack.length - 1] === "{") { 29 | this.stack.pop(); 30 | } 31 | this.closedObject = true; 32 | break; 33 | case "[": 34 | this.stack.push(char); 35 | this.closedArray = false; 36 | break; 37 | case "]": 38 | if (this.stack[this.stack.length - 1] === "[") { 39 | this.stack.pop(); 40 | } 41 | this.closedArray = true; 42 | break; 43 | case '"': 44 | if (this.stack[this.stack.length - 1] === '"') { 45 | this.stack.pop(); 46 | } else { 47 | this.stack.push(char); 48 | } 49 | break; 50 | default: 51 | break; 52 | } 53 | } 54 | } 55 | 56 | closeJson(): string { 57 | let closeBuffer = this.buffer.trim(); 58 | 59 | for (const char of [...this.stack].reverse()) { 60 | switch (char) { 61 | case "{": 62 | if (closeBuffer[closeBuffer.length - 1] === ",") { 63 | closeBuffer = closeBuffer.slice(0, -1); 64 | } 65 | closeBuffer += "}"; 66 | break; 67 | case "[": 68 | if (closeBuffer[closeBuffer.length - 1] === ",") { 69 | closeBuffer = closeBuffer.slice(0, -1); 70 | } 71 | closeBuffer += "]"; 72 | break; 73 | case '"': 74 | if (this.mode === StreamMode.StreamObjectKeyValueTokens) { 75 | closeBuffer += '"'; 76 | } 77 | break; 78 | default: 79 | break; 80 | } 81 | } 82 | 83 | return closeBuffer; 84 | } 85 | 86 | parse(): [boolean, any, any[]] { 87 | try { 88 | const closedJson = this.closeJson(); 89 | const jsonRes = JSON.parse(closedJson); 90 | 91 | const size = JSON.stringify(jsonRes).length; 92 | 93 | let hasChanged = false; 94 | if (size > this.prevSize) { 95 | this.prevSize = size; 96 | hasChanged = true; 97 | } 98 | // Do not process twice if the array and the object get closed. 99 | // XOR operation to check if either the array or object get closed but not both 100 | else if (this.closedObject !== this.closedArray) { 101 | // If the object have been closed consider it as a change 102 | // If the array have been close the object have been closed too 103 | // No need to consider it as a change 104 | // This is to avoid processing twice the same completion 105 | hasChanged = this.closedObject; 106 | this.closedObject = false; 107 | } 108 | 109 | return [hasChanged, jsonRes, this.stack]; 110 | } catch (error) { 111 | return [false, null, this.stack]; 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/src/openAiHandler.ts: -------------------------------------------------------------------------------- 1 | import { StreamMode, StreamResponseWrapper, Status } from "./utils"; 2 | import { StreamParser } from "./streamParser"; 3 | 4 | export class OpenAiHandler { 5 | private itemIdx = 0; 6 | private noStreamBufferList: any = []; 7 | private parser: StreamParser; 8 | private mode: StreamMode; 9 | 10 | constructor(mode = StreamMode.StreamObjectKeyValueTokens) { 11 | this.mode = mode; 12 | this.parser = new StreamParser(this.mode); 13 | } 14 | 15 | async *process( 16 | stream: any, 17 | ): AsyncGenerator { 18 | for await (const msg of stream) { 19 | let content = ""; 20 | 21 | content = msg.choices[0].delta.content; 22 | 23 | // If no content, check function calling content 24 | if (!content) { 25 | content = msg.choices[0]?.delta?.function_call?.arguments; 26 | } 27 | 28 | // Check tools 29 | if (!content) { 30 | content = msg.choices[0]?.delta?.tool_calls?.[0]?.function?.arguments; 31 | } 32 | 33 | if (content === undefined) { 34 | continue; 35 | } 36 | 37 | // TODO: Handle this in the stream parser 38 | // The stream parser should be able to return multi responses. 39 | const chunks = content.split(/(?<={|})/); 40 | 41 | for (const chunk of chunks) { 42 | const res = this.parser.parse(chunk); 43 | 44 | if ( 45 | this.mode === StreamMode.NoStream || 46 | this.mode === StreamMode.Batch 47 | ) { 48 | if (res) { 49 | this.noStreamBufferList.push(res); 50 | } 51 | } else if (res) { 52 | yield res; 53 | } 54 | } 55 | } 56 | 57 | if (this.mode === StreamMode.NoStream) { 58 | for (const item of this.noStreamBufferList) { 59 | yield item; 60 | } 61 | } else if (this.mode === StreamMode.Batch) { 62 | const streamRes: StreamResponseWrapper = { 63 | index: this.itemIdx, 64 | status: Status.COMPLETED, 65 | data: this.noStreamBufferList.map((item: any) => item.data), 66 | }; 67 | yield streamRes; 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/src/streamParser.ts: -------------------------------------------------------------------------------- 1 | import { StreamMode } from "./utils"; 2 | import { JsonCloser } from "./jsonCloser"; 3 | import { Status, StreamResponseWrapper, ErrorResponse } from "./utils"; 4 | 5 | export class StreamParser { 6 | private jsonCloser: JsonCloser; 7 | private mode: StreamMode; 8 | private entityIndex: number = 0; 9 | 10 | constructor(mode: StreamMode = StreamMode.StreamObject) { 11 | this.mode = mode; 12 | this.jsonCloser = new JsonCloser(mode); 13 | } 14 | 15 | // Write to the buffer 16 | // Return a value if the parsing is possible 17 | // if not return empty or null 18 | // Output only if there was a change 19 | // Return based on the mode 20 | parse(chunk: string): StreamResponseWrapper | null { 21 | let index = this.entityIndex; 22 | let completed = false; 23 | let outputEntity: any = null; 24 | let end = chunk.indexOf("}"); 25 | let error = null; 26 | 27 | this.jsonCloser.append(chunk); 28 | 29 | const [hasChanged, resJson, stack] = this.jsonCloser.parse(); 30 | 31 | // If an object have been closed 32 | // Check if an array is open or if the stack is empty 33 | // Meaning the object is completed and a new entity can be created 34 | if ( 35 | end !== -1 && 36 | ("[" === stack[stack.length - 1] || stack.length === 0) 37 | ) { 38 | this.entityIndex += 1; 39 | completed = true; 40 | } 41 | 42 | if (hasChanged && resJson) { 43 | outputEntity = resJson; 44 | } else { 45 | outputEntity = null; 46 | } 47 | 48 | if ( 49 | completed === false && 50 | (this.mode === StreamMode.StreamObject || 51 | this.mode === StreamMode.NoStream) 52 | ) { 53 | return null; 54 | } 55 | 56 | if (outputEntity) { 57 | const streamRes: StreamResponseWrapper = { 58 | index: index, 59 | status: completed ? Status.COMPLETED : Status.PARTIAL, 60 | data: outputEntity, 61 | }; 62 | return streamRes; 63 | } else if (error) { 64 | const streamRes: StreamResponseWrapper = { 65 | index: index, 66 | status: Status.FAILED, 67 | data: error, 68 | }; 69 | return streamRes; 70 | } 71 | return null; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/src/utils.ts: -------------------------------------------------------------------------------- 1 | export enum StreamMode { 2 | Batch = "Batch", 3 | NoStream = "NoStream", 4 | StreamObject = "StreamObject", 5 | StreamObjectKeyValue = "StreamObjectKeyValue", 6 | StreamObjectKeyValueTokens = "StreamObjectKeyValueTokens", 7 | } 8 | 9 | export enum Status { 10 | COMPLETED = "COMPLETED", 11 | PARTIAL = "PARTIAL", 12 | FAILED = "FAILED", 13 | } 14 | 15 | export type StreamResponseWrapper = { 16 | index: number; 17 | status: Status; 18 | data: Record; 19 | }; 20 | 21 | export type ParsedResponse = { 22 | entity: K; 23 | index: number; 24 | status: Status; 25 | data: T; 26 | }; 27 | 28 | export type ErrorResponse = { 29 | code: string; 30 | error: string; 31 | message: string; 32 | }; 33 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/tests/jsonCloser.test.ts: -------------------------------------------------------------------------------- 1 | // External Libraries 2 | import { test, expect } from "@jest/globals"; 3 | 4 | // Internal Modules 5 | import { JsonCloser } from "../src/jsonCloser"; 6 | import { StreamMode } from "../src/utils"; 7 | 8 | let jsonCloser: JsonCloser; 9 | 10 | beforeEach(() => { 11 | jsonCloser = new JsonCloser(); 12 | }); 13 | 14 | const runAndExpectClose = (input: string[], expected: string) => { 15 | input.forEach((data) => jsonCloser.append(data)); 16 | let closedJson = jsonCloser.closeJson(); 17 | expect(closedJson).toBe(expected); 18 | }; 19 | 20 | test("handles complete JSON", async () => { 21 | runAndExpectClose([`{"a": 1, "b": 2, "c": 3}`], `{"a": 1, "b": 2, "c": 3}`); 22 | }); 23 | 24 | test("handles partial JSON with missing parenthese", async () => { 25 | runAndExpectClose([`{"a": 1, "b": 2`], `{"a": 1, "b": 2}`); 26 | }); 27 | 28 | test("handles partial JSON with missing quote", async () => { 29 | runAndExpectClose([`{"a": 1, "b`], `{"a": 1, "b}`); 30 | }); 31 | 32 | test("closes multi-token partial JSON", async () => { 33 | runAndExpectClose([`{"a": 1, "b`, `": 2}`], `{"a": 1, "b": 2}`); 34 | }); 35 | 36 | test("closes empty JSON object", async () => { 37 | runAndExpectClose(["{", "}"], `{}`); 38 | }); 39 | 40 | test("closes empty JSON array", async () => { 41 | runAndExpectClose(["[", "]"], `[]`); 42 | }); 43 | 44 | test("closes multi-token JSON with incomplete string", async () => { 45 | runAndExpectClose([`{"a": 1, "b": "ok`], `{"a": 1, "b": "ok}`); 46 | }); 47 | 48 | test("closes multi-token JSON with StreamMode", async () => { 49 | jsonCloser = new JsonCloser(StreamMode.StreamObjectKeyValueTokens); 50 | runAndExpectClose([`{"a": 1, "b": "ok`], `{"a": 1, "b": "ok"}`); 51 | }); 52 | 53 | test("closes nested multi-token JSON with StreamMode", async () => { 54 | jsonCloser = new JsonCloser(StreamMode.StreamObjectKeyValueTokens); 55 | runAndExpectClose( 56 | [`{"a": 1, "b": {"ok": "super`], 57 | `{"a": 1, "b": {"ok": "super"}}`, 58 | ); 59 | }); 60 | 61 | test("closes partial JSON array with nested objects", async () => { 62 | runAndExpectClose( 63 | [`[{"a": 1,`, ` "b":`, ` {"ok": "super"`], 64 | `[{"a": 1, "b": {"ok": "super"}}]`, 65 | ); 66 | }); 67 | 68 | test("close partial JSON many appends", async () => { 69 | runAndExpectClose( 70 | [ 71 | `{`, 72 | `"`, 73 | `a"`, 74 | `: 1`, 75 | `, "`, 76 | `b"`, 77 | `: {"o`, 78 | `k`, 79 | `": "super`, 80 | `"`, 81 | `}}`, 82 | ], 83 | `{"a": 1, "b": {"ok": "super"}}`, 84 | ); 85 | }); 86 | 87 | // PARSE 88 | 89 | test("parse complete JSON", async () => { 90 | const json = `{"a": 1, "b": 2, "c": 3}`; 91 | jsonCloser.append(json); 92 | const [hasChanged, resJson] = jsonCloser.parse(); 93 | expect(hasChanged).toBe(true); 94 | expect(resJson).toEqual(JSON.parse(json)); 95 | }); 96 | 97 | test("parse partial JSON", async () => { 98 | const json = `{"a": 1, "b": 2`; 99 | const expected = { a: 1, b: 2 }; 100 | jsonCloser.append(json); 101 | const [hasChanged, resJson] = jsonCloser.parse(); 102 | expect(hasChanged).toBe(true); 103 | expect(resJson).toMatchObject(expected); 104 | }); 105 | 106 | test("parse partial JSON with missing quote", async () => { 107 | const json = `{"a": 1, "b`; 108 | jsonCloser.append(json); 109 | const [hasChanged, resJson] = jsonCloser.parse(); 110 | expect(hasChanged).toBe(false); 111 | expect(resJson).toBeNull(); 112 | }); 113 | 114 | test("parse partial JSON multiple parse", async () => { 115 | const json = `{"a": 1, "b": "ok"`; 116 | jsonCloser.append(json); 117 | const expected = { a: 1, b: "ok" }; 118 | 119 | const [hasChanged, resJson] = jsonCloser.parse(); 120 | expect(hasChanged).toBe(true); 121 | expect(resJson).toMatchObject(expected); 122 | 123 | jsonCloser.append(`}`); 124 | 125 | const [hasChanged2, resJson2] = jsonCloser.parse(); 126 | expect(hasChanged2).toBe(true); // TODO: Need to improve the parser 127 | expect(resJson2).toMatchObject(expected); 128 | }); 129 | 130 | // TODO: Should return null, need to improve the parser, malfroemd JSON 131 | test("parse partial JSON with malformed json", async () => { 132 | runAndExpectClose([`{"a": 1, "b": "o\"k"`], `{"a": 1, "b": "o\"k"}`); 133 | }); 134 | 135 | test("parse partial JSON with escape quote", async () => { 136 | runAndExpectClose([`{"a": 1, "b": "o\"k"`], `{"a": 1, "b": "o\"k"}`); 137 | }); 138 | 139 | test("parse partial JSON with escape quote", async () => { 140 | runAndExpectClose( 141 | [`{"a": 1, "b": "o\\"sup\\"k"`], 142 | `{"a": 1, "b": "o\\"sup\\"k"}`, 143 | ); 144 | }); 145 | 146 | // TODO: Need to improve the parser, fix the test 147 | test.skip("parse partial JSON with escape quote", async () => { 148 | runAndExpectClose([`{"a": 1, "b": "o\\"k"}`], `{"a": 1, "b": "o\\"k"}`); 149 | }); 150 | 151 | test("parse partial JSON with escape quote", async () => { 152 | let jsonCloser = new JsonCloser(StreamMode.StreamObjectKeyValue); 153 | const json = `{"a": 1, "b": "o\\"k"`; 154 | jsonCloser.append(json); 155 | const expected = { a: 1, b: 'o"k' }; 156 | const [hasChanged, resJson] = jsonCloser.parse(); 157 | expect(hasChanged).toBe(true); 158 | expect(resJson).toMatchObject(expected); 159 | }); 160 | 161 | // TODO: Need to improve the parser, fix the test 162 | // Current closed json: {"a": 1, "b": "o\"k""} 163 | test.skip("parse partial JSON with escape quote", async () => { 164 | let jsonCloser = new JsonCloser(StreamMode.StreamObjectKeyValueTokens); 165 | const json = `{"a": 1, "b": "o\\"k"`; 166 | jsonCloser.append(json); 167 | const expected = { a: 1, b: 'o"k' }; 168 | const [hasChanged, resJson] = jsonCloser.parse(); 169 | expect(hasChanged).toBe(true); 170 | expect(resJson).toMatchObject(expected); 171 | }); 172 | 173 | test("close array", async () => { 174 | let jsonCloser = new JsonCloser(StreamMode.StreamObjectKeyValueTokens); 175 | const json = `{"a": 1, "b": ["ok", "super"`; 176 | jsonCloser.append(json); 177 | const expected = { a: 1, b: ["ok", "super"] }; 178 | const [hasChanged, resJson] = jsonCloser.parse(); 179 | expect(hasChanged).toBe(true); 180 | expect(resJson).toMatchObject(expected); 181 | }); 182 | 183 | test("parse array and object", async () => { 184 | let jsonCloser = new JsonCloser(StreamMode.StreamObjectKeyValueTokens); 185 | const json = `{"a": 1, "b": ["ok", "super"], "c": {"ok": "super"}`; 186 | jsonCloser.append(json); 187 | const expected = { a: 1, b: ["ok", "super"], c: { ok: "super" } }; 188 | const [hasChanged, resJson] = jsonCloser.parse(); 189 | expect(hasChanged).toBe(true); 190 | expect(resJson).toMatchObject(expected); 191 | }); 192 | 193 | test("parse array and object", async () => { 194 | let jsonCloser = new JsonCloser(StreamMode.StreamObjectKeyValueTokens); 195 | const json = `{"a": 1, "b": ["ok", "super", {"ok": "super`; 196 | jsonCloser.append(json); 197 | const expected = { a: 1, b: ["ok", "super", { ok: "super" }] }; 198 | const [hasChanged, resJson] = jsonCloser.parse(); 199 | expect(hasChanged).toBe(true); 200 | expect(resJson).toMatchObject(expected); 201 | }); 202 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/tests/streamParser.test.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from "@jest/globals"; 2 | 3 | import { StreamMode, StreamResponseWrapper } from "../src/utils"; 4 | import { StreamParser } from "../src/streamParser"; 5 | import { z } from "zod"; 6 | import { Entity } from "../src/entity"; 7 | 8 | test("stream parser", async () => { 9 | const streamParser = new StreamParser(StreamMode.StreamObject); 10 | 11 | let content = [`{"a":`, ` 1, `, `"b": 2`, `, "c": `, `3}`]; 12 | 13 | let results: (StreamResponseWrapper | null)[] = []; 14 | 15 | for (const item of content) { 16 | const res = streamParser.parse(item); 17 | results.push(res); 18 | } 19 | 20 | const expected = [ 21 | null, 22 | null, 23 | null, 24 | null, 25 | { index: 0, status: "COMPLETED", data: { a: 1, b: 2, c: 3 } }, 26 | ]; 27 | 28 | expect(results).toEqual(expected); 29 | }); 30 | 31 | test("stream partial parser", async () => { 32 | const streamParser = new StreamParser(StreamMode.StreamObjectKeyValue); 33 | 34 | let content = [`{"a":`, ` 1, `, `"b": 2`, `, "c": `, `3}`]; 35 | 36 | let results: (StreamResponseWrapper | null)[] = []; 37 | 38 | for (const item of content) { 39 | results.push(streamParser.parse(item)); 40 | } 41 | 42 | const expected = [ 43 | null, 44 | { index: 0, status: "PARTIAL", data: { a: 1 } }, 45 | { index: 0, status: "PARTIAL", data: { a: 1, b: 2 } }, 46 | null, 47 | { index: 0, status: "COMPLETED", data: { a: 1, b: 2, c: 3 } }, 48 | ]; 49 | 50 | expect(results).toEqual(expected); 51 | }); 52 | 53 | test("stream partial parser tokens", async () => { 54 | const streamParser = new StreamParser( 55 | StreamMode.StreamObjectKeyValueTokens, 56 | ); 57 | 58 | let content = [`{"a":`, ` "ok`, ` su`, `per",`, `"b": 2`, `, "c": `, `3}`]; 59 | 60 | let results: (StreamResponseWrapper | null)[] = []; 61 | 62 | for (const item of content) { 63 | results.push(streamParser.parse(item)); 64 | } 65 | 66 | const expected = [ 67 | null, 68 | { index: 0, status: "PARTIAL", data: { a: "ok" } }, 69 | { index: 0, status: "PARTIAL", data: { a: "ok su" } }, 70 | { index: 0, status: "PARTIAL", data: { a: "ok super" } }, 71 | { index: 0, status: "PARTIAL", data: { a: "ok super", b: 2 } }, 72 | null, 73 | { index: 0, status: "COMPLETED", data: { a: "ok super", b: 2, c: 3 } }, 74 | ]; 75 | 76 | expect(results).toEqual(expected); 77 | }); 78 | 79 | async function* arrayToGenerator(arr: T[]): AsyncGenerator { 80 | for await (const item of arr) { 81 | yield item; 82 | } 83 | } 84 | 85 | const ColorSchema = z.object({ 86 | hex: z.string().optional(), 87 | name: z.string().optional(), 88 | description: z.string().optional(), 89 | }); 90 | 91 | const COLORS_INPUT = { 92 | colors: [ 93 | { 94 | hex: "#cd5268", 95 | name: "Passionate Pink", 96 | description: 97 | "This is a vibrant, dramatic pink with a romantic charm. It adds a fashionable and bold accent to your designs.", 98 | }, 99 | { 100 | hex: "#287d8e", 101 | name: "Serene Ocean", 102 | description: 103 | "A calming yet energetic shade of blue that represents the ocean. It provides a sense of tranquility and inspiration.", 104 | }, 105 | { 106 | hex: "#fcdab7", 107 | name: "Soft Peach", 108 | description: 109 | "A delicate and warm peach color. It carries a sense of freshness, gentleness, and positivity.", 110 | }, 111 | ], 112 | }; 113 | 114 | const EXPECTED_COLORS = [ 115 | { 116 | index: 0, 117 | status: "COMPLETED", 118 | data: COLORS_INPUT.colors[0], 119 | entity: "colors", 120 | }, 121 | { 122 | index: 1, 123 | status: "COMPLETED", 124 | data: COLORS_INPUT.colors[1], 125 | entity: "colors", 126 | }, 127 | { 128 | index: 2, 129 | status: "COMPLETED", 130 | data: COLORS_INPUT.colors[2], 131 | entity: "colors", 132 | }, 133 | ]; 134 | 135 | test("stream partial array", async () => { 136 | const jsonInputs = JSON.stringify(COLORS_INPUT); 137 | const streamParser = new StreamParser(StreamMode.StreamObject); 138 | const entityColors = new Entity("colors", ColorSchema); 139 | 140 | const results = [...jsonInputs].map((item) => streamParser.parse(item)); 141 | 142 | const streamGen = arrayToGenerator(results); 143 | const colorEntityStream = entityColors.genParseArray(streamGen); 144 | 145 | const colors: any[] = []; 146 | for await (const item of colorEntityStream) { 147 | if (item) { 148 | colors.push(item); 149 | } 150 | } 151 | 152 | expect(colors).toHaveLength(3); 153 | colors.forEach((color, idx) => { 154 | expect(color).toEqual(EXPECTED_COLORS[idx]); 155 | }); 156 | }); 157 | 158 | // TODO: Improve the parer to handle this case 159 | test("stream partial", async () => { 160 | const inputs = [ 161 | `{ 162 | "hello": "world", 163 | `, 164 | ` 165 | "name": "jack", 166 | `, 167 | ` 168 | "address": { 169 | "street": "123 Fake St", 170 | "city": "Springfield", 171 | "state": "NY" 172 | }, 173 | `, 174 | ` 175 | "age": 30 176 | }`, 177 | ]; 178 | const streamParser = new StreamParser(StreamMode.StreamObjectKeyValue); 179 | 180 | const results = inputs.map((item) => streamParser.parse(item)); 181 | 182 | const expected = [ 183 | { index: 0, status: "PARTIAL", data: { hello: "world" } }, 184 | { 185 | index: 0, 186 | status: "PARTIAL", 187 | data: { hello: "world", name: "jack" }, 188 | }, 189 | { 190 | index: 0, 191 | status: "PARTIAL", 192 | data: { 193 | hello: "world", 194 | name: "jack", 195 | address: { 196 | street: "123 Fake St", 197 | city: "Springfield", 198 | state: "NY", 199 | }, 200 | }, 201 | }, 202 | { 203 | index: 0, 204 | status: "COMPLETED", 205 | data: { 206 | hello: "world", 207 | name: "jack", 208 | address: { 209 | street: "123 Fake St", 210 | city: "Springfield", 211 | state: "NY", 212 | }, 213 | age: 30, 214 | }, 215 | }, 216 | ]; 217 | 218 | expect(results).toEqual(expected); 219 | }); 220 | 221 | // TODO: Improve the parer to handle this case 222 | // Parser should be able to return an array when their is mutli object closed 223 | // In the same chunk 224 | test.skip("stream partial longer tokens", async () => { 225 | const inputs = [ 226 | `{ "colors": [{"hex": "#FF7F50`, 227 | `"},{`, 228 | `"he`, 229 | `x": "#465`, 230 | `2B4"}] }`, 231 | ]; 232 | 233 | // TODO: Add another case 234 | // const inputs = [ 235 | // ` 236 | // { "colors": [{"a":1},{"b":2},{"c":3`, 237 | // `}`, 238 | // `] }`, 239 | // ]; 240 | 241 | const streamParser = new StreamParser(StreamMode.StreamObject); 242 | 243 | const results = inputs.map((item) => streamParser.parse(item)); 244 | 245 | const expected = [ 246 | null, 247 | [ 248 | { 249 | index: 0, 250 | status: "COMPLETED", 251 | data: { colors: [{ hex: "#FF7F50" }] }, 252 | }, 253 | { 254 | index: 1, 255 | status: "PARTIAL", 256 | data: { colors: [{ hex: "#FF7F50" }, {}] }, 257 | }, 258 | ], 259 | null, 260 | null, 261 | { 262 | index: 1, 263 | status: "COMPLETED", 264 | data: { colors: [{ hex: "#FF7F50" }, { hex: "#4652B4" }] }, 265 | }, 266 | ]; 267 | 268 | results.forEach((result) => { 269 | console.log(result?.index, result?.status, result?.data); 270 | }); 271 | 272 | expect(results).toEqual(expected); 273 | }); 274 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/tests/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from "@jest/globals"; 2 | import { Status } from "../src/utils"; 3 | 4 | test("Status", async () => { 5 | expect(Status.COMPLETED).toBe("COMPLETED"); 6 | expect(Status.PARTIAL).toBe("PARTIAL"); 7 | expect(Status.FAILED).toBe("FAILED"); 8 | }); 9 | -------------------------------------------------------------------------------- /packages/openai-partial-stream/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "esModuleInterop": true, 4 | "forceConsistentCasingInFileNames": true, 5 | "isolatedModules": true, 6 | "moduleResolution": "node", 7 | "preserveWatchOutput": true, 8 | "skipLibCheck": true, 9 | "noEmit": true, 10 | "strict": true 11 | }, 12 | "include": [ 13 | "src/**/*.ts", 14 | "tests/**/*.ts", 15 | ], 16 | "exclude": [ 17 | "node_modules", 18 | ".turbo" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /spec/CHALLENGES.md: -------------------------------------------------------------------------------- 1 | # Challenges 2 | 3 | A set of challenges that need to be solve to implement the specification. In a perfomant and reliable way. 4 | 5 | 6 | 7 | - Parse a stream of JSON data coming as a stream of tokens. 8 | - JSON Closer, close a JSON object that is not closed yet. 9 | - Detect if inside a key or inside a value. 10 | - Wait for the whole key. 11 | - Early true/false detection. 12 | - Early key detection. Based on schema. Key uniqueness. 13 | - Key only, set value on null on waiting for the value. 14 | - Providate delta between the previous data and the current data. In a valid format. 15 | - Detect if the data is valid JSON. 16 | - Path detection. Get the current data path of the current entity. 17 | - COMPLETED status detection. Return the status on the correct message. 18 | - Index increment. Increment an index for each entity. 19 | - Progressive mode return the Key/Value 20 | - Realtime mode return the key and partial value. 21 | - One by one mode return the whole entity. 22 | - Batch mode return the whole payload. 23 | - Handle escaped characters. 24 | - Raw text to JSON. Convert a raw text to valid JSON entity. 25 | -------------------------------------------------------------------------------- /spec/THE_SPEC.md: -------------------------------------------------------------------------------- 1 | # Partial Stream Spec 2 | 3 | Turn a stream of token into a parsable JSON object as soon as possible. 4 | 5 | ## Introduction 6 | 7 | Partial Stream Spec is a specification for a stream of raw text or structured JSON that can be partially parsed and return early results for an early consumption. 8 | 9 | Use cases are: 10 | - LLM stream of token as JSON format. 11 | - OpenAI Function calling, handling stream of data. 12 | - Improve UI/UX by showing partial results to the end user. 13 | 14 | ## Objectives 15 | 16 | Return early results as soon as possible into a parsable JSON foramt. 17 | With the raise of LLM and API that produce a stream of text data and where new use case are emerging like OpenAI Function Calling. 18 | Even if the stream of the data is enable on those API the data still need to be entirely arrived to be able to parse it for using it. 19 | 20 | Because it's a stream of token the JSON payload arrive in chunks that are not valid JSON until the end of the stream. 21 | If we accumuluate the chunks and try to parse it it will fails due to malformated JSON because still incomplete. 22 | 23 | The goal if to be able to use the JSON object when some part are ready to be used. 24 | 25 | For example some keys and values could have arrived already and we can already use it to display it to the end user or to perform any action. 26 | 27 | This early processing can be enable by always returning a valid JSON object even if it's **not data complete**. 28 | 29 | 30 | ## Example 31 | 32 | Let's say we have a function that return the name of a file to create with it's content generated by the LLM. 33 | In that case the content can be long to arrive, but we can already create the file in the meantime. 34 | 35 | ``` 36 | // The entire payload we're expecting to receive 37 | { 38 | "filename": "my_file.txt", 39 | "content": "Hello World" 40 | } 41 | 42 | // The stream of tokens: 43 | // Part 1 -- Malformed JSON -- No Keys and Values available 44 | // Nothing can be done with it yet 45 | { 46 | "filena 47 | 48 | 49 | // Part 2 -- Malformed JSON -- Key: Filename and it's value is available 50 | // We can already create the file 51 | { 52 | "filename": "my_file.txt", 53 | "conte 54 | 55 | 56 | // Part 3 -- Malformed JSON -- Key: Content and partial of it's value available. 57 | // We can start writting to the file 58 | { 59 | "filename": "my_file.txt", 60 | "content": "Hello 61 | 62 | 63 | // Part 4 -- Well formated JSON -- Everything is available 64 | // We can close the file 65 | { 66 | "filename": "my_file.txt", 67 | "content": "Hello World" 68 | } 69 | ``` 70 | 71 | The goal is to be able to consume this partial JSON. 72 | To enable this we will always return a valid JSON object even if the data is not complete yet. 73 | And we will progressively provide it when it become available. 74 | 75 | 76 | The objective is to simplify the developer experience by not having to deal with the stream of token and the parsing of it. 77 | 78 | 79 | As a stream of text when it is to be read by a human a character/words/token is fine to be read and understood. But when we want to base some action on it and be handled by a function or other system it need to be parsable and some how correct in certain extend. 80 | 81 | Let take the example of the file creation again. 82 | 83 | If we are based on the JSON keys to perform some action we need to have the keys available to be able to perform the action. 84 | 85 | For example if the payload is only 86 | ```jsonc 87 | { 88 | "filena 89 | 90 | ``` 91 | We can not be sure it will be the correct key for the action or even so if it's in the middle of the filename. 92 | ```jsonc 93 | { 94 | "filename": "my_fil 95 | ``` 96 | We can not create the file because we don't know yet which file to create. 97 | 98 | So we need different level of parsing to be able to perform certain action. 99 | 100 | Because the key is the principal component we want to wait to have at least 101 | the entire key before returning a partial JSON. 102 | 103 | For example we can return this: 104 | 105 | ```json 106 | { 107 | "filename": "my_file.txt" 108 | } 109 | ``` 110 | Where we now have the key and value to create the file. But we don't have the content yet. but this is fine because we can already create the file. 111 | And this mode is the **Progressive** mode. 112 | 113 | When the content arrive we can add it to the payload, same we will need the entire key beforing providing the valid JSON. 114 | ```json 115 | { 116 | "filename": "my_file.txt", 117 | "content": "Hello world" 118 | } 119 | ``` 120 | Now we can start writting to the file. 121 | 122 | 123 | As well another mode it the realtime mode where the content is streamed as well but in a valid JSON. 124 | 125 | For example it is useful to start writting to the file as soon as the content is available. 126 | 127 | ```jsonc 128 | 129 | // Partial content 130 | { 131 | "filename": "my_file.txt", 132 | "content": "Hello w" 133 | } 134 | ``` 135 | 136 | ```jsonc 137 | // Completed content 138 | { 139 | "filename": "my_file.txt", 140 | "content": "Hello world" 141 | } 142 | ``` 143 | 144 | This mode is the **Realtime** mode 145 | 146 | 147 | 148 | Another mode is the One-by-one mode where each entity (json object) is returned one by one, it waits to be completed with all the keys and values before returning it. 149 | 150 | For example if we have an array of object we can return each object one by one when it's completed. 151 | 152 | This is the overall payload: 153 | ```json 154 | [ 155 | { 156 | "filename": "my_file.txt", 157 | "content": "Hello world" 158 | }, 159 | { 160 | "filename": "my_file2.txt", 161 | "content": "Awesomeness" 162 | } 163 | ] 164 | ``` 165 | 166 | This is the first object returned: 167 | ```json 168 | { 169 | "filename": "my_file.txt", 170 | "content": "Hello world" 171 | } 172 | ``` 173 | 174 | This is the second object returned: 175 | ```json 176 | { 177 | "filename": "my_file2.txt", 178 | "content": "Awesomeness" 179 | } 180 | ``` 181 | 182 | This mode is the **One-by-one** mode 183 | 184 | 185 | 186 | 187 | 188 | We can as well have a mode where we still want all the data to arrive together or even as a batch. 189 | 190 | 191 | ## Specification 192 | 193 | 1. Return a valid JSON 194 | 195 | Always return a valid JSON object even if the data is not complete yet. 196 | 197 | 2. Return the entire key, not partial keys. 198 | 199 | The key is an element that is not supposed to be partial, it's either there or not. 200 | So we will wait to have the entire key before returning a partial JSON. Or we will return a valid JSON without the key that is progressivly completed. 201 | 202 | 3. Return the zero value of the value type. 203 | 204 | The value is an element that can be partial, so we will return the zero value of the value type. if any character of the value have not arrived yet. 205 | For a string it will be an **empty string**, for a number it will be **0**, for a boolean it will be **false**, for an array it will be an **empty array**, for an object it will be an **empty object**. 206 | 207 | It certain case better to not return the key until the entire value is available, it will depend on the usecase where the zero value can cause a wrong behavior. 208 | 209 | 4. Partial value 210 | 211 | Depending on the mode chosen the value can be returned as partial or not. 212 | - For a string it will be the beging of the string: 213 | - For a number it will be the beging of the number, 214 | - For a boolean it will be either the zero value or based on the first letter or true/false or it can be based on the first letter that arrive: t/f 215 | - For an array it can containts the first element. 216 | - For an object it can containts the first key or/and value. 217 | 218 | 5. Status 219 | 220 | The status of the data returned should be available to know if the data is complete or not. 221 | Status: 222 | - **PARTIAL** - In the case the data is not complete yet the status will be. 223 | - **COMPLETED** - In the case the data is complete the status will be. 224 | - **ERROR** - In the case the data is not valid JSON or if the data is not valid for the schema provided an error will be returned. 225 | 226 | 6. Entity 227 | 228 | When the data is an array of object we can return each object one by one when it's completed. Mark the data as an entity type. 229 | 230 | Include an `entity` field. 231 | Where the entity is a string. 232 | e.g.: `"entity": "file"` 233 | 234 | 6. Index 235 | 236 | If the data is an array of object we can return the index of the object in the array. 237 | 238 | Include an `index` field. 239 | 240 | Where the index is a number starting at 0. 241 | e.g.: `"index": 0` 242 | 243 | 244 | 7. Mode 245 | 246 | The mode is the way the data is returned. 247 | - **`BATCH`** - The data is returned as a batch when the entire payload have arrived. As an array of entities. 248 | - **`ALL-TOGETHER`** - The data is returned all together when the entire payload have arrived but with indivudal entities. 249 | - **`ONE-BY-ONE`** - The data is returned as a whole entity all the keys and values of one entity at once. 250 | - **`PROGRESSIVE`** - The data is returned as completed Key/Value 251 | - **`REALTIME`** - The data is returned as soon as it's available with a completed key but partial value. 252 | 253 | 8. Error 254 | 255 | If the data is not valid JSON or if the data is not valid for the schema provided an error will be returned. 256 | 257 | Include an `error` field. 258 | With an error code `code` and an error message `message`. 259 | 260 | Where the code is a uppercase string: 261 | - `INVALID_JSON` -- The data is not valid JSON 262 | - `INVALID_SCHEMA` -- The data is not valid for the schema provided 263 | 264 | Where the error is a string. 265 | 266 | 9. Delta 267 | 268 | The delta is the difference between the previous data and the current data. 269 | 270 | Include a `delta` field. 271 | 272 | Where the delta is the new data that have arrived. 273 | 274 | 10. Completed 275 | 276 | Detect the completion of an entity. 277 | When the entity have been completed the status will be `COMPLETED`. 278 | And return the entire entity. 279 | 280 | 11. Only when changes 281 | 282 | Only return the data when it changes. 283 | If the data is the same as the previous one don't return it. 284 | 285 | 286 | 287 | ## Feature 288 | 289 | - Stream of chunk of json 290 | - Json can have any shape 291 | - Process partial json 292 | - Iterate over array 293 | - Each element of an array is an individual entity 294 | - Detect when an entity is COMPLETED 295 | - Detect when the entity have an ERROR 296 | - Produce only when changes 297 | - Always valid JSON 298 | 299 | 300 | ## SSE Spec 301 | 302 | Send the data as a stream of SSE. Do not send empty data. 303 | 304 | Each event is of type message. 305 | ```sse 306 | 307 | data: {"index":4,"status":"PARTIAL","data":{"hex":"#9400D3","name":"Dark Violet"},"entity":"colors"} 308 | 309 | data: {"index":4,"status":"PARTIAL","data":{"hex":"#9400D3","name":"Dark Violet","description":"A deep, rich"},"entity":"colors"} 310 | ``` 311 | 312 | With 2 newlines at the end of each data. 313 | 314 | Close the SSE connection when the data is completed. 315 | With an event of type `CLOSE` and followed with [DONE] as data. 316 | ``` 317 | event: CLOSE 318 | data: [DONE] 319 | ``` 320 | -------------------------------------------------------------------------------- /turbo.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://turbo.build/schema.json", 3 | "pipeline": { 4 | "build": { 5 | "dependsOn": [ 6 | "^build" 7 | ], 8 | "outputs": [ 9 | "dist/**" 10 | ] 11 | }, 12 | "dev": { 13 | "cache": false, 14 | "persistent": true 15 | }, 16 | "lint": {}, 17 | "format": { 18 | "cache": true 19 | }, 20 | "test": {}, 21 | "test:watch": { 22 | "cache": false 23 | } 24 | } 25 | } 26 | --------------------------------------------------------------------------------