├── .gitignore ├── js-client ├── bun.lockb ├── README.md ├── .prettierrc ├── src │ ├── index.ts │ ├── wrapper.test.ts │ ├── internal.ts │ ├── wrapper.ts │ ├── runs.test.ts │ ├── logger.ts │ ├── events.ts │ ├── types.ts │ └── client.ts ├── .prettierignore ├── tsconfig.json ├── package.json ├── CHANGELOG.md └── .gitignore ├── rustfmt.toml ├── proxy ├── src │ ├── providers │ │ ├── fixtures │ │ │ ├── ollama_text.json │ │ │ ├── anthropic_text.json │ │ │ ├── openai_text.json │ │ │ ├── groq_text.json │ │ │ ├── anthropic_tools_response_nonstreaming.json │ │ │ ├── ollama_request.sh │ │ │ ├── groq_request.sh │ │ │ ├── groq_tools_response_nonstreaming.json │ │ │ ├── openai_request.sh │ │ │ ├── anthropic_request.sh │ │ │ ├── anthropic_tools.json │ │ │ ├── openai_tools.json │ │ │ ├── groq_tools.json │ │ │ ├── openai_tools_response_nonstreaming.json │ │ │ ├── openai_text_response_nonstreaming.json │ │ │ ├── anthropic_text_response_nonstreaming.json │ │ │ ├── groq_tools_response_streaming.txt │ │ │ ├── ollama_text_response_nonstreaming.json │ │ │ ├── groq_text_response_nonstreaming.json │ │ │ ├── anthropic_tools_response_streaming.txt │ │ │ ├── openai_tools_response_streaming.txt │ │ │ └── anthropic_text_and_tools_response_streaming.txt │ │ ├── mistral.rs │ │ ├── together.rs │ │ ├── anyscale.rs │ │ ├── deepinfra.rs │ │ ├── fireworks.rs │ │ └── custom.rs │ ├── snapshots │ │ ├── chronicle_proxy__test__call_provider.snap │ │ ├── chronicle_proxy__test__call_provider_nonstreaming.snap │ │ ├── chronicle_proxy__test__call_provider_streaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@openai_text_streaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@groq_tool_calls_nonstreaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@openai_tool_calls_nonstreaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@groq_tool_calls_streaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@anthropic_tool_calls_nonstreaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@openai_tool_calls_streaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@openai_text_nonstreaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@anthropic_tool_calls_streaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@anthropic_text_nonstreaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@groq_text_nonstreaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@anthropic_text_streaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@ollama_text_nonstreaming.snap │ │ ├── chronicle_proxy__testing__fixture_response@ollama_text_streaming.snap │ │ └── chronicle_proxy__testing__fixture_response@groq_text_streaming.snap │ ├── error.rs │ ├── database.rs │ ├── streaming.rs │ ├── config.rs │ ├── database │ │ ├── testing.rs │ │ └── logging.rs │ └── response.rs ├── migrations │ ├── 20240424_chronicle_proxy_data_tables_sqlite.sql │ ├── 20240424_chronicle_proxy_data_tables_postgresql.sql │ ├── 20240419_chronicle_proxy_init_sqlite.sql │ ├── 20240419_chronicle_proxy_init_postgresql.sql │ ├── 20240625_chronicle_proxy_steps_sqlite.sql │ └── 20240625_chronicle_proxy_steps_postgresql.sql ├── CHANGELOG.md └── Cargo.toml ├── justfile ├── api ├── .sqlx │ ├── query-e0f41147ec6888ccade24a344ffc3d3f6c155b3bf7fd0b88f5823f3848dfa32e.json │ ├── query-10df9013515179bad2258e1455c1df5112ec80d8e60ae29637d29ae2dd749aff.json │ ├── query-086d7f008bc18ec7176861561171259cb302127f77ea432b8106a76573bed9b3.json │ ├── query-4cd3527be088e50a26aea445ae9f54bafdad413dcb54b4a101d79d54fe3f2f7d.json │ ├── query-99057bfc0d631cf49bc9f6fa28fa984b59789ca4965d9ef04da8ea29307c24c2.json │ ├── query-06ffa4ce61920b3edc549d3d8bc542e8353d13497b9729449d348e63fee21449.json │ ├── query-2b54620a64e6e9620b0e72b016281aa775f49f4566911ac1fcbd9b968233cea3.json │ ├── query-3af44c3f763a1e3ebe07172cd79b20c434098c8336f6de8da1496e97f5f26faa.json │ ├── query-cc1f79b44a5437c0a3f4086a0c1d27bee85f38006b5ee6e2cce0a2034cfc4069.json │ ├── query-ed7888aeb1ef8a84d5e17c81abae30405f36fd2c52248ed187fb6621b748338d.json │ ├── query-b46633aa2e966bc746c2e39838239a114165666fbb62bd41bb6ee73ca7126b92.json │ ├── query-d6d06dea325d3ef8acb61fa50281511bbc3bcd4aa734106a5500dd8c4daa75a3.json │ ├── query-679f767fa0712716d1ed6e556db43f5b9aef432185b3bc883df870aa6de1d80d.json │ ├── query-91dff4cfd5df0be650efdd9bbf3a8196fd9c2033293aca47d6507c5b26bf404f.json │ ├── query-e70e28f56e8a998627c0829f6f26926ce45c3c914052a3f075339bb86c11405e.json │ ├── query-4be1576cf369ef497786f42c6268468ac977712714a9f0fb1f2b1c3d183cc45e.json │ ├── query-be01a8e882fddcc5b5365018ed8e28182325664583c161679fd7e4f9513e302f.json │ ├── query-9889a4ad0b25bb3621e29eb881ec86c9c17e05b19ee4998c93fdb007ae2b6743.json │ ├── query-73d78f5181b3b0753b208041520696fc7131f3ca864e2d793f98c64b7f8f6fa6.json │ ├── query-6a2e15ca052183363cf32696ea30c50fb69730433700de8a85876470c9f663b5.json │ ├── query-dd98c73a917d0b2014b27bd8f6d7462bf22fd0b4970aff97d3bae2e29d6c8501.json │ ├── query-840ef63ec6eb5ea426c6ae8af10c8206cbdfa39d83c183a9fdceacc527ae8765.json │ ├── query-7354dc5ce855c4a218b2df038e421aa2d2d0358e553c3f10124742dfee3973a3.json │ ├── query-905d0c0ef057084be737d421c298a1a686392c01f3b52d17f3ee1db1a1159789.json │ ├── query-eb1ed7eab6b4b36c88887e9d0873d167422864b878fb193e80b2d674d7fe4909.json │ ├── query-6358d020be9dda3b63dcdb3fac3f98ff75fa13d011e3bf920955862906c8dc81.json │ ├── query-a5b1858fc31b80561b6be6738c8e6debf75e4a56bfbbec1f8091c496806bd719.json │ ├── query-299c2b0e94a77372ab8acbabccc118b17e8da37a8577c698fa8daf3fbb18b66b.json │ ├── query-80049063dbf162f954233d825e0b0ba1a2a3850c557a1d9350512a159d9a7d19.json │ ├── query-5881c058a486308ea83071f1bd0281f967e429cf2abf0f1214f3243a60d2e113.json │ ├── query-8904afb292c1e2d3d2f0760b29a3438f2b30677dc864a52a99bacc217dd12435.json │ └── query-636c15e0d8b3dab82c819c974a1547c78b6afde540013c699caf2f827d99115c.json ├── CHANGELOG.md ├── Cargo.toml └── src │ ├── events.rs │ ├── config.rs │ ├── error.rs │ ├── proxy.rs │ └── main.rs ├── Cargo.toml ├── Dockerfile └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .env 3 | .env.* 4 | *.sqlite3 5 | *.sqlite3* 6 | test.db* 7 | -------------------------------------------------------------------------------- /js-client/bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimfeld/chronicle/HEAD/js-client/bun.lockb -------------------------------------------------------------------------------- /js-client/README.md: -------------------------------------------------------------------------------- 1 | # chronicle-proxy 2 | 3 | This library is a client SDK for the Chronicle LLM Proxy. 4 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | imports_granularity = "Crate" 3 | group_imports = "StdExternalCrate" 4 | 5 | -------------------------------------------------------------------------------- /js-client/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "useTabs": false, 3 | "singleQuote": true, 4 | "trailingComma": "es5", 5 | "printWidth": 100 6 | } 7 | -------------------------------------------------------------------------------- /js-client/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './client.js'; 2 | export * from './events.js'; 3 | export * from './runs.js'; 4 | export * from './types.js'; 5 | export * from './wrapper.js'; 6 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/ollama_text.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "llama3", 3 | "max_tokens": 1024, 4 | "messages": [ 5 | { "role": "user", "content": "Who was Lassie the dog?"} 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/anthropic_text.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "claude-3-haiku-20240307", 3 | "max_tokens": 1024, 4 | "messages": [ 5 | { "role": "user", "content": "Who was Lassie the dog?"} 6 | ] 7 | } 8 | 9 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/openai_text.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "gpt-3.5-turbo", 3 | "max_tokens": 1024, 4 | "messages": [ 5 | { "role": "user", "content": "Who was Lassie the dog?"} 6 | ] 7 | } 8 | 9 | 10 | -------------------------------------------------------------------------------- /js-client/.prettierignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | /package 5 | .env 6 | .env.* 7 | !.env.example 8 | 9 | # Ignore files for PNPM, NPM and YARN 10 | pnpm-lock.yaml 11 | package-lock.json 12 | yarn.lock 13 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/groq_text.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "llama3-8b-8192", 3 | "max_tokens": 1024, 4 | "messages": [ 5 | { "role": "user", "content": "Who was Lassie the dog?"} 6 | ] 7 | } 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | _list: 2 | @just --list 3 | 4 | run: 5 | cd api/web && bun run build 6 | cd api && cargo run --release serve 7 | 8 | filigree: 9 | cd api && ../../filigree/target/debug/filigree write 10 | 11 | prepare: 12 | cd api/web && bun install && bun run build 13 | 14 | dev-api: 15 | cd api && cargo watch -d 0.1 -x 'lrun serve --dev' 16 | 17 | dev-web: 18 | cd api/web && bun run dev 19 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/anthropic_tools_response_nonstreaming.json: -------------------------------------------------------------------------------- 1 | {"id":"msg_01YQtguCgRjBPW8oJtt5DKqk","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[{"type":"tool_use","id":"toolu_018FXg1i7nbT3stUdoysiffi","name":"get_characteristics","input":{"name":"Daniel","hair_color":"brown"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":459,"output_tokens":52}} -------------------------------------------------------------------------------- /api/.sqlx/query-e0f41147ec6888ccade24a344ffc3d3f6c155b3bf7fd0b88f5823f3848dfa32e.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM user_sessions WHERE expires_at < now()", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [] 8 | }, 9 | "nullable": [] 10 | }, 11 | "hash": "e0f41147ec6888ccade24a344ffc3d3f6c155b3bf7fd0b88f5823f3848dfa32e" 12 | } 13 | -------------------------------------------------------------------------------- /api/.sqlx/query-10df9013515179bad2258e1455c1df5112ec80d8e60ae29637d29ae2dd749aff.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM user_sessions WHERE user_id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "10df9013515179bad2258e1455c1df5112ec80d8e60ae29637d29ae2dd749aff" 14 | } 15 | -------------------------------------------------------------------------------- /api/.sqlx/query-086d7f008bc18ec7176861561171259cb302127f77ea432b8106a76573bed9b3.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM user_sessions WHERE id = $1 and hash = $2", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "086d7f008bc18ec7176861561171259cb302127f77ea432b8106a76573bed9b3" 15 | } 16 | -------------------------------------------------------------------------------- /api/.sqlx/query-4cd3527be088e50a26aea445ae9f54bafdad413dcb54b4a101d79d54fe3f2f7d.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO email_logins (user_id, email, verified)\n VALUES ($1, $2, $3)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Text", 10 | "Bool" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "4cd3527be088e50a26aea445ae9f54bafdad413dcb54b4a101d79d54fe3f2f7d" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-99057bfc0d631cf49bc9f6fa28fa984b59789ca4965d9ef04da8ea29307c24c2.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM api_keys\n WHERE\n api_key_id = $1\n AND organization_id = $2", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "99057bfc0d631cf49bc9f6fa28fa984b59789ca4965d9ef04da8ea29307c24c2" 15 | } 16 | -------------------------------------------------------------------------------- /api/.sqlx/query-06ffa4ce61920b3edc549d3d8bc542e8353d13497b9729449d348e63fee21449.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM organization_members\n WHERE\n organization_id = $1\n AND user_id = $2", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "06ffa4ce61920b3edc549d3d8bc542e8353d13497b9729449d348e63fee21449" 15 | } 16 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/ollama_request.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat ./$1.json | \ 4 | jq '. += { stream: false }' | \ 5 | curl http://localhost:11434/api/chat \ 6 | -v -X POST \ 7 | --header "Content-Type: application/json" \ 8 | --data-binary @- > $1_response_nonstreaming.json 9 | 10 | cat ./$1.json | \ 11 | jq '. += { stream: true }' | \ 12 | curl http://localhost:11434/api/chat \ 13 | -v -X POST \ 14 | --header "Content-Type: application/json" \ 15 | --data-binary @- > $1_response_streaming.txt 16 | 17 | 18 | -------------------------------------------------------------------------------- /api/.sqlx/query-2b54620a64e6e9620b0e72b016281aa775f49f4566911ac1fcbd9b968233cea3.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO organization_members\n (organization_id, user_id)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "2b54620a64e6e9620b0e72b016281aa775f49f4566911ac1fcbd9b968233cea3" 15 | } 16 | -------------------------------------------------------------------------------- /api/.sqlx/query-3af44c3f763a1e3ebe07172cd79b20c434098c8336f6de8da1496e97f5f26faa.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE api_keys\n SET active = $3\n WHERE api_key_id = $1\n AND organization_id = $2", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "Bool" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "3af44c3f763a1e3ebe07172cd79b20c434098c8336f6de8da1496e97f5f26faa" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-cc1f79b44a5437c0a3f4086a0c1d27bee85f38006b5ee6e2cce0a2034cfc4069.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE email_logins\n SET reset_token = $2,\n reset_expires_at = now() + '1 hour'::interval\n WHERE email = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Text", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "cc1f79b44a5437c0a3f4086a0c1d27bee85f38006b5ee6e2cce0a2034cfc4069" 15 | } 16 | -------------------------------------------------------------------------------- /api/.sqlx/query-ed7888aeb1ef8a84d5e17c81abae30405f36fd2c52248ed187fb6621b748338d.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO oauth_logins\n (user_id, oauth_provider, oauth_account_id)\n VALUES\n ($1, $2, $3)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Text", 10 | "Text" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "ed7888aeb1ef8a84d5e17c81abae30405f36fd2c52248ed187fb6621b748338d" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-b46633aa2e966bc746c2e39838239a114165666fbb62bd41bb6ee73ca7126b92.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n INSERT INTO user_sessions (id, user_id, hash, expires_at) VALUES\n ($1, $2, $3, now() + $4)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "Uuid", 11 | "Interval" 12 | ] 13 | }, 14 | "nullable": [] 15 | }, 16 | "hash": "b46633aa2e966bc746c2e39838239a114165666fbb62bd41bb6ee73ca7126b92" 17 | } 18 | -------------------------------------------------------------------------------- /api/.sqlx/query-d6d06dea325d3ef8acb61fa50281511bbc3bcd4aa734106a5500dd8c4daa75a3.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE organization_members\n SET active = $3\n WHERE\n organization_id = $1\n AND user_id = $2", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "Bool" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "d6d06dea325d3ef8acb61fa50281511bbc3bcd4aa734106a5500dd8c4daa75a3" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-679f767fa0712716d1ed6e556db43f5b9aef432185b3bc883df870aa6de1d80d.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE email_logins\n SET passwordless_login_token = $2,\n passwordless_login_expires_at = now() + interval '1 hour'\n WHERE email = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Text", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "679f767fa0712716d1ed6e556db43f5b9aef432185b3bc883df870aa6de1d80d" 15 | } 16 | -------------------------------------------------------------------------------- /api/.sqlx/query-91dff4cfd5df0be650efdd9bbf3a8196fd9c2033293aca47d6507c5b26bf404f.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n DELETE FROM user_roles\n WHERE\n organization_id = $1\n AND user_id = $2\n AND role_id = ANY($3)\n ", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "UuidArray" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "91dff4cfd5df0be650efdd9bbf3a8196fd9c2033293aca47d6507c5b26bf404f" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-e70e28f56e8a998627c0829f6f26926ce45c3c914052a3f075339bb86c11405e.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n DELETE FROM permissions\n WHERE\n organization_id = $1\n AND actor_id = $2\n AND permission = ANY($3)\n ", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "TextArray" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "e70e28f56e8a998627c0829f6f26926ce45c3c914052a3f075339bb86c11405e" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-4be1576cf369ef497786f42c6268468ac977712714a9f0fb1f2b1c3d183cc45e.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n INSERT INTO user_roles (organization_id, user_id, role_id)\n (\n SELECT $1, $2, role_id FROM UNNEST($3::uuid[]) role_id\n )\n ON CONFLICT DO NOTHING\n ", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "UuidArray" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "4be1576cf369ef497786f42c6268468ac977712714a9f0fb1f2b1c3d183cc45e" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-be01a8e882fddcc5b5365018ed8e28182325664583c161679fd7e4f9513e302f.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n INSERT INTO permissions (organization_id, actor_id, permission)\n (\n SELECT $1, $2, permission FROM UNNEST($3::text[]) permission\n )\n ON CONFLICT DO NOTHING\n ", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "TextArray" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "be01a8e882fddcc5b5365018ed8e28182325664583c161679fd7e4f9513e302f" 16 | } 17 | -------------------------------------------------------------------------------- /api/.sqlx/query-9889a4ad0b25bb3621e29eb881ec86c9c17e05b19ee4998c93fdb007ae2b6743.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT user_id FROM oauth_logins\n WHERE oauth_provider = $1 AND oauth_account_id = $2", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "user_id", 9 | "type_info": "Uuid" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [ 14 | "Text", 15 | "Text" 16 | ] 17 | }, 18 | "nullable": [ 19 | false 20 | ] 21 | }, 22 | "hash": "9889a4ad0b25bb3621e29eb881ec86c9c17e05b19ee4998c93fdb007ae2b6743" 23 | } 24 | -------------------------------------------------------------------------------- /api/.sqlx/query-73d78f5181b3b0753b208041520696fc7131f3ca864e2d793f98c64b7f8f6fa6.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE user_sessions\n SET expires_at = now() + $1\n WHERE id=$2 and hash=$3\n -- Prevent unnecessary updates\n AND (expires_at < now() + $1 - '1 minute'::interval)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Interval", 9 | "Uuid", 10 | "Uuid" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "73d78f5181b3b0753b208041520696fc7131f3ca864e2d793f98c64b7f8f6fa6" 16 | } 17 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/groq_request.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ../../../../.env 3 | 4 | cat ./$1.json | \ 5 | curl https://api.groq.com/openai/v1/chat/completions \ 6 | -v -X POST \ 7 | --header "Authorization: Bearer ${GROQ_API_KEY}" \ 8 | --header "Content-Type: application/json" \ 9 | --data-binary @- > $1_response_nonstreaming.json 10 | 11 | cat ./$1.json | \ 12 | jq '. += { stream: true }' | \ 13 | curl https://api.groq.com/openai/v1/chat/completions \ 14 | -v -X POST \ 15 | --header "Authorization: Bearer ${GROQ_API_KEY}" \ 16 | --header "Content-Type: application/json" \ 17 | --data-binary @- > $1_response_streaming.txt 18 | 19 | 20 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/groq_tools_response_nonstreaming.json: -------------------------------------------------------------------------------- 1 | {"id":"chatcmpl-140967f7-5999-4589-9103-ee8142fb56a5","object":"chat.completion","created":1717287190,"model":"llama3-8b-8192","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_2tj9","type":"function","function":{"name":"get_characteristics","arguments":"{\"hair_color\":\"brown\",\"name\":\"Daniel\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":828,"prompt_time":0.225026111,"completion_tokens":81,"completion_time":0.064292381,"total_tokens":909,"total_time":0.289318492},"system_fingerprint":"fp_dadc9d6142","x_groq":{"id":"req_01hzb4nzb6e24t5bcbaxs35dak"}} 2 | -------------------------------------------------------------------------------- /api/.sqlx/query-6a2e15ca052183363cf32696ea30c50fb69730433700de8a85876470c9f663b5.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n INSERT INTO user_roles (organization_id, user_id, role_id)\n (\n SELECT $1, $2, default_role as role_id\n FROM organizations\n WHERE id = $1 AND default_role IS NOT NULL\n )\n ON CONFLICT DO NOTHING\n ", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "6a2e15ca052183363cf32696ea30c50fb69730433700de8a85876470c9f663b5" 15 | } 16 | -------------------------------------------------------------------------------- /api/.sqlx/query-dd98c73a917d0b2014b27bd8f6d7462bf22fd0b4970aff97d3bae2e29d6c8501.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO oauth_authorization_sessions\n (key, provider, add_to_user_id, redirect_to, pkce_verifier, expires_at)\n VALUES\n ($1, $2, $3, $4, $5, now() + '10 minutes'::interval)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Text", 9 | "Text", 10 | "Uuid", 11 | "Text", 12 | "Text" 13 | ] 14 | }, 15 | "nullable": [] 16 | }, 17 | "hash": "dd98c73a917d0b2014b27bd8f6d7462bf22fd0b4970aff97d3bae2e29d6c8501" 18 | } 19 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/openai_request.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ../../../../.env 3 | 4 | cat ./$1.json | \ 5 | curl https://api.openai.com/v1/chat/completions \ 6 | -v -X POST \ 7 | --header "Authorization: Bearer ${OPENAI_API_KEY}" \ 8 | --header "Content-Type: application/json" \ 9 | --data-binary @- > $1_response_nonstreaming.json 10 | 11 | cat ./$1.json | \ 12 | jq '. += { stream: true, stream_options: { include_usage: true } }' | \ 13 | curl https://api.openai.com/v1/chat/completions \ 14 | -v -X POST \ 15 | --header "Authorization: Bearer ${OPENAI_API_KEY}" \ 16 | --header "Content-Type: application/json" \ 17 | --data-binary @- > $1_response_streaming.txt 18 | 19 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__test__call_provider.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/lib.rs 3 | expression: result 4 | --- 5 | { 6 | "created": 1, 7 | "model": null, 8 | "system_fingerprint": null, 9 | "choices": [ 10 | { 11 | "index": 0, 12 | "message": { 13 | "role": "assistant", 14 | "content": "hello" 15 | }, 16 | "finish_reason": "stop" 17 | } 18 | ], 19 | "usage": { 20 | "prompt_tokens": 1, 21 | "completion_tokens": 1, 22 | "total_tokens": 2 23 | }, 24 | "meta": { 25 | "id": "00000000-0000-0000-0000-000000000000", 26 | "provider": "test", 27 | "response_meta": null, 28 | "was_rate_limited": false 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "api", 4 | "proxy" 5 | ] 6 | resolver = "2" 7 | 8 | [workspace.package] 9 | authors = ["Daniel Imfeld "] 10 | license = "Apache-2.0" 11 | readme = "README.md" 12 | repository = "https://github.com/dimfeld/chronicle" 13 | keywords = ["llm", "proxy", "logging", "observability"] 14 | 15 | [workspace.dependencies] 16 | filigree = { version = "0.4.1", features = ["tracing", "tracing_export"] } 17 | sqlx = "0.8.0" 18 | sqlx-transparent-json-decode = "3.0.0" 19 | #filigree = { "path" = "../filigree/filigree", features = ["tracing", "tracing_export"] } 20 | 21 | [profile.dev.package.insta] 22 | opt-level = 3 23 | 24 | [profile.dev.package.similar] 25 | opt-level = 3 26 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/anthropic_request.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | source ../../../../.env 4 | 5 | cat ./$1.json | \ 6 | curl https://api.anthropic.com/v1/messages \ 7 | -v -X POST \ 8 | --header "x-api-key: ${ANTHROPIC_API_KEY}" \ 9 | --header "Content-Type: application/json" \ 10 | --header "anthropic-version: 2023-06-01" \ 11 | --data-binary @- > $1_response_nonstreaming.json 12 | 13 | cat ./$1.json | \ 14 | jq '. += { stream: true }' | \ 15 | curl https://api.anthropic.com/v1/messages \ 16 | -v -X POST \ 17 | --header "x-api-key: ${ANTHROPIC_API_KEY}" \ 18 | --header "Content-Type: application/json" \ 19 | --header "anthropic-version: 2023-06-01" \ 20 | --data-binary @- > $1_response_streaming.txt 21 | -------------------------------------------------------------------------------- /api/.sqlx/query-840ef63ec6eb5ea426c6ae8af10c8206cbdfa39d83c183a9fdceacc527ae8765.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n UPDATE api_keys\n SET\n description = COALESCE($4, description),\n active = COALESCE($5, active)\n WHERE\n api_key_id = $1\n AND organization_id = $2\n AND user_id IS NOT DISTINCT FROM $3\n ", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "Uuid", 11 | "Text", 12 | "Bool" 13 | ] 14 | }, 15 | "nullable": [] 16 | }, 17 | "hash": "840ef63ec6eb5ea426c6ae8af10c8206cbdfa39d83c183a9fdceacc527ae8765" 18 | } 19 | -------------------------------------------------------------------------------- /api/.sqlx/query-7354dc5ce855c4a218b2df038e421aa2d2d0358e553c3f10124742dfee3973a3.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO user_invites (email, organization_id, token, token_expires_at)\n VALUES ($1, NULL, $2, now() + interval '1 hour')\n ON CONFLICT(email, organization_id)\n DO UPDATE SET invite_sent_at = now(),\n token = $2,\n token_expires_at = now() + interval '1 hour'", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Text", 9 | "Uuid" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "7354dc5ce855c4a218b2df038e421aa2d2d0358e553c3f10124742dfee3973a3" 15 | } 16 | -------------------------------------------------------------------------------- /js-client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "dist", 4 | // Enable latest features 5 | "lib": ["ESNext", "DOM"], 6 | "target": "ESNext", 7 | "module": "Node16", 8 | "moduleDetection": "force", 9 | 10 | "moduleResolution": "Node16", 11 | "sourceMap": true, 12 | "declaration": true, 13 | "declarationMap": true, 14 | 15 | // Best practices 16 | "strict": true, 17 | "skipLibCheck": true, 18 | "noFallthroughCasesInSwitch": true, 19 | 20 | // Some stricter flags (disabled by default) 21 | "noUnusedLocals": false, 22 | "noUnusedParameters": false, 23 | "noPropertyAccessFromIndexSignature": false 24 | }, 25 | "include": ["src/**/*"], 26 | "exclude": ["src/**/*.test.ts"] 27 | } 28 | -------------------------------------------------------------------------------- /api/.sqlx/query-905d0c0ef057084be737d421c298a1a686392c01f3b52d17f3ee1db1a1159789.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM user_invites\n WHERE email=$1 AND organization_id IS NULL\n RETURNING token, token_expires_at", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "token", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "token_expires_at", 14 | "type_info": "Timestamptz" 15 | } 16 | ], 17 | "parameters": { 18 | "Left": [ 19 | "Text" 20 | ] 21 | }, 22 | "nullable": [ 23 | false, 24 | false 25 | ] 26 | }, 27 | "hash": "905d0c0ef057084be737d421c298a1a686392c01f3b52d17f3ee1db1a1159789" 28 | } 29 | -------------------------------------------------------------------------------- /api/.sqlx/query-eb1ed7eab6b4b36c88887e9d0873d167422864b878fb193e80b2d674d7fe4909.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO api_keys\n (api_key_id,\n organization_id,\n user_id,\n hash,\n inherits_user_permissions,\n description,\n active,\n expires_at)\n VALUES\n ($1, $2, $3, $4, $5, $6, $7, $8)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid", 9 | "Uuid", 10 | "Uuid", 11 | "Bytea", 12 | "Bool", 13 | "Text", 14 | "Bool", 15 | "Timestamptz" 16 | ] 17 | }, 18 | "nullable": [] 19 | }, 20 | "hash": "eb1ed7eab6b4b36c88887e9d0873d167422864b878fb193e80b2d674d7fe4909" 21 | } 22 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/anthropic_tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "claude-3-haiku-20240307", 3 | "max_tokens": 1024, 4 | "messages": [ 5 | { "role": "user", "content": "Given this information: Daniel has brown hair.\nWhat are the characteristics??"} 6 | ], 7 | "tools": [ 8 | { 9 | "name": "get_characteristics", 10 | "description": "Use this tool to extract the physical characteristics of a person.", 11 | "input_schema": { 12 | "type": "object", 13 | "properties": { 14 | "name": { "type": "string" }, 15 | "hair_color": { "type": "string" } 16 | }, 17 | "required": ["name", "hair", "color"] 18 | } 19 | } 20 | ], 21 | "tool_choice": { 22 | "type": "tool", 23 | "name": "get_characteristics" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /js-client/src/wrapper.test.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from 'bun:test'; 2 | import { fetchChronicle } from './wrapper.js'; 3 | import OpenAI from 'openai'; 4 | 5 | test('wrap OpenAI', async () => { 6 | let client = new OpenAI({ 7 | apiKey: '', 8 | fetch: fetchChronicle({ 9 | defaults: { 10 | metadata: { 11 | application: 'chronicle-test', 12 | environment: 'test', 13 | workflow_id: 'wrap OpenAI', 14 | }, 15 | }, 16 | }), 17 | }); 18 | 19 | let result = await client.chat.completions.create({ 20 | model: 'groq/llama3-8b-8192', 21 | max_tokens: 128, 22 | temperature: 0, 23 | messages: [ 24 | { 25 | role: 'user', 26 | content: 'Hello', 27 | }, 28 | ], 29 | }); 30 | 31 | console.dir(result, { depth: null }); 32 | }); 33 | -------------------------------------------------------------------------------- /proxy/src/providers/mistral.rs: -------------------------------------------------------------------------------- 1 | use super::custom::{CustomProvider, OpenAiRequestFormatOptions, ProviderRequestFormat}; 2 | use crate::config::CustomProviderConfig; 3 | 4 | pub struct Mistral; 5 | 6 | impl Mistral { 7 | pub fn new(client: reqwest::Client, token: Option) -> CustomProvider { 8 | let config = CustomProviderConfig { 9 | name: "mistral".into(), 10 | label: Some("Mistral".to_string()), 11 | url: "https://api.mistral.ai/v1/chat/completions".into(), 12 | format: ProviderRequestFormat::OpenAi(OpenAiRequestFormatOptions::default()), 13 | api_key: None, 14 | api_key_source: None, 15 | headers: Default::default(), 16 | prefix: Some("mistral/".to_string()), 17 | } 18 | .with_token_or_env(token, "MISTRAL_API_KEY"); 19 | 20 | CustomProvider::new(config, client) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/openai_tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "gpt-3.5-turbo", 3 | "max_tokens": 1024, 4 | "messages": [ 5 | { "role": "user", "content": "Given this information: Daniel has brown hair.\nWhat are the characteristics??"} 6 | ], 7 | "tools": [ 8 | { 9 | "type": "function", 10 | "function": { 11 | "name": "get_characteristics", 12 | "description": "Use this tool to extract the physical characteristics of a person.", 13 | "parameters": { 14 | "type": "object", 15 | "properties": { 16 | "name": { "type": "string" }, 17 | "hair_color": { "type": "string" } 18 | }, 19 | "required": ["name", "hair_color"] 20 | } 21 | } 22 | } 23 | ], 24 | "tool_choice": { 25 | "type": "function", 26 | "function": { 27 | "name": "get_characteristics" 28 | } 29 | } 30 | } 31 | 32 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/groq_tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "llama3-8b-8192", 3 | "max_tokens": 1024, 4 | "messages": [ 5 | { "role": "user", "content": "Given this information: Daniel has brown hair.\nWhat are the characteristics??"} 6 | ], 7 | "tools": [ 8 | { 9 | "type": "function", 10 | "function": { 11 | "name": "get_characteristics", 12 | "description": "Use this tool to extract the physical characteristics of a person.", 13 | "parameters": { 14 | "type": "object", 15 | "properties": { 16 | "name": { "type": "string" }, 17 | "hair_color": { "type": "string" } 18 | } 19 | }, 20 | "required": ["name", "hair", "color"] 21 | } 22 | } 23 | ], 24 | "tool_choice": { 25 | "type": "function", 26 | "function": { 27 | "name": "get_characteristics" 28 | } 29 | } 30 | } 31 | 32 | 33 | -------------------------------------------------------------------------------- /proxy/src/providers/together.rs: -------------------------------------------------------------------------------- 1 | use super::custom::{CustomProvider, OpenAiRequestFormatOptions, ProviderRequestFormat}; 2 | use crate::config::CustomProviderConfig; 3 | 4 | pub struct Together; 5 | 6 | impl Together { 7 | pub fn new(client: reqwest::Client, token: Option) -> CustomProvider { 8 | let config = CustomProviderConfig { 9 | name: "together".into(), 10 | label: Some("together.ai".to_string()), 11 | url: "https://api.together.xyz/v1/chat/completions".into(), 12 | format: ProviderRequestFormat::OpenAi(OpenAiRequestFormatOptions::default()), 13 | api_key: None, 14 | api_key_source: None, 15 | headers: Default::default(), 16 | prefix: Some("together/".to_string()), 17 | } 18 | .with_token_or_env(token, "TOGETHER_API_KEY"); 19 | 20 | CustomProvider::new(config, client) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /js-client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@dimfeld/chronicle", 3 | "description": "Client SDK for the Chronicle LLM Proxy", 4 | "version": "0.4.1", 5 | "module": "dist/index.js", 6 | "license": "Apache-2.0", 7 | "type": "module", 8 | "exports": { 9 | ".": { 10 | "types": "./dist/index.d.ts", 11 | "default": "./dist/index.js" 12 | } 13 | }, 14 | "scripts": { 15 | "prepare": "npm run build", 16 | "build": "rm -rf dist && tsc && publint", 17 | "dev": "tsc --watch" 18 | }, 19 | "files": [ 20 | "dist", 21 | "package.json" 22 | ], 23 | "devDependencies": { 24 | "@honeycombio/opentelemetry-node": "^0.7.2", 25 | "@types/node": "*", 26 | "publint": "^0.2.7", 27 | "typescript": "^5.0.0" 28 | }, 29 | "dependencies": { 30 | "@opentelemetry/api": "^1.8.0", 31 | "@opentelemetry/core": "^1.24.0", 32 | "openai": "^4", 33 | "uuidv7": "^1.0.1" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /proxy/src/providers/anyscale.rs: -------------------------------------------------------------------------------- 1 | use super::custom::{CustomProvider, OpenAiRequestFormatOptions, ProviderRequestFormat}; 2 | use crate::config::CustomProviderConfig; 3 | 4 | pub struct Anyscale; 5 | 6 | impl Anyscale { 7 | pub fn new(client: reqwest::Client, token: Option) -> CustomProvider { 8 | let config = CustomProviderConfig { 9 | name: "anyscale".into(), 10 | label: Some("Anyscale".to_string()), 11 | url: "https://api.endpoints.anyscale.com/v1/chat/completions".into(), 12 | format: ProviderRequestFormat::OpenAi(OpenAiRequestFormatOptions::default()), 13 | api_key: None, 14 | api_key_source: None, 15 | headers: Default::default(), 16 | prefix: Some("anyscale/".to_string()), 17 | } 18 | .with_token_or_env(token, "ANYSCALE_API_KEY"); 19 | 20 | CustomProvider::new(config, client) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /proxy/src/providers/deepinfra.rs: -------------------------------------------------------------------------------- 1 | use super::custom::{CustomProvider, OpenAiRequestFormatOptions, ProviderRequestFormat}; 2 | use crate::config::CustomProviderConfig; 3 | 4 | pub struct DeepInfra; 5 | 6 | impl DeepInfra { 7 | pub fn new(client: reqwest::Client, token: Option) -> CustomProvider { 8 | let config = CustomProviderConfig { 9 | name: "deepinfra".into(), 10 | label: Some("DeepInfra".to_string()), 11 | url: "https://api.deepinfra.com/v1/openai/chat/completions".into(), 12 | format: ProviderRequestFormat::OpenAi(OpenAiRequestFormatOptions::default()), 13 | api_key: None, 14 | api_key_source: None, 15 | headers: Default::default(), 16 | prefix: Some("deepinfra/".to_string()), 17 | } 18 | .with_token_or_env(token, "DEEPINFRA_API_KEY"); 19 | 20 | CustomProvider::new(config, client) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /proxy/src/providers/fireworks.rs: -------------------------------------------------------------------------------- 1 | use super::custom::{CustomProvider, OpenAiRequestFormatOptions, ProviderRequestFormat}; 2 | use crate::config::CustomProviderConfig; 3 | 4 | pub struct Fireworks; 5 | 6 | impl Fireworks { 7 | pub fn new(client: reqwest::Client, token: Option) -> CustomProvider { 8 | let config = CustomProviderConfig { 9 | name: "fireworks".into(), 10 | label: Some("fireworks.ai".to_string()), 11 | url: "https://api.fireworks.ai/inference/v1/chat/completions".into(), 12 | format: ProviderRequestFormat::OpenAi(OpenAiRequestFormatOptions::default()), 13 | api_key: None, 14 | api_key_source: None, 15 | headers: Default::default(), 16 | prefix: Some("fireworks/".to_string()), 17 | } 18 | .with_token_or_env(token, "FIREWORKS_API_KEY"); 19 | 20 | CustomProvider::new(config, client) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /api/.sqlx/query-6358d020be9dda3b63dcdb3fac3f98ff75fa13d011e3bf920955862906c8dc81.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "WITH sel AS (\n SELECT user_id, (reset_token IS NOT DISTINCT FROM $2 AND reset_expires_at > now()) AS matches\n FROM email_logins\n WHERE email = $1\n ),\n upd_el AS (\n -- Always clear the token\n UPDATE email_logins\n SET reset_token = null, reset_expires_at = null\n WHERE email = $1 AND reset_token IS NOT NULL\n )\n UPDATE users\n SET password_hash = $3\n FROM sel\n WHERE users.id = sel.user_id AND sel.matches", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Text", 9 | "Uuid", 10 | "Text" 11 | ] 12 | }, 13 | "nullable": [] 14 | }, 15 | "hash": "6358d020be9dda3b63dcdb3fac3f98ff75fa13d011e3bf920955862906c8dc81" 16 | } 17 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/openai_tools_response_nonstreaming.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "chatcmpl-9VE3hM6XGZvE6YEA82QjEe6rcO84Z", 3 | "object": "chat.completion", 4 | "created": 1717229237, 5 | "model": "gpt-3.5-turbo-0125", 6 | "choices": [ 7 | { 8 | "index": 0, 9 | "message": { 10 | "role": "assistant", 11 | "content": null, 12 | "tool_calls": [ 13 | { 14 | "id": "call_UedDYyaORb1eAjaUkCeIBSHw", 15 | "type": "function", 16 | "function": { 17 | "name": "get_characteristics", 18 | "arguments": "{\"name\":\"Daniel\",\"hair_color\":\"brown\"}" 19 | } 20 | } 21 | ] 22 | }, 23 | "logprobs": null, 24 | "finish_reason": "stop" 25 | } 26 | ], 27 | "usage": { 28 | "prompt_tokens": 80, 29 | "completion_tokens": 10, 30 | "total_tokens": 90 31 | }, 32 | "system_fingerprint": null 33 | } 34 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__test__call_provider_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/lib.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "test", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "a-test-model" 16 | }, 17 | "was_streaming": false, 18 | "num_chunks": 1, 19 | "response": { 20 | "created": 1, 21 | "model": null, 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "hello" 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 1, 35 | "completion_tokens": 1, 36 | "total_tokens": 2 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__test__call_provider_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/lib.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "test", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "a_model" 16 | }, 17 | "was_streaming": true, 18 | "num_chunks": 2, 19 | "response": { 20 | "created": 1, 21 | "model": "a_model", 22 | "system_fingerprint": "abbadada", 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "hello and hello again" 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 1, 35 | "completion_tokens": 1, 36 | "total_tokens": 2 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /api/.sqlx/query-a5b1858fc31b80561b6be6738c8e6debf75e4a56bfbbec1f8091c496806bd719.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT user_id as \"user_id: UserId\", password_hash, email_logins.verified\n FROM email_logins\n JOIN users ON users.id = email_logins.user_id\n WHERE email_logins.email = $1", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "user_id: UserId", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "password_hash", 14 | "type_info": "Text" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "verified", 19 | "type_info": "Bool" 20 | } 21 | ], 22 | "parameters": { 23 | "Left": [ 24 | "Text" 25 | ] 26 | }, 27 | "nullable": [ 28 | false, 29 | true, 30 | false 31 | ] 32 | }, 33 | "hash": "a5b1858fc31b80561b6be6738c8e6debf75e4a56bfbbec1f8091c496806bd719" 34 | } 35 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/openai_text_response_nonstreaming.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "chatcmpl-9VE2R0BxEW9IIrOIcV0sXD8U2YwQg", 3 | "object": "chat.completion", 4 | "created": 1717229159, 5 | "model": "gpt-3.5-turbo-0125", 6 | "choices": [ 7 | { 8 | "index": 0, 9 | "message": { 10 | "role": "assistant", 11 | "content": "Lassie was a fictional character, a Rough Collie dog, who was featured in a long-running series of movies, TV shows, and books. The character first appeared in a short story by Eric Knight in 1938 and became popular through the classic television series \"Lassie\" which aired from 1954 to 1973. Lassie is known for her loyalty, bravery, and intelligence, and is considered one of the most iconic dogs in popular culture." 12 | }, 13 | "logprobs": null, 14 | "finish_reason": "stop" 15 | } 16 | ], 17 | "usage": { 18 | "prompt_tokens": 15, 19 | "completion_tokens": 97, 20 | "total_tokens": 112 21 | }, 22 | "system_fingerprint": null 23 | } 24 | -------------------------------------------------------------------------------- /proxy/migrations/20240424_chronicle_proxy_data_tables_sqlite.sql: -------------------------------------------------------------------------------- 1 | -- Data tables. These are optional and only needed if you want to store and load configuration in the database. 2 | CREATE TABLE IF NOT EXISTS chronicle_custom_providers ( 3 | id integer PRIMARY KEY, 4 | name text NOT NULL, 5 | label text, 6 | url text NOT NULL, 7 | api_key text, 8 | api_key_source text, 9 | format text NOT NULL, 10 | headers text, 11 | prefix text 12 | ); 13 | 14 | CREATE TABLE IF NOT EXISTS chronicle_aliases ( 15 | id integer PRIMARY KEY, 16 | name text NOT NULL, 17 | random_order bool NOT NULL DEFAULT FALSE 18 | ); 19 | 20 | CREATE TABLE IF NOT EXISTS chronicle_alias_providers ( 21 | id integer PRIMARY KEY, 22 | alias_id bigint REFERENCES chronicle_aliases (id) ON DELETE CASCADE, 23 | sort int NOT NULL DEFAULT 0, 24 | model text NOT NULL, 25 | provider text NOT NULL, 26 | api_key_name text 27 | ); 28 | 29 | CREATE TABLE IF NOT EXISTS chronicle_api_keys ( 30 | id integer PRIMARY KEY, 31 | name text, 32 | source text, 33 | value text 34 | ); 35 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/anthropic_text_response_nonstreaming.json: -------------------------------------------------------------------------------- 1 | {"id":"msg_014BoUBuFM9xgAdkJGRmxtC2","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[{"type":"text","text":"Lassie was a fictional dog character that appeared in a series of Hollywood films, television shows, and books starting in the 1940s.\n\nThe original Lassie was a female Rough Collie owned by a trainer named Rudd Weatherwax. Lassie first appeared in the 1943 film Lassie Come Home, starring Roddy McDowall. \n\nOver the years, the role of Lassie was played by several different Rough Collies, but they were all trained to portray the iconic character. Lassie was known for her intelligence, loyalty, and heroic deeds in rescuing and helping her human companions, particularly young boys like Timmy.\n\nThe Lassie character became hugely popular, starring in multiple feature films as well as a long-running TV series that aired from 1954 to 1973. Lassie is considered one of the most famous and beloved dog characters in entertainment history."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":16,"output_tokens":217}} -------------------------------------------------------------------------------- /proxy/migrations/20240424_chronicle_proxy_data_tables_postgresql.sql: -------------------------------------------------------------------------------- 1 | -- Data tables. These are optional and only needed if you want to store and load configuration in the database. 2 | CREATE TABLE IF NOT EXISTS chronicle_custom_providers ( 3 | id bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, 4 | name text, 5 | label text, 6 | url text NOT NULL, 7 | api_key text, 8 | api_key_source text, 9 | format jsonb NOT NULL, 10 | headers jsonb, 11 | prefix text 12 | ); 13 | 14 | CREATE TABLE IF NOT EXISTS chronicle_aliases ( 15 | id bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, 16 | name text NOT NULL UNIQUE, 17 | random_order bool NOT NULL DEFAULT FALSE 18 | ); 19 | 20 | CREATE TABLE IF NOT EXISTS chronicle_alias_providers ( 21 | id bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, 22 | alias_id bigint REFERENCES chronicle_aliases (id), 23 | sort int NOT NULL DEFAULT 0, 24 | model text NOT NULL, 25 | provider text NOT NULL, 26 | api_key_name text 27 | ); 28 | 29 | CREATE TABLE IF NOT EXISTS chronicle_api_keys ( 30 | id bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, 31 | name text, 32 | source text, 33 | value text 34 | ); 35 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef 2 | ENV SQLX_OFFLINE=true 3 | ENV SQLX_OFFLINE_DIR=/app/.sqlx 4 | RUN apt-get update && apt-get install -y pkg-config libssl-dev 5 | WORKDIR /app 6 | 7 | FROM chef AS planner 8 | COPY . . 9 | ENV SQLX_OFFLINE=true 10 | # Make sure sqlx always looks in this directory, even when building other crates. 11 | ENV SQLX_OFFLINE_DIR=/app/api/.sqlx 12 | RUN cargo chef prepare --recipe-path recipe.json 13 | 14 | FROM chef AS api-builder 15 | COPY --from=planner /app/recipe.json recipe.json 16 | COPY api/.sqlx ./api/.sqlx/ 17 | RUN cargo chef cook --release --recipe-path recipe.json 18 | COPY . . 19 | RUN cargo build --release --bin chronicle 20 | 21 | ##### Final image 22 | FROM debian:bookworm-slim AS runtime 23 | RUN apt-get update && apt-get install -y pkg-config libssl-dev ca-certificates && apt-get clean 24 | RUN update-ca-certificates 25 | WORKDIR /app 26 | ARG TARGETARCH 27 | 28 | RUN mkdir -p /data 29 | COPY --from=api-builder /app/target/release/chronicle /app/chronicle 30 | 31 | ENV HOST=::0 32 | ENV ENV=production 33 | 34 | # Primary server port 35 | ENV PORT=9782 36 | EXPOSE 9782/tcp 37 | 38 | ENTRYPOINT [ "/app/chronicle" ] 39 | CMD [ "serve" ] 40 | -------------------------------------------------------------------------------- /api/.sqlx/query-299c2b0e94a77372ab8acbabccc118b17e8da37a8577c698fa8daf3fbb18b66b.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM oauth_authorization_sessions\n WHERE key = $1\n RETURNING provider, expires_at, pkce_verifier, add_to_user_id, redirect_to", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "provider", 9 | "type_info": "Text" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "expires_at", 14 | "type_info": "Timestamptz" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "pkce_verifier", 19 | "type_info": "Text" 20 | }, 21 | { 22 | "ordinal": 3, 23 | "name": "add_to_user_id", 24 | "type_info": "Uuid" 25 | }, 26 | { 27 | "ordinal": 4, 28 | "name": "redirect_to", 29 | "type_info": "Text" 30 | } 31 | ], 32 | "parameters": { 33 | "Left": [ 34 | "Text" 35 | ] 36 | }, 37 | "nullable": [ 38 | false, 39 | false, 40 | true, 41 | true, 42 | true 43 | ] 44 | }, 45 | "hash": "299c2b0e94a77372ab8acbabccc118b17e8da37a8577c698fa8daf3fbb18b66b" 46 | } 47 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@openai_text_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "openai", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "gpt-3.5-turbo-0125" 16 | }, 17 | "was_streaming": true, 18 | "num_chunks": 65, 19 | "response": { 20 | "created": 0, 21 | "model": "gpt-3.5-turbo-0125", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "Lassie was a fictional Collie dog that originated in a short story called \"Lassie Come-Home\" written by Eric Knight in 1938. The character of Lassie has since appeared in numerous movies, television shows, and books, becoming a beloved and iconic character in popular culture." 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 15, 35 | "completion_tokens": 62, 36 | "total_tokens": 77 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/groq_tools_response_streaming.txt: -------------------------------------------------------------------------------- 1 | data: {"id":"chatcmpl-596b956c-9fc3-4f29-a1fe-e27b243e53a8","object":"chat.completion.chunk","created":1717287190,"model":"llama3-8b-8192","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_01hzb4nzzxfyw9vhhkz0czb3d0"}} 2 | 3 | data: {"id":"chatcmpl-596b956c-9fc3-4f29-a1fe-e27b243e53a8","object":"chat.completion.chunk","created":1717287190,"model":"llama3-8b-8192","system_fingerprint":"fp_179b0f92c9","choices":[{"index":0,"delta":{"tool_calls":[{"id":"call_w5s5","type":"function","function":{"name":"get_characteristics","arguments":"{\"hair_color\":\"brown\",\"name\":\"Daniel\"}"},"index":0}]},"logprobs":null,"finish_reason":null}]} 4 | 5 | data: {"id":"chatcmpl-596b956c-9fc3-4f29-a1fe-e27b243e53a8","object":"chat.completion.chunk","created":1717287190,"model":"llama3-8b-8192","system_fingerprint":"fp_179b0f92c9","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"x_groq":{"id":"req_01hzb4nzzxfyw9vhhkz0czb3d0","usage":{"queue_time":0.07776507800000002,"prompt_tokens":828,"prompt_time":0.227004479,"completion_tokens":74,"completion_time":0.05917994,"total_tokens":902,"total_time":0.286184419}}} 6 | 7 | data: [DONE] 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # chronicle 2 | 3 | Chronicle is a proxy for language model API calls which 4 | 5 | - Presents an OpenAI-compatible API regardless of the underlying provider. Switching providers is as easy as changing the model string in your request. 6 | - Provides retries and optionally falls back to other providers on a failed call 7 | - Records each call in a database, and sends OpenTelemetry events 8 | - Lets you switch model provider APIs without changing your request format. 9 | - Supports both SQLite and PostgreSQL databases 10 | - Comes with a drop-in fetch function that will redirect OpenAI SDK calls to Chronicle instead. 11 | - Supports logging "runs" and "steps" for multi-step workflows 12 | 13 | [See the roadmap](https://imfeld.dev/notes/projects_chronicle) for the current status and other notes. 14 | 15 | The project contains both a Rust crate named [chronicle-proxy](https://crates.io/crates/chronicle-proxy) in the `proxy` directory for embedding in applications, and a [turnkey server](https://crates.io/crates/chronicle-api) in the `api` directory which can be run directly. 16 | 17 | See the [CHANGELOG](./api/CHANGELOG.md) for latest changes. 18 | 19 | ## Supported Providers 20 | 21 | - OpenAI 22 | - Anthropic 23 | - AWS Bedrock 24 | - Groq 25 | - Ollama 26 | - AnyScale 27 | - DeepInfra 28 | - Fireworks 29 | - Together 30 | -------------------------------------------------------------------------------- /api/.sqlx/query-80049063dbf162f954233d825e0b0ba1a2a3850c557a1d9350512a159d9a7d19.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "\n UPDATE email_logins upd\n SET passwordless_login_token = null,\n passwordless_login_expires_at = null,\n verified = upd.verified OR\n (upd.passwordless_login_token = $2 AND upd.passwordless_login_expires_at > now())\n -- self-join since it lets us get the token even while we clear it in the UPDATE\n FROM email_logins old\n WHERE old.email = upd.email\n AND upd.email = $1\n AND upd.passwordless_login_token IS NOT NULL\n RETURNING old.user_id AS \"user_id: UserId\",\n (old.passwordless_login_token = $2 AND old.passwordless_login_expires_at > now()) AS valid\n ", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "user_id: UserId", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "valid", 14 | "type_info": "Bool" 15 | } 16 | ], 17 | "parameters": { 18 | "Left": [ 19 | "Text", 20 | "Uuid" 21 | ] 22 | }, 23 | "nullable": [ 24 | false, 25 | null 26 | ] 27 | }, 28 | "hash": "80049063dbf162f954233d825e0b0ba1a2a3850c557a1d9350512a159d9a7d19" 29 | } 30 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@groq_tool_calls_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "groq", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "llama3-8b-8192" 16 | }, 17 | "was_streaming": false, 18 | "num_chunks": 1, 19 | "response": { 20 | "created": 0, 21 | "model": "llama3-8b-8192", 22 | "system_fingerprint": "fp_dadc9d6142", 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": null, 29 | "tool_calls": [ 30 | { 31 | "id": "call_2tj9", 32 | "type": "function", 33 | "function": { 34 | "name": "get_characteristics", 35 | "arguments": "{\"hair_color\":\"brown\",\"name\":\"Daniel\"}" 36 | } 37 | } 38 | ] 39 | }, 40 | "finish_reason": "tool_calls" 41 | } 42 | ], 43 | "usage": { 44 | "prompt_tokens": 828, 45 | "completion_tokens": 81, 46 | "total_tokens": 909 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /js-client/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # chronicle-proxy JavaScript client changelog 2 | 3 | ## 0.4.1 4 | 5 | - Make `run_id` and `step_id` optional in `GenericEvent` since they will be filled in automatically when possible. 6 | 7 | ## 0.4.0 8 | 9 | - When running a step outside of a run, automatically wrap it in a run. 10 | - Allow disabling all event logging in an application. 11 | - Allow explicitly passing a run context to `runStep`, in case it can not be retrieved from the normal `AsyncLocalStorage` context. 12 | - The client is now also an `EventEmitter`, and will emit any events passed to `client.event()`. 13 | - Add a `withMetadata` function to the client, which returns a child client with updated default metadata values in the requests. This client shares the same `EventEmitter` and other settings with its parent. 14 | - Allow passing a custom Chronicle client instance to `runStep`. 15 | - Allow passing metadata to `startRun` or `runStep`, which will automatically create a child Chronicle client using `withMetadata`. 16 | 17 | ## 0.3.0 18 | 19 | - Support submitting runs and step trace data to Chronicle 20 | - Add an event queue to ensure that events are submitted in the order they occur. 21 | 22 | ## 0.2.0 23 | 24 | - Support streaming 25 | 26 | ## 0.1.1 27 | 28 | - Allow sending arbitrary events to the Chronicle proxy 29 | 30 | ## 0.1.0 31 | 32 | - Initial release 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@openai_tool_calls_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "openai", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "gpt-3.5-turbo-0125" 16 | }, 17 | "was_streaming": false, 18 | "num_chunks": 1, 19 | "response": { 20 | "created": 0, 21 | "model": "gpt-3.5-turbo-0125", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": null, 29 | "tool_calls": [ 30 | { 31 | "id": "call_UedDYyaORb1eAjaUkCeIBSHw", 32 | "type": "function", 33 | "function": { 34 | "name": "get_characteristics", 35 | "arguments": "{\"name\":\"Daniel\",\"hair_color\":\"brown\"}" 36 | } 37 | } 38 | ] 39 | }, 40 | "finish_reason": "stop" 41 | } 42 | ], 43 | "usage": { 44 | "prompt_tokens": 80, 45 | "completion_tokens": 10, 46 | "total_tokens": 90 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@groq_tool_calls_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "groq", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "llama3-8b-8192" 16 | }, 17 | "was_streaming": true, 18 | "num_chunks": 3, 19 | "response": { 20 | "created": 0, 21 | "model": "llama3-8b-8192", 22 | "system_fingerprint": "fp_179b0f92c9", 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "", 29 | "tool_calls": [ 30 | { 31 | "index": 0, 32 | "id": "call_w5s5", 33 | "type": "function", 34 | "function": { 35 | "name": "get_characteristics", 36 | "arguments": "{\"hair_color\":\"brown\",\"name\":\"Daniel\"}" 37 | } 38 | } 39 | ] 40 | }, 41 | "finish_reason": "tool_calls" 42 | } 43 | ], 44 | "usage": { 45 | "prompt_tokens": 828, 46 | "completion_tokens": 74, 47 | "total_tokens": 902 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@anthropic_tool_calls_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "anthropic", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "claude-3-haiku-20240307" 16 | }, 17 | "was_streaming": false, 18 | "num_chunks": 1, 19 | "response": { 20 | "created": 0, 21 | "model": "claude-3-haiku-20240307", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": null, 29 | "tool_calls": [ 30 | { 31 | "id": "toolu_018FXg1i7nbT3stUdoysiffi", 32 | "type": "function", 33 | "function": { 34 | "name": "get_characteristics", 35 | "arguments": "{\"hair_color\":\"brown\",\"name\":\"Daniel\"}" 36 | } 37 | } 38 | ] 39 | }, 40 | "finish_reason": "tool_calls" 41 | } 42 | ], 43 | "usage": { 44 | "prompt_tokens": 459, 45 | "completion_tokens": 52, 46 | "total_tokens": null 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /js-client/src/internal.ts: -------------------------------------------------------------------------------- 1 | import { context } from '@opentelemetry/api'; 2 | import { W3CTraceContextPropagator } from '@opentelemetry/core'; 3 | 4 | export function propagateSpan(req: Request) { 5 | let propagator = new W3CTraceContextPropagator(); 6 | 7 | const setter = { 8 | set: (req: Request, headerName: string, headerValue: string) => { 9 | req.headers.set(headerName, headerValue); 10 | }, 11 | }; 12 | 13 | propagator.inject(context.active(), req, setter); 14 | } 15 | 16 | export function proxyUrl(configured?: string | URL, path = '/chat') { 17 | let url = new URL(configured || process.env.CHRONICLE_PROXY_URL || 'http://localhost:9782'); 18 | if (url.pathname.length <= 1) { 19 | url.pathname = path; 20 | } 21 | 22 | return url; 23 | } 24 | 25 | export async function handleError(res: Response) { 26 | let message = ''; 27 | const err = await res.text(); 28 | try { 29 | const { error } = JSON.parse(err); 30 | 31 | let errorBody = error?.details.body; 32 | if (errorBody?.error) { 33 | errorBody = errorBody.error; 34 | } 35 | 36 | if (errorBody) { 37 | message = typeof errorBody === 'string' ? errorBody : JSON.stringify(errorBody); 38 | } 39 | } catch (e) { 40 | message = err; 41 | } 42 | 43 | // TODO The api returns a bunch of other error details, so integrate them here. 44 | return message || 'An error occurred'; 45 | } 46 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@openai_tool_calls_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "openai", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "gpt-3.5-turbo-0125" 16 | }, 17 | "was_streaming": true, 18 | "num_chunks": 13, 19 | "response": { 20 | "created": 0, 21 | "model": "gpt-3.5-turbo-0125", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": null, 29 | "tool_calls": [ 30 | { 31 | "index": 0, 32 | "id": "call_YoAFuD3iHKfA5C7Gcifn74Nj", 33 | "type": "function", 34 | "function": { 35 | "name": "get_characteristics", 36 | "arguments": "{\"name\":\"Daniel\",\"hair_color\":\"brown\"}" 37 | } 38 | } 39 | ] 40 | }, 41 | "finish_reason": "stop" 42 | } 43 | ], 44 | "usage": { 45 | "prompt_tokens": 80, 46 | "completion_tokens": 10, 47 | "total_tokens": 90 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@openai_text_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "openai", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "gpt-3.5-turbo-0125" 16 | }, 17 | "was_streaming": false, 18 | "num_chunks": 1, 19 | "response": { 20 | "created": 0, 21 | "model": "gpt-3.5-turbo-0125", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "Lassie was a fictional character, a Rough Collie dog, who was featured in a long-running series of movies, TV shows, and books. The character first appeared in a short story by Eric Knight in 1938 and became popular through the classic television series \"Lassie\" which aired from 1954 to 1973. Lassie is known for her loyalty, bravery, and intelligence, and is considered one of the most iconic dogs in popular culture." 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 15, 35 | "completion_tokens": 97, 36 | "total_tokens": 112 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@anthropic_tool_calls_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "anthropic", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "claude-3-haiku-20240307" 16 | }, 17 | "was_streaming": true, 18 | "num_chunks": 10, 19 | "response": { 20 | "created": 0, 21 | "model": "claude-3-haiku-20240307", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "", 29 | "tool_calls": [ 30 | { 31 | "index": 0, 32 | "id": "toolu_01HrgV1j1HYwPNSroDY8XZHK", 33 | "type": "function", 34 | "function": { 35 | "name": "get_characteristics", 36 | "arguments": "{\"name\": \"Daniel\", \"hair_color\": \"brown\"}" 37 | } 38 | } 39 | ] 40 | }, 41 | "finish_reason": "tool_calls" 42 | } 43 | ], 44 | "usage": { 45 | "prompt_tokens": 459, 46 | "completion_tokens": 52, 47 | "total_tokens": null 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/ollama_text_response_nonstreaming.json: -------------------------------------------------------------------------------- 1 | {"model":"llama3","created_at":"2024-06-02T06:58:55.489964Z","message":{"role":"assistant","content":"A beloved topic!\n\nLassie is a fictional Rough Collie dog character created by Eric Knight in his 1940 novel \"Lassie Come-Home.\" The story follows a loyal and courageous golden-colored collie named Lassie, who loves her human family, particularly Timmy Martin. After being separated from her family during a tragic accident, Lassie embarks on an incredible journey to find them.\n\nThe character has since become a cultural icon through various adaptations:\n\n1. **Novels**: Eric Knight wrote six more \"Lassie\" novels between 1940 and 1957.\n2. **Films**: The first film, \"Lassie Come-Home,\" was released in 1943, followed by over 30 more movies featuring Lassie from the 1940s to the 1990s.\n3. **Television**: A popular TV series aired from 1954 to 1974, with a total of 574 episodes. The show starred Lassie as a loyal and heroic canine companion, often helping her human friends in need.\n4. **Radio**: Lassie had her own radio show, \"The Lassie Story,\" which ran from 1947 to 1953.\n\nThroughout the years, Lassie has become synonymous with loyalty, courage, and friendship, making her one of the most beloved and recognizable canine characters in popular culture."},"done_reason":"stop","done":true,"total_duration":5057938084,"load_duration":3278875,"prompt_eval_duration":162740000,"eval_count":287,"eval_duration":4889146000} -------------------------------------------------------------------------------- /proxy/migrations/20240419_chronicle_proxy_init_sqlite.sql: -------------------------------------------------------------------------------- 1 | -- basic tables required for general proxy use 2 | INSERT INTO chronicle_meta ( 3 | key, 4 | value) 5 | VALUES ( 6 | 'migration_version', 7 | '1'); 8 | 9 | CREATE TABLE chronicle_pricing_plans ( 10 | id text PRIMARY KEY, 11 | provider text, 12 | start_date bigint, 13 | end_date bigint, 14 | per_input_token numeric, 15 | per_output_token numeric, 16 | per_request numeric 17 | ); 18 | 19 | CREATE TABLE IF NOT EXISTS chronicle_events ( 20 | id text PRIMARY KEY, 21 | event_type text, 22 | organization_id text, 23 | project_id text, 24 | user_id text, 25 | chat_request text, 26 | chat_response text, 27 | error text, 28 | provider text, 29 | model text, 30 | application text, 31 | environment text, 32 | request_organization_id text, 33 | request_project_id text, 34 | request_user_id text, 35 | workflow_id text, 36 | workflow_name text, 37 | run_id text, 38 | step text, 39 | step_index int, 40 | prompt_id text, 41 | prompt_version int, 42 | meta text, 43 | response_meta text, 44 | retries int, 45 | rate_limited bool, 46 | request_latency_ms int, 47 | total_latency_ms int, 48 | pricing_plan bigint REFERENCES chronicle_pricing_plans (id), 49 | created_at int NOT NULL 50 | ); 51 | 52 | CREATE INDEX chronicle_events_workflow_id_idx ON chronicle_events (workflow_id); 53 | 54 | CREATE INDEX chronicle_events_run_id_idx ON chronicle_events (run_id); 55 | -------------------------------------------------------------------------------- /api/.sqlx/query-5881c058a486308ea83071f1bd0281f967e429cf2abf0f1214f3243a60d2e113.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "WITH\n email_lookup AS (\n SELECT user_id\n FROM email_logins\n WHERE email = $1\n ),\n oauth_lookup AS (\n SELECT user_id\n FROM oauth_logins\n WHERE oauth_provider = $2 AND oauth_account_id = $3\n )\n SELECT COALESCE(email_lookup.user_id, oauth_lookup.user_id) AS user_id,\n email_lookup.user_id IS NOT NULL AS \"email_exists!\",\n oauth_lookup.user_id IS NOT NULL AS \"oauth_exists!\"\n FROM email_lookup\n FULL JOIN oauth_lookup USING (user_id)", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "user_id", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "email_exists!", 14 | "type_info": "Bool" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "oauth_exists!", 19 | "type_info": "Bool" 20 | } 21 | ], 22 | "parameters": { 23 | "Left": [ 24 | "Text", 25 | "Text", 26 | "Text" 27 | ] 28 | }, 29 | "nullable": [ 30 | null, 31 | null, 32 | null 33 | ] 34 | }, 35 | "hash": "5881c058a486308ea83071f1bd0281f967e429cf2abf0f1214f3243a60d2e113" 36 | } 37 | -------------------------------------------------------------------------------- /proxy/migrations/20240419_chronicle_proxy_init_postgresql.sql: -------------------------------------------------------------------------------- 1 | -- basic tables required for general proxy use 2 | INSERT INTO chronicle_meta ( 3 | key, 4 | value) 5 | VALUES ( 6 | 'migration_version', 7 | '1' ::jsonb); 8 | 9 | CREATE TABLE chronicle_pricing_plans ( 10 | id uuid PRIMARY KEY, 11 | provider uuid, 12 | start_date date, 13 | end_date date, 14 | per_input_token numeric, 15 | per_output_token numeric, 16 | per_request numeric 17 | ); 18 | 19 | CREATE TABLE IF NOT EXISTS chronicle_events ( 20 | id uuid PRIMARY KEY, 21 | event_type text, 22 | organization_id text, 23 | project_id text, 24 | user_id text, 25 | chat_request jsonb NOT NULL, 26 | chat_response jsonb, 27 | error jsonb, 28 | provider text, 29 | model text, 30 | application text, 31 | environment text, 32 | request_organization_id text, 33 | request_project_id text, 34 | request_user_id text, 35 | workflow_id text, 36 | workflow_name text, 37 | run_id text, 38 | step text, 39 | step_index int, 40 | prompt_id text, 41 | prompt_version int, 42 | meta jsonb, 43 | response_meta jsonb, 44 | retries int, 45 | rate_limited bool, 46 | request_latency_ms int, 47 | total_latency_ms int, 48 | pricing_plan uuid REFERENCES chronicle_pricing_plans (id), 49 | created_at timestamptz NOT NULL DEFAULT now() 50 | ); 51 | 52 | CREATE INDEX chronicle_events_workflow_id_idx ON chronicle_events (workflow_id); 53 | 54 | CREATE INDEX chronicle_events_run_id_idx ON chronicle_events (run_id); 55 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/groq_text_response_nonstreaming.json: -------------------------------------------------------------------------------- 1 | {"id":"chatcmpl-3bd751b3-3284-4b8e-b374-40ae4a485bae","object":"chat.completion","created":1717287124,"model":"llama3-8b-8192","choices":[{"index":0,"message":{"role":"assistant","content":"Lassie is a legendary canine character who has been a beloved global icon since the 1940s. She is a Rough Collie, a breed of dog known for its intelligence, loyalty, and protective nature.\n\nIn the beginning, Lassie was a real-life Rough Collie named Pal, played by a male dog named Rudd Weatherwax, who starred in the 1943 film \"Lassie Come-Home.\" The story was written by Eric Knight and tells the tale of a young boy who buys a smooth collie puppy from a street vendor and later loses him. The novel was a huge success and was adapted into several films, television shows, and even a theme park attraction.\n\nOver time, Lassie has become a symbol of loyalty, courage, and faithfulness. She is often depicted as a female dog, although the original Pal, who played Lassie in the films, was a male. The character has been portrayed by several dogs over the years, with the most well-known being Pal, Sue, and Baby.\n\nLassie has appeared in numerous films, television shows, and books throughout the years. The character has also been featured in various media, such as comic books, video games, and even a stage show."},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":18,"prompt_time":0.005628405,"completion_tokens":255,"completion_time":0.210188341,"total_tokens":273,"total_time":0.215816746},"system_fingerprint":"fp_33d61fdfc3","x_groq":{"id":"req_01hzb4kz98fxqvprcqq9wf2a6n"}} 2 | -------------------------------------------------------------------------------- /proxy/migrations/20240625_chronicle_proxy_steps_sqlite.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE chronicle_runs ( 2 | id text PRIMARY KEY, 3 | name text NOT NULL, 4 | description text, 5 | application text, 6 | environment text, 7 | input text, 8 | output text, 9 | status text NOT NULL, 10 | trace_id text, 11 | span_id text, 12 | tags text, 13 | info text, 14 | updated_at int NOT NULL, 15 | created_at int NOT NULL 16 | ); 17 | 18 | CREATE INDEX chronicle_runs_name_created_at_idx ON chronicle_runs (name, created_at DESC); 19 | 20 | CREATE INDEX chronicle_runs_name_updated_at_idx ON chronicle_runs (name, updated_at DESC); 21 | 22 | CREATE INDEX chronicle_runs_env_app_created_at_idx ON chronicle_runs (environment, application, 23 | created_at DESC); 24 | 25 | CREATE INDEX chronicle_runs_env_app_updated_at_idx ON chronicle_runs (environment, application, 26 | updated_at DESC); 27 | 28 | CREATE INDEX chronicle_runs_updated_at_idx ON chronicle_runs (updated_at DESC); 29 | 30 | CREATE INDEX chronicle_runs_created_at_idx ON chronicle_runs (created_at DESC); 31 | 32 | CREATE TABLE chronicle_steps ( 33 | id text PRIMARY KEY, 34 | run_id text NOT NULL, 35 | type text NOT NULL, 36 | parent_step text, 37 | name text, 38 | input text, 39 | output text, 40 | status text NOT NULL, 41 | span_id text, 42 | tags text, 43 | info text, 44 | start_time int NOT NULL, 45 | end_time int 46 | ); 47 | 48 | CREATE INDEX chronicle_steps_run_id_idx ON chronicle_steps (run_id); 49 | 50 | CREATE INDEX chronicle_events_run_id_created_at_idx ON chronicle_events (run_id, created_at DESC); 51 | 52 | DROP INDEX chronicle_events_run_id_idx; 53 | 54 | ALTER TABLE chronicle_events RENAME COLUMN step TO step_id; 55 | -------------------------------------------------------------------------------- /api/.sqlx/query-8904afb292c1e2d3d2f0760b29a3438f2b30677dc864a52a99bacc217dd12435.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT api_key_id,\n organization_id,\n user_id AS \"user_id: UserId\",\n inherits_user_permissions,\n description,\n active,\n expires_at\n FROM api_keys\n WHERE\n organization_id = $1\n AND user_id IS NOT DISTINCT FROM $2", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "api_key_id", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "organization_id", 14 | "type_info": "Uuid" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "user_id: UserId", 19 | "type_info": "Uuid" 20 | }, 21 | { 22 | "ordinal": 3, 23 | "name": "inherits_user_permissions", 24 | "type_info": "Bool" 25 | }, 26 | { 27 | "ordinal": 4, 28 | "name": "description", 29 | "type_info": "Text" 30 | }, 31 | { 32 | "ordinal": 5, 33 | "name": "active", 34 | "type_info": "Bool" 35 | }, 36 | { 37 | "ordinal": 6, 38 | "name": "expires_at", 39 | "type_info": "Timestamptz" 40 | } 41 | ], 42 | "parameters": { 43 | "Left": [ 44 | "Uuid", 45 | "Uuid" 46 | ] 47 | }, 48 | "nullable": [ 49 | false, 50 | false, 51 | true, 52 | false, 53 | false, 54 | false, 55 | false 56 | ] 57 | }, 58 | "hash": "8904afb292c1e2d3d2f0760b29a3438f2b30677dc864a52a99bacc217dd12435" 59 | } 60 | -------------------------------------------------------------------------------- /api/.sqlx/query-636c15e0d8b3dab82c819c974a1547c78b6afde540013c699caf2f827d99115c.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT api_key_id,\n organization_id,\n user_id AS \"user_id: UserId\",\n inherits_user_permissions,\n description,\n active,\n expires_at\n FROM api_keys\n WHERE\n api_key_id = $1\n AND hash = $2\n AND active\n AND expires_at > now()", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "api_key_id", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "organization_id", 14 | "type_info": "Uuid" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "user_id: UserId", 19 | "type_info": "Uuid" 20 | }, 21 | { 22 | "ordinal": 3, 23 | "name": "inherits_user_permissions", 24 | "type_info": "Bool" 25 | }, 26 | { 27 | "ordinal": 4, 28 | "name": "description", 29 | "type_info": "Text" 30 | }, 31 | { 32 | "ordinal": 5, 33 | "name": "active", 34 | "type_info": "Bool" 35 | }, 36 | { 37 | "ordinal": 6, 38 | "name": "expires_at", 39 | "type_info": "Timestamptz" 40 | } 41 | ], 42 | "parameters": { 43 | "Left": [ 44 | "Uuid", 45 | "Bytea" 46 | ] 47 | }, 48 | "nullable": [ 49 | false, 50 | false, 51 | true, 52 | false, 53 | false, 54 | false, 55 | false 56 | ] 57 | }, 58 | "hash": "636c15e0d8b3dab82c819c974a1547c78b6afde540013c699caf2f827d99115c" 59 | } 60 | -------------------------------------------------------------------------------- /proxy/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # chronicle-proxy changelog 2 | 3 | ## Unreleased 4 | 5 | - Support Ollama 0.3.0 tool calls 6 | - Support `strict` field in function calls with GPT4o 7 | 8 | ## 0.4.3 9 | 10 | - Allow customizing the initial status in a `run:start` event. 11 | - When receiving a `run:start` event for an already-existing run, update the status instead of ignoring it. 12 | - Fix Anthropic tool use results and streaming 13 | - Add support for AWS Bedrock 14 | - Change `finish_reason` to be an enum instead of a string. This also standardizes the field across providers. 15 | 16 | ## 0.4.2 17 | 18 | - When using a PostgreSQL database, send notifications for each run that has an update 19 | 20 | ## 0.4.1 21 | 22 | - Don't require `tags` on "run:start" event 23 | - Don't require `input` on "step:start" event 24 | - Remove foreign key constraint of step run_id, in case events arrive out of order. 25 | - Don't require `application` or `environment` in run SQLite table schema 26 | 27 | ## 0.4.0 28 | 29 | - Add "runs" and "steps", with events to manage them 30 | 31 | ## 0.3.2 32 | 33 | - Anthropic provider was omitting the system message 34 | 35 | ## 0.3.1 36 | 37 | - Provide a `max_token` value to Anthropic when the request omits it. 38 | - Add Mistral provider 39 | 40 | ## 0.3.0 41 | 42 | - Streaming support for OpenAI-compatible providers, Anthropic, and Groq 43 | 44 | ## 0.2.0 45 | 46 | - Support Anthropic `tool_choice` field 47 | - Recover from Groq error when it fails to parse an actually-valid tool call response 48 | - Support both SQLite and PostgreSQL without recompiling. 49 | 50 | ## 0.1.5 51 | 52 | - Add function for recording non-LLM events 53 | 54 | ## 0.1.4 55 | 56 | - Support tool calling 57 | 58 | ## 0.1.3 59 | 60 | - Added support for Anyscale, DeepInfra, Fireworks, and Together.ai 61 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@anthropic_text_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "anthropic", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "claude-3-haiku-20240307" 16 | }, 17 | "was_streaming": false, 18 | "num_chunks": 1, 19 | "response": { 20 | "created": 0, 21 | "model": "claude-3-haiku-20240307", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "Lassie was a fictional dog character that appeared in a series of Hollywood films, television shows, and books starting in the 1940s.\n\nThe original Lassie was a female Rough Collie owned by a trainer named Rudd Weatherwax. Lassie first appeared in the 1943 film Lassie Come Home, starring Roddy McDowall. \n\nOver the years, the role of Lassie was played by several different Rough Collies, but they were all trained to portray the iconic character. Lassie was known for her intelligence, loyalty, and heroic deeds in rescuing and helping her human companions, particularly young boys like Timmy.\n\nThe Lassie character became hugely popular, starring in multiple feature films as well as a long-running TV series that aired from 1954 to 1973. Lassie is considered one of the most famous and beloved dog characters in entertainment history." 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 16, 35 | "completion_tokens": 217, 36 | "total_tokens": null 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /api/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # chronicle-api changelog 2 | 3 | ## Unreleased 4 | 5 | - Support Ollama 0.3.0 tool calls 6 | - Support `strict` field in function calls with GPT4o 7 | 8 | ## 0.4.3 9 | 10 | - Allow customizing the initial status in a `run:start` event. 11 | - When receiving a `run:start` event for an already-existing run, update the status instead of ignoring it. 12 | - Fix Anthropic tool use results and streaming 13 | - Add support for AWS Bedrock 14 | - Standardize the `finish_reason` field values across providers. 15 | 16 | ## 0.4.2 17 | 18 | - When using a PostgreSQL database, send notifications for each run that has an update 19 | 20 | ## 0.4.1 21 | 22 | - Better validation of step and run events with invalid payloads 23 | - Don't require `tags` on "run:start" event 24 | - Don't require `input` on "step:start" event 25 | - Remove foreign key constraint of step run_id, in case events arrive out of order. 26 | - Don't require `application` or `environment` in run SQLite table schema 27 | 28 | ## 0.4.0 29 | 30 | - Add "runs" and "steps", with events to manage them 31 | 32 | ## 0.3.2 33 | 34 | - Anthropic provider was omitting the system message 35 | 36 | ## 0.3.1 37 | 38 | - Fix reading `.env` files associated with global configs. 39 | - Provide a `max_token` value to Anthropic when the request omits it. 40 | - Add Mistral provider 41 | - Handle the `/` route. This just returns `{ status: 'ok' }` without doing anything. 42 | 43 | ## 0.3.0 44 | 45 | - Streaming support for OpenAI-compatible providers, Anthropic, and Groq 46 | 47 | ## 0.2.0 48 | 49 | - Removed the version of the API which was designed to eventually have a full web app. The API-only binary is the only one available for now. A web app will probably return at some point in some other form. 50 | - Support configuring SQLite or PostgreSQL at runtime. 51 | 52 | ## 0.1.1 53 | 54 | - Allow sending arbitrary events to the Chronicle proxy 55 | 56 | ## 0.1.0 57 | 58 | - Initial release 59 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/anthropic_tools_response_streaming.txt: -------------------------------------------------------------------------------- 1 | event: message_start 2 | data: {"type":"message_start","message":{"id":"msg_014jZxtKZiPCDENpooV6YAH6","type":"message","role":"assistant","model":"claude-3-haiku-20240307","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":459,"output_tokens":10}} } 3 | 4 | event: content_block_start 5 | data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01HrgV1j1HYwPNSroDY8XZHK","name":"get_characteristics","input":{}} } 6 | 7 | event: ping 8 | data: {"type": "ping"} 9 | 10 | event: content_block_delta 11 | data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":""} } 12 | 13 | event: content_block_delta 14 | data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"name\": "} } 15 | 16 | event: content_block_delta 17 | data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"Daniel\""} } 18 | 19 | event: content_block_delta 20 | data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":", "} } 21 | 22 | event: content_block_delta 23 | data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"hai"} } 24 | 25 | event: content_block_delta 26 | data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"r_color\": "} } 27 | 28 | event: content_block_delta 29 | data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"brown\"}"} } 30 | 31 | event: content_block_stop 32 | data: {"type":"content_block_stop","index":0 } 33 | 34 | event: message_delta 35 | data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":52} } 36 | 37 | event: message_stop 38 | data: {"type":"message_stop" } 39 | 40 | -------------------------------------------------------------------------------- /api/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "chronicle-api" 3 | description = "The Chronicle LLM proxy packaged as an API" 4 | version = "0.4.3" 5 | edition = "2021" 6 | authors.workspace = true 7 | license.workspace = true 8 | readme.workspace = true 9 | repository.workspace = true 10 | keywords.workspace = true 11 | 12 | [[bin]] 13 | name = "chronicle" 14 | path = "src/main.rs" 15 | 16 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 17 | 18 | [dependencies] 19 | async-trait = "0.1.75" 20 | axum = { version = "0.7.3", features = ["tokio", "http1", "macros"] } 21 | bytes = "1.5.0" 22 | chronicle-proxy = { path = "../proxy", version = "0.4.3", default-features = false, features = ["sqlite", "postgres", "filigree", "aws-bedrock"] } 23 | chrono = "0.4.33" 24 | clap = { version = "4.4.11", features = ["env", "derive"] } 25 | dotenvy = "0.15.7" 26 | error-stack = { version = "0.5.0", features = ["spantrace"] } 27 | etcetera = "0.8.0" 28 | eyre = "0.6.11" 29 | filigree.workspace = true 30 | futures = "0.3.30" 31 | http = "1.0.0" 32 | hyper = { version = "1.1.0", features = ["server", "http1", "http2"] } 33 | itertools = "0.12.1" 34 | opentelemetry = "0.21.0" 35 | percent-encoding = "2.3.1" 36 | reqwest = { version = "0.11.23", features = ["cookies", "json"] } 37 | rust-embed = "8.1.0" 38 | serde = { version = "1.0.193", features = ["derive"] } 39 | serde_json = "1.0.113" 40 | serde_with = { version = "3.6.1", features = ["json", "schemars_0_8"] } 41 | smallvec = { version = "1.13.2", features = ["serde", "union"] } 42 | sqlx = { version = "0.8.0", features = ["chrono", "sqlite"] } 43 | sqlx-transparent-json-decode.workspace = true 44 | thiserror = "1.0.56" 45 | tokio = { version = "1.36.0", features = ["full"] } 46 | tokio-stream = "0.1.15" 47 | toml = "0.8.12" 48 | tower = "0.4.13" 49 | tower-http = { version = "0.5.1", features = ["full"] } 50 | tracing = "0.1.40" 51 | tracing-opentelemetry = "0.22.0" 52 | tracing-subscriber = { version = "0.3.18", features = ["chrono"] } 53 | url = "2.5.0" 54 | uuid = "1.6.1" 55 | 56 | [dev-dependencies] 57 | temp-dir = "0.1.13" 58 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@groq_text_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "groq", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "llama3-8b-8192" 16 | }, 17 | "was_streaming": false, 18 | "num_chunks": 1, 19 | "response": { 20 | "created": 0, 21 | "model": "llama3-8b-8192", 22 | "system_fingerprint": "fp_33d61fdfc3", 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "Lassie is a legendary canine character who has been a beloved global icon since the 1940s. She is a Rough Collie, a breed of dog known for its intelligence, loyalty, and protective nature.\n\nIn the beginning, Lassie was a real-life Rough Collie named Pal, played by a male dog named Rudd Weatherwax, who starred in the 1943 film \"Lassie Come-Home.\" The story was written by Eric Knight and tells the tale of a young boy who buys a smooth collie puppy from a street vendor and later loses him. The novel was a huge success and was adapted into several films, television shows, and even a theme park attraction.\n\nOver time, Lassie has become a symbol of loyalty, courage, and faithfulness. She is often depicted as a female dog, although the original Pal, who played Lassie in the films, was a male. The character has been portrayed by several dogs over the years, with the most well-known being Pal, Sue, and Baby.\n\nLassie has appeared in numerous films, television shows, and books throughout the years. The character has also been featured in various media, such as comic books, video games, and even a stage show." 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 18, 35 | "completion_tokens": 255, 36 | "total_tokens": 273 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@anthropic_text_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "anthropic", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "claude-3-haiku-20240307" 16 | }, 17 | "was_streaming": true, 18 | "num_chunks": 263, 19 | "response": { 20 | "created": 0, 21 | "model": "claude-3-haiku-20240307", 22 | "system_fingerprint": null, 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "Lassie was a famous fictional dog character that starred in a long-running media franchise.\n\nThe key facts about Lassie:\n\n- Lassie was originally created as the main character in a short story published in 1940 called \"Lassie Come-Home\" by Eric Knight.\n\n- The character then became the star of a popular film series starting in 1943, with the first movie titled \"Lassie Come Home.\"\n\n- In the 1950s, Lassie became the star of a hugely popular CBS television series called \"The Adventures of Lassie\" that ran for 19 seasons until 1973.\n\n- The Lassie character was portrayed by a series of female collies over the years, with the original Lassie dog being a dog named Pal.\n\n- Lassie was known for being a loyal, heroic, and highly intelligent collie dog who would often save the day and help her human companions in times of trouble.\n\n- The Lassie franchise, including the films and TV shows, became one of the most successful and iconic dog character stories in popular culture.\n\nSo in summary, Lassie was a legendary fictional collie dog character that became a beloved entertainment icon across multiple mediums for decades." 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 16, 35 | "completion_tokens": 284, 36 | "total_tokens": null 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /proxy/migrations/20240625_chronicle_proxy_steps_postgresql.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE chronicle_runs ( 2 | id uuid PRIMARY KEY, 3 | name text, 4 | description text, 5 | application text, 6 | environment text, 7 | input jsonb, 8 | output jsonb, 9 | status text NOT NULL, 10 | trace_id text, 11 | span_id text, 12 | tags text[], 13 | info jsonb, 14 | updated_at timestamp with time zone NOT NULL DEFAULT now(), 15 | created_at timestamp with time zone NOT NULL DEFAULT now() 16 | ); 17 | 18 | CREATE INDEX chronicle_runs_name_created_at_idx ON chronicle_runs (name, created_at DESC); 19 | 20 | CREATE INDEX chronicle_runs_name_updated_at_idx ON chronicle_runs (name, updated_at DESC); 21 | 22 | CREATE INDEX chronicle_runs_app_env_created_at_idx ON chronicle_runs (application, environment, 23 | created_at DESC); 24 | 25 | CREATE INDEX chronicle_runs_app_env_updated_at_idx ON chronicle_runs (application, environment, 26 | updated_at DESC); 27 | 28 | CREATE INDEX chronicle_runs_updated_at_idx ON chronicle_runs (updated_at DESC); 29 | 30 | CREATE INDEX chronicle_runs_created_at_idx ON chronicle_runs (created_at DESC); 31 | 32 | CREATE INDEX chronicle_runs_tags_idx ON chronicle_runs USING gin (tags); 33 | 34 | CREATE TABLE chronicle_steps ( 35 | id uuid PRIMARY KEY, 36 | run_id uuid NOT NULL, 37 | type text NOT NULL, 38 | parent_step uuid, 39 | name text, 40 | input jsonb, 41 | output jsonb, 42 | status text NOT NULL, 43 | tags text[], 44 | info jsonb, 45 | span_id text, 46 | start_time timestamp with time zone NOT NULL DEFAULT now(), 47 | end_time timestamp with time zone 48 | ); 49 | 50 | CREATE INDEX chronicle_steps_run_id_idx ON chronicle_steps (run_id); 51 | 52 | CREATE INDEX chronicle_steps_tags_idx ON chronicle_steps USING gin (tags); 53 | 54 | ALTER TABLE chronicle_events 55 | ALTER COLUMN run_id TYPE uuid 56 | USING run_id::uuid; 57 | 58 | ALTER TABLE chronicle_events 59 | ALTER COLUMN step TYPE uuid 60 | USING step::uuid; 61 | 62 | ALTER TABLE chronicle_events RENAME COLUMN step TO step_id; 63 | 64 | CREATE INDEX chronicle_events_run_id_created_at_idx ON chronicle_events (run_id, created_at DESC); 65 | 66 | DROP INDEX chronicle_events_run_id_idx; 67 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@ollama_text_nonstreaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "ollama", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": { 15 | "eval_duration": 4889146000, 16 | "load_duration": 3278875, 17 | "prompt_eval_duration": 162740000 18 | }, 19 | "model": "llama3" 20 | }, 21 | "was_streaming": false, 22 | "num_chunks": 1, 23 | "response": { 24 | "created": 0, 25 | "model": "llama3", 26 | "system_fingerprint": null, 27 | "choices": [ 28 | { 29 | "index": 0, 30 | "message": { 31 | "role": "assistant", 32 | "content": "A beloved topic!\n\nLassie is a fictional Rough Collie dog character created by Eric Knight in his 1940 novel \"Lassie Come-Home.\" The story follows a loyal and courageous golden-colored collie named Lassie, who loves her human family, particularly Timmy Martin. After being separated from her family during a tragic accident, Lassie embarks on an incredible journey to find them.\n\nThe character has since become a cultural icon through various adaptations:\n\n1. **Novels**: Eric Knight wrote six more \"Lassie\" novels between 1940 and 1957.\n2. **Films**: The first film, \"Lassie Come-Home,\" was released in 1943, followed by over 30 more movies featuring Lassie from the 1940s to the 1990s.\n3. **Television**: A popular TV series aired from 1954 to 1974, with a total of 574 episodes. The show starred Lassie as a loyal and heroic canine companion, often helping her human friends in need.\n4. **Radio**: Lassie had her own radio show, \"The Lassie Story,\" which ran from 1947 to 1953.\n\nThroughout the years, Lassie has become synonymous with loyalty, courage, and friendship, making her one of the most beloved and recognizable canine characters in popular culture." 33 | }, 34 | "finish_reason": "stop" 35 | } 36 | ], 37 | "usage": { 38 | "prompt_tokens": null, 39 | "completion_tokens": 287, 40 | "total_tokens": null 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@ollama_text_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "ollama", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": { 15 | "eval_duration": 4726492000, 16 | "load_duration": 743750, 17 | "prompt_eval_duration": 17083000 18 | }, 19 | "model": "llama3" 20 | }, 21 | "was_streaming": true, 22 | "num_chunks": 278, 23 | "response": { 24 | "created": 0, 25 | "model": "llama3", 26 | "system_fingerprint": null, 27 | "choices": [ 28 | { 29 | "index": 0, 30 | "message": { 31 | "role": "assistant", 32 | "content": "A classic!\n\nLassie is a fictional Rough Collie dog character created by Eric Knight in his 1940 novel \"Lassie Come-Home\". The story follows a collie dog named Lassie, who is a loyal companion to a young girl named Ruth Fielding. The book was a huge success, and the character of Lassie has since become an iconic symbol of courage, loyalty, and friendship.\n\nThe first television show featuring Lassie premiered in 1954 and ran for 17 seasons until 1974. The series followed the adventures of a heroic collie dog named Lassie, who lives on a ranch with her best friend Timmy Martin (played by Jon Provost) and his family. Lassie was known for her incredible intelligence, bravery, and ability to help those in need.\n\nOver the years, there have been several reboots and spin-offs of the original TV series, including a 1997-2001 revival starring Christopher Knight as Jonathan \"Jesse\" Harper, Timmy's older brother. More recently, a new Lassie TV series premiered on The Hallmark Channel in 2019.\n\nLassie has become an enduring cultural phenomenon, with numerous films, TV shows, books, and merchandise featuring the beloved character. She remains one of the most iconic and beloved dogs in American pop culture!" 33 | }, 34 | "finish_reason": "stop" 35 | } 36 | ], 37 | "usage": { 38 | "prompt_tokens": null, 39 | "completion_tokens": 278, 40 | "total_tokens": null 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /proxy/src/error.rs: -------------------------------------------------------------------------------- 1 | /// Proxy errors 2 | #[derive(thiserror::Error, Debug)] 3 | pub enum Error { 4 | /// A specified model provider was not found 5 | #[error("Unknown model provider {0}")] 6 | UnknownProvider(String), 7 | 8 | /// No provider was specified, and no default could be inferred from the model name 9 | #[error("No default provider for model {0}")] 10 | NoDefault(String), 11 | 12 | /// An alias references a provider that doesn't exist 13 | #[error("Alias {0} references nonexistent provider {1}")] 14 | NoAliasProvider(String, String), 15 | 16 | /// An alias does not reference any models 17 | #[error("Alias {0} has no associated models")] 18 | AliasEmpty(String), 19 | 20 | /// An alias references an API key that doesn't exist 21 | #[error("Alias {0} references nonexistent API key {1}")] 22 | NoAliasApiKey(String, String), 23 | 24 | /// The requested API key does not exist 25 | #[error("Unknown API key name {0}")] 26 | NoApiKey(String), 27 | 28 | /// The request is missing the model name 29 | #[error("Request is missing model name")] 30 | ModelNotSpecified, 31 | 32 | /// The model provider returned an error 33 | #[error("Model provider returned an error")] 34 | ModelError, 35 | 36 | /// The API key was not provided 37 | #[error("API key not provided")] 38 | MissingApiKey, 39 | 40 | /// The environment variable for the API key was not found 41 | #[error("Did not find environment variable {1} for API key {0}")] 42 | MissingApiKeyEnv(String, String), 43 | 44 | /// Failed to parse the model provider's output 45 | #[error("Failed to parse model provider's output")] 46 | ResultParseError, 47 | 48 | /// Failed to read the configuration file 49 | #[error("Failed to read configuration file")] 50 | ReadingConfig, 51 | 52 | /// Failed to load data from the database 53 | #[error("Failed to load from the database")] 54 | LoadingDatabase, 55 | 56 | /// Failed to parse a header value 57 | #[error("Failed to parse header value {0}: Expected a {1}")] 58 | ReadingHeader(String, &'static str), 59 | 60 | /// A required piece of information was missing from the response stream 61 | #[error("Did not see {0} in response stream")] 62 | MissingStreamInformation(&'static str), 63 | } 64 | -------------------------------------------------------------------------------- /proxy/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "chronicle-proxy" 3 | version = "0.4.3" 4 | edition = "2021" 5 | description = "LLM Provider Abstraction and Logging" 6 | documentation = "https://docs.rs/chronicle-proxy" 7 | license.workspace = true 8 | authors.workspace = true 9 | readme.workspace = true 10 | repository.workspace = true 11 | keywords.workspace = true 12 | 13 | [dependencies] 14 | ahash = "0.8.11" 15 | async-trait = "0.1.80" 16 | aws-config = { version = "1.1.7", features = ["behavior-version-latest"], optional = true } 17 | aws-sdk-bedrockruntime = { version = "1.38.0", optional = true } 18 | aws-smithy-types = { version = "1.2.0", optional = true } 19 | backon = "0.4.4" 20 | bytes = "1.6.0" 21 | chrono = { version = "0.4.38", features = ["serde"] } 22 | error-stack = "0.5.0" 23 | eventsource-stream = "0.2.3" 24 | filigree = { workspace = true, optional = true } 25 | flume = "0.11.0" 26 | futures = "0.3.30" 27 | http = "1.1.0" 28 | itertools = "0.12.1" 29 | rand = "0.8.5" 30 | reqwest = { version = "0.12.3", features = ["json", "stream"] } 31 | schemars = { version = "0.8.16", optional = true } 32 | serde = { version = "1.0.198", features = ["derive"] } 33 | serde_json = "1.0.116" 34 | serde_path_to_error = "0.1.16" 35 | serde_with = "3.8.1" 36 | smallvec = { version = "1.13.2", features = ["union", "const_generics", "serde"] } 37 | sqlx = { version = "0.8.0", features = ["chrono", "json", "uuid"] } 38 | sqlx-transparent-json-decode.workspace = true 39 | thiserror = "1.0.58" 40 | tokio = { version = "1.37.0", features = ["fs", "macros", "time"] } 41 | tokio-util = { version = "0.7.11", features = ["io"] } 42 | toml = "0.8.12" 43 | tracing = "0.1.40" 44 | url = "2.5.0" 45 | uuid = { version = "1.8.0", features = ["v4", "v7", "serde"] } 46 | 47 | [dev-dependencies] 48 | dotenvy = "0.15.7" 49 | filigree = { workspace = true } 50 | insta = { version = "1.38.0", features = ["json", "redactions"] } 51 | tokio = { version = "1.37.0", features = ["fs", "macros", "rt", "test-util", "time"] } 52 | wiremock = "0.6.0" 53 | 54 | [features] 55 | default = ["postgres", "sqlite"] 56 | postgres = ["sqlx/postgres"] 57 | sqlite = ["sqlx/sqlite"] 58 | filigree = ["dep:filigree"] 59 | schemars = ["dep:schemars"] 60 | aws-bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"] 61 | 62 | live-test = ["live-test-anthropic", "live-test-aws-bedrock"] 63 | live-test-anthropic = [] 64 | live-test-aws-bedrock = ["aws-bedrock"] 65 | 66 | [package.metadata.docs.rs] 67 | all-features = true 68 | -------------------------------------------------------------------------------- /proxy/src/snapshots/chronicle_proxy__testing__fixture_response@groq_text_streaming.snap: -------------------------------------------------------------------------------- 1 | --- 2 | source: proxy/src/testing.rs 3 | expression: response 4 | --- 5 | { 6 | "request_info": { 7 | "id": "00000000-0000-0000-0000-000000000000", 8 | "provider": "groq", 9 | "model": "me/a-test-model", 10 | "num_retries": 0, 11 | "was_rate_limited": false 12 | }, 13 | "response_info": { 14 | "meta": null, 15 | "model": "llama3-8b-8192" 16 | }, 17 | "was_streaming": true, 18 | "num_chunks": 351, 19 | "response": { 20 | "created": 0, 21 | "model": "llama3-8b-8192", 22 | "system_fingerprint": "fp_af05557ca2", 23 | "choices": [ 24 | { 25 | "index": 0, 26 | "message": { 27 | "role": "assistant", 28 | "content": "Lassie is a legendary Rough Collie dog that was the star of a series of classic movies, television shows, and books. The character was created by Eric Knight, an American writer, in his 1940 novel \"Lassie Come-Home.\"\n\nIn the novel and subsequent film adaptations, Lassie is a charming and intelligent Collie who is separated from her young owner, Joe Carraclough, when he is forced to give her up due to financial difficulties. The novel and films follow Lassie's journey as she travels back to England to find Joe and be reunited with him.\n\nThe most famous Lassie was Pal, a male Rough Collie who played the title role in the 1943 film \"Lassie Come-Home\" and its sequels. Pal went on to become an iconic Hollywood star, earning $250,000 per year (approximately $4.5 million today) and receiving the American Humane Association's \"Pets in Film Award.\"\n\nThe Lassie franchise expanded to television in the 1950s, with the popular television series \"Lassie\" (1954-1974) starring a female Rough Collie named Pal's son, Lassie Jr., and his descendants. Over the years, several dogs played the role of Lassie on television, including three generations of the same family of Collins dogs.\n\nLassie became an international symbol of loyalty, courage, and devotion, inspiring numerous adaptations, parodies, and references in popular culture. Despite the death of Pal in 1958, the legend of Lassie continues to endure, with new generations of dogs playing the role and the character remaining a beloved and iconic figure in popular culture." 29 | }, 30 | "finish_reason": "stop" 31 | } 32 | ], 33 | "usage": { 34 | "prompt_tokens": 18, 35 | "completion_tokens": 349, 36 | "total_tokens": 367 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /proxy/src/database.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::BTreeMap, sync::Arc}; 2 | 3 | use error_stack::Report; 4 | use logging::ProxyLogEntry; 5 | 6 | use crate::{ 7 | config::{AliasConfig, ApiKeyConfig, CustomProviderConfig}, 8 | providers::custom::ProviderRequestFormat, 9 | Error, 10 | }; 11 | 12 | pub mod logging; 13 | #[cfg(feature = "postgres")] 14 | pub mod postgres; 15 | #[cfg(feature = "sqlite")] 16 | pub mod sqlite; 17 | #[cfg(test)] 18 | mod testing; 19 | 20 | /// A DBMS-agnostic interface to a database 21 | #[async_trait::async_trait] 22 | pub trait ProxyDatabase: std::fmt::Debug + Send + Sync { 23 | /// Load provider configuration from the database 24 | async fn load_providers_from_database( 25 | &self, 26 | providers_table: &str, 27 | ) -> Result, Report>; 28 | 29 | /// Load alias configuration from the database 30 | async fn load_aliases_from_database( 31 | &self, 32 | alias_table: &str, 33 | providers_table: &str, 34 | ) -> Result, Report>; 35 | 36 | /// Load API key configuration from the database 37 | async fn load_api_key_configs_from_database( 38 | &self, 39 | table: &str, 40 | ) -> Result, Report>; 41 | 42 | /// Write a batch of log entries to the database 43 | async fn write_log_batch(&self, items: Vec) -> Result<(), sqlx::Error>; 44 | } 45 | 46 | /// A [ProxyDatabase] wrapped in an [Arc] 47 | pub type Database = Arc; 48 | 49 | /// A provider configuration loaded from the database 50 | #[derive(sqlx::FromRow)] 51 | pub struct DbProvider { 52 | name: String, 53 | label: Option, 54 | url: String, 55 | api_key: Option, 56 | format: sqlx::types::Json, 57 | headers: Option>>, 58 | prefix: Option, 59 | api_key_source: Option, 60 | } 61 | 62 | /// Load provider configuration from the database 63 | pub async fn load_providers_from_database( 64 | db: &dyn ProxyDatabase, 65 | providers_table: &str, 66 | ) -> Result, Report> { 67 | let rows = db.load_providers_from_database(providers_table).await?; 68 | let providers = rows 69 | .into_iter() 70 | .map(|row| CustomProviderConfig { 71 | name: row.name, 72 | label: row.label, 73 | url: row.url, 74 | api_key: row.api_key, 75 | format: row.format.0, 76 | headers: row.headers.unwrap_or_default().0, 77 | prefix: row.prefix, 78 | api_key_source: row.api_key_source, 79 | }) 80 | .collect(); 81 | Ok(providers) 82 | } 83 | -------------------------------------------------------------------------------- /js-client/src/wrapper.ts: -------------------------------------------------------------------------------- 1 | /** Functions for wrapping the OpenAI SDK client */ 2 | 3 | import { ChronicleClientOptions } from './client.js'; 4 | import { proxyUrl, propagateSpan } from './internal.js'; 5 | 6 | /** Return a custom fetch function that calls Chronicle instead. This can be passed to 7 | * the OpenAI client's `fetch` parameter. */ 8 | export function fetchChronicle(options?: ChronicleClientOptions) { 9 | let thisFetch = options?.fetch ?? globalThis.fetch; 10 | const url = proxyUrl(options?.url); 11 | const { token, defaults } = options ?? {}; 12 | 13 | const headers = [ 14 | ['x-chronicle-provider-api-key', defaults?.api_key], 15 | ['x-chronicle-provider', defaults?.provider], 16 | ['x-chronicle-model', defaults?.model], 17 | ['x-chronicle-override-url', defaults?.override_url], 18 | ['x-chronicle-api-key', defaults?.api_key], 19 | ['x-chronicle-models', JSON.stringify(defaults?.models)], 20 | ['x-chronicle-random-choice', defaults?.random_choice], 21 | ['x-chronicle-retry', JSON.stringify(defaults?.retry)], 22 | ['x-chronicle-timeout', defaults?.timeout], 23 | ['x-chronicle-application', defaults?.metadata?.application], 24 | ['x-chronicle-environment', defaults?.metadata?.environment], 25 | ['x-chronicle-organization-id', defaults?.metadata?.organization_id], 26 | ['x-chronicle-project-id', defaults?.metadata?.project_id], 27 | ['x-chronicle-user-id', defaults?.metadata?.user_id], 28 | ['x-chronicle-workflow-id', defaults?.metadata?.workflow_id], 29 | ['x-chronicle-workflow-name', defaults?.metadata?.workflow_name], 30 | ['x-chronicle-run-id', defaults?.metadata?.run_id], 31 | ['x-chronicle-step-id', defaults?.metadata?.step_id], 32 | ['x-chronicle-step-index', defaults?.metadata?.step_index], 33 | ['x-chronicle-prompt-id', defaults?.metadata?.prompt_id], 34 | ['x-chronicle-prompt-version', defaults?.metadata?.prompt_version], 35 | ['x-chronicle-extra-meta', JSON.stringify(defaults?.metadata?.extra)], 36 | ] 37 | .filter(([_, v]) => v !== undefined) 38 | .map(([k, v]) => [k, v!.toString()]) as [string, string][]; 39 | 40 | return async function (requestInfo: RequestInfo, init?: RequestInit) { 41 | // First just coalesce requestInfo and init into a single request 42 | let req = new Request(requestInfo, init); 43 | // If Chronicle updates to support other types of endpoints then we should look at the URL to decide 44 | // which endpoint it's trying to call. 45 | req = new Request(url, req); 46 | 47 | propagateSpan(req); 48 | 49 | for (const [k, v] of headers) { 50 | req.headers.set(k, v); 51 | } 52 | 53 | if (token) { 54 | req.headers.set('Authorization', `Bearer ${token}`); 55 | } 56 | 57 | return thisFetch(req); 58 | }; 59 | } 60 | -------------------------------------------------------------------------------- /js-client/.gitignore: -------------------------------------------------------------------------------- 1 | # Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore 2 | 3 | # Logs 4 | 5 | logs 6 | _.log 7 | npm-debug.log_ 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | .pnpm-debug.log* 12 | 13 | # Caches 14 | 15 | .cache 16 | 17 | # Diagnostic reports (https://nodejs.org/api/report.html) 18 | 19 | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json 20 | 21 | # Runtime data 22 | 23 | pids 24 | _.pid 25 | _.seed 26 | *.pid.lock 27 | 28 | # Directory for instrumented libs generated by jscoverage/JSCover 29 | 30 | lib-cov 31 | 32 | # Coverage directory used by tools like istanbul 33 | 34 | coverage 35 | *.lcov 36 | 37 | # nyc test coverage 38 | 39 | .nyc_output 40 | 41 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 42 | 43 | .grunt 44 | 45 | # Bower dependency directory (https://bower.io/) 46 | 47 | bower_components 48 | 49 | # node-waf configuration 50 | 51 | .lock-wscript 52 | 53 | # Compiled binary addons (https://nodejs.org/api/addons.html) 54 | 55 | build/Release 56 | 57 | # Dependency directories 58 | 59 | node_modules/ 60 | jspm_packages/ 61 | 62 | # Snowpack dependency directory (https://snowpack.dev/) 63 | 64 | web_modules/ 65 | 66 | # TypeScript cache 67 | 68 | *.tsbuildinfo 69 | 70 | # Optional npm cache directory 71 | 72 | .npm 73 | 74 | # Optional eslint cache 75 | 76 | .eslintcache 77 | 78 | # Optional stylelint cache 79 | 80 | .stylelintcache 81 | 82 | # Microbundle cache 83 | 84 | .rpt2_cache/ 85 | .rts2_cache_cjs/ 86 | .rts2_cache_es/ 87 | .rts2_cache_umd/ 88 | 89 | # Optional REPL history 90 | 91 | .node_repl_history 92 | 93 | # Output of 'npm pack' 94 | 95 | *.tgz 96 | 97 | # Yarn Integrity file 98 | 99 | .yarn-integrity 100 | 101 | # dotenv environment variable files 102 | 103 | .env 104 | .env.development.local 105 | .env.test.local 106 | .env.production.local 107 | .env.local 108 | 109 | # parcel-bundler cache (https://parceljs.org/) 110 | 111 | .parcel-cache 112 | 113 | # Next.js build output 114 | 115 | .next 116 | out 117 | 118 | # Nuxt.js build / generate output 119 | 120 | .nuxt 121 | dist 122 | 123 | # Gatsby files 124 | 125 | # Comment in the public line in if your project uses Gatsby and not Next.js 126 | 127 | # https://nextjs.org/blog/next-9-1#public-directory-support 128 | 129 | # public 130 | 131 | # vuepress build output 132 | 133 | .vuepress/dist 134 | 135 | # vuepress v2.x temp and cache directory 136 | 137 | .temp 138 | 139 | # Docusaurus cache and generated files 140 | 141 | .docusaurus 142 | 143 | # Serverless directories 144 | 145 | .serverless/ 146 | 147 | # FuseBox cache 148 | 149 | .fusebox/ 150 | 151 | # DynamoDB Local files 152 | 153 | .dynamodb/ 154 | 155 | # TernJS port file 156 | 157 | .tern-port 158 | 159 | # Stores VSCode versions used for testing VSCode extensions 160 | 161 | .vscode-test 162 | 163 | # yarn v2 164 | 165 | .yarn/cache 166 | .yarn/unplugged 167 | .yarn/build-state.yml 168 | .yarn/install-state.gz 169 | .pnp.* 170 | 171 | # IntelliJ based IDEs 172 | .idea 173 | 174 | # Finder (MacOS) folder config 175 | .DS_Store 176 | chronicle-test.db* 177 | -------------------------------------------------------------------------------- /js-client/src/runs.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from 'bun:test'; 2 | import { asStep, recordStepInfo, runStep, startRun } from './runs.js'; 3 | import { uuidv7 } from 'uuidv7'; 4 | import { flushEvents } from './logger.js'; 5 | 6 | const autoStep = asStep(async function autoStep(n: number) { 7 | recordStepInfo({ addend: 1 }); 8 | return await namedStep(n + 1); 9 | }); 10 | 11 | const namedStep = asStep( 12 | async (n: number) => { 13 | recordStepInfo({ more_info: true }); 14 | return n + 2; 15 | }, 16 | { 17 | type: 'adder', 18 | name: 'addTwo', 19 | info: { 20 | addend: 2, 21 | }, 22 | tags: ['adder', 'math'], 23 | } 24 | ); 25 | 26 | const errorStep = asStep(async function errorStep() { 27 | throw new Error('test error'); 28 | }); 29 | 30 | test('runs and steps ', async () => { 31 | const runId = uuidv7(); 32 | let retVal = await startRun( 33 | { 34 | name: 'Test Run', 35 | application: 'chronicle-test', 36 | environment: 'test', 37 | description: 'This is a test run', 38 | runId, 39 | info: { 40 | testing: true, 41 | }, 42 | input: { 43 | value: 1, 44 | }, 45 | tags: ['test'], 46 | }, 47 | async () => { 48 | return await runStep( 49 | { 50 | name: 'outer step', 51 | type: 'outer', 52 | }, 53 | async (ctx) => { 54 | expect(ctx.runId).toEqual(runId); 55 | try { 56 | await errorStep(); 57 | throw new Error(`Failed to propagate error from step`); 58 | } catch (e) { 59 | expect(e.message).toEqual('test error'); 60 | } 61 | 62 | ctx.recordRunInfo({ addedRunInfo: true }); 63 | return await autoStep(1); 64 | } 65 | ); 66 | } 67 | ); 68 | 69 | expect(retVal).toEqual({ 70 | id: runId, 71 | info: { 72 | addedRunInfo: true, 73 | }, 74 | output: 4, 75 | }); 76 | 77 | await flushEvents(); 78 | }); 79 | 80 | test('run with custom finish status', async () => { 81 | const runId = uuidv7(); 82 | await startRun( 83 | { 84 | name: 'Test with custom finish status', 85 | runId, 86 | info: { 87 | testing: true, 88 | }, 89 | input: { 90 | value: 1, 91 | }, 92 | tags: ['test'], 93 | }, 94 | async (ctx) => { 95 | ctx.setRunFinishStatus('skipped'); 96 | return 1; 97 | } 98 | ); 99 | 100 | await flushEvents(); 101 | }); 102 | 103 | test('run error', async () => { 104 | const runId = uuidv7(); 105 | try { 106 | await startRun( 107 | { 108 | runId, 109 | name: 'error test run', 110 | }, 111 | async () => { 112 | throw new Error('test error'); 113 | } 114 | ); 115 | 116 | throw new Error(`Failed to propagate error from run`); 117 | } catch (e) { 118 | expect(e.message).toEqual('test error'); 119 | } finally { 120 | await flushEvents(); 121 | } 122 | }); 123 | 124 | test('implicit run around a step', async () => { 125 | try { 126 | let output = await autoStep(1); 127 | expect(output).toEqual(4); 128 | } finally { 129 | await flushEvents(); 130 | } 131 | }); 132 | -------------------------------------------------------------------------------- /js-client/src/logger.ts: -------------------------------------------------------------------------------- 1 | import EventEmitter from 'node:events'; 2 | import { ChronicleEvent } from './events.js'; 3 | import { handleError } from './internal.js'; 4 | 5 | /** Map of event URL to logging queue. */ 6 | const loggerMap = new Map(); 7 | 8 | type QueueState = 'idle' | 'waiting' | 'writing'; 9 | 10 | const QUEUE_THRESHOLD = 500; 11 | const DEBOUNCE_TIME = 50; 12 | 13 | /** Collect logs in order and send them out in batches periodically */ 14 | export class Logger { 15 | url: string; 16 | queue_state: QueueState = 'idle'; 17 | event_queue: ChronicleEvent[] = []; 18 | 19 | flushed = new EventEmitter<{ flush: [] }>(); 20 | 21 | constructor(url: string) { 22 | this.url = url; 23 | } 24 | 25 | enqueue(event: ChronicleEvent | ChronicleEvent[]) { 26 | if (Array.isArray(event)) { 27 | this.event_queue.push(...event); 28 | } else { 29 | this.event_queue.push(event); 30 | } 31 | 32 | if (this.event_queue.length > QUEUE_THRESHOLD && this.queue_state !== 'writing') { 33 | this.writeEvents(); 34 | } else if (this.queue_state === 'idle') { 35 | this.queue_state = 'waiting'; 36 | setTimeout(() => this.writeEvents(), DEBOUNCE_TIME); 37 | } 38 | } 39 | 40 | async writeEvents() { 41 | let thisBatch = this.event_queue; 42 | this.event_queue = []; 43 | 44 | this.queue_state = 'writing'; 45 | 46 | try { 47 | let req = new Request(this.url, { 48 | method: 'POST', 49 | headers: { 50 | 'content-type': 'application/json; charset=utf-8', 51 | }, 52 | body: JSON.stringify({ events: thisBatch }), 53 | }); 54 | let res = await fetch(req); 55 | if (!res.ok) { 56 | throw new Error(await handleError(res)); 57 | } 58 | } catch (e) { 59 | // TODO log error to somewhere real 60 | console.error(e); 61 | console.error('Writing', thisBatch); 62 | } finally { 63 | if (this.event_queue.length) { 64 | const overThreshold = this.event_queue.length > QUEUE_THRESHOLD; 65 | let nextTime = overThreshold ? 0 : DEBOUNCE_TIME; 66 | this.queue_state = overThreshold ? 'writing' : 'waiting'; 67 | setTimeout(() => this.writeEvents(), nextTime); 68 | } else { 69 | this.flushed.emit('flush'); 70 | this.queue_state = 'idle'; 71 | } 72 | } 73 | } 74 | 75 | /** Wait for all existing events to be flushed. */ 76 | flushEvents() { 77 | if (!this.event_queue.length) { 78 | return; 79 | } 80 | 81 | return new Promise((resolve) => { 82 | this.flushed.once('flush', () => resolve()); 83 | }); 84 | } 85 | } 86 | 87 | export function getLogger(url: string | URL): Logger { 88 | url = url.toString(); 89 | let existing = loggerMap.get(url); 90 | if (existing) { 91 | return existing; 92 | } 93 | 94 | let logger = new Logger(url); 95 | loggerMap.set(url, logger); 96 | return logger; 97 | } 98 | 99 | /** Wait for all loggers to flush their events. This can be used to ensure that the 100 | * process stays alive while events are still being sent, in some environments that 101 | * may not stay alive such as Bun tests. */ 102 | export async function flushEvents() { 103 | const flushes = Array.from(loggerMap.values()).map((l) => l.flushEvents()); 104 | await Promise.all(flushes); 105 | } 106 | -------------------------------------------------------------------------------- /proxy/src/streaming.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use error_stack::{Report, ResultExt}; 4 | use eventsource_stream::{Event, Eventsource}; 5 | use futures::StreamExt; 6 | 7 | use crate::{ 8 | format::{ResponseInfo, StreamingChatResponse, StreamingResponse, StreamingResponseSender}, 9 | providers::{ProviderError, ProviderErrorKind}, 10 | }; 11 | 12 | /// Stream an SSE response to the channel 13 | /// 14 | /// `start_time` - the time the request was started 15 | /// `response` - the response to stream 16 | /// `chunk_tx` - the channel to send the chunks to 17 | /// `map_chunk` - a function to map the event to a standard chat response. 18 | /// 19 | /// `map_chunk` can return Ok(None) if the event should be skipped, as with Anthropic's 20 | /// ping event. 21 | pub fn stream_sse_to_channel( 22 | response: reqwest::Response, 23 | chunk_tx: StreamingResponseSender, 24 | mut mapper: impl StreamingChunkMapper, 25 | ) -> tokio::task::JoinHandle<()> { 26 | tokio::task::spawn(async move { 27 | let mut stream = response.bytes_stream().eventsource(); 28 | let mut model: Option = None; 29 | 30 | while let Some(event) = stream.next().await { 31 | match event { 32 | Ok(event) => { 33 | let chunk = mapper.process_chunk(&event); 34 | tracing::trace!(chunk = ?chunk); 35 | match chunk { 36 | Ok(None) => continue, 37 | Ok(Some(chunk)) => { 38 | if model.is_none() { 39 | model = chunk.model.clone(); 40 | } 41 | 42 | let result = chunk_tx 43 | .send_async(Ok(StreamingResponse::Chunk(chunk))) 44 | .await; 45 | if result.is_err() { 46 | // Channel was closed 47 | tracing::warn!("channel closed early"); 48 | return; 49 | } 50 | } 51 | Err(e) => { 52 | chunk_tx.send_async(Err(e)).await.ok(); 53 | return; 54 | } 55 | } 56 | } 57 | Err(e) => { 58 | chunk_tx 59 | .send_async(Err(e).change_context(ProviderError { 60 | kind: ProviderErrorKind::ProviderClosedConnection, 61 | status_code: None, 62 | body: None, 63 | latency: Duration::ZERO, 64 | })) 65 | .await 66 | .ok(); 67 | return; 68 | } 69 | } 70 | } 71 | 72 | chunk_tx 73 | .send_async(Ok(StreamingResponse::ResponseInfo(ResponseInfo { 74 | meta: None, 75 | model: model.unwrap_or_default(), 76 | }))) 77 | .await 78 | .ok(); 79 | }) 80 | } 81 | 82 | /// Process an SSE Event and optionally return a chat response 83 | pub trait StreamingChunkMapper: Send + Sync + 'static { 84 | fn process_chunk( 85 | &mut self, 86 | event: &Event, 87 | ) -> Result, Report>; 88 | } 89 | -------------------------------------------------------------------------------- /proxy/src/providers/custom.rs: -------------------------------------------------------------------------------- 1 | //! Handle custom provider configurations that look close enough to an existing provider 2 | //! that we can declare them in data. 3 | 4 | use error_stack::Report; 5 | use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | use super::{openai::send_openai_request, ChatModelProvider, ProviderError, SendRequestOptions}; 9 | use crate::{ 10 | config::CustomProviderConfig, 11 | format::{ChatRequestTransformation, StreamingResponseSender}, 12 | }; 13 | 14 | #[derive(Debug, Clone)] 15 | pub struct CustomProvider { 16 | pub config: CustomProviderConfig, 17 | pub client: reqwest::Client, 18 | pub headers: HeaderMap, 19 | } 20 | 21 | #[derive(Serialize, Deserialize, Default, PartialEq, Eq, Debug, Clone)] 22 | #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] 23 | pub struct OpenAiRequestFormatOptions { 24 | pub transforms: ChatRequestTransformation<'static>, 25 | } 26 | 27 | /// The format that this proider uses for requests 28 | /// todo move this somewhere else 29 | #[derive(Serialize, Deserialize, Eq, PartialEq, Debug, Clone)] 30 | #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] 31 | #[serde(tag = "type", rename_all = "snake_case")] 32 | pub enum ProviderRequestFormat { 33 | /// OpenAI format 34 | OpenAi(OpenAiRequestFormatOptions), 35 | } 36 | 37 | sqlx_transparent_json_decode::sqlx_json_decode!(ProviderRequestFormat); 38 | 39 | impl Default for ProviderRequestFormat { 40 | fn default() -> Self { 41 | Self::OpenAi(OpenAiRequestFormatOptions::default()) 42 | } 43 | } 44 | 45 | impl CustomProvider { 46 | pub fn new(mut config: CustomProviderConfig, client: reqwest::Client) -> Self { 47 | let headers = std::mem::take(&mut config.headers); 48 | let headers: HeaderMap = headers 49 | .into_iter() 50 | .filter_map(|(k, v)| { 51 | let k = HeaderName::from_bytes(k.as_bytes()).ok()?; 52 | let v = HeaderValue::from_str(v.as_str()).ok()?; 53 | Some((k, v)) 54 | }) 55 | .collect(); 56 | Self { 57 | config, 58 | client, 59 | headers, 60 | } 61 | } 62 | } 63 | 64 | #[async_trait::async_trait] 65 | impl ChatModelProvider for CustomProvider { 66 | fn name(&self) -> &str { 67 | &self.config.name 68 | } 69 | 70 | fn label(&self) -> &str { 71 | self.config.label.as_deref().unwrap_or(&self.config.name) 72 | } 73 | 74 | async fn send_request( 75 | &self, 76 | options: SendRequestOptions, 77 | chunk_tx: StreamingResponseSender, 78 | ) -> Result<(), Report> { 79 | match &self.config.format { 80 | ProviderRequestFormat::OpenAi(OpenAiRequestFormatOptions { transforms }) => { 81 | send_openai_request( 82 | &self.client, 83 | &self.config.url, 84 | Some(&self.headers), 85 | self.config.api_key.as_deref(), 86 | chunk_tx, 87 | &transforms, 88 | options, 89 | ) 90 | .await 91 | } 92 | } 93 | } 94 | 95 | fn is_default_for_model(&self, model: &str) -> bool { 96 | self.config 97 | .prefix 98 | .as_deref() 99 | .map(|s| model.starts_with(s)) 100 | .unwrap_or(false) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /api/src/events.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use axum::{extract::State, response::IntoResponse, Json}; 4 | use chronicle_proxy::{ 5 | workflow_events::{EventPayload, WorkflowEvent}, 6 | ProxyRequestMetadata, 7 | }; 8 | use error_stack::ResultExt; 9 | use http::{HeaderMap, StatusCode}; 10 | use serde::Deserialize; 11 | use smallvec::{smallvec, SmallVec}; 12 | 13 | use crate::{error::Error, proxy::ServerState}; 14 | 15 | async fn record_event( 16 | State(state): State>, 17 | headers: HeaderMap, 18 | Json(mut body): Json, 19 | ) -> Result { 20 | match &mut body { 21 | WorkflowEvent::RunStart(event) => { 22 | let mut metadata = ProxyRequestMetadata::default(); 23 | metadata 24 | .merge_request_headers(&headers) 25 | .change_context(Error::InvalidProxyHeader)?; 26 | event.merge_metadata(&metadata); 27 | } 28 | _ => {} 29 | } 30 | 31 | state.proxy.record_event_batch(smallvec![body]).await; 32 | 33 | Ok(StatusCode::ACCEPTED) 34 | } 35 | 36 | #[derive(Deserialize)] 37 | struct EventsPayload { 38 | events: SmallVec<[WorkflowEvent; 1]>, 39 | } 40 | 41 | fn check_invalid_fixed_payload(e: &EventPayload) -> Result<(), Error> { 42 | match e.typ.as_str() { 43 | // Catch events that use the fixed event types but didn't serialize properly 44 | "run:start" | "run:update" | "step:start" | "step:end" | "step:error" | "step:state" => { 45 | tracing::warn!(event=?e, "Invalid fixed payload"); 46 | Err(Error::InvalidEventPayload( 47 | e.typ.clone(), 48 | e.data.clone().unwrap_or_default(), 49 | )) 50 | } 51 | _ => Ok(()), 52 | } 53 | } 54 | 55 | async fn record_events( 56 | State(state): State>, 57 | headers: HeaderMap, 58 | Json(mut body): Json, 59 | ) -> Result { 60 | let mut metadata = ProxyRequestMetadata::default(); 61 | metadata 62 | .merge_request_headers(&headers) 63 | .change_context(Error::InvalidProxyHeader)?; 64 | 65 | for event in &mut body.events { 66 | match event { 67 | WorkflowEvent::RunStart(event) => { 68 | event.merge_metadata(&metadata); 69 | } 70 | WorkflowEvent::Event(e) => check_invalid_fixed_payload(e)?, 71 | _ => {} 72 | } 73 | } 74 | 75 | state.proxy.record_event_batch(body.events).await; 76 | 77 | Ok(StatusCode::ACCEPTED) 78 | } 79 | 80 | pub fn create_routes() -> axum::Router> { 81 | axum::Router::new() 82 | .route( 83 | "/", 84 | axum::routing::get(|| async { axum::Json(serde_json::json!({ "status": "ok" })) }), 85 | ) 86 | .route("/event", axum::routing::post(record_event)) 87 | .route("/events", axum::routing::post(record_events)) 88 | .route("/v1/event", axum::routing::post(record_event)) 89 | .route("/v1/events", axum::routing::post(record_events)) 90 | .route( 91 | "/healthz", 92 | axum::routing::get(|| async { axum::Json(serde_json::json!({ "status": "ok" })) }), 93 | ) 94 | } 95 | 96 | #[cfg(test)] 97 | mod test { 98 | use chronicle_proxy::workflow_events::EventPayload; 99 | use uuid::Uuid; 100 | 101 | use super::check_invalid_fixed_payload; 102 | 103 | #[test] 104 | fn bad_event() { 105 | let bad_event = EventPayload { 106 | typ: "step:start".to_string(), 107 | data: Some(serde_json::json!({"test": true})), 108 | run_id: Uuid::new_v4(), 109 | step_id: Uuid::new_v4(), 110 | error: None, 111 | time: None, 112 | internal_metadata: None, 113 | }; 114 | 115 | check_invalid_fixed_payload(&bad_event).unwrap_err(); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/openai_tools_response_streaming.txt: -------------------------------------------------------------------------------- 1 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_YoAFuD3iHKfA5C7Gcifn74Nj","type":"function","function":{"name":"get_characteristics","arguments":""}}]},"logprobs":null,"finish_reason":null}],"usage":null} 2 | 3 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}],"usage":null} 4 | 5 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"name"}}]},"logprobs":null,"finish_reason":null}],"usage":null} 6 | 7 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}],"usage":null} 8 | 9 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"Daniel"}}]},"logprobs":null,"finish_reason":null}],"usage":null} 10 | 11 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\",\""}}]},"logprobs":null,"finish_reason":null}],"usage":null} 12 | 13 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"hair"}}]},"logprobs":null,"finish_reason":null}],"usage":null} 14 | 15 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"_color"}}]},"logprobs":null,"finish_reason":null}],"usage":null} 16 | 17 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}],"usage":null} 18 | 19 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"brown"}}]},"logprobs":null,"finish_reason":null}],"usage":null} 20 | 21 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}],"usage":null} 22 | 23 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} 24 | 25 | data: {"id":"chatcmpl-9VE3hUCEJv2Gsml6UQkGl9sjyO0M4","object":"chat.completion.chunk","created":1717229237,"model":"gpt-3.5-turbo-0125","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":80,"completion_tokens":10,"total_tokens":90}} 26 | 27 | data: [DONE] 28 | 29 | -------------------------------------------------------------------------------- /proxy/src/providers/fixtures/anthropic_text_and_tools_response_streaming.txt: -------------------------------------------------------------------------------- 1 | event: message_start 2 | data: {"type":"message_start","message":{"id":"msg_014p7gG3wDgGV9EUtLvnow3U","type":"message","role":"assistant","model":"claude-3-haiku-20240307","stop_sequence":null,"usage":{"input_tokens":472,"output_tokens":2},"content":[],"stop_reason":null}} 3 | 4 | event: content_block_start 5 | data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} 6 | 7 | event: ping 8 | data: {"type": "ping"} 9 | 10 | event: content_block_delta 11 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Okay"}} 12 | 13 | event: content_block_delta 14 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} 15 | 16 | event: content_block_delta 17 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" let"}} 18 | 19 | event: content_block_delta 20 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"}} 21 | 22 | event: content_block_delta 23 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" check"}} 24 | 25 | event: content_block_delta 26 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" the"}} 27 | 28 | event: content_block_delta 29 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" weather"}} 30 | 31 | event: content_block_delta 32 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" for"}} 33 | 34 | event: content_block_delta 35 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" San"}} 36 | 37 | event: content_block_delta 38 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Francisco"}} 39 | 40 | event: content_block_delta 41 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} 42 | 43 | event: content_block_delta 44 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" CA"}} 45 | 46 | event: content_block_delta 47 | data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":":"}} 48 | 49 | event: content_block_stop 50 | data: {"type":"content_block_stop","index":0} 51 | 52 | event: content_block_start 53 | data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01T1x1fJ34qAmk2tNTrN7Up6","name":"get_weather","input":{}}} 54 | 55 | event: content_block_delta 56 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}} 57 | 58 | event: content_block_delta 59 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"location\":"}} 60 | 61 | event: content_block_delta 62 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" \"San"}} 63 | 64 | event: content_block_delta 65 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" Francisc"}} 66 | 67 | event: content_block_delta 68 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"o,"}} 69 | 70 | event: content_block_delta 71 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":" CA\""}} 72 | 73 | event: content_block_delta 74 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":", "}} 75 | 76 | event: content_block_delta 77 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"unit\": \"fah"}} 78 | 79 | event: content_block_delta 80 | data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"renheit\"}"}} 81 | 82 | event: content_block_stop 83 | data: {"type":"content_block_stop","index":1} 84 | 85 | event: message_delta 86 | data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":89}} 87 | 88 | event: message_stop 89 | data: {"type":"message_stop"} 90 | 91 | -------------------------------------------------------------------------------- /api/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::path::{Path, PathBuf}; 2 | 3 | use chronicle_proxy::config::ProxyConfig; 4 | use error_stack::{Report, ResultExt}; 5 | use etcetera::BaseStrategy; 6 | use serde::Deserialize; 7 | 8 | use crate::error::Error; 9 | 10 | #[derive(Deserialize)] 11 | pub struct LocalConfig { 12 | #[serde(flatten)] 13 | pub server_config: LocalServerConfig, 14 | 15 | #[serde(flatten)] 16 | pub proxy_config: ProxyConfig, 17 | } 18 | 19 | #[derive(Deserialize)] 20 | pub struct LocalServerConfig { 21 | /// The path or URL to the database, if a database should be used. 22 | /// This can either be a file path, an sqlite:// URL, or a postgresql:// URL 23 | pub database: Option, 24 | 25 | /// The port to listen on 26 | pub port: Option, 27 | 28 | /// The IP to bind to. 29 | pub host: Option, 30 | 31 | /// Set to false to skip loading the .env file alongside this config. 32 | pub dotenv: Option, 33 | } 34 | 35 | pub fn merge_server_config(configs: &Configs) -> LocalServerConfig { 36 | let mut output = LocalServerConfig { 37 | database: None, 38 | port: None, 39 | host: None, 40 | dotenv: None, 41 | }; 42 | 43 | // Apply the global configs, then the CWD configs on top of those 44 | for config in configs.global.iter().chain(configs.cwd.iter()) { 45 | if let Some(path) = &config.1.server_config.database { 46 | let full_path = config.0.join(path); 47 | output.database = Some(full_path.to_string_lossy().to_string()); 48 | } 49 | 50 | if let Some(host) = &config.1.server_config.host { 51 | output.host = Some(host.clone()); 52 | } 53 | 54 | if let Some(port) = &config.1.server_config.port { 55 | output.port = Some(*port); 56 | } 57 | 58 | if let Some(dotenv) = &config.1.server_config.dotenv { 59 | output.dotenv = Some(*dotenv); 60 | } 61 | } 62 | 63 | output 64 | } 65 | 66 | pub struct Configs { 67 | /// Global config directories that have a chronicle.toml 68 | pub global: Vec<(PathBuf, LocalConfig)>, 69 | /// Directories starting from the root directory up to the CWD that have a chronicle.toml and 70 | /// maybe a .env 71 | pub cwd: Vec<(PathBuf, LocalConfig)>, 72 | } 73 | 74 | pub fn find_configs(directory: Option) -> Result> { 75 | if let Some(directory) = directory { 76 | let path = PathBuf::from(directory); 77 | let config = read_config(&path, path.is_dir()).change_context(Error::Config)?; 78 | 79 | let Some(config) = config else { 80 | return Err(Report::new(Error::Config)) 81 | .attach_printable_lazy(|| format!("No config found in path {}", path.display())); 82 | }; 83 | 84 | return Ok(Configs { 85 | cwd: vec![config], 86 | global: vec![], 87 | }); 88 | } 89 | 90 | let default_configs = find_default_configs()?; 91 | let cwd_configs = find_current_dir_configs()?; 92 | 93 | Ok(Configs { 94 | cwd: cwd_configs, 95 | global: default_configs, 96 | }) 97 | } 98 | 99 | fn find_default_configs() -> Result, Report> { 100 | // search for configs in the .config/chronicle directory, and looking up from the current 101 | // directory 102 | let etc = etcetera::base_strategy::choose_native_strategy().unwrap(); 103 | 104 | [ 105 | etc.home_dir().join(".config").join("chronicle"), 106 | etc.config_dir().join("chronicle"), 107 | ] 108 | .into_iter() 109 | .filter_map(|dir| read_config(&dir, true).transpose()) 110 | .collect::, Report>>() 111 | } 112 | 113 | fn find_current_dir_configs() -> Result, Report> { 114 | let Ok(current_dir) = std::env::current_dir() else { 115 | return Ok(Vec::new()); 116 | }; 117 | 118 | let mut configs = Vec::new(); 119 | let mut search_dir = Some(current_dir.as_path()); 120 | 121 | while let Some(dir) = search_dir { 122 | let config = read_config(dir, true)?; 123 | 124 | if let Some(config) = config { 125 | configs.push(config); 126 | } 127 | 128 | search_dir = dir.parent(); 129 | } 130 | 131 | // Reverse the order so that we'll apply the innermost directory last. 132 | configs.reverse(); 133 | 134 | Ok(configs) 135 | } 136 | 137 | fn read_config( 138 | path: &Path, 139 | is_directory: bool, 140 | ) -> Result, Report> { 141 | let config_path = if is_directory { 142 | path.join("chronicle.toml") 143 | } else { 144 | path.to_path_buf() 145 | }; 146 | 147 | let config_dir = if is_directory { 148 | path 149 | } else { 150 | let Some(p) = config_path.parent() else { 151 | return Ok(None); 152 | }; 153 | 154 | p 155 | }; 156 | 157 | let Ok(buf) = std::fs::read_to_string(&config_path) else { 158 | return Ok(None); 159 | }; 160 | 161 | let config = toml::from_str::(&buf) 162 | .change_context(Error::Config) 163 | .attach_printable_lazy(|| format!("Error in config file {}", config_path.display()))?; 164 | tracing::info!("Loaded config at {}", config_path.display()); 165 | Ok(Some((PathBuf::from(config_dir), config))) 166 | } 167 | -------------------------------------------------------------------------------- /proxy/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::BTreeMap, time::Duration}; 2 | 3 | use serde::{Deserialize, Serialize}; 4 | 5 | use crate::providers::custom::{CustomProvider, ProviderRequestFormat}; 6 | 7 | /// Configuration for the proxy 8 | #[derive(Serialize, Deserialize, Debug, Clone, Default)] 9 | pub struct ProxyConfig { 10 | /// Model providers that the proxy should use 11 | #[serde(default)] 12 | pub providers: Vec, 13 | /// Aliases that map to providers and models 14 | #[serde(default)] 15 | pub aliases: Vec, 16 | /// API keys that the proxy should use 17 | #[serde(default)] 18 | pub api_keys: Vec, 19 | /// The default timeout for requests 20 | pub default_timeout: Option, 21 | /// Whether to log to the database or not. 22 | pub log_to_database: Option, 23 | /// The user agent to use when making requests 24 | pub user_agent: Option, 25 | } 26 | 27 | /// An alias configuration mape a single name to a list of provider-model pairs 28 | #[derive(Serialize, Deserialize, Debug, Clone, sqlx::FromRow)] 29 | pub struct AliasConfig { 30 | /// A name for this alias 31 | pub name: String, 32 | /// If true, start from a random provider. 33 | /// If false, always start with the first provider, and only use later providers on retry. 34 | #[serde(default)] 35 | pub random_order: bool, 36 | /// The providers and models that this alias represents. 37 | pub models: Vec, 38 | } 39 | 40 | /// A provider and model to use in an alias 41 | #[derive(Serialize, Deserialize, Debug, Clone, sqlx::FromRow)] 42 | pub struct AliasConfigProvider { 43 | /// The model to use 44 | pub model: String, 45 | /// The provider to use 46 | pub provider: String, 47 | /// An API key configuration to use 48 | pub api_key_name: Option, 49 | } 50 | 51 | sqlx_transparent_json_decode::sqlx_json_decode!(AliasConfigProvider); 52 | 53 | /// An API key, or where to find one 54 | #[derive(Serialize, Deserialize, Clone, sqlx::FromRow)] 55 | pub struct ApiKeyConfig { 56 | /// A name for this key 57 | pub name: String, 58 | /// If "env", the key is an environment variable name to read, rather than the key itself. 59 | /// Eventually this will support other pluggable sources. 60 | pub source: String, 61 | /// The key itself, or if `source` is "env", the name of the environment variable to read. 62 | pub value: String, 63 | } 64 | 65 | impl std::fmt::Debug for ApiKeyConfig { 66 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 67 | f.debug_struct("ApiKeyConfig") 68 | .field("name", &self.name) 69 | .field("source", &self.source) 70 | .field( 71 | "value", 72 | if self.source == "env" { 73 | &self.value 74 | } else { 75 | &"***" 76 | }, 77 | ) 78 | .finish_non_exhaustive() 79 | } 80 | } 81 | 82 | /// A declarative definition of a model provider 83 | #[derive(Serialize, Deserialize, Debug, Clone)] 84 | pub struct CustomProviderConfig { 85 | /// The name of the provider, as referenced in proxy requests 86 | pub name: String, 87 | /// A human-readable name for the provider 88 | pub label: Option, 89 | /// The url to use 90 | pub url: String, 91 | /// The API token to pass along 92 | pub api_key: Option, 93 | /// Where to retrieve the value for `api_key`. 94 | /// If `api_key_source` is "env" then `api_key` is an environment variable. 95 | /// If it is empty, then `api_key` is assumed to be the token itself, if provided. 96 | /// In the future the key sources will be pluggable, to support external secret sources. 97 | pub api_key_source: Option, 98 | /// What kind of request format this provider uses. Defaults to OpenAI-compatible 99 | #[serde(default)] 100 | pub format: ProviderRequestFormat, 101 | /// Extra headers to pass with the request 102 | #[serde(default)] 103 | pub headers: BTreeMap, 104 | /// Models starting with this prefix will use this provider by default. 105 | pub prefix: Option, 106 | } 107 | 108 | impl CustomProviderConfig { 109 | /// Generate a [CustomProvider] object from the configuration 110 | pub fn into_provider(mut self, client: reqwest::Client) -> CustomProvider { 111 | if self.api_key_source.as_deref().unwrap_or_default() == "env" { 112 | if let Some(token) = self 113 | .api_key 114 | .as_deref() 115 | .and_then(|var| std::env::var(&var).ok()) 116 | { 117 | self.api_key = Some(token); 118 | } 119 | } 120 | 121 | CustomProvider::new(self, client) 122 | } 123 | 124 | /// Add an API token to the [CustomProviderConfig], or if one is not provided, then configure 125 | /// it to read from the given environment variable. 126 | pub fn with_token_or_env(mut self, token: Option, env: &str) -> Self { 127 | match token { 128 | Some(token) => { 129 | self.api_key = Some(token); 130 | self.api_key_source = None; 131 | } 132 | None => { 133 | self.api_key = Some(env.to_string()); 134 | self.api_key_source = Some("env".to_string()); 135 | } 136 | } 137 | 138 | self 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /proxy/src/database/testing.rs: -------------------------------------------------------------------------------- 1 | use chrono::{TimeZone, Utc}; 2 | use serde_json::json; 3 | use uuid::Uuid; 4 | 5 | use crate::{ 6 | database::logging::{ProxyLogEntry, ProxyLogEvent}, 7 | workflow_events::{ 8 | ErrorData, EventPayload, RunStartEvent, RunUpdateEvent, StepEndData, StepEventData, 9 | StepStartData, WorkflowEvent, 10 | }, 11 | }; 12 | 13 | pub const TEST_STEP1_ID: Uuid = Uuid::from_u128(1); 14 | pub const TEST_STEP2_ID: Uuid = Uuid::from_u128(2); 15 | pub const TEST_RUN_ID: Uuid = Uuid::from_u128(100); 16 | pub const TEST_EVENT1_ID: Uuid = Uuid::from_u128(5); 17 | 18 | pub fn test_events() -> Vec { 19 | vec![ 20 | ProxyLogEntry::Workflow(WorkflowEvent::RunStart(RunStartEvent { 21 | id: TEST_RUN_ID, 22 | name: "test run".to_string(), 23 | description: Some("test description".to_string()), 24 | application: Some("test application".to_string()), 25 | environment: Some("test environment".to_string()), 26 | status: None, 27 | input: Some(json!({"query":"abc"})), 28 | trace_id: Some("0123456789abcdef".to_string()), 29 | span_id: Some("12345678".to_string()), 30 | tags: vec!["tag1".to_string(), "tag2".to_string()], 31 | info: Some(json!({ 32 | "info1": "value1", 33 | "info2": "value2" 34 | })), 35 | time: Some(Utc.timestamp_opt(1, 0).unwrap()), 36 | })), 37 | ProxyLogEntry::Workflow(WorkflowEvent::StepStart(StepEventData { 38 | step_id: TEST_STEP1_ID, 39 | run_id: TEST_RUN_ID, 40 | time: Some(Utc.timestamp_opt(2, 0).unwrap()), 41 | data: StepStartData { 42 | name: Some("source_node1".to_string()), 43 | typ: "step_type".to_string(), 44 | parent_step: None, 45 | span_id: Some("11111111".to_string()), 46 | info: Some(json!({ "model": "a_model" })), 47 | tags: vec!["dag".to_string(), "node".to_string()], 48 | input: json!({ "task_param": "value" }), 49 | }, 50 | })), 51 | ProxyLogEntry::Workflow(WorkflowEvent::StepStart(StepEventData { 52 | step_id: TEST_STEP2_ID, 53 | run_id: TEST_RUN_ID, 54 | time: Some(Utc.timestamp_opt(3, 0).unwrap()), 55 | data: StepStartData { 56 | name: Some("source_node2".to_string()), 57 | typ: "llm".to_string(), 58 | parent_step: Some(TEST_STEP1_ID), 59 | span_id: Some("22222222".to_string()), 60 | info: Some(json!({ "model": "a_model" })), 61 | tags: vec![], 62 | input: json!({ "task_param2": "value" }), 63 | }, 64 | })), 65 | ProxyLogEntry::Proxied(Box::new(ProxyLogEvent { 66 | id: TEST_EVENT1_ID, 67 | event_type: std::borrow::Cow::Borrowed("query"), 68 | timestamp: Utc.timestamp_opt(4, 0).unwrap(), 69 | request: None, 70 | response: None, 71 | latency: None, 72 | total_latency: None, 73 | was_rate_limited: Some(false), 74 | num_retries: Some(0), 75 | error: None, 76 | options: crate::ProxyRequestOptions { 77 | metadata: crate::ProxyRequestMetadata { 78 | step_id: Some(TEST_STEP2_ID), 79 | run_id: Some(TEST_RUN_ID), 80 | extra: Some( 81 | json!({ 82 | "some_key": "some_value", 83 | }) 84 | .as_object() 85 | .unwrap() 86 | .clone(), 87 | ), 88 | ..Default::default() 89 | }, 90 | ..Default::default() 91 | }, 92 | })), 93 | ProxyLogEntry::Workflow(WorkflowEvent::Event(EventPayload { 94 | typ: "an_event".to_string(), 95 | data: Some(json!({ 96 | "key": "value", 97 | })), 98 | error: Some(json!({ 99 | "message": "something went wrong" 100 | })), 101 | step_id: TEST_STEP2_ID, 102 | run_id: TEST_RUN_ID, 103 | time: Some(Utc.timestamp_opt(5, 0).unwrap()), 104 | internal_metadata: None, 105 | })), 106 | ProxyLogEntry::Workflow(WorkflowEvent::StepError(StepEventData { 107 | step_id: TEST_STEP2_ID, 108 | run_id: TEST_RUN_ID, 109 | time: Some(Utc.timestamp_opt(5, 0).unwrap()), 110 | data: ErrorData { 111 | error: json!({"message": "an error"}), 112 | }, 113 | })), 114 | ProxyLogEntry::Workflow(WorkflowEvent::StepEnd(StepEventData { 115 | step_id: TEST_STEP1_ID, 116 | run_id: TEST_RUN_ID, 117 | time: Some(Utc.timestamp_opt(5, 0).unwrap()), 118 | data: StepEndData { 119 | output: json!({ "result": "success" }), 120 | info: Some(json!({ "info3": "value3" })), 121 | }, 122 | })), 123 | ProxyLogEntry::Workflow(WorkflowEvent::RunUpdate(RunUpdateEvent { 124 | id: TEST_RUN_ID, 125 | status: Some("finished".to_string()), 126 | output: Some(json!({ "result": "success" })), 127 | info: Some(json!({ "info2": "new_value", "info3": "value3"})), 128 | time: Some(Utc.timestamp_opt(5, 0).unwrap()), 129 | })), 130 | ] 131 | } 132 | -------------------------------------------------------------------------------- /api/src/error.rs: -------------------------------------------------------------------------------- 1 | use axum::response::{IntoResponse, Response}; 2 | use error_stack::Report; 3 | use filigree::errors::HttpError; 4 | use http::StatusCode; 5 | use serde_json::json; 6 | use thiserror::Error; 7 | 8 | /// The top-level error type from the platform 9 | #[derive(Debug, Error)] 10 | pub enum Error { 11 | /// Failed to initialize database 12 | #[error("Failed to initialize database")] 13 | DbInit, 14 | /// Database error not otherwise handled 15 | #[error("Database error")] 16 | Db, 17 | /// Configuration error 18 | #[error("Configuration error")] 19 | Config, 20 | /// Failed to start the HTTP server 21 | #[error("Failed to start server")] 22 | ServerStart, 23 | /// Failure while shutting down 24 | #[error("Encountered error while shutting down")] 25 | Shutdown, 26 | /// The requested item was not found 27 | #[error("{0} not found")] 28 | NotFound(&'static str), 29 | /// A wrapper around a Report to let it be returned from an Axum handler, since we can't 30 | /// implement IntoResponse on Report 31 | #[error("{0}")] 32 | WrapReport(Report), 33 | #[error("Missing Model")] 34 | MissingModel, 35 | #[error("Missing provider for model {0}")] 36 | MissingProvider(String), 37 | 38 | #[error("Invalid event payload for type {0}")] 39 | InvalidEventPayload(String, serde_json::Value), 40 | 41 | #[error("Model provider error")] 42 | Proxy, 43 | #[error("Failed to build proxy")] 44 | BuildingProxy, 45 | #[error("Failed to read proxy request options")] 46 | InvalidProxyHeader, 47 | } 48 | 49 | impl From> for Error { 50 | fn from(value: Report) -> Self { 51 | Error::WrapReport(value) 52 | } 53 | } 54 | 55 | impl Error { 56 | /// If this Error contains a Report, find an inner HttpError whose error data we may want to use. 57 | fn find_downstack_error_code(&self) -> Option { 58 | let Error::WrapReport(report) = self else { 59 | return None; 60 | }; 61 | 62 | report.frames().find_map(|frame| { 63 | filigree::downref_report_frame!( 64 | frame, 65 | |e| e.status_code(), 66 | chronicle_proxy::providers::ProviderError 67 | ) 68 | }) 69 | } 70 | 71 | /// If this Error contains a Report, find an inner HttpError whose error data we may want to use. 72 | fn find_downstack_error_kind(&self) -> Option<&'static str> { 73 | let Error::WrapReport(report) = self else { 74 | return None; 75 | }; 76 | 77 | report.frames().find_map(|frame| { 78 | filigree::downref_report_frame!( 79 | frame, 80 | |e| e.error_kind(), 81 | chronicle_proxy::providers::ProviderError 82 | ) 83 | }) 84 | } 85 | 86 | fn error_detail_body(&self) -> Option { 87 | match self { 88 | Error::InvalidEventPayload(_, body) => Some(body.clone()), 89 | Error::WrapReport(report) => report.frames().find_map(|frame| { 90 | frame 91 | .downcast_ref::() 92 | .and_then(|e| e.body.clone()) 93 | }), 94 | _ => None, 95 | } 96 | } 97 | } 98 | 99 | impl HttpError for Error { 100 | type Detail = serde_json::Value; 101 | 102 | fn error_kind(&self) -> &'static str { 103 | if let Some(error_kind) = self.find_downstack_error_kind() { 104 | return error_kind; 105 | } 106 | 107 | match self { 108 | Error::WrapReport(e) => e.current_context().error_kind(), 109 | Error::DbInit => "db_init", 110 | Error::Db => "db", 111 | Error::ServerStart => "server_start", 112 | Error::NotFound(_) => "not_found", 113 | Error::Shutdown => "shutdown", 114 | Error::MissingModel => "missing_model", 115 | Error::MissingProvider(_) => "missing_provider", 116 | Error::Proxy => "proxy", 117 | Error::BuildingProxy => "building_proxy", 118 | Error::InvalidProxyHeader => "invalid_proxy_headers", 119 | Error::Config => "config", 120 | Error::InvalidEventPayload(_, _) => "invalid_event_payload", 121 | } 122 | } 123 | 124 | fn status_code(&self) -> StatusCode { 125 | if let Some(status_code) = self.find_downstack_error_code() { 126 | return status_code; 127 | } 128 | 129 | match self { 130 | Error::WrapReport(e) => e.current_context().status_code(), 131 | Error::DbInit => StatusCode::INTERNAL_SERVER_ERROR, 132 | Error::Db => StatusCode::INTERNAL_SERVER_ERROR, 133 | Error::ServerStart => StatusCode::INTERNAL_SERVER_ERROR, 134 | Error::NotFound(_) => StatusCode::NOT_FOUND, 135 | Error::Shutdown => StatusCode::INTERNAL_SERVER_ERROR, 136 | Error::Config => StatusCode::INTERNAL_SERVER_ERROR, 137 | Error::MissingModel => StatusCode::BAD_REQUEST, 138 | Error::MissingProvider(_) => StatusCode::BAD_REQUEST, 139 | Error::InvalidProxyHeader => StatusCode::UNPROCESSABLE_ENTITY, 140 | Error::Proxy => StatusCode::INTERNAL_SERVER_ERROR, 141 | Error::BuildingProxy => StatusCode::INTERNAL_SERVER_ERROR, 142 | Error::InvalidEventPayload(_, _) => StatusCode::UNPROCESSABLE_ENTITY, 143 | } 144 | } 145 | 146 | fn error_detail(&self) -> serde_json::Value { 147 | let body = self.error_detail_body(); 148 | 149 | let error_details = match self { 150 | Error::WrapReport(e) => e.error_detail().into(), 151 | _ => serde_json::Value::Null, 152 | }; 153 | 154 | json!({ 155 | "body": body, 156 | "details": error_details, 157 | }) 158 | } 159 | } 160 | 161 | impl IntoResponse for Error { 162 | fn into_response(self) -> Response { 163 | self.to_response() 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /proxy/src/database/logging.rs: -------------------------------------------------------------------------------- 1 | //! Logging events to the database 2 | use std::{borrow::Cow, time::Duration}; 3 | 4 | use chrono::Utc; 5 | use smallvec::SmallVec; 6 | use tracing::instrument; 7 | use uuid::Uuid; 8 | 9 | use super::{Database, ProxyDatabase}; 10 | use crate::{ 11 | format::{ChatRequest, ResponseInfo, SingleChatResponse}, 12 | workflow_events::{EventPayload, WorkflowEvent}, 13 | ProxyRequestOptions, 14 | }; 15 | 16 | /// An event from the proxy. 17 | #[derive(Debug)] 18 | pub struct ProxyLogEvent { 19 | /// A unique ID for this event 20 | pub id: Uuid, 21 | /// The type of event 22 | pub event_type: Cow<'static, str>, 23 | /// The timestamp of the event 24 | pub timestamp: chrono::DateTime, 25 | /// The request that was proxied 26 | pub request: Option, 27 | /// The response from the model provider 28 | pub response: Option, 29 | /// The latency of the request that succeeded 30 | pub latency: Option, 31 | /// The total latency of the request, including retries. 32 | pub total_latency: Option, 33 | /// Whether the request was rate limited 34 | pub was_rate_limited: Option, 35 | /// The number of retries 36 | pub num_retries: Option, 37 | /// The error that occurred, if any. 38 | pub error: Option, 39 | /// The options that were used for the request 40 | pub options: ProxyRequestOptions, 41 | } 42 | 43 | impl ProxyLogEvent { 44 | /// Create a new event from a submitted payload 45 | pub fn from_payload(id: Uuid, payload: EventPayload) -> Self { 46 | let extra = match payload.data { 47 | Some(serde_json::Value::Object(m)) => Some(m), 48 | _ => None, 49 | }; 50 | 51 | ProxyLogEvent { 52 | id, 53 | event_type: Cow::Owned(payload.typ), 54 | timestamp: payload.time.unwrap_or_else(|| Utc::now()), 55 | request: None, 56 | response: None, 57 | total_latency: None, 58 | latency: None, 59 | was_rate_limited: None, 60 | num_retries: None, 61 | error: payload.error, 62 | options: ProxyRequestOptions { 63 | metadata: crate::ProxyRequestMetadata { 64 | extra, 65 | step_id: Some(payload.step_id), 66 | run_id: Some(payload.run_id), 67 | ..Default::default() 68 | }, 69 | internal_metadata: payload.internal_metadata.unwrap_or_default(), 70 | ..Default::default() 71 | }, 72 | } 73 | } 74 | } 75 | 76 | /// A response from the model provider, collected into a single body if it was streamed 77 | #[derive(Debug)] 78 | pub struct CollectedProxiedResult { 79 | /// The response itself 80 | pub body: SingleChatResponse, 81 | /// Other information about the response 82 | pub info: ResponseInfo, 83 | /// The provider which was used for the successful response. 84 | pub provider: String, 85 | } 86 | 87 | /// An event to be logged 88 | #[derive(Debug)] 89 | pub enum ProxyLogEntry { 90 | /// The result of a proxied model request 91 | Proxied(Box), 92 | /// An update from a workflow step or run 93 | Workflow(WorkflowEvent), 94 | } 95 | 96 | /// A channel on which log events can be sent. 97 | pub type LogSender = flume::Sender>; 98 | 99 | /// Start the database logger task 100 | pub fn start_database_logger( 101 | db: Database, 102 | batch_size: usize, 103 | debounce_time: Duration, 104 | ) -> (LogSender, tokio::task::JoinHandle<()>) { 105 | let (log_tx, log_rx) = flume::unbounded(); 106 | 107 | let task = tokio::task::spawn(database_logger_task(db, log_rx, batch_size, debounce_time)); 108 | 109 | (log_tx, task) 110 | } 111 | 112 | async fn database_logger_task( 113 | db: Database, 114 | rx: flume::Receiver>, 115 | batch_size: usize, 116 | debounce_time: Duration, 117 | ) { 118 | let mut batch = Vec::with_capacity(batch_size); 119 | 120 | loop { 121 | tokio::select! { 122 | item = rx.recv_async() => { 123 | let Ok(item) = item else { 124 | // channel closed so we're done 125 | break; 126 | }; 127 | 128 | tracing::debug!(num_items=item.len(), "Received items"); 129 | batch.extend(item); 130 | 131 | if batch.len() >= batch_size { 132 | let send_batch = std::mem::replace(&mut batch, Vec::with_capacity(batch_size)); 133 | write_batch(db.as_ref(), send_batch).await; 134 | } 135 | 136 | } 137 | _ = tokio::time::sleep(debounce_time), if !batch.is_empty() => { 138 | let send_batch = std::mem::replace(&mut batch, Vec::with_capacity(batch_size)); 139 | write_batch(db.as_ref(), send_batch).await; 140 | } 141 | } 142 | } 143 | tracing::debug!("Closing database logger"); 144 | 145 | if !batch.is_empty() { 146 | write_batch(db.as_ref(), batch).await; 147 | } 148 | } 149 | 150 | pub(super) const EVENT_INSERT_PREFIX: &str = 151 | "INSERT INTO chronicle_events 152 | (id, event_type, organization_id, project_id, user_id, chat_request, chat_response, 153 | error, provider, model, application, environment, request_organization_id, request_project_id, 154 | request_user_id, workflow_id, workflow_name, run_id, step_id, step_index, 155 | prompt_id, prompt_version, 156 | meta, response_meta, retries, rate_limited, request_latency_ms, 157 | total_latency_ms, created_at) VALUES\n"; 158 | 159 | #[instrument(level = "trace", parent=None, skip(db, items), fields(chronicle.db_batch.num_items = items.len()))] 160 | async fn write_batch(db: &dyn ProxyDatabase, items: Vec) { 161 | let result = db.write_log_batch(items).await; 162 | 163 | if let Err(e) = result { 164 | tracing::error!(error = ?e, "Failed to write logs to database"); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /js-client/src/events.ts: -------------------------------------------------------------------------------- 1 | import { SpanContext } from '@opentelemetry/api'; 2 | 3 | /** Represents a UUID as a string */ 4 | export type Uuid = string; 5 | 6 | /** Starts a run in a workflow */ 7 | export interface RunStartEvent { 8 | /** The type of the event */ 9 | type: 'run:start'; 10 | /** The unique identifier for the run. UUIDv7 recommended */ 11 | id: Uuid; 12 | /** The name of the run */ 13 | name: string; 14 | /** Optional description of the run */ 15 | description?: string; 16 | /** Optional application associated with the run */ 17 | application?: string; 18 | /** Optional environment in which the run is executed */ 19 | environment?: string; 20 | /** Customize the initial status of the run. The default value is 'started'. */ 21 | status?: string; 22 | /** Optional input data for the run */ 23 | input?: unknown; 24 | /** OpenTelemetry trace ID for distributed tracing */ 25 | trace_id?: string; 26 | /** OpenTelemetry span ID for distributed tracing */ 27 | span_id?: string; 28 | /** Array of tags associated with the run */ 29 | tags?: string[]; 30 | /** Optional additional information about the run */ 31 | info?: object; 32 | /** Optional timestamp of when the event occurred */ 33 | time?: Date; 34 | } 35 | 36 | /** Updates a run in a workflow */ 37 | export interface RunUpdateEvent { 38 | /** The type of the event */ 39 | type: 'run:update'; 40 | /** The unique identifier for the run */ 41 | id: Uuid; 42 | /** Optional new status of the run */ 43 | status?: string; 44 | /** Optional output data from the run */ 45 | output?: unknown; 46 | /** Optional additional information about the run */ 47 | info?: object; 48 | /** Optional timestamp of when the event occurred */ 49 | time?: Date; 50 | } 51 | 52 | /** Step event in a workflow */ 53 | export interface StepEventData { 54 | /** The type of the step event */ 55 | type: TYPE; 56 | /** The unique identifier for this step */ 57 | step_id?: Uuid; 58 | /** The unique identifier for the run containing this step */ 59 | run_id?: Uuid; 60 | /** Optional timestamp of when the event occurred */ 61 | time?: Date; 62 | /** The data associated with this step event */ 63 | data: T; 64 | } 65 | 66 | /** Data for the start of a step */ 67 | export interface StepStartData { 68 | /** The type of the step */ 69 | type: string; 70 | /** Optional name of the step */ 71 | name?: string; 72 | /** Optional unique identifier of the parent step */ 73 | parent_step?: Uuid; 74 | /** Optional span ID for distributed tracing */ 75 | span_id?: string; 76 | /** Array of tags associated with the step */ 77 | tags?: string[]; 78 | /** Optional additional information about the step */ 79 | info?: object; 80 | /** Input data for the step */ 81 | input: unknown; 82 | } 83 | 84 | /** Data for the end of a step */ 85 | export interface StepEndData { 86 | /** Output data from the step */ 87 | output: unknown; 88 | /** Optional additional information about the step completion. This will be merged with the info from the step start event */ 89 | info?: object; 90 | } 91 | 92 | /** Data for an error in a step */ 93 | export interface ErrorData { 94 | /** Error information */ 95 | error: object; 96 | } 97 | 98 | /** Data for updating the state of a step */ 99 | export interface StepStateData { 100 | /** The current state of the step */ 101 | state: string; 102 | } 103 | 104 | /** Represents a step start event */ 105 | export type StepStartEvent = StepEventData<'step:start', StepStartData>; 106 | 107 | /** Represents a step end event */ 108 | export type StepEndEvent = StepEventData<'step:end', StepEndData>; 109 | 110 | /** Represents a step error event */ 111 | export type StepErrorEvent = StepEventData<'step:error', ErrorData>; 112 | 113 | /** Represents a step state change event */ 114 | export type StepStateEvent = StepEventData<'step:state', StepStateData>; 115 | 116 | export type WorkflowEventTypes = 117 | | 'run:start' 118 | | 'run:update' 119 | | 'step:start' 120 | | 'step:end' 121 | | 'step:error' 122 | | 'step:state'; 123 | 124 | /** Represents a generic event in the system */ 125 | export interface GenericEvent< 126 | TYPE extends Omit = Omit, 127 | DATA = object | undefined, 128 | > { 129 | /** The type of the event */ 130 | type: TYPE; 131 | /** Data associated with the event */ 132 | data: DATA; 133 | /** Optional error information */ 134 | error?: object; 135 | /** The ID for the run associated with this event. If not supplied, this will 136 | * be filled in from the context. */ 137 | run_id?: Uuid; 138 | /** The ID for the step associated with this event. If not supplied, this wil 139 | * be filled in from the context. */ 140 | step_id?: Uuid; 141 | /** Timestamp of when the event occurred. If not supplied, `new Date()` will be used. */ 142 | time?: Date; 143 | } 144 | 145 | export type ChronicleWorkflowEvent = 146 | | RunStartEvent 147 | | RunUpdateEvent 148 | | StepStartEvent 149 | | StepEndEvent 150 | | StepErrorEvent 151 | | StepStateEvent; 152 | 153 | /** Represents any type of event that can be submitted to Chronicle */ 154 | export type ChronicleEvent = ChronicleWorkflowEvent | GenericEvent; 155 | 156 | export function isWorkflowEvent(event: ChronicleEvent): event is ChronicleWorkflowEvent { 157 | // If the event type is not any of the known types, it's generic. 158 | return ( 159 | event.type === 'run:start' || 160 | event.type === 'run:update' || 161 | event.type === 'step:start' || 162 | event.type === 'step:end' || 163 | event.type === 'step:error' || 164 | event.type === 'step:state' 165 | ); 166 | } 167 | 168 | const NIL_UUID = '00000000-0000-0000-0000-000000000000'; 169 | 170 | /** Fill in information from events that may have been omitted. */ 171 | export function fillInEvents( 172 | events: ChronicleEvent[], 173 | runId: Uuid | undefined, 174 | stepId: Uuid | undefined, 175 | spanCtx: SpanContext | undefined, 176 | now: Date 177 | ) { 178 | for (let event of events) { 179 | if (!event.time) { 180 | event.time = now; 181 | } 182 | 183 | if (isWorkflowEvent(event)) { 184 | switch (event.type) { 185 | case 'step:start': 186 | event.run_id ??= runId ?? NIL_UUID; 187 | event.step_id ??= stepId ?? NIL_UUID; 188 | event.data.span_id ??= spanCtx?.spanId; 189 | break; 190 | case 'step:end': 191 | case 'step:error': 192 | case 'step:state': 193 | event.run_id ??= runId ?? NIL_UUID; 194 | event.step_id ??= stepId ?? NIL_UUID; 195 | break; 196 | case 'run:update': 197 | event.id ??= runId ?? NIL_UUID; 198 | break; 199 | case 'run:start': 200 | event.span_id ??= spanCtx?.spanId; 201 | event.trace_id ??= spanCtx?.traceId; 202 | } 203 | } else { 204 | event.run_id ??= runId ?? NIL_UUID; 205 | event.step_id ??= stepId ?? NIL_UUID; 206 | } 207 | } 208 | } 209 | 210 | let loggingEnabled = true; 211 | 212 | /** Return if logging is enabled, application-wide. */ 213 | export function getLoggingEnabled() { 214 | return loggingEnabled; 215 | } 216 | 217 | /** Enable or disable run, step, and event logging globally. */ 218 | export function setLoggingEnabled(enabled: boolean) { 219 | loggingEnabled = enabled; 220 | } 221 | -------------------------------------------------------------------------------- /js-client/src/types.ts: -------------------------------------------------------------------------------- 1 | import type * as openai from 'openai'; 2 | 3 | /** A chat request. This is the same as the arguments to OpenAI's chat.completions.create function. */ 4 | export type ChronicleChatRequestStreaming = 5 | openai.OpenAI.Chat.ChatCompletionCreateParamsStreaming & { max_tokens: number }; 6 | export type ChronicleChatRequestNonStreaming = 7 | openai.OpenAI.Chat.ChatCompletionCreateParamsNonStreaming & { max_tokens: number }; 8 | 9 | export type ChronicleChatRequest = ChronicleChatRequestStreaming | ChronicleChatRequestNonStreaming; 10 | 11 | export interface ChronicleModelAndProvider { 12 | /** The model to use */ 13 | model: string; 14 | /** The name of the provider to use. */ 15 | provider: string; 16 | /** An API key to pass to the provider. */ 17 | api_key?: string; 18 | /** A reference to an API key object that was configured in the proxy. */ 19 | api_key_name?: string; 20 | } 21 | 22 | export type ChronicleRepeatBackoffBehavior = 23 | | { 24 | /** Use the initial backoff duration for additional retries as well. */ 25 | type: 'constant'; 26 | } 27 | | { 28 | /** Increase backoff time exponentially. */ 29 | type: 'exponential'; 30 | /** Multiply the previous backoff time by this number on every retry. */ 31 | multiplier: number; 32 | } 33 | | { 34 | /** Add a fixed amount to the backoff time, in milliseconds, on every retry. */ 35 | type: 'additive'; 36 | /** The number of milliseconds to add. */ 37 | amount: number; 38 | }; 39 | 40 | export interface ChronicleRetryOptions { 41 | /** The amount of time to wait on the first backoff, in milliseconds. Defaults to 500ms */ 42 | initial_backoff?: number; 43 | /** How to increase the backoff time on multiple retries. The default is to multiply the previous backoff time by 2. */ 44 | increase?: ChronicleRepeatBackoffBehavior; 45 | /** The number of times to try the request, including the first try. Defaults to 4. */ 46 | max_tries?: number; 47 | /** The maximum amount of jitter to add to the backoff time, in milliseconds. Defaults to 100ms. */ 48 | jitter?: number; 49 | /** The maximum amount of time to wait between tries, in milliseconds. Defaults to 5000ms. */ 50 | max_backoff?: number; 51 | /** If a rate limit response asks us to wait longer than `max_backoff`, just fail instead of waiting 52 | * if we don't have any other models to fall back to. 53 | * Defaults to true. */ 54 | fail_if_rate_limit_exceeds_max_backoff?: boolean; 55 | } 56 | 57 | export interface ChronicleRequestMetadata { 58 | /** application making this call. This can also be set by passing the 59 | chronicle-application HTTP header. */ 60 | application?: string; 61 | /** The environment the application is running in. This can also be set by passing the 62 | x-chronicle-environment HTTP header. */ 63 | environment?: string; 64 | /** The organization related to the request. This can also be set by passing the 65 | x-chronicle-organization-id HTTP header. */ 66 | organization_id?: string; 67 | /** The project related to the request. This can also be set by passing the 68 | x-chronicle-project-id HTTP header. */ 69 | project_id?: string; 70 | /** The id of the user that triggered the request. This can also be set by passing the 71 | x-chronicle-user-id HTTP header. */ 72 | user_id?: string; 73 | /** The id of the workflow that this request belongs to. This can also be set by passing the 74 | x-chronicle-workflow-id HTTP header. */ 75 | workflow_id?: string; 76 | /** A readable name of the workflow that this request belongs to. This can also be set by 77 | passing the x-chronicle-workflow-name HTTP header. */ 78 | workflow_name?: string; 79 | /** The id of of the specific run that this request belongs to. This can also be set by 80 | passing the x-chronicle-run-id HTTP header. */ 81 | run_id?: string; 82 | /** The name of the workflow step. This can also be set by passing the 83 | x-chronicle-step-id HTTP header. */ 84 | step_id?: string; 85 | /** The index of the step within the workflow. This can also be set by passing the 86 | x-chronicle-step-index HTTP header. */ 87 | step_index?: number; 88 | /** A unique ID for this prompt. This can also be set by passing the 89 | x-chronicle-prompt-id HTTP header. */ 90 | prompt_id?: string; 91 | /** The version of this prompt. This can also be set by passing the 92 | x-chronicle-prompt-version HTTP header. */ 93 | prompt_version?: number; 94 | 95 | /** Any other metadata to include. When passing this in the request body, any unknown fields 96 | are collected here. This can also be set by passing a JSON object to the 97 | x-chronicle-extra-meta HTTP header. */ 98 | extra?: Record; 99 | } 100 | 101 | export interface ChronicleRequestOptions { 102 | /** Override the model from the request body or select an alias. 103 | This can also be set by passing the x-chronicle-model HTTP header. */ 104 | model?: string; 105 | /** Choose a specific provider to use. This can also be set by passing the 106 | x-chronicle-provider HTTP header. */ 107 | provider?: string; 108 | /** Force the provider to use a specific URL instead of its default. This can also be set 109 | by passing the x-chronicle-override-url HTTP header. */ 110 | override_url?: string; 111 | /** An API key to pass to the provider. This can also be set by passing the 112 | x-chronicle-provider-api-key HTTP header. */ 113 | api_key?: string; 114 | /** Supply multiple provider/model choices, which will be tried in order. 115 | If this is provided, the `model`, `provider`, and `api_key` fields are ignored in favor of those given here. 116 | This field can not reference model aliases. 117 | This can also be set by passing the x-chronicle-models HTTP header using JSON syntax. */ 118 | models?: Array; 119 | /** When using `models` to supply multiple choices, start at a random choice instead of the 120 | first one. This can also be set by passing the x-chronicle-random-choice HTTP header. */ 121 | random_choice?: boolean; 122 | /** The timeout, in milliseconds. 123 | This can also be set by passing the x-chronicle-timeout HTTP header. */ 124 | timeout?: number; 125 | /** Customize the retry behavior. This can also be set by passing the 126 | x-chronicle-retry HTTP header. */ 127 | retry?: ChronicleRetryOptions; 128 | 129 | metadata?: ChronicleRequestMetadata; 130 | 131 | /** An AbortSignal for this request */ 132 | signal?: AbortSignal; 133 | } 134 | 135 | export interface ChronicleResponseMeta { 136 | /** A UUID assigned by Chronicle to the request, which is linked to the logged information. 137 | This is different from the `id` returned at the top level of the `ChronicleChatResponse`, which 138 | comes from the provider itself. */ 139 | id: string; 140 | /** Which provider was useed for the request. */ 141 | provider: string; 142 | /** Any provider-specific metadata returned from the provider that doesn't fit in with 143 | * the regular fields. */ 144 | response_meta?: object; 145 | /** True if this request had to retry or fallback from the default model due to rate limiting. */ 146 | was_rate_limited: boolean; 147 | } 148 | 149 | export interface ChronicleChatResponseNonStreaming extends openai.OpenAI.Chat.ChatCompletion { 150 | meta: ChronicleResponseMeta; 151 | } 152 | 153 | export interface ChronicleChatResponseStreaming extends openai.OpenAI.Chat.ChatCompletionChunk { 154 | meta?: ChronicleResponseMeta; 155 | } 156 | 157 | export type ChronicleChatResponseStream = AsyncIterable; 158 | 159 | export type ChronicleChatResponse = STREAMING extends true 160 | ? ChronicleChatResponseStream 161 | : ChronicleChatResponseNonStreaming; 162 | -------------------------------------------------------------------------------- /js-client/src/client.ts: -------------------------------------------------------------------------------- 1 | import { Attributes, trace } from '@opentelemetry/api'; 2 | import { proxyUrl, propagateSpan, handleError } from './internal.js'; 3 | import { Stream } from './streaming.js'; 4 | import type { 5 | ChronicleChatRequest, 6 | ChronicleChatRequestNonStreaming, 7 | ChronicleChatRequestStreaming, 8 | ChronicleChatResponseStream, 9 | ChronicleRequestOptions, 10 | ChronicleChatResponseNonStreaming, 11 | ChronicleChatResponseStreaming, 12 | ChronicleRequestMetadata, 13 | } from './types.js'; 14 | import { ChronicleEvent, fillInEvents, isWorkflowEvent, getLoggingEnabled } from './events.js'; 15 | import { getEventContext } from './runs.js'; 16 | import { getLogger } from './logger.js'; 17 | import EventEmitter from 'node:events'; 18 | 19 | export interface ChronicleClientOptions { 20 | /** Replace the normal fetch function with this one */ 21 | fetch?: typeof fetch; 22 | /** Set the url of the proxy. If omitted, the client will try to use the `CHRONICLE_PROXY_URL` environment variable, 23 | * or default to http://localhost:9782. */ 24 | url?: string; 25 | /** If the Chronicle proxy is behind a system that requires authentication, a bearer token to use. */ 26 | token?: string; 27 | 28 | /** Set default options for requests made by this client. */ 29 | defaults?: Omit, 'signal'>; 30 | } 31 | 32 | export type NonStreamingClientFn = ( 33 | chat: ChronicleChatRequestNonStreaming & Partial, 34 | options?: ChronicleRequestOptions 35 | ) => Promise; 36 | export type StreamingClientFn = ( 37 | chat: ChronicleChatRequestStreaming & Partial, 38 | options?: ChronicleRequestOptions 39 | ) => Promise; 40 | 41 | export type ChronicleEventFn = (event: ChronicleEvent | ChronicleEvent[]) => Promise; 42 | export type ChronicleClient = NonStreamingClientFn & 43 | StreamingClientFn & { 44 | /** Send an event to Chronicle. All events will also be re-emitted from the client using the EventEmitter interface. 45 | * where the event name is 'event' and the data is the event passed to this function. */ 46 | event: ChronicleEventFn; 47 | /** Create a child client which shares the same settings and EventEmitter, but with new metadata merged 48 | * over the existing metadata values. */ 49 | withMetadata: (newDefaults: ChronicleRequestMetadata) => ChronicleClient; 50 | /** The request options used by this client. */ 51 | requestOptions: Partial; 52 | } & EventEmitter<{ event: [ChronicleEvent] }>; 53 | 54 | /** Create a Chronicle proxy client. This returns a function which will call the Chronicle proxy */ 55 | export function createChronicleClient(options?: ChronicleClientOptions): ChronicleClient { 56 | let { fetch = globalThis.fetch, token, defaults = {} } = options ?? {}; 57 | let url = proxyUrl(options?.url); 58 | let eventUrl = new URL('/events', url); 59 | 60 | let emitter = new EventEmitter<{ event: [ChronicleEvent] }>(); 61 | 62 | const client = async ( 63 | chat: ChronicleChatRequest & Partial, 64 | options?: ChronicleRequestOptions 65 | ) => { 66 | let { signal, ...reqOptions } = options ?? {}; 67 | 68 | let body = { 69 | ...client.requestOptions, 70 | ...chat, 71 | ...reqOptions, 72 | metadata: { 73 | ...client.requestOptions?.metadata, 74 | ...chat.metadata, 75 | ...reqOptions.metadata, 76 | }, 77 | }; 78 | 79 | if (!body.metadata.run_id || !body.metadata.step_id) { 80 | const context = getEventContext(); 81 | body.metadata.run_id ??= context?.runId; 82 | body.metadata.step_id ??= context?.stepId ?? undefined; 83 | } 84 | 85 | let req = new Request(url, { 86 | method: 'POST', 87 | headers: { 88 | 'content-type': 'application/json; charset=utf-8', 89 | accept: body.stream 90 | ? 'text/event-stream; charset=utf-8' 91 | : 'application/json; charset=utf-8', 92 | }, 93 | body: JSON.stringify(body), 94 | signal, 95 | }); 96 | 97 | if (token) { 98 | req.headers.set('Authorization', `Bearer ${token}`); 99 | } 100 | 101 | propagateSpan(req); 102 | 103 | let res = await fetch(req); 104 | if (res.ok) { 105 | if (chat.stream) { 106 | return Stream.fromSSEResponse(res, options?.signal); 107 | } else { 108 | return (await res.json()) as ChronicleChatResponseNonStreaming; 109 | } 110 | } else { 111 | throw new Error(await handleError(res)); 112 | } 113 | }; 114 | 115 | client.requestOptions = defaults; 116 | 117 | // Wire through the event emitter 118 | client.on = emitter.on.bind(emitter); 119 | client.once = emitter.once.bind(emitter); 120 | client.emit = emitter.emit.bind(emitter); 121 | 122 | client.event = (event: ChronicleEvent | ChronicleEvent[]) => { 123 | return sendEvent(eventUrl, event, emitter); 124 | }; 125 | 126 | function updateWithMetadata( 127 | client: ChronicleClient, 128 | newMetadata: ChronicleRequestMetadata 129 | ): ChronicleClient { 130 | return { 131 | ...client, 132 | requestOptions: { 133 | ...client.requestOptions, 134 | metadata: { 135 | ...client.requestOptions.metadata, 136 | ...newMetadata, 137 | }, 138 | }, 139 | } as ChronicleClient; 140 | } 141 | 142 | client.withMetadata = (defaults: ChronicleRequestMetadata): ChronicleClient => { 143 | let newClient = updateWithMetadata(client as ChronicleClient, defaults); 144 | newClient.withMetadata = (defaults: ChronicleRequestMetadata): ChronicleClient => { 145 | return updateWithMetadata(newClient, defaults); 146 | }; 147 | 148 | return newClient; 149 | }; 150 | 151 | // @ts-expect-error 152 | return client; 153 | } 154 | 155 | function sendEvent( 156 | url: string | URL | undefined, 157 | body: ChronicleEvent | ChronicleEvent[], 158 | emitter: EventEmitter<{ event: [ChronicleEvent] }> 159 | ) { 160 | const payload = Array.isArray(body) ? body : [body]; 161 | 162 | const span = trace.getActiveSpan(); 163 | if (span?.isRecording() && !Array.isArray(body) && !isWorkflowEvent(body)) { 164 | let eventAttributes: Attributes = {}; 165 | 166 | if (body.data) { 167 | for (let k in body.data) { 168 | let value = body.data[k as keyof typeof body.data]; 169 | 170 | let attrKey = `llm.meta.${k}`; 171 | if (value && typeof value === 'object' && !Array.isArray(value)) { 172 | eventAttributes[attrKey] = JSON.stringify(value); 173 | } else { 174 | eventAttributes[attrKey] = value; 175 | } 176 | } 177 | } 178 | 179 | if (body.error) { 180 | eventAttributes['error'] = 181 | typeof body.error === 'object' ? JSON.stringify(body.error) : body.error; 182 | } 183 | 184 | span.addEvent(body.type as string, eventAttributes); 185 | } 186 | 187 | let eventContext = getEventContext(); 188 | let spanCtx = span?.isRecording() ? span.spanContext() : undefined; 189 | fillInEvents(payload, eventContext?.runId, eventContext?.stepId, spanCtx, new Date()); 190 | 191 | for (let event of payload) { 192 | emitter.emit('event', event); 193 | } 194 | 195 | if (!getLoggingEnabled()) { 196 | return; 197 | } 198 | 199 | let logger = getLogger(proxyUrl(url, '/events')); 200 | logger.enqueue(payload); 201 | } 202 | 203 | let defaultClient: ChronicleClient | undefined; 204 | 205 | /** Initialize the default client with custom options. */ 206 | export function createDefaultClient(options: ChronicleClientOptions) { 207 | defaultClient = createChronicleClient(options); 208 | return defaultClient; 209 | } 210 | 211 | /** Return the default client, or create one if it doesn't exist. This is primarily 212 | * used by the run auto-instrumentation functions. */ 213 | export function getDefaultClient() { 214 | if (!defaultClient) { 215 | defaultClient = createChronicleClient(); 216 | } 217 | return defaultClient; 218 | } 219 | -------------------------------------------------------------------------------- /api/src/proxy.rs: -------------------------------------------------------------------------------- 1 | use std::{future::ready, sync::Arc}; 2 | 3 | use axum::{ 4 | extract::State, 5 | http::HeaderMap, 6 | response::{sse, IntoResponse, Response, Sse}, 7 | Json, 8 | }; 9 | use chronicle_proxy::{ 10 | collect_response, 11 | database::Database, 12 | format::{ 13 | ChatRequest, RequestInfo, SingleChatResponse, StreamingChatResponse, StreamingResponse, 14 | }, 15 | Proxy, ProxyRequestOptions, 16 | }; 17 | use error_stack::{Report, ResultExt}; 18 | use futures::StreamExt; 19 | use serde::{Deserialize, Serialize}; 20 | 21 | use crate::{config::Configs, Error}; 22 | 23 | pub async fn build_proxy(db: Option, configs: Configs) -> Result> { 24 | let mut builder = Proxy::builder(); 25 | 26 | if let Some(db) = db { 27 | builder = builder 28 | .with_database(db) 29 | .log_to_database(true) 30 | .load_config_from_database(true); 31 | } 32 | 33 | for (_, config) in configs.global.into_iter().chain(configs.cwd.into_iter()) { 34 | builder = builder.with_config(config.proxy_config); 35 | } 36 | 37 | builder.build().await.change_context(Error::BuildingProxy) 38 | } 39 | 40 | pub struct ServerState { 41 | pub proxy: Proxy, 42 | } 43 | 44 | #[derive(Deserialize, Debug)] 45 | struct ProxyRequestPayload { 46 | #[serde(flatten)] 47 | request: ChatRequest, 48 | 49 | #[serde(flatten)] 50 | options: ProxyRequestOptions, 51 | } 52 | 53 | #[derive(Debug, Serialize)] 54 | struct ProxyRequestNonstreamingResult { 55 | #[serde(flatten)] 56 | response: SingleChatResponse, 57 | meta: RequestInfo, 58 | } 59 | 60 | #[derive(Serialize)] 61 | struct DeltaWithRequestInfo { 62 | #[serde(flatten)] 63 | data: StreamingChatResponse, 64 | meta: RequestInfo, 65 | } 66 | 67 | #[derive(Serialize)] 68 | struct OpenAiSseError { 69 | error: Option, 70 | message: String, 71 | } 72 | 73 | async fn proxy_request( 74 | State(state): State>, 75 | headers: HeaderMap, 76 | Json(mut body): Json, 77 | ) -> Result { 78 | body.options 79 | .merge_request_headers(&headers) 80 | .change_context(Error::InvalidProxyHeader)?; 81 | 82 | let n = body.request.n.unwrap_or(1) as usize; 83 | let stream = body.request.stream; 84 | let result = state 85 | .proxy 86 | .send(body.options, body.request) 87 | .await 88 | .change_context(Error::Proxy)?; 89 | 90 | if stream { 91 | // The first item will always be a RequestInfo or an error. We pull it off here so that if the 92 | // model provider returned an error we can catch it in advance and return a proper error. 93 | let request_info = result 94 | .recv_async() 95 | .await 96 | .change_context(Error::Proxy) 97 | .attach_printable("Connection terminated unexpectedly")? 98 | .change_context(Error::Proxy)?; 99 | 100 | let request_info = match request_info { 101 | StreamingResponse::RequestInfo(info) => Some(info), 102 | _ => { 103 | tracing::error!("First stream item was not a RequestInfo"); 104 | None 105 | } 106 | }; 107 | 108 | let stream = result 109 | .into_stream() 110 | .scan(request_info, |request_info, chunk| { 111 | let result = match chunk { 112 | Ok(StreamingResponse::Chunk(chunk)) => { 113 | if let Some(info) = request_info.take() { 114 | // Attach RequestInfo to the chunk if we have it 115 | let chunk = DeltaWithRequestInfo { 116 | data: chunk, 117 | meta: info, 118 | }; 119 | Some(sse::Event::default().json_data(chunk)) 120 | } else { 121 | Some(sse::Event::default().json_data(chunk)) 122 | } 123 | } 124 | Ok(StreamingResponse::Single(chunk)) => { 125 | // Attach RequestInfo to the chunk if we have it 126 | let chunk = StreamingChatResponse::from(chunk); 127 | if let Some(info) = request_info.take() { 128 | let chunk = DeltaWithRequestInfo { 129 | data: chunk, 130 | meta: info, 131 | }; 132 | Some(sse::Event::default().json_data(chunk)) 133 | } else { 134 | Some(sse::Event::default().json_data(chunk)) 135 | } 136 | } 137 | Ok(StreamingResponse::RequestInfo(_)) => { 138 | // This should never happen since we already received it above. 139 | debug_assert!(false, "got multiple RequestInfo"); 140 | None 141 | } 142 | Ok(StreamingResponse::ResponseInfo(_)) => { 143 | // Need to figure out if there's some way we can send this along with the 144 | // deltas as metadata, but it's difficult since the OpenAI format uses 145 | // data-only SSE so we can't just define a new event type or something. 146 | // Might work to send an extra chunk with an empty choices but need to see if 147 | // that messes things up. Not a big deal though since the ResponseInfo 148 | // doesn't contain much important, and it gets logged anyway. 149 | None 150 | } 151 | Err(e) => { 152 | let err = e.current_context(); 153 | let err_payload = if let Some(body) = &err.body { 154 | // See if the error body looks like the OpenAI format, and if so just use it. 155 | let message = &body["message"]; 156 | let error = &body["error"]; 157 | 158 | if let serde_json::Value::String(message) = message { 159 | OpenAiSseError { 160 | error: Some(error.clone()), 161 | message: message.clone(), 162 | } 163 | } else { 164 | OpenAiSseError { 165 | error: Some(body.clone()), 166 | message: err.to_string(), 167 | } 168 | } 169 | } else { 170 | OpenAiSseError { 171 | error: None, 172 | message: err.to_string(), 173 | } 174 | }; 175 | 176 | Some(sse::Event::default().event("error").json_data(err_payload)) 177 | } 178 | }; 179 | 180 | // We're really just using `scan` to attach `request_info` as a persistent piece of 181 | // state. We don't actually want to end the stream, so wrap the value in Some so 182 | // `scan` won't end things. The `filter_map` in the next stage will filter out the 183 | // None values that come from the match statement. 184 | ready(Some(result)) 185 | }) 186 | .filter_map(|x| ready(x)) 187 | .chain(futures::stream::once(ready(Ok( 188 | // Mimic OpenAI's [DONE] message 189 | sse::Event::default().data("[DONE]"), 190 | )))); 191 | 192 | Ok(Sse::new(stream).into_response()) 193 | } else { 194 | let result = collect_response(result, n) 195 | .await 196 | .change_context(Error::Proxy)?; 197 | Ok(Json(ProxyRequestNonstreamingResult { 198 | response: result.response, 199 | meta: result.request_info, 200 | }) 201 | .into_response()) 202 | } 203 | } 204 | 205 | pub fn create_routes() -> axum::Router> { 206 | axum::Router::new() 207 | .route("/chat", axum::routing::post(proxy_request)) 208 | // We don't use the wildcard path, but allow calling with any path for compatibility with clients 209 | // that always append an API path to a base url. 210 | .route("/chat/*path", axum::routing::post(proxy_request)) 211 | .route("/v1/chat/*path", axum::routing::post(proxy_request)) 212 | } 213 | -------------------------------------------------------------------------------- /proxy/src/response.rs: -------------------------------------------------------------------------------- 1 | use error_stack::{Report, ResultExt}; 2 | use serde::Serialize; 3 | use serde_json::json; 4 | use smallvec::smallvec; 5 | use tracing::Span; 6 | 7 | use crate::{ 8 | database::logging::{CollectedProxiedResult, LogSender, ProxyLogEntry, ProxyLogEvent}, 9 | format::{ 10 | RequestInfo, ResponseInfo, SingleChatResponse, StreamingResponse, 11 | StreamingResponseReceiver, StreamingResponseSender, 12 | }, 13 | request::TryModelChoicesResult, 14 | Error, 15 | }; 16 | 17 | pub async fn handle_response( 18 | current_span: Span, 19 | log_entry: ProxyLogEvent, 20 | global_start: tokio::time::Instant, 21 | request_n: usize, 22 | meta: TryModelChoicesResult, 23 | chunk_rx: StreamingResponseReceiver, 24 | output_tx: StreamingResponseSender, 25 | log_tx: Option, 26 | ) { 27 | let response = collect_stream( 28 | current_span.clone(), 29 | log_entry, 30 | global_start, 31 | request_n, 32 | &meta, 33 | chunk_rx, 34 | output_tx, 35 | log_tx.as_ref(), 36 | ) 37 | .await; 38 | let Ok((response, info, mut log_entry)) = response else { 39 | // Errors were already handled by collect_stream. 40 | return; 41 | }; 42 | let global_send_time = global_start.elapsed(); 43 | let this_send_time = meta.start_time.elapsed(); 44 | log_entry.latency = Some(this_send_time); 45 | 46 | // In case of retries, this might be meaningfully different from the main latency. 47 | current_span.record("llm.total_latency", global_send_time.as_millis()); 48 | 49 | current_span.record( 50 | "llm.completions", 51 | response 52 | .choices 53 | .iter() 54 | .filter_map(|c| c.message.content.as_deref()) 55 | .collect::>() 56 | .join("\n\n"), 57 | ); 58 | current_span.record( 59 | "llm.completions.raw", 60 | serde_json::to_string(&response.choices).ok(), 61 | ); 62 | current_span.record("llm.vendor", &meta.provider); 63 | current_span.record("llm.response.model", &response.model); 64 | current_span.record("llm.latency", this_send_time.as_millis()); 65 | current_span.record("llm.retries", meta.num_retries); 66 | current_span.record("llm.rate_limited", meta.was_rate_limited); 67 | 68 | let usage = response.usage.clone().unwrap_or_default(); 69 | 70 | current_span.record("llm.usage.prompt_tokens", usage.prompt_tokens); 71 | current_span.record( 72 | "llm.finish_reason", 73 | response.choices.get(0).map(|c| c.finish_reason.as_str()), 74 | ); 75 | current_span.record("llm.usage.completion_tokens", usage.completion_tokens); 76 | let total_tokens = usage 77 | .total_tokens 78 | .unwrap_or_else(|| usage.prompt_tokens.unwrap_or(0) + usage.completion_tokens.unwrap_or(0)); 79 | current_span.record("llm.usage.total_tokens", total_tokens); 80 | 81 | if let Some(log_tx) = log_tx { 82 | log_entry.total_latency = Some(global_send_time); 83 | log_entry.num_retries = Some(meta.num_retries); 84 | log_entry.was_rate_limited = Some(meta.was_rate_limited); 85 | log_entry.response = Some(CollectedProxiedResult { 86 | body: response, 87 | info, 88 | provider: meta.provider, 89 | }); 90 | 91 | log_tx 92 | .send_async(smallvec![ProxyLogEntry::Proxied(Box::new(log_entry))]) 93 | .await 94 | .ok(); 95 | } 96 | } 97 | 98 | /// Internal stream collection that saves the information for logging. 99 | async fn collect_stream( 100 | current_span: Span, 101 | log_entry: ProxyLogEvent, 102 | global_start: tokio::time::Instant, 103 | request_n: usize, 104 | meta: &TryModelChoicesResult, 105 | chunk_rx: StreamingResponseReceiver, 106 | output_tx: StreamingResponseSender, 107 | log_tx: Option<&LogSender>, 108 | ) -> Result<(SingleChatResponse, ResponseInfo, ProxyLogEvent), ()> { 109 | let mut response = SingleChatResponse::new_for_collection(request_n); 110 | 111 | let mut res_stats = ResponseInfo { 112 | model: String::new(), 113 | meta: None, 114 | }; 115 | 116 | // Collect the message chunks so we can log the result, while also passing them on to the output channel. 117 | while let Some(chunk) = chunk_rx.recv_async().await.ok() { 118 | tracing::info!(?chunk, "Got chunk"); 119 | match &chunk { 120 | Ok(StreamingResponse::Chunk(chunk)) => { 121 | response.merge_delta(chunk); 122 | } 123 | Ok(StreamingResponse::ResponseInfo(i)) => { 124 | res_stats = i.clone(); 125 | } 126 | Ok(StreamingResponse::RequestInfo(_)) => { 127 | // Don't need to handle RequestInfo since we've already incorporated its 128 | // information into `log_entry`. 129 | } 130 | Ok(StreamingResponse::Single(res)) => { 131 | response = res.clone(); 132 | } 133 | Err(e) => { 134 | record_error( 135 | log_entry, 136 | e, 137 | global_start, 138 | meta.num_retries, 139 | meta.was_rate_limited, 140 | current_span, 141 | log_tx, 142 | ) 143 | .await; 144 | output_tx.send_async(chunk).await.ok(); 145 | return Err(()); 146 | } 147 | } 148 | 149 | tracing::debug!(?chunk, "Sending chunk"); 150 | output_tx.send_async(chunk).await.ok(); 151 | } 152 | 153 | Ok((response, res_stats, log_entry)) 154 | } 155 | 156 | pub async fn record_error( 157 | mut log_entry: ProxyLogEvent, 158 | error: E, 159 | send_start: tokio::time::Instant, 160 | num_retries: u32, 161 | was_rate_limited: bool, 162 | current_span: Span, 163 | log_tx: Option<&LogSender>, 164 | ) { 165 | tracing::error!(error.full=?error, "Request failed"); 166 | 167 | current_span.record("error", error.to_string()); 168 | current_span.record("llm.retries", num_retries); 169 | current_span.record("llm.rate_limited", was_rate_limited); 170 | 171 | if let Some(log_tx) = log_tx { 172 | log_entry.total_latency = Some(send_start.elapsed()); 173 | log_entry.num_retries = Some(num_retries); 174 | log_entry.was_rate_limited = Some(was_rate_limited); 175 | log_entry.error = Some(json!(format!("{:?}", error))); 176 | log_tx 177 | .send_async(smallvec![ProxyLogEntry::Proxied(Box::new(log_entry))]) 178 | .await 179 | .ok(); 180 | } 181 | } 182 | 183 | #[derive(Serialize, Debug)] 184 | pub struct CollectedResponse { 185 | pub request_info: RequestInfo, 186 | pub response_info: ResponseInfo, 187 | pub was_streaming: bool, 188 | pub num_chunks: usize, 189 | pub response: SingleChatResponse, 190 | } 191 | 192 | /// Collect a stream contents into a single response 193 | pub async fn collect_response( 194 | receiver: StreamingResponseReceiver, 195 | request_n: usize, 196 | ) -> Result> { 197 | let mut request_info = None; 198 | let mut response_info = None; 199 | let mut was_streaming = false; 200 | 201 | let mut num_chunks = 0; 202 | let mut response = SingleChatResponse::new_for_collection(request_n); 203 | 204 | while let Ok(res) = receiver.recv_async().await { 205 | tracing::debug!(?res, "Got response chunk"); 206 | match res.change_context(Error::ModelError)? { 207 | StreamingResponse::RequestInfo(info) => { 208 | debug_assert!(request_info.is_none(), "Saw multiple RequestInfo objects"); 209 | debug_assert_eq!(num_chunks, 0, "RequestInfo was not the first chunk"); 210 | request_info = Some(info); 211 | } 212 | StreamingResponse::ResponseInfo(info) => { 213 | debug_assert!(response_info.is_none(), "Saw multiple ResponseInfo objects"); 214 | response_info = Some(info); 215 | } 216 | StreamingResponse::Single(res) => { 217 | debug_assert_eq!(num_chunks, 0, "Saw more than one non-streaming chunk"); 218 | num_chunks += 1; 219 | response = res; 220 | } 221 | StreamingResponse::Chunk(res) => { 222 | was_streaming = true; 223 | num_chunks += 1; 224 | response.merge_delta(&res); 225 | } 226 | } 227 | } 228 | 229 | let request_info = request_info.ok_or(Error::MissingStreamInformation("request info"))?; 230 | Ok(CollectedResponse { 231 | response_info: response_info.unwrap_or_else(|| ResponseInfo { 232 | meta: None, 233 | model: request_info.model.clone(), 234 | }), 235 | request_info, 236 | was_streaming, 237 | num_chunks, 238 | response, 239 | }) 240 | } 241 | -------------------------------------------------------------------------------- /api/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::{IpAddr, SocketAddr}, 3 | sync::Arc, 4 | time::Duration, 5 | }; 6 | 7 | use axum::Router; 8 | use chronicle_proxy::database::Database; 9 | use clap::Parser; 10 | use config::{Configs, LocalServerConfig}; 11 | use database::init_database; 12 | use error_stack::{Report, ResultExt}; 13 | use filigree::{ 14 | errors::panic_handler, 15 | propagate_http_span::extract_request_parent, 16 | tracing_config::{configure_tracing, create_tracing_config, teardown_tracing, TracingProvider}, 17 | }; 18 | use futures::Future; 19 | use tower::ServiceBuilder; 20 | use tower_http::{ 21 | compression::CompressionLayer, 22 | trace::{DefaultOnFailure, DefaultOnRequest, TraceLayer}, 23 | }; 24 | use tracing::{Level, Span}; 25 | use tracing_opentelemetry::OpenTelemetrySpanExt; 26 | 27 | use crate::{ 28 | config::{find_configs, merge_server_config}, 29 | proxy::ServerState, 30 | }; 31 | 32 | mod config; 33 | mod database; 34 | mod error; 35 | mod events; 36 | mod proxy; 37 | 38 | use error::Error; 39 | 40 | #[derive(Debug, Parser)] 41 | #[command(version, about)] 42 | pub(crate) struct Cli { 43 | /// The path to the configuration file or a directory containing it. If omitted, 44 | /// the default configuration path will be checked. 45 | #[clap(long, short = 'c')] 46 | config: Option, 47 | 48 | /// Do not read the .env file 49 | #[clap(long)] 50 | no_dotenv: bool, 51 | 52 | /// The SQLite or PostgreSQL database to use, if any. This can also be set in the configuration file. 53 | /// Takes a file path for SQLite or a connection string for PostgreSQL 54 | #[clap(long = "db", env = "DATABASE_URL")] 55 | database: Option, 56 | 57 | /// The IP host to bind to 58 | #[clap(long, env = "HOST")] 59 | host: Option, 60 | 61 | /// The TCP port to listen on 62 | #[clap(long, env = "PORT")] 63 | port: Option, 64 | } 65 | 66 | pub(crate) async fn run(cmd: Cli) -> Result<(), Report> { 67 | let configs = find_configs(cmd.config.clone())?; 68 | let mut server_config = merge_server_config(&configs); 69 | 70 | // Must load configs and run dotenv before starting tracing, so that they can set destination and 71 | // service name. 72 | if !cmd.no_dotenv { 73 | let mut loaded_env = false; 74 | for (dir, config) in configs.cwd.iter().rev().chain(configs.global.iter().rev()) { 75 | if config.server_config.dotenv.unwrap_or(true) { 76 | dotenvy::from_path(dir.join(".env")).ok(); 77 | loaded_env = true; 78 | } 79 | } 80 | 81 | if server_config.dotenv.unwrap_or(true) { 82 | dotenvy::dotenv().ok(); 83 | loaded_env = true; 84 | } 85 | 86 | if loaded_env { 87 | // Reread with the environment variables in place 88 | let cmd = Cli::parse(); 89 | 90 | if cmd.database.is_some() { 91 | server_config.database = cmd.database; 92 | } 93 | 94 | if cmd.host.is_some() { 95 | server_config.host = cmd.host; 96 | } 97 | 98 | if cmd.port.is_some() { 99 | server_config.port = cmd.port; 100 | } 101 | } 102 | } 103 | 104 | let tracing_config = create_tracing_config( 105 | "", 106 | "CHRONICLE_", 107 | TracingProvider::None, 108 | Some("chronicle".to_string()), 109 | None, 110 | ) 111 | .change_context(Error::ServerStart)?; 112 | 113 | configure_tracing( 114 | "CHRONICLE_", 115 | tracing_config, 116 | tracing_subscriber::fmt::time::ChronoUtc::rfc_3339(), 117 | std::io::stdout, 118 | ) 119 | .change_context(Error::ServerStart)?; 120 | 121 | for (dir, _) in configs.global.iter().chain(configs.cwd.iter()) { 122 | tracing::info!("Loaded config from {}", dir.display()); 123 | } 124 | let db = init_database(server_config.database.clone()) 125 | .await 126 | .change_context(Error::Db)?; 127 | 128 | let shutdown_signal = filigree::server::shutdown_signal(); 129 | serve(server_config, configs, db, shutdown_signal).await 130 | } 131 | 132 | pub(crate) async fn serve( 133 | server_config: LocalServerConfig, 134 | all_configs: Configs, 135 | db: Option, 136 | shutdown: impl Future + Send + 'static, 137 | ) -> Result<(), Report> { 138 | let proxy = proxy::build_proxy(db, all_configs).await?; 139 | 140 | let mut state = Arc::new(ServerState { proxy }); 141 | 142 | let app = Router::new() 143 | .merge(events::create_routes()) 144 | .merge(proxy::create_routes()) 145 | .with_state(state.clone()) 146 | .layer( 147 | ServiceBuilder::new() 148 | .layer(panic_handler(false)) 149 | .layer( 150 | TraceLayer::new_for_http() 151 | .make_span_with(|req: &axum::extract::Request| { 152 | let method = req.method(); 153 | let uri = req.uri(); 154 | 155 | // Add the matched route to the span 156 | let route = req 157 | .extensions() 158 | .get::() 159 | .map(|matched_path| matched_path.as_str()); 160 | 161 | let host = req.headers().get("host").and_then(|s| s.to_str().ok()); 162 | 163 | let request_id = req 164 | .headers() 165 | .get("X-Request-Id") 166 | .and_then(|s| s.to_str().ok()) 167 | .unwrap_or(""); 168 | 169 | let span = tracing::info_span!("request", 170 | request_id, 171 | http.host=host, 172 | http.method=%method, 173 | http.uri=%uri, 174 | http.route=route, 175 | http.status_code = tracing::field::Empty, 176 | error = tracing::field::Empty 177 | ); 178 | 179 | let context = extract_request_parent(req); 180 | span.set_parent(context); 181 | 182 | span 183 | }) 184 | .on_response(|res: &http::Response<_>, latency: Duration, span: &Span| { 185 | let status = res.status(); 186 | span.record("http.status_code", status.as_u16()); 187 | if status.is_client_error() || status.is_server_error() { 188 | span.record("error", "true"); 189 | } 190 | 191 | tracing::info!( 192 | latency = %format!("{} ms", latency.as_millis()), 193 | http.status_code = status.as_u16(), 194 | "finished processing request" 195 | ); 196 | }) 197 | .on_request(DefaultOnRequest::new().level(Level::INFO)) 198 | .on_failure(DefaultOnFailure::new().level(Level::ERROR)), 199 | ) 200 | .layer(CompressionLayer::new()) 201 | .into_inner(), 202 | ); 203 | 204 | let bind_ip = server_config 205 | .host 206 | .as_deref() 207 | .unwrap_or("::1") 208 | .parse::() 209 | .change_context(Error::ServerStart)?; 210 | let port = server_config.port.unwrap_or(9782); 211 | let bind_addr = SocketAddr::from((bind_ip, port)); 212 | let listener = tokio::net::TcpListener::bind(bind_addr) 213 | .await 214 | .change_context(Error::ServerStart)?; 215 | let actual_addr = listener.local_addr().change_context(Error::ServerStart)?; 216 | let port = actual_addr.port(); 217 | let host = actual_addr.ip().to_string(); 218 | tracing::info!("Listening on {host}:{port}"); 219 | 220 | axum::serve( 221 | listener, 222 | app.into_make_service_with_connect_info::(), 223 | ) 224 | .with_graceful_shutdown(shutdown) 225 | .await 226 | .change_context(Error::ServerStart)?; 227 | 228 | tracing::info!("Shutting down proxy"); 229 | Arc::get_mut(&mut state) 230 | .ok_or(Error::Shutdown) 231 | .attach_printable("Failed to get proxy reference for shutdown")? 232 | .proxy 233 | .shutdown() 234 | .await; 235 | 236 | tracing::info!("Exporting remaining traces"); 237 | teardown_tracing().await.change_context(Error::Shutdown)?; 238 | tracing::info!("Trace shut down complete"); 239 | 240 | Ok(()) 241 | } 242 | 243 | fn main() -> Result<(), Report> { 244 | tokio::runtime::Builder::new_multi_thread() 245 | .enable_all() 246 | .build() 247 | .unwrap() 248 | .block_on(actual_main()) 249 | } 250 | 251 | pub async fn actual_main() -> Result<(), Report> { 252 | error_stack::Report::set_color_mode(error_stack::fmt::ColorMode::None); 253 | let cli = Cli::parse(); 254 | run(cli).await?; 255 | Ok(()) 256 | } 257 | --------------------------------------------------------------------------------