├── llm ├── api │ ├── __init__.py │ ├── anthropicapi.py │ └── openaiapi.py ├── utils │ ├── __init__.py │ ├── parsing.py │ └── apikeys.py ├── __init__.py └── main.py ├── MANIFEST.in ├── .github └── workflows │ └── tests.yml ├── requirements.txt ├── setup.py ├── tests ├── test_multi_stream.py ├── fixtures │ ├── anthropic │ │ ├── test_completion.yaml │ │ ├── test_chat.yaml │ │ ├── test_simple_chat.yaml │ │ ├── test_streaming_chat_delta.yaml │ │ └── test_streaming_chat.yaml │ ├── openai │ │ ├── test_completion.yaml │ │ ├── test_completion_chat_model.yaml │ │ ├── test_chat.yaml │ │ ├── test_simple_chat.yaml │ │ ├── test_streaming_chat_method_full.yaml │ │ └── test_streaming_chat_method_delta.yaml │ └── multi_stream │ │ └── test_multistream.yaml ├── test_anthropic.py └── test_openai.py ├── LICENSE ├── README.md └── .gitignore /llm/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llm/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt -------------------------------------------------------------------------------- /llm/__init__.py: -------------------------------------------------------------------------------- 1 | from llm.main import complete, chat, stream_chat, multi_stream_chat, set_api_key 2 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Python Test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | name: Run tests 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - name: Checkout code 12 | uses: actions/checkout@v2 13 | 14 | - name: Set up Python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: 3.10.9 18 | 19 | - name: Install dependencies 20 | run: pip install -r requirements.txt 21 | 22 | - name: Run tests 23 | run: python -m unittest discover -s tests 24 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.8.3 2 | aiosignal==1.3.1 3 | anthropic==0.2.8 4 | anyio==3.6.2 5 | async-timeout==4.0.2 6 | asynctest==0.13.0 7 | attrs==23.1.0 8 | certifi==2023.5.7 9 | charset-normalizer==2.1.1 10 | distlib==0.3.6 11 | filelock==3.12.0 12 | frozenlist==1.3.3 13 | h11==0.14.0 14 | httpcore==0.17.1 15 | httpx==0.24.1 16 | idna==3.4 17 | multidict==6.0.4 18 | openai==0.27.2 19 | platformdirs==3.5.1 20 | python-dotenv==1.0.0 21 | PyYAML==5.4.1 22 | regex==2023.5.5 23 | requests==2.30.0 24 | six==1.16.0 25 | sniffio==1.3.0 26 | tiktoken==0.4.0 27 | tokenizers==0.13.3 28 | tqdm==4.65.0 29 | urllib3==1.26.14 30 | vcrpy==4.2.1 31 | virtualenv==20.19.0 32 | wrapt==1.15.0 33 | yarl==1.9.2 34 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_namespace_packages 2 | 3 | with open('requirements.txt') as f: 4 | requirements = f.read().splitlines() 5 | 6 | with open('README.md') as f: 7 | long_description = f.read() 8 | 9 | setup( 10 | name='python-llm', 11 | version='0.1.9', 12 | author='Daniel Gross', 13 | author_email='d@dcgross.com', 14 | description='An LLM wrapper for Humans', 15 | long_description=long_description, 16 | long_description_content_type="text/markdown", 17 | packages=find_namespace_packages(include=['llm', 'llm.*']), 18 | url="https://github.com/danielgross/python-llm", 19 | classifiers=[ 20 | ], 21 | python_requires='>=3.6', 22 | install_requires=requirements, 23 | ) 24 | -------------------------------------------------------------------------------- /tests/test_multi_stream.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import llm 3 | import os 4 | import vcr 5 | import asynctest 6 | import collections 7 | 8 | 9 | class TestAnthropicStreaming(asynctest.TestCase): 10 | 11 | @vcr.use_cassette("tests/fixtures/multi_stream/test_multistream.yaml", filter_headers=['authorization', 'x-api-key']) 12 | async def test_multistream(self): 13 | results = collections.defaultdict(list) 14 | async for engine, response in llm.multi_stream_chat(["Write a poem about the number pi."], engines=["anthropic:claude-instant-v1", "openai:gpt-3.5-turbo"], max_tokens=10): 15 | results[engine].append(response) 16 | final = {k: "".join(v[-1]) for k, v in results.items()} 17 | self.assertTrue(final["anthropic:claude-instant-v1"].startswith( 18 | "Here is a poem I wrote about the number pi"), final["anthropic:claude-instant-v1"]) 19 | self.assertTrue( 20 | final["openai:gpt-3.5-turbo"].startswith("Pi, the circle’s constant"), final["openai:gpt-3.5-turbo"]) 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Daniel Gross 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/fixtures/anthropic/test_completion.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"prompt": "Hello, my name is", "model": "claude-v1", "max_tokens_to_sample": 4 | 1}' 5 | headers: 6 | Accept: 7 | - '*/*' 8 | Accept-Encoding: 9 | - gzip, deflate, br 10 | Connection: 11 | - keep-alive 12 | Content-Length: 13 | - '80' 14 | Content-Type: 15 | - application/json 16 | User-Agent: 17 | - python-requests/2.28.1 18 | method: POST 19 | uri: https://api.anthropic.com/v1/complete 20 | response: 21 | body: 22 | string: '{"completion":" Claude","stop":null,"stop_reason":"max_tokens","truncated":false,"log_id":"5470beac3a9dc758a58a173ca102f741","model":"claude-v1","exception":null}' 23 | headers: 24 | Alt-Svc: 25 | - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 26 | Content-Length: 27 | - '162' 28 | content-type: 29 | - application/json 30 | date: 31 | - Sat, 20 May 2023 13:10:28 GMT 32 | request-id: 33 | - 5470beac3a9dc758a58a173ca102f741 34 | server: 35 | - Google Frontend 36 | via: 37 | - 1.1 google 38 | x-cloud-trace-context: 39 | - 5470beac3a9dc758a58a173ca102f741 40 | status: 41 | code: 200 42 | message: OK 43 | version: 1 44 | -------------------------------------------------------------------------------- /tests/fixtures/anthropic/test_chat.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"prompt": "\n\nHuman: what is your favorite cat breed?\n\nAssistant: I 4 | like tabbies.\n\nHuman: And what about dogs? Reply with punctuation.\n\nAssistant:", 5 | "stop_sequences": ["\n\nHuman:"], "model": "claude-v1", "max_tokens_to_sample": 6 | 30}' 7 | headers: 8 | Accept: 9 | - application/json 10 | Accept-Encoding: 11 | - gzip, deflate, br 12 | Client: 13 | - anthropic-python/0.2.8 14 | Connection: 15 | - keep-alive 16 | Content-Length: 17 | - '240' 18 | Content-Type: 19 | - application/json 20 | User-Agent: 21 | - python-requests/2.28.1 22 | method: POST 23 | uri: https://api.anthropic.com/v1/complete 24 | response: 25 | body: 26 | string: '{"completion":" When it comes to dogs, I have a soft spot for Labrador 27 | Retrievers; they''re such friendly, energetic, and playful companions.","stop":null,"stop_reason":"max_tokens","truncated":false,"log_id":"704c9cc898badc213b242e67832dcc35","model":"claude-v1","exception":null}' 28 | headers: 29 | Alt-Svc: 30 | - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 31 | Content-Length: 32 | - '280' 33 | content-type: 34 | - application/json 35 | date: 36 | - Sat, 20 May 2023 13:10:28 GMT 37 | request-id: 38 | - 704c9cc898badc213b242e67832dcc35 39 | server: 40 | - Google Frontend 41 | via: 42 | - 1.1 google 43 | x-cloud-trace-context: 44 | - 704c9cc898badc213b242e67832dcc35 45 | status: 46 | code: 200 47 | message: OK 48 | version: 1 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-llm: A LLM API for Humans 2 | 3 | ![Tests](https://github.com/danielgross/python-llm/actions/workflows/tests.yml/badge.svg) 4 | 5 | The simplicity and elegance of python-requests, but for LLMs. This library supports models from OpenAI and Anthropic. I will try to add more when I have the time, and am warmly accepting pull requests if that's of interest. 6 | 7 | ## Usage 8 | 9 | ```python 10 | import llm 11 | llm.set_api_key(openai="sk-...", anthropic="sk-...") 12 | 13 | # Chat 14 | llm.chat("what is 2+2") # 4. Uses GPT-3 by default if key is provided. 15 | llm.chat("what is 2+2", engine="anthropic:claude-instant-v1") # 4. 16 | 17 | # Completion 18 | llm.complete("hello, I am") # A GPT model. 19 | llm.complete("hello, I am", engine="openai:gpt-4") # A big GPT model. 20 | llm.complete("hello, I am ", engine="anthropic:claude-instant-v1") # Claude. 21 | 22 | # Back-and-forth chat [human, assistant, human] 23 | llm.chat(["hi", "hi there, how are you?", "good, tell me a joke"]) # Why did chicken cross road? 24 | 25 | # Streaming chat 26 | llm.stream_chat(["what is 2+2"]) # 4. 27 | llm.multi_stream_chat(["what is 2+2"], 28 | engines= 29 | ["anthropic:claude-instant-v1", 30 | "openai:gpt-3.5-turbo"]) 31 | # Results will stream back to you from both models at the same time like this: 32 | # ["anthropic:claude-instant-v1", "hi"], ["openai:gpt-3.5-turbo", "howdy"], 33 | # ["anthropic:claude-instant-v1", " there"] ["openai:gpt-3.5-turbo", " my friend"] 34 | 35 | # Engines are in the provider:model format, as in openai:gpt-4, or anthropic:claude-instant-v1. 36 | ``` 37 | 38 | ## Multi Stream Chat In Action 39 | Given this feature is very lively, I've included a video of it in action. 40 | 41 | https://github.com/danielgross/python-llm/assets/279531/d68eb843-7a32-4ffe-8ac2-b06b81e764b0 42 | 43 | ## Installation 44 | 45 | To install `python-llm`, use pip: ```pip install python-llm```. 46 | 47 | ## Configuration 48 | You can set API keys in a few ways: 49 | 1. Through environment variables (you can also set a `.env` file). 50 | ```bash 51 | export OPENAI_API_KEY=sk_... 52 | export ANTHROPIC_API_KEY=sk_... 53 | ``` 54 | 2. By calling the method manually: 55 | ```python 56 | import llm 57 | llm.set_api_key(openai="sk-...", anthropic="sk-...") 58 | ``` 59 | 3. By passing a JSON file like this: 60 | ```python 61 | llm.set_api_key("path/to/api_keys.json") 62 | ``` 63 | The JSON should look like: 64 | ```json 65 | { 66 | "openai": "sk-...", 67 | "anthropic": "sk-..." 68 | } 69 | ``` 70 | 71 | ## TODO 72 | - [ ] Caching! 73 | - [ ] More LLM vendors! 74 | - [ ] More tests! 75 | 76 | -------------------------------------------------------------------------------- /llm/utils/parsing.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | 4 | def parse_args(engine, **kwargs): 5 | """Parse args.""" 6 | assert ":" in engine, "Engine must be in the format 'engine:args', as in openai:text-davinci-003" 7 | service, engine = engine.split(':', 1) 8 | if service == "anthropic": 9 | kwargs["max_tokens_to_sample"] = kwargs.pop("max_tokens", 100) 10 | return collections.namedtuple("Args", ["service", "engine", "kwargs"])(service, engine, kwargs) 11 | 12 | 13 | def structure_chat(messages): 14 | """Structure chat messages.""" 15 | # if we got a plan string, wrap it in a list 16 | if isinstance(messages, str): 17 | messages = [messages] 18 | if isinstance(messages[0], str): 19 | # recreate as dictionary where "role" is either "assistant" or "user" 20 | messages = [{"role": "user" if i % 2 == 0 else "assistant", 21 | "content": text} for i, text in enumerate(messages)] 22 | elif isinstance(messages[0], str): 23 | raise ValueError("Chat messages must be strings or dictionaries.") 24 | else: 25 | messages = [{"role": message.get( 26 | "role", "user"), "content": message["content"]} for message in messages] 27 | return messages 28 | 29 | 30 | def format_streaming_output(raw_tokens, stream_method, service, output_cache): 31 | if stream_method == "default": 32 | # Don't even bother building a cache. 33 | return raw_tokens, output_cache 34 | elif stream_method == "full" and service == "anthropic": 35 | output_cache.append(raw_tokens) 36 | return raw_tokens, output_cache 37 | elif stream_method == "delta" and service == "openai": 38 | output_cache.append(raw_tokens) 39 | return raw_tokens, output_cache 40 | elif stream_method == "delta" and service == "anthropic": 41 | # "raw_tokens" contain the string repeated from start, and we 42 | # want to yield only the new part. 43 | # output_cache is a list of raw_tokens 44 | streaming_output = raw_tokens[len(''.join(output_cache)):] 45 | output_cache = [raw_tokens] 46 | return streaming_output, output_cache 47 | elif stream_method == "full" and service == "openai": 48 | # Here the situation is reversed: raw_tokens contain only the 49 | # new part, and we want to yield the full string. 50 | streaming_output = ''.join(output_cache) + raw_tokens 51 | output_cache.append(raw_tokens) 52 | return streaming_output, output_cache 53 | else: 54 | raise ValueError( 55 | f"Stream method {stream_method} is not supported for service {service}.") 56 | -------------------------------------------------------------------------------- /tests/fixtures/openai/test_completion.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"prompt": "2+2 is how much? Good question, I think 2+2=", "max_tokens": 4 | 1}' 5 | headers: 6 | Accept: 7 | - '*/*' 8 | Accept-Encoding: 9 | - gzip, deflate 10 | Connection: 11 | - keep-alive 12 | Content-Length: 13 | - '75' 14 | Content-Type: 15 | - application/json 16 | User-Agent: 17 | - OpenAI/v1 PythonBindings/0.27.2 18 | X-OpenAI-Client-User-Agent: 19 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 20 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 21 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 22 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 23 | method: POST 24 | uri: https://api.openai.com/v1/engines/text-davinci-002/completions 25 | response: 26 | body: 27 | string: !!binary | 28 | H4sIAAAAAAAAA0SOzUrEMBRG9z7Gt06hI+NMzQs4/owIrqxISZNrGyfNjc2tDJS+u4wI3Z5zFmeG 29 | d9CwQwrF/v74Nkl9nB6l/m4fnk8vdcxDezCvd9snKHD7RVagIXSWxvKQAonnCAU7khFy0Jtdtd1t 30 | ytuqUhjYUfjPC2d+fLS+KMvrS9+zt5Sh3+c/DY0bKPjo6AxdKgTu0shtho5TCAqfPvrcNyOZzBEa 31 | gWInPZYPhSmbjqBnpJGHJI3wiWK+rCiskytWEBYTVrBflqtfAAAA//8DALLL02cIAQAA 32 | headers: 33 | CF-Cache-Status: 34 | - DYNAMIC 35 | CF-RAY: 36 | - 7ca6fd95d9f0d9fd-MIA 37 | Cache-Control: 38 | - no-cache, must-revalidate 39 | Connection: 40 | - keep-alive 41 | Content-Encoding: 42 | - gzip 43 | Content-Type: 44 | - application/json 45 | Date: 46 | - Sat, 20 May 2023 19:29:48 GMT 47 | Server: 48 | - cloudflare 49 | Transfer-Encoding: 50 | - chunked 51 | access-control-allow-origin: 52 | - '*' 53 | alt-svc: 54 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 55 | openai-model: 56 | - text-davinci-002 57 | openai-organization: 58 | - dgorg 59 | openai-processing-ms: 60 | - '98' 61 | openai-version: 62 | - '2020-10-01' 63 | strict-transport-security: 64 | - max-age=15724800; includeSubDomains 65 | x-ratelimit-limit-requests: 66 | - '3000' 67 | x-ratelimit-limit-tokens: 68 | - '250000' 69 | x-ratelimit-remaining-requests: 70 | - '2999' 71 | x-ratelimit-remaining-tokens: 72 | - '249999' 73 | x-ratelimit-reset-requests: 74 | - 20ms 75 | x-ratelimit-reset-tokens: 76 | - 0s 77 | x-request-id: 78 | - 8a6f61d50bb9623515264625277efbfc 79 | status: 80 | code: 200 81 | message: OK 82 | version: 1 83 | -------------------------------------------------------------------------------- /tests/fixtures/openai/test_completion_chat_model.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, 4 | my name is"}], "max_tokens": 1}' 5 | headers: 6 | Accept: 7 | - '*/*' 8 | Accept-Encoding: 9 | - gzip, deflate 10 | Connection: 11 | - keep-alive 12 | Content-Length: 13 | - '107' 14 | Content-Type: 15 | - application/json 16 | User-Agent: 17 | - OpenAI/v1 PythonBindings/0.27.2 18 | X-OpenAI-Client-User-Agent: 19 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 20 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 21 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 22 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 23 | method: POST 24 | uri: https://api.openai.com/v1/chat/completions 25 | response: 26 | body: 27 | string: !!binary | 28 | H4sIAAAAAAAAA0SOzUrEMBRG9z7Gt06Hho5jzXIWgqAMMiCKyJBJr000fzS3Wih9dxlQuj1wDmeG 29 | 66BgrGYTsq9u7h9fx719mI7251s+5xc+Pk3T3SFI6QgC6fxJhv+MjUkhe2KXIgTMQJqpg5K7druT 30 | 9W3bCoTUkYdCn7lqNtcVj8M5VXVTSwiMRfcENSMPKWQ+cfqiWKBkI7CmVyzAibVfwXYRMDY5QwXq 31 | bUag8p8ckico6FJcYR35MpgiU7zMHzJFLAIfLrpiTwPpkiIUPMWeLQRc7GiCqpf35eoXAAD//wMA 32 | OTKZfyMBAAA= 33 | headers: 34 | CF-Cache-Status: 35 | - DYNAMIC 36 | CF-RAY: 37 | - 7ca6fd973d4b223f-MIA 38 | Cache-Control: 39 | - no-cache, must-revalidate 40 | Connection: 41 | - keep-alive 42 | Content-Encoding: 43 | - gzip 44 | Content-Type: 45 | - application/json 46 | Date: 47 | - Sat, 20 May 2023 19:29:49 GMT 48 | Server: 49 | - cloudflare 50 | Transfer-Encoding: 51 | - chunked 52 | access-control-allow-origin: 53 | - '*' 54 | alt-svc: 55 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 56 | openai-model: 57 | - gpt-3.5-turbo-0301 58 | openai-organization: 59 | - dgorg 60 | openai-processing-ms: 61 | - '545' 62 | openai-version: 63 | - '2020-10-01' 64 | strict-transport-security: 65 | - max-age=15724800; includeSubDomains 66 | x-ratelimit-limit-requests: 67 | - '3500' 68 | x-ratelimit-limit-tokens: 69 | - '90000' 70 | x-ratelimit-remaining-requests: 71 | - '3499' 72 | x-ratelimit-remaining-tokens: 73 | - '89993' 74 | x-ratelimit-reset-requests: 75 | - 17ms 76 | x-ratelimit-reset-tokens: 77 | - 4ms 78 | x-request-id: 79 | - 605206d633ee63332d6575440f56f243 80 | status: 81 | code: 200 82 | message: OK 83 | version: 1 84 | -------------------------------------------------------------------------------- /llm/api/anthropicapi.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import anthropic 3 | 4 | from llm.utils.apikeys import api_keys 5 | 6 | 7 | BASE_URL = "https://api.anthropic.com" 8 | 9 | 10 | def client(): 11 | return anthropic.Client(api_keys["anthropic"]) 12 | 13 | 14 | def format_chat_messages(messages): 15 | prompt = [] 16 | for message in messages: 17 | if message["role"] == "user": 18 | prompt.append(f"{anthropic.HUMAN_PROMPT} {message['content']}") 19 | elif message["role"] == "assistant": 20 | prompt.append(f"{anthropic.AI_PROMPT} {message['content']}") 21 | else: 22 | raise ValueError(f"Unknown role {message['role']}") 23 | if messages[-1]["role"] != "user": 24 | raise ValueError("Last message must be from the user.") 25 | # Add a final AI prompt 26 | prompt.append(anthropic.AI_PROMPT) 27 | return "".join(prompt) 28 | 29 | 30 | def complete(prompt, engine="claude-v1", prompt_overflow=None, max_tokens_to_sample=10, **kwargs): 31 | # TODO implement max_prompt_tokens 32 | 33 | headers = { 34 | "x-api-key": api_keys.get("anthropic"), 35 | "Content-Type": "application/json", 36 | } 37 | 38 | data = { 39 | "prompt": prompt, 40 | "model": engine, 41 | "max_tokens_to_sample": max_tokens_to_sample, 42 | **kwargs, 43 | } 44 | 45 | response = requests.post( 46 | f"{BASE_URL}/v1/complete", 47 | headers=headers, 48 | json=data, 49 | ) 50 | 51 | if response.status_code != 200: 52 | raise Exception(f"Error {response.status_code}: {response.text}") 53 | 54 | result = response.json() 55 | return result["completion"] 56 | 57 | 58 | def chat(messages, engine="claude-v1", system=None, prompt_overflow=None, max_tokens_to_sample=30, **kwargs): 59 | if system is not None: 60 | raise NotImplementedError( 61 | "System messages are not yet implemented for Claude.") 62 | 63 | c = client() 64 | prompt = format_chat_messages(messages) 65 | resp = c.completion( 66 | prompt=prompt, 67 | stop_sequences=[anthropic.HUMAN_PROMPT], 68 | model=engine, 69 | max_tokens_to_sample=max_tokens_to_sample, 70 | **kwargs 71 | ) 72 | return resp["completion"] 73 | 74 | 75 | async def stream_chat(messages, engine="claude-v1", system=None, prompt_overflow=None, max_tokens_to_sample=100, **kwargs): 76 | c = client() 77 | prompt = format_chat_messages(messages) 78 | response = await c.acompletion_stream( 79 | prompt=prompt, 80 | stop_sequences=[anthropic.HUMAN_PROMPT], 81 | max_tokens_to_sample=max_tokens_to_sample, 82 | model=engine, 83 | stream=True, 84 | ) 85 | async for data in response: 86 | # return a generator of responses 87 | yield data["completion"] 88 | -------------------------------------------------------------------------------- /tests/fixtures/openai/test_chat.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "hello"}, 4 | {"role": "assistant", "content": "how are you?"}, {"role": "user", "content": 5 | "I''m good, thanks."}]}' 6 | headers: 7 | Accept: 8 | - '*/*' 9 | Accept-Encoding: 10 | - gzip, deflate 11 | Connection: 12 | - keep-alive 13 | Content-Length: 14 | - '178' 15 | Content-Type: 16 | - application/json 17 | User-Agent: 18 | - OpenAI/v1 PythonBindings/0.27.2 19 | X-OpenAI-Client-User-Agent: 20 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 21 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 22 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 23 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 24 | method: POST 25 | uri: https://api.openai.com/v1/chat/completions 26 | response: 27 | body: 28 | string: !!binary | 29 | H4sIAAAAAAAAA0SOQUvDQBQG7/6K9bt42ZTEtrHdiyBoqehBlIqKlO3mmaxN9oXsK7WU/Hcpar0O 30 | zDB7+AIGrrLimrZOLub3L9HN1gu/eLzh51vadq8PdzO6ug7lCBq8+iQnv8bAcdPWJJ4DNFxHVqiA 31 | yfLJKM/S6STXaLigGgZlK8lwME5k0604SYdpBo1NtCXB7NF23LSyFF5TiDDnU43/9BFnuYaw2PpI 32 | RuNew1XsHUWYtz0ain/NjmuCgY3RR7FBDocchMLh/qmychZVeVhWwqoi252qeVRSUUfKhp1UPpRq 33 | rpwN6qehdrxRWy+VEi7s7hK9xocPPlbLjmzkAIMo3ELDh4K+YNL+vT/5BgAA//8DAKOpRt9iAQAA 34 | headers: 35 | CF-Cache-Status: 36 | - DYNAMIC 37 | CF-RAY: 38 | - 7ca6fd87cb540a3e-MIA 39 | Cache-Control: 40 | - no-cache, must-revalidate 41 | Connection: 42 | - keep-alive 43 | Content-Encoding: 44 | - gzip 45 | Content-Type: 46 | - application/json 47 | Date: 48 | - Sat, 20 May 2023 19:29:48 GMT 49 | Server: 50 | - cloudflare 51 | Transfer-Encoding: 52 | - chunked 53 | access-control-allow-origin: 54 | - '*' 55 | alt-svc: 56 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 57 | openai-model: 58 | - gpt-3.5-turbo-0301 59 | openai-organization: 60 | - dgorg 61 | openai-processing-ms: 62 | - '2048' 63 | openai-version: 64 | - '2020-10-01' 65 | strict-transport-security: 66 | - max-age=15724800; includeSubDomains 67 | x-ratelimit-limit-requests: 68 | - '3500' 69 | x-ratelimit-limit-tokens: 70 | - '90000' 71 | x-ratelimit-remaining-requests: 72 | - '3499' 73 | x-ratelimit-remaining-tokens: 74 | - '89971' 75 | x-ratelimit-reset-requests: 76 | - 17ms 77 | x-ratelimit-reset-tokens: 78 | - 18ms 79 | x-request-id: 80 | - 486e7317ffcf6e0dcd289cfd261278d5 81 | status: 82 | code: 200 83 | message: OK 84 | version: 1 85 | -------------------------------------------------------------------------------- /tests/fixtures/anthropic/test_simple_chat.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"prompt": "\n\nHuman: What is 2+2? Reply with just one number and no punctuation.\n\nAssistant:", 4 | "stop_sequences": ["\n\nHuman:"], "model": "claude-v1", "max_tokens_to_sample": 5 | 100}' 6 | headers: 7 | Accept: 8 | - application/json 9 | Accept-Encoding: 10 | - gzip, deflate, br 11 | Client: 12 | - anthropic-python/0.2.8 13 | Connection: 14 | - keep-alive 15 | Content-Length: 16 | - '183' 17 | Content-Type: 18 | - application/json 19 | User-Agent: 20 | - python-requests/2.28.1 21 | method: POST 22 | uri: https://api.anthropic.com/v1/complete 23 | response: 24 | body: 25 | string: '{"completion":" 4","stop":"\n\nHuman:","stop_reason":"stop_sequence","truncated":false,"log_id":"449dfaaebe392ea4b2109708d6738b38","model":"claude-v1","exception":null}' 26 | headers: 27 | Alt-Svc: 28 | - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 29 | Content-Length: 30 | - '168' 31 | content-type: 32 | - application/json 33 | date: 34 | - Sat, 20 May 2023 19:13:11 GMT 35 | request-id: 36 | - 449dfaaebe392ea4b2109708d6738b38 37 | server: 38 | - Google Frontend 39 | via: 40 | - 1.1 google 41 | x-cloud-trace-context: 42 | - 449dfaaebe392ea4b2109708d6738b38 43 | status: 44 | code: 200 45 | message: OK 46 | - request: 47 | body: '{"prompt": "\n\nHuman: What is 2+2? Reply with just one number and no punctuation.\n\nAssistant:", 48 | "stop_sequences": ["\n\nHuman:"], "model": "claude-v1", "max_tokens_to_sample": 49 | 100}' 50 | headers: 51 | Accept: 52 | - application/json 53 | Accept-Encoding: 54 | - gzip, deflate, br 55 | Client: 56 | - anthropic-python/0.2.8 57 | Connection: 58 | - keep-alive 59 | Content-Length: 60 | - '183' 61 | Content-Type: 62 | - application/json 63 | User-Agent: 64 | - python-requests/2.28.1 65 | method: POST 66 | uri: https://api.anthropic.com/v1/complete 67 | response: 68 | body: 69 | string: '{"completion":" 4","stop":"\n\nHuman:","stop_reason":"stop_sequence","truncated":false,"log_id":"254871ca9e297d19df904b8d09d7f98e","model":"claude-v1","exception":null}' 70 | headers: 71 | Alt-Svc: 72 | - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 73 | Content-Length: 74 | - '168' 75 | content-type: 76 | - application/json 77 | date: 78 | - Sat, 20 May 2023 19:13:12 GMT 79 | request-id: 80 | - 254871ca9e297d19df904b8d09d7f98e 81 | server: 82 | - Google Frontend 83 | via: 84 | - 1.1 google 85 | x-cloud-trace-context: 86 | - 254871ca9e297d19df904b8d09d7f98e 87 | status: 88 | code: 200 89 | message: OK 90 | version: 1 91 | -------------------------------------------------------------------------------- /llm/api/openaiapi.py: -------------------------------------------------------------------------------- 1 | """Implement the OpenAI API.""" 2 | 3 | from llm.utils.parsing import structure_chat 4 | import openai 5 | import tiktoken 6 | import logging 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | MODEL_CONTEXT_SIZE_LIMITS = { 11 | "gpt-4": 8192, 12 | "gpt-4-0314": 8192, 13 | "gpt-4-32k": 32768, 14 | "gpt-4-32k-0314": 32768, 15 | "gpt-3.5-turbo": 4096, 16 | "gpt-3.5-turbo-0301": 4096, 17 | "text-davinci-003": 4097, 18 | "text-davinci-002": 4097, 19 | "code-davinci-002": 8001, 20 | } 21 | 22 | 23 | def _trim_overflow(text, model): 24 | """Trim text to max_tokens.""" 25 | enc = tiktoken.encoding_for_model(model) 26 | tokens = enc.encode(text) 27 | model_limit = MODEL_CONTEXT_SIZE_LIMITS[model] 28 | # TODO I should set based on how much requested by user, for now make it 10% of possible size. 29 | response_buffer = int(model_limit * 0.1) 30 | logger.debug(f"Trimming overflow for {model} (from {len(tokens)} to {model_limit - response_buffer})") 31 | total_size = len(tokens) + response_buffer 32 | if total_size > model_limit: 33 | tokens = tokens[:model_limit - response_buffer] 34 | return enc.decode(tokens) 35 | return text 36 | 37 | 38 | def complete(prompt, engine, prompt_overflow=None, **kwargs): 39 | """Complete text using the OpenAI API.""" 40 | if prompt_overflow == "trim": 41 | logger.debug(f"Trimming overflow for {engine}") 42 | prompt = _trim_overflow(prompt, engine) 43 | 44 | if engine.startswith("gpt-3.5") or engine.startswith("gpt-4"): 45 | return chat(structure_chat([prompt]), engine=engine, **kwargs) 46 | logger.debug(f"Completing with {engine} using prompt: {prompt}") 47 | return openai.Completion.create( 48 | engine=engine, 49 | prompt=prompt, 50 | **kwargs 51 | ).choices[0].text 52 | 53 | 54 | def chat(messages, engine, system=None, prompt_overflow=None, **kwargs): 55 | """Chat with the OpenAI API.""" 56 | if prompt_overflow == "trim": 57 | logger.debug(f"Trimming overflow for {engine}") 58 | # TODO For now, just trim the last message. 59 | messages[-1]['content'] = _trim_overflow( 60 | messages[-1]['content'], engine) 61 | 62 | if engine.startswith("text-"): 63 | raise ValueError( 64 | f"Cannot issue a ChatCompletion with engine {engine}.") 65 | if system is not None: 66 | messages = [{"role": "system", "content": system}] + messages 67 | logger.debug(f"Chatting with {engine} using messages: {messages}") 68 | response = openai.ChatCompletion.create( 69 | model=engine, 70 | messages=messages, 71 | **kwargs 72 | ) 73 | return response.choices[0].message.content 74 | 75 | 76 | async def stream_chat(messages, engine, system=None, prompt_overflow=None, **kwargs): 77 | """Chat with the OpenAI API.""" 78 | if prompt_overflow == "trim": 79 | logger.debug(f"Trimming overflow for {engine}") 80 | # TODO For now, just trim the last message. 81 | messages[-1]['content'] = _trim_overflow( 82 | messages[-1]['content'], engine) 83 | 84 | if system is not None: 85 | messages = [{"role": "system", "content": system}] + messages 86 | logger.debug(f"Chatting with {engine} using messages: {messages}") 87 | 88 | result = openai.ChatCompletion.create( 89 | model=engine, 90 | messages=messages, 91 | stream=True, 92 | **kwargs 93 | ) 94 | for chunk in result: 95 | if chunk.choices[0].delta.get('content') != None: 96 | yield chunk.choices[0].delta.content 97 | -------------------------------------------------------------------------------- /tests/test_anthropic.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import llm 3 | import os 4 | import vcr 5 | import asynctest 6 | 7 | # Only get API keys if they exist 8 | _key = open(os.path.expanduser("~/.anthropic")).read().strip() if os.path.exists( 9 | os.path.expanduser("~/.anthropic")) else 'test' 10 | llm.set_api_key(anthropic=_key) 11 | 12 | 13 | class TestClaudeCompletions(unittest.TestCase): 14 | 15 | @vcr.use_cassette("tests/fixtures/anthropic/test_completion.yaml", filter_headers=['authorization', 'x-api-key']) 16 | def test_completion(self): 17 | prompt = "Hello, my name is" 18 | completion = llm.complete( 19 | prompt, engine="anthropic:claude-v1", max_tokens_to_sample=1).strip() 20 | self.assertEqual(completion, "Claude") 21 | 22 | @vcr.use_cassette("tests/fixtures/anthropic/test_simple_chat.yaml", filter_headers=['authorization', 'x-api-key']) 23 | def test_simple_chat(self): 24 | self.assertEqual(llm.chat( 25 | "What is 2+2? Reply with just one number and no punctuation.", engine="anthropic:claude-v1"), "4") 26 | self.assertEqual(llm.chat( 27 | ["What is 2+2? Reply with just one number and no punctuation."], engine="anthropic:claude-v1"), "4") 28 | 29 | @vcr.use_cassette("tests/fixtures/anthropic/test_chat.yaml", filter_headers=['authorization', 'x-api-key']) 30 | def test_chat(self): 31 | messages = ["what is your favorite cat breed?", "I like tabbies.", 32 | "And what about dogs? Reply with punctuation."] 33 | response = llm.chat(messages, engine="anthropic:claude-v1") 34 | self.assertTrue(response[0] != " ", 35 | "Response should not start with a space.") 36 | # Make sure the response is sorta directionally correct, e.g. has more than 3 words and some punctuation. 37 | self.assertTrue(len(response.split(' ')) > 3) 38 | self.assertTrue( 39 | '.' in response or '!' in response or '?' in response or ',' in response, response) 40 | 41 | 42 | class TestAnthropicStreaming(asynctest.TestCase): 43 | 44 | # TODO: I am not sure this really works? It passes but I'm a little suspicious. 45 | @vcr.use_cassette("tests/fixtures/anthropic/test_streaming_chat.yaml", filter_headers=['authorization', 'x-api-key']) 46 | async def test_streaming_chat(self): 47 | messages = ["what is your favorite cat breed?", "I like tabbies.", 48 | "And what about dogs? Reply with punctuation."] 49 | responses = [] 50 | async for response in llm.stream_chat(messages, engine="anthropic:claude-v1"): 51 | responses.append(response) 52 | final_response = responses[-1] 53 | self.assertEqual( 54 | final_response, "I do not actually have any favorite animal breeds, as I am an AI assistant without personal preferences.") 55 | 56 | @vcr.use_cassette("tests/fixtures/anthropic/test_streaming_chat_delta.yaml", filter_headers=['authorization', 'x-api-key']) 57 | async def test_streaming_chat_delta(self): 58 | messages = ["what is your favorite cat breed?", "I like tabbies.", 59 | "And what about dogs? Reply with punctuation."] 60 | responses = [] 61 | async for response in llm.stream_chat(messages, engine="anthropic:claude-v1", stream_method="delta"): 62 | responses.append(response) 63 | self.assertEqual(responses[0], "I'm") 64 | self.assertEqual(responses[1], " an") 65 | self.assertTrue("".join(responses).startswith( 66 | "I'm an AI assistant created by Anthropic to be helpful, harmless, and honest."), "".join(responses)) 67 | -------------------------------------------------------------------------------- /llm/utils/apikeys.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | 5 | from dotenv import load_dotenv 6 | 7 | # Useful os variables 8 | py_llm_dir = os.path.join(os.path.expanduser('~'), ".python-llm") 9 | env_path = os.path.join(py_llm_dir, "apikeys.env") 10 | 11 | # Supported apis and memory 12 | supported_apis = ["openai", "anthropic"] 13 | api_keys = {} 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | # --- public interface --- 19 | 20 | def configure_api_keys(*args, **kwargs) -> None: 21 | """Configure the API keys. Call me like this: 22 | - llm.set_api_keys(openai="sk-...") 23 | - llm.set_api_keys("path/to/api_keys.json") 24 | """ 25 | 26 | if args and len(args) > 1 and not isinstance(args[0], str): 27 | raise ValueError( 28 | "Only one positional argument (str) is allowed for filepath.") 29 | 30 | # load api keys from a json filepath 31 | elif args and len(args) == 1 and isinstance(args[0], str): 32 | filepath = args[0] 33 | if not os.path.exists(filepath): 34 | raise ValueError(f"Filepath {filepath} does not exist.") 35 | elif filepath.endswith(".json"): 36 | with open(filepath) as f: 37 | _update_keys_memory(json.load(f)) 38 | else: 39 | raise ValueError( 40 | f"File type of {filepath} is not supported (only .json)") 41 | 42 | # load api keys from keyword arguments 43 | elif set(kwargs.keys()).issubset(set(supported_apis)): 44 | _update_keys_memory(kwargs) 45 | else: 46 | raise ValueError(f"Only {supported_apis} apis are supported.") 47 | 48 | # save to cache and sync 49 | _save_keys_to_cache() 50 | _sync_keys_with_apis() 51 | 52 | 53 | def load_keys_from_cache() -> int: 54 | """Load API keys from the disk cache.""" 55 | load_dotenv(env_path) # from cache to os environment 56 | num_keys = _load_keys_from_env() #  load keys from os env 57 | logger.debug(f"Loaded {num_keys} api keys from cache" 58 | if num_keys else "api keys env cache does not exist") 59 | 60 | 61 | # --- private related to key management --- 62 | 63 | def _update_keys_memory(keys: dict) -> None: 64 | """Update the API keys memory.""" 65 | _verify_keys(keys) #  check if keys are supported 66 | api_keys.update(keys) #  then save to live memory 67 | 68 | 69 | def _verify_keys(keys_from_env: dict) -> bool: 70 | """Verify that the API keys are supported.""" 71 | for (api, key) in keys_from_env.items(): 72 | if api not in supported_apis: 73 | raise ValueError(f"API {api} not supported.") 74 | 75 | 76 | def _sync_keys_with_apis() -> None: 77 | """Update the API keys from the environment.""" 78 | import openai 79 | logger.debug(f"Setting OpenAI API key to {api_keys.get('openai')}") 80 | openai.api_key = api_keys.get("openai") 81 | 82 | 83 | # --- private related to key cache --- 84 | 85 | def _save_keys_to_cache(filepath: str = env_path) -> None: 86 | """Save API keys to the disk cache.""" 87 | _verify_keys(api_keys) 88 | os.makedirs(os.path.dirname(filepath), exist_ok=True) 89 | with open(filepath, 'w') as f: 90 | for api, key in api_keys.items(): 91 | f.write(f"{api.upper()}=\"{key}\"\n") 92 | 93 | 94 | def _load_keys_from_env() -> int: 95 | """Load API keys from os environment.""" 96 | keys_from_cache = { 97 | key: os.environ.get(key.upper()) 98 | for key in supported_apis 99 | if os.environ.get(key.upper()) 100 | } 101 | _update_keys_memory(keys_from_cache) 102 | _sync_keys_with_apis() 103 | return len(keys_from_cache) 104 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | .vscode/ 162 | .DS_Store 163 | -------------------------------------------------------------------------------- /tests/fixtures/anthropic/test_streaming_chat_delta.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"stream": true, "prompt": "\n\nHuman: what is your favorite cat breed?\n\nAssistant: 4 | I like tabbies.\n\nHuman: And what about dogs? Reply with punctuation.\n\nAssistant:", 5 | "stop_sequences": ["\n\nHuman:"], "max_tokens_to_sample": 30, "model": "claude-v1"}' 6 | headers: 7 | Accept: 8 | - application/json 9 | Client: 10 | - anthropic-python/0.2.8 11 | Content-Type: 12 | - application/json 13 | method: post 14 | uri: https://api.anthropic.com/v1/complete 15 | response: 16 | body: 17 | string: "data: {\"completion\": \" I'm\", \"stop\": null, \"stop_reason\": null,\ 18 | \ \"truncated\": false, \"log_id\": \"e6ad1d223f39e4186eabd2b0753f50ee\",\ 19 | \ \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 20 | : \" I'm an\", \"stop\": null, \"stop_reason\": null, \"truncated\": false,\ 21 | \ \"log_id\": \"e6ad1d223f39e4186eabd2b0753f50ee\", \"model\": \"claude-v1\"\ 22 | , \"exception\": null}\r\n\r\ndata: {\"completion\": \" I'm an AI assistant\ 23 | \ created by An\", \"stop\": null, \"stop_reason\": null, \"truncated\": false,\ 24 | \ \"log_id\": \"e6ad1d223f39e4186eabd2b0753f50ee\", \"model\": \"claude-v1\"\ 25 | , \"exception\": null}\r\n\r\ndata: {\"completion\": \" I'm an AI assistant\ 26 | \ created by Anthropic to be helpful\", \"stop\": null, \"stop_reason\": null,\ 27 | \ \"truncated\": false, \"log_id\": \"e6ad1d223f39e4186eabd2b0753f50ee\",\ 28 | \ \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 29 | : \" I'm an AI assistant created by Anthropic to be helpful, harmless, and\ 30 | \ honest\", \"stop\": null, \"stop_reason\": null, \"truncated\": false, \"\ 31 | log_id\": \"e6ad1d223f39e4186eabd2b0753f50ee\", \"model\": \"claude-v1\",\ 32 | \ \"exception\": null}\r\n\r\ndata: {\"completion\": \" I'm an AI assistant\ 33 | \ created by Anthropic to be helpful, harmless, and honest. I don't actually\"\ 34 | , \"stop\": null, \"stop_reason\": null, \"truncated\": false, \"log_id\"\ 35 | : \"e6ad1d223f39e4186eabd2b0753f50ee\", \"model\": \"claude-v1\", \"exception\"\ 36 | : null}\r\n\r\ndata: {\"completion\": \" I'm an AI assistant created by Anthropic\ 37 | \ to be helpful, harmless, and honest. I don't actually have personal\", \"\ 38 | stop\": null, \"stop_reason\": null, \"truncated\": false, \"log_id\": \"\ 39 | e6ad1d223f39e4186eabd2b0753f50ee\", \"model\": \"claude-v1\", \"exception\"\ 40 | : null}\r\n\r\ndata: {\"completion\": \" I'm an AI assistant created by Anthropic\ 41 | \ to be helpful, harmless, and honest. I don't actually have personal preferences\ 42 | \ when it comes to\", \"stop\": null, \"stop_reason\": null, \"truncated\"\ 43 | : false, \"log_id\": \"e6ad1d223f39e4186eabd2b0753f50ee\", \"model\": \"claude-v1\"\ 44 | , \"exception\": null}\r\n\r\ndata: {\"completion\": \" I'm an AI assistant\ 45 | \ created by Anthropic to be helpful, harmless, and honest. I don't actually\ 46 | \ have personal preferences when it comes to\", \"stop\": null, \"stop_reason\"\ 47 | : \"max_tokens\", \"truncated\": false, \"log_id\": \"e6ad1d223f39e4186eabd2b0753f50ee\"\ 48 | , \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: [DONE]\r\n\r\ 49 | \n" 50 | headers: 51 | Alt-Svc: 52 | - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 53 | Cache-Control: 54 | - no-cache 55 | Content-Type: 56 | - text/event-stream; charset=utf-8 57 | Date: 58 | - Sat, 20 May 2023 13:19:44 GMT 59 | Server: 60 | - Google Frontend 61 | Transfer-Encoding: 62 | - chunked 63 | Via: 64 | - 1.1 google 65 | request-id: 66 | - e6ad1d223f39e4186eabd2b0753f50ee 67 | x-accel-buffering: 68 | - 'no' 69 | status: 70 | code: 200 71 | message: OK 72 | url: https://api.anthropic.com/v1/complete 73 | version: 1 74 | -------------------------------------------------------------------------------- /tests/fixtures/anthropic/test_streaming_chat.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"stream": true, "prompt": "\n\nHuman: what is your favorite cat breed?\n\nAssistant: 4 | I like tabbies.\n\nHuman: And what about dogs? Reply with punctuation.\n\nAssistant:", 5 | "stop_sequences": ["\n\nHuman:"], "max_tokens_to_sample": 30, "model": "claude-v1"}' 6 | headers: 7 | Accept: 8 | - application/json 9 | Client: 10 | - anthropic-python/0.2.8 11 | Content-Type: 12 | - application/json 13 | method: post 14 | uri: https://api.anthropic.com/v1/complete 15 | response: 16 | body: 17 | string: "data: {\"completion\": \" I do\", \"stop\": null, \"stop_reason\":\ 18 | \ null, \"truncated\": false, \"log_id\": \"14f945b66b10a12390686cb206a254bf\"\ 19 | , \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 20 | : \" I do not actually have any favorite\", \"stop\": null, \"stop_reason\"\ 21 | : null, \"truncated\": false, \"log_id\": \"14f945b66b10a12390686cb206a254bf\"\ 22 | , \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 23 | : \" I do not actually have any favorite animal\", \"stop\": null, \"stop_reason\"\ 24 | : null, \"truncated\": false, \"log_id\": \"14f945b66b10a12390686cb206a254bf\"\ 25 | , \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 26 | : \" I do not actually have any favorite animal breeds,\", \"stop\": null,\ 27 | \ \"stop_reason\": null, \"truncated\": false, \"log_id\": \"14f945b66b10a12390686cb206a254bf\"\ 28 | , \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 29 | : \" I do not actually have any favorite animal breeds, as\", \"stop\": null,\ 30 | \ \"stop_reason\": null, \"truncated\": false, \"log_id\": \"14f945b66b10a12390686cb206a254bf\"\ 31 | , \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 32 | : \" I do not actually have any favorite animal breeds, as I am an\", \"stop\"\ 33 | : null, \"stop_reason\": null, \"truncated\": false, \"log_id\": \"14f945b66b10a12390686cb206a254bf\"\ 34 | , \"model\": \"claude-v1\", \"exception\": null}\r\n\r\ndata: {\"completion\"\ 35 | : \" I do not actually have any favorite animal breeds, as I am an AI assistant\ 36 | \ without\", \"stop\": null, \"stop_reason\": null, \"truncated\": false,\ 37 | \ \"log_id\": \"14f945b66b10a12390686cb206a254bf\", \"model\": \"claude-v1\"\ 38 | , \"exception\": null}\r\n\r\ndata: {\"completion\": \" I do not actually\ 39 | \ have any favorite animal breeds, as I am an AI assistant without personal\"\ 40 | , \"stop\": null, \"stop_reason\": null, \"truncated\": false, \"log_id\"\ 41 | : \"14f945b66b10a12390686cb206a254bf\", \"model\": \"claude-v1\", \"exception\"\ 42 | : null}\r\n\r\ndata: {\"completion\": \" I do not actually have any favorite\ 43 | \ animal breeds, as I am an AI assistant without personal preferences.\",\ 44 | \ \"stop\": null, \"stop_reason\": null, \"truncated\": false, \"log_id\"\ 45 | : \"14f945b66b10a12390686cb206a254bf\", \"model\": \"claude-v1\", \"exception\"\ 46 | : null}\r\n\r\ndata: {\"completion\": \" I do not actually have any favorite\ 47 | \ animal breeds, as I am an AI assistant without personal preferences.\",\ 48 | \ \"stop\": \"\\n\\nHuman:\", \"stop_reason\": \"stop_sequence\", \"truncated\"\ 49 | : false, \"log_id\": \"14f945b66b10a12390686cb206a254bf\", \"model\": \"claude-v1\"\ 50 | , \"exception\": null}\r\n\r\ndata: [DONE]\r\n\r\n" 51 | headers: 52 | Alt-Svc: 53 | - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 54 | Cache-Control: 55 | - no-cache 56 | Content-Type: 57 | - text/event-stream; charset=utf-8 58 | Date: 59 | - Sat, 20 May 2023 13:10:26 GMT 60 | Server: 61 | - Google Frontend 62 | Transfer-Encoding: 63 | - chunked 64 | Via: 65 | - 1.1 google 66 | request-id: 67 | - 14f945b66b10a12390686cb206a254bf 68 | x-accel-buffering: 69 | - 'no' 70 | status: 71 | code: 200 72 | message: OK 73 | url: https://api.anthropic.com/v1/complete 74 | version: 1 75 | -------------------------------------------------------------------------------- /tests/test_openai.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import asynctest 3 | import os 4 | import vcr 5 | import llm 6 | import llm.api.openaiapi 7 | import logging 8 | 9 | # Only get API keys if they exist 10 | _key = open(os.path.expanduser("~/.openai")).read().strip() if os.path.exists( 11 | os.path.expanduser("~/.openai")) else 'test' 12 | llm.set_api_key(openai=_key) 13 | 14 | 15 | class TestOpenAICompletions(unittest.TestCase): 16 | 17 | @vcr.use_cassette("tests/fixtures/openai/test_completion.yaml", filter_headers=['authorization']) 18 | def test_completion(self): 19 | prompt = "2+2 is how much? Good question, I think 2+2=" 20 | completion = llm.complete( 21 | prompt, engine="openai:text-davinci-002", max_tokens=1) 22 | # Assert result is 4 or 5 because sometimes OpenAI returns 2+2=5 23 | self.assertTrue(completion in ["4", "5"], completion) 24 | 25 | @vcr.use_cassette("tests/fixtures/openai/test_simple_chat.yaml", filter_headers=['authorization', 'x-api-key']) 26 | def test_simple_chat(self): 27 | self.assertEqual(llm.chat( 28 | "What is 2+2? Reply with just one number and no punctuaction.", engine="openai:gpt-3.5-turbo"), "4") 29 | self.assertEqual(llm.chat( 30 | ["What is 2+2? Reply with just one number and no punctuaction."], engine="openai:gpt-3.5-turbo"), "4") 31 | 32 | @vcr.use_cassette("tests/fixtures/openai/test_completion_chat_model.yaml", filter_headers=['authorization']) 33 | def test_completion_chat_model(self): 34 | """Test that we can use a chat model for completion.""" 35 | prompt = "Hello, my name is" 36 | completion = llm.complete( 37 | prompt, engine="openai:gpt-3.5-turbo", max_tokens=1) 38 | # Assert we only got one extra word that is capitalized 39 | self.assertEqual(len(completion.split(' ')), 1) 40 | self.assertTrue(completion[0].isupper(), completion) 41 | 42 | @vcr.use_cassette("tests/fixtures/openai/test_chat.yaml", filter_headers=['authorization']) 43 | def test_chat(self): 44 | messages = ["hello", "how are you?", "I'm good, thanks."] 45 | response = llm.chat(messages, engine="openai:gpt-3.5-turbo") 46 | # Make sure the response is sorta directionally correct, e.g. has more than 3 words and some punctuation. 47 | self.assertTrue(len(response.split(' ')) > 3) 48 | self.assertTrue( 49 | '.' in response or '!' in response or '?' in response, response) 50 | 51 | 52 | @vcr.use_cassette("tests/fixtures/openai/test_token_overflow.yaml", filter_headers=['authorization']) 53 | def test_token_overflow(self): 54 | """Test that we can handle a token overflow.""" 55 | prompt = "Answer basic math question. 2+2=X Then multiply X by 100. Final answer is?" 56 | normal_completion = llm.complete(prompt=prompt, engine="openai:gpt-3.5-turbo") 57 | chat_completion = llm.chat(prompt, engine="openai:gpt-3.5-turbo") 58 | self.assertTrue("400" in normal_completion, normal_completion) 59 | self.assertTrue("400" in chat_completion, chat_completion) 60 | llm.api.openaiapi.MODEL_CONTEXT_SIZE_LIMITS["gpt-3.5-turbo"] = 10 61 | normal_completion = llm.complete(prompt=prompt, engine="openai:gpt-3.5-turbo", prompt_overflow="trim") 62 | chat_completion = llm.chat(prompt, engine="openai:gpt-3.5-turbo", prompt_overflow="trim") 63 | self.assertTrue("4" in normal_completion and "400" not in normal_completion, normal_completion) 64 | self.assertTrue("4" in chat_completion and "400" not in chat_completion, chat_completion) 65 | 66 | 67 | class TestOpenAIStreaming(asynctest.TestCase): 68 | 69 | @vcr.use_cassette("tests/fixtures/openai/test_streaming_chat_method_full.yaml", filter_headers=['authorization', 'x-api-key']) 70 | async def test_streaming_chat_method_full(self): 71 | messages = ["what is your favorite cat breed?", "I like tabbies.", 72 | "And what about dogs?"] 73 | responses = [] 74 | async for response in llm.stream_chat(messages, engine="openai:gpt-3.5-turbo", stream_method="full"): 75 | responses.append(response) 76 | final_response = responses[-1] 77 | self.assertEqual("As an AI language model, I do not have preferences or emotions. However, I can provide you some information about different dog breeds if you would like to know.", 78 | final_response, final_response) 79 | 80 | @vcr.use_cassette("tests/fixtures/openai/test_streaming_chat_method_delta.yaml", filter_headers=['authorization', 'x-api-key']) 81 | async def test_streaming_chat_method_delta(self): 82 | messages = ["what is your favorite cat breed?", "I like tabbies.", 83 | "And what about dogs?"] 84 | responses = [] 85 | async for response in llm.stream_chat(messages, engine="openai:gpt-3.5-turbo", stream_method="delta"): 86 | responses.append(response) 87 | self.assertEqual(responses[0], 'As') 88 | self.assertEqual(responses[1], ' an') 89 | -------------------------------------------------------------------------------- /llm/main.py: -------------------------------------------------------------------------------- 1 | """Simple LLM API for Python.""" 2 | 3 | # Usage: 4 | # import llm 5 | # llm.complete("hello, I am an animal called") --> "cat" # Uses GPT-3 by default if key is provided 6 | # llm.complete("hello, I am an animal called", engine="huggingface/roberta-base") --> "cat" 7 | # llm.chat(["hello", "hi", "how are you?"], system="Behave like a goat.") 8 | # Also try: 9 | # llm.setup_cache() and all requests will be cached 10 | 11 | from llm.utils.parsing import parse_args, structure_chat, format_streaming_output 12 | from llm.utils.apikeys import load_keys_from_cache, configure_api_keys 13 | from llm.api import anthropicapi, openaiapi 14 | import asyncio 15 | 16 | # Try loading keys from cache 17 | load_keys_from_cache() 18 | 19 | 20 | def complete(prompt, engine="openai:text-davinci-003", **kwargs): 21 | args = parse_args(engine, **kwargs) 22 | if args.service == "openai": 23 | result = openaiapi.complete(prompt, args.engine, **args.kwargs) 24 | elif args.service == "anthropic": 25 | result = anthropicapi.complete(prompt, args.engine, **args.kwargs) 26 | else: 27 | raise ValueError(f"Engine {engine} is not supported.") 28 | return result.strip() 29 | 30 | 31 | # Can also pass in system="Behave like a bunny rabbit" for system message. 32 | def chat(messages, engine="openai:gpt-3.5-turbo", **kwargs): 33 | """Chat with the LLM API.""" 34 | args = parse_args(engine, **kwargs) 35 | messages = structure_chat(messages) 36 | if args.service == "openai": 37 | result = openaiapi.chat(messages, args.engine, **args.kwargs) 38 | elif args.service == "anthropic": 39 | result = anthropicapi.chat(messages, args.engine, **args.kwargs) 40 | else: 41 | raise ValueError(f"Engine {engine} is not supported.") 42 | return result.strip() 43 | 44 | 45 | async def stream_chat(messages, engine="openai:gpt-3.5-turbo", stream_method="default", **kwargs): 46 | """Chat with the LLM API. 47 | stream_method="default" uses the default streaming method for the engine. 48 | For OpenAI this will be "delta" and for Anthropic (which restreams the 49 | output on every `yield`, it would be "full". 50 | """ 51 | args = parse_args(engine, **kwargs) 52 | messages = structure_chat(messages) 53 | if args.service == "openai": 54 | result = openaiapi.stream_chat(messages, args.engine, **args.kwargs) 55 | elif args.service == "anthropic": 56 | result = anthropicapi.stream_chat(messages, args.engine, **args.kwargs) 57 | else: 58 | raise ValueError(f"Engine {engine} is not supported.") 59 | 60 | output_cache = [] 61 | initial_response_done = False 62 | async for raw_token in result: 63 | token, output_cache = format_streaming_output( 64 | raw_token, stream_method, args.service, output_cache) 65 | if args.service == "anthropic": 66 | # First token of Anthropic output always starts with an empty space. 67 | # If we're in delta mode, strip the space only if this is the first token out. 68 | # If we're in full mode, strip the space every time. 69 | if stream_method == "delta" and not initial_response_done: 70 | token = token.lstrip() 71 | elif stream_method == "default" or stream_method == "full": 72 | token = token.lstrip() 73 | initial_response_done = True 74 | yield token 75 | 76 | 77 | async def multi_stream_chat(messages, engines=["anthropic:claude-instant-v1", "openai:gpt-3.5-turbo"], **kwargs): 78 | """Chat with multiple LLM APIs simultaneously. 79 | 80 | engines should be a list of engine strings, e.g. 81 | ["openai:gpt-3.5-turbo", "anthropic:claude-v1"] 82 | 83 | The responses will be yielded in the order they are received from the engines. 84 | Each response is returned as a tuple of the form (engine_name, response). 85 | """ 86 | if 'engine' in kwargs: 87 | raise ValueError("multi_stream_chat does not accept an engine argument, use stream_chat instead.") 88 | if 'stream_method' in kwargs: 89 | raise ValueError("multi_stream_chat does not accept a stream_method argument, it is set to 'full' for now.") 90 | # Create a list of streams, each one labeled with the name of the engine 91 | streams = [(engine, stream_chat(messages, engine, "full", **kwargs)) 92 | for engine in engines] 93 | 94 | while streams: # While there are still streams left 95 | for i, (engine, stream) in enumerate(streams): 96 | try: 97 | result = await stream.__anext__() 98 | # Yield the result along with the engine name 99 | yield (engine, result) 100 | except StopAsyncIteration: # The stream has ended 101 | del streams[i] # Remove it from the list 102 | 103 | 104 | def set_api_key(*args, **kwargs): 105 | """Set the OpenAI API key. Call me like this: 106 | - llm.set_api_keys(openai="sk-...") 107 | - llm.set_api_keys("path/to/api_keys.json") 108 | """ 109 | return configure_api_keys(*args, **kwargs) 110 | -------------------------------------------------------------------------------- /tests/fixtures/openai/test_simple_chat.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What 4 | is 2+2? Reply with just one number and no punctuaction."}]}' 5 | headers: 6 | Accept: 7 | - '*/*' 8 | Accept-Encoding: 9 | - gzip, deflate 10 | Connection: 11 | - keep-alive 12 | Content-Length: 13 | - '133' 14 | Content-Type: 15 | - application/json 16 | User-Agent: 17 | - OpenAI/v1 PythonBindings/0.27.2 18 | X-OpenAI-Client-User-Agent: 19 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 20 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 21 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 22 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 23 | method: POST 24 | uri: https://api.openai.com/v1/chat/completions 25 | response: 26 | body: 27 | string: !!binary | 28 | H4sIAAAAAAAAA0SOwUrDQBRF937GXU9KYmvazk7oolUEXShGKWUyeW2mSeYNmRcRQv5dCmq3l3MP 29 | Z4SroGFrI7YLbbLcPRVfhdu8P38U293rOQ8vD0N5n741zeMRClyeycrvY2a5Cy2JYw8F25MRqqCz 30 | fLXIs3S9Wit0XFELjVOQZD67S2ToS07SeZpBYYjmRNAjQs9dkINwQz5C3+YKV/X/nCkIi2mv3HJS 31 | sDU7SxH6c0RH8U/Zc0vQMDG6KMbLJZC9kL/ELzApHJ13sT70ZCJ7aEThAAXnK/qGTqf9dPMDAAD/ 32 | /wMAuVynkh4BAAA= 33 | headers: 34 | CF-Cache-Status: 35 | - DYNAMIC 36 | CF-RAY: 37 | - 7ca6fd9bce2767e7-MIA 38 | Cache-Control: 39 | - no-cache, must-revalidate 40 | Connection: 41 | - keep-alive 42 | Content-Encoding: 43 | - gzip 44 | Content-Type: 45 | - application/json 46 | Date: 47 | - Sat, 20 May 2023 19:29:50 GMT 48 | Server: 49 | - cloudflare 50 | Transfer-Encoding: 51 | - chunked 52 | access-control-allow-origin: 53 | - '*' 54 | alt-svc: 55 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 56 | openai-model: 57 | - gpt-3.5-turbo-0301 58 | openai-organization: 59 | - dgorg 60 | openai-processing-ms: 61 | - '602' 62 | openai-version: 63 | - '2020-10-01' 64 | strict-transport-security: 65 | - max-age=15724800; includeSubDomains 66 | x-ratelimit-limit-requests: 67 | - '3500' 68 | x-ratelimit-limit-tokens: 69 | - '90000' 70 | x-ratelimit-remaining-requests: 71 | - '3499' 72 | x-ratelimit-remaining-tokens: 73 | - '89968' 74 | x-ratelimit-reset-requests: 75 | - 17ms 76 | x-ratelimit-reset-tokens: 77 | - 21ms 78 | x-request-id: 79 | - 4efd9e88b2679fd6b577d60f53e297ad 80 | status: 81 | code: 200 82 | message: OK 83 | - request: 84 | body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What 85 | is 2+2? Reply with just one number and no punctuaction."}]}' 86 | headers: 87 | Accept: 88 | - '*/*' 89 | Accept-Encoding: 90 | - gzip, deflate 91 | Connection: 92 | - keep-alive 93 | Content-Length: 94 | - '133' 95 | Content-Type: 96 | - application/json 97 | User-Agent: 98 | - OpenAI/v1 PythonBindings/0.27.2 99 | X-OpenAI-Client-User-Agent: 100 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 101 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 102 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 103 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 104 | method: POST 105 | uri: https://api.openai.com/v1/chat/completions 106 | response: 107 | body: 108 | string: !!binary | 109 | H4sIAAAAAAAAA0SOwUrDQBRF937GXU9KYmJiZ12QQuvOiIiU6eSZjM3MGzKvqIT8uxTUbi/nHs4M 110 | 10HDDkasj2PWbPcvn/3ztt2UD1VJ35udb82jNNKapx0U+PhBVn4fK8s+jiSOAxTsREaogy7q+6ou 111 | 8vU6V/Dc0QiNPkpWru4yOU9HzvIyL6BwTqYn6BlxYh/lIHyikKBva4Wr+n8uFITFjFeuWRTswM5S 112 | gn6d4Sn9KSceCRomJZfEBLkEchAKl/gKi8K7Cy4Nh4lM4gCNJByh4EJHX9D58rbc/AAAAP//AwAX 113 | FG/6HgEAAA== 114 | headers: 115 | CF-Cache-Status: 116 | - DYNAMIC 117 | CF-RAY: 118 | - 7ca6fda01e5867e7-MIA 119 | Cache-Control: 120 | - no-cache, must-revalidate 121 | Connection: 122 | - keep-alive 123 | Content-Encoding: 124 | - gzip 125 | Content-Type: 126 | - application/json 127 | Date: 128 | - Sat, 20 May 2023 19:29:50 GMT 129 | Server: 130 | - cloudflare 131 | Transfer-Encoding: 132 | - chunked 133 | access-control-allow-origin: 134 | - '*' 135 | alt-svc: 136 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 137 | openai-model: 138 | - gpt-3.5-turbo-0301 139 | openai-organization: 140 | - dgorg 141 | openai-processing-ms: 142 | - '576' 143 | openai-version: 144 | - '2020-10-01' 145 | strict-transport-security: 146 | - max-age=15724800; includeSubDomains 147 | x-ratelimit-limit-requests: 148 | - '3500' 149 | x-ratelimit-limit-tokens: 150 | - '90000' 151 | x-ratelimit-remaining-requests: 152 | - '3499' 153 | x-ratelimit-remaining-tokens: 154 | - '89967' 155 | x-ratelimit-reset-requests: 156 | - 17ms 157 | x-ratelimit-reset-tokens: 158 | - 21ms 159 | x-request-id: 160 | - 3fe588f6a5696eee6017dd0f94dfee4b 161 | status: 162 | code: 200 163 | message: OK 164 | version: 1 165 | -------------------------------------------------------------------------------- /tests/fixtures/multi_stream/test_multistream.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"stream": true, "prompt": "\n\nHuman: Write a poem about the number pi.\n\nAssistant:", 4 | "stop_sequences": ["\n\nHuman:"], "max_tokens_to_sample": 10, "model": "claude-instant-v1"}' 5 | headers: 6 | Accept: 7 | - application/json 8 | Client: 9 | - anthropic-python/0.2.8 10 | Content-Type: 11 | - application/json 12 | method: post 13 | uri: https://api.anthropic.com/v1/complete 14 | response: 15 | body: 16 | string: "data: {\"completion\": \" Here is a poem\", \"stop\": null, \"stop_reason\"\ 17 | : null, \"truncated\": false, \"log_id\": \"f1ef089ce4dfdca9d54e51f1b239b445\"\ 18 | , \"model\": \"claude-instant-v1\", \"exception\": null}\r\n\r\ndata: {\"\ 19 | completion\": \" Here is a poem I\", \"stop\": null, \"stop_reason\": null,\ 20 | \ \"truncated\": false, \"log_id\": \"f1ef089ce4dfdca9d54e51f1b239b445\",\ 21 | \ \"model\": \"claude-instant-v1\", \"exception\": null}\r\n\r\ndata: {\"\ 22 | completion\": \" Here is a poem I wrote\", \"stop\": null, \"stop_reason\"\ 23 | : null, \"truncated\": false, \"log_id\": \"f1ef089ce4dfdca9d54e51f1b239b445\"\ 24 | , \"model\": \"claude-instant-v1\", \"exception\": null}\r\n\r\ndata: {\"\ 25 | completion\": \" Here is a poem I wrote about the\", \"stop\": null, \"stop_reason\"\ 26 | : null, \"truncated\": false, \"log_id\": \"f1ef089ce4dfdca9d54e51f1b239b445\"\ 27 | , \"model\": \"claude-instant-v1\", \"exception\": null}\r\n\r\ndata: {\"\ 28 | completion\": \" Here is a poem I wrote about the number pi\", \"stop\": null,\ 29 | \ \"stop_reason\": null, \"truncated\": false, \"log_id\": \"f1ef089ce4dfdca9d54e51f1b239b445\"\ 30 | , \"model\": \"claude-instant-v1\", \"exception\": null}\r\n\r\ndata: {\"\ 31 | completion\": \" Here is a poem I wrote about the number pi\", \"stop\": null,\ 32 | \ \"stop_reason\": \"max_tokens\", \"truncated\": false, \"log_id\": \"f1ef089ce4dfdca9d54e51f1b239b445\"\ 33 | , \"model\": \"claude-instant-v1\", \"exception\": null}\r\n\r\ndata: [DONE]\r\ 34 | \n\r\n" 35 | headers: 36 | Alt-Svc: 37 | - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 38 | Cache-Control: 39 | - no-cache 40 | Content-Type: 41 | - text/event-stream; charset=utf-8 42 | Date: 43 | - Sat, 20 May 2023 16:25:51 GMT 44 | Server: 45 | - Google Frontend 46 | Transfer-Encoding: 47 | - chunked 48 | Via: 49 | - 1.1 google 50 | request-id: 51 | - f1ef089ce4dfdca9d54e51f1b239b445 52 | x-accel-buffering: 53 | - 'no' 54 | status: 55 | code: 200 56 | message: OK 57 | url: https://api.anthropic.com/v1/complete 58 | - request: 59 | body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Write 60 | a poem about the number pi."}], "stream": true, "max_tokens": 10}' 61 | headers: 62 | Accept: 63 | - '*/*' 64 | Accept-Encoding: 65 | - gzip, deflate 66 | Connection: 67 | - keep-alive 68 | Content-Length: 69 | - '140' 70 | Content-Type: 71 | - application/json 72 | User-Agent: 73 | - OpenAI/v1 PythonBindings/0.27.2 74 | X-OpenAI-Client-User-Agent: 75 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 76 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 77 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 78 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 79 | method: POST 80 | uri: https://api.openai.com/v1/chat/completions 81 | response: 82 | body: 83 | string: "data: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 84 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 85 | ,\"choices\":[{\"delta\":{\"role\":\"assistant\"},\"index\":0,\"finish_reason\"\ 86 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 87 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 88 | ,\"choices\":[{\"delta\":{\"content\":\"Pi\"},\"index\":0,\"finish_reason\"\ 89 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 90 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 91 | ,\"choices\":[{\"delta\":{\"content\":\",\"},\"index\":0,\"finish_reason\"\ 92 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 93 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 94 | ,\"choices\":[{\"delta\":{\"content\":\" the\"},\"index\":0,\"finish_reason\"\ 95 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 96 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 97 | ,\"choices\":[{\"delta\":{\"content\":\" circle\"},\"index\":0,\"finish_reason\"\ 98 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 99 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 100 | ,\"choices\":[{\"delta\":{\"content\":\"\u2019s\"},\"index\":0,\"finish_reason\"\ 101 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 102 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 103 | ,\"choices\":[{\"delta\":{\"content\":\" constant\"},\"index\":0,\"finish_reason\"\ 104 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 105 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 106 | ,\"choices\":[{\"delta\":{\"content\":\" friend\"},\"index\":0,\"finish_reason\"\ 107 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 108 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 109 | ,\"choices\":[{\"delta\":{\"content\":\",\\n\"},\"index\":0,\"finish_reason\"\ 110 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 111 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 112 | ,\"choices\":[{\"delta\":{\"content\":\"A\"},\"index\":0,\"finish_reason\"\ 113 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 114 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 115 | ,\"choices\":[{\"delta\":{\"content\":\" number\"},\"index\":0,\"finish_reason\"\ 116 | :null}]}\n\ndata: {\"id\":\"chatcmpl-7IJgurkSTpTFaZZ5ToMkHDMwYU0h4\",\"object\"\ 117 | :\"chat.completion.chunk\",\"created\":1684599952,\"model\":\"gpt-3.5-turbo-0301\"\ 118 | ,\"choices\":[{\"delta\":{},\"index\":0,\"finish_reason\":\"length\"}]}\n\n\ 119 | data: [DONE]\n\n" 120 | headers: 121 | CF-Cache-Status: 122 | - DYNAMIC 123 | CF-RAY: 124 | - 7ca5f022885a67d2-MIA 125 | Cache-Control: 126 | - no-cache, must-revalidate 127 | Connection: 128 | - keep-alive 129 | Content-Type: 130 | - text/event-stream 131 | Date: 132 | - Sat, 20 May 2023 16:25:52 GMT 133 | Server: 134 | - cloudflare 135 | Transfer-Encoding: 136 | - chunked 137 | access-control-allow-origin: 138 | - '*' 139 | alt-svc: 140 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 141 | openai-model: 142 | - gpt-3.5-turbo-0301 143 | openai-organization: 144 | - dgorg 145 | openai-processing-ms: 146 | - '606' 147 | openai-version: 148 | - '2020-10-01' 149 | strict-transport-security: 150 | - max-age=15724800; includeSubDomains 151 | x-ratelimit-limit-requests: 152 | - '3500' 153 | x-ratelimit-limit-tokens: 154 | - '90000' 155 | x-ratelimit-remaining-requests: 156 | - '3499' 157 | x-ratelimit-remaining-tokens: 158 | - '89979' 159 | x-ratelimit-reset-requests: 160 | - 17ms 161 | x-ratelimit-reset-tokens: 162 | - 13ms 163 | x-request-id: 164 | - a1a41913e4cf3b3af476ced0c16f02a1 165 | status: 166 | code: 200 167 | message: OK 168 | version: 1 169 | -------------------------------------------------------------------------------- /tests/fixtures/openai/test_streaming_chat_method_full.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "what 4 | is your favorite cat breed?"}, {"role": "assistant", "content": "I like tabbies."}, 5 | {"role": "user", "content": "And what about dogs?"}], "stream": true}' 6 | headers: 7 | Accept: 8 | - '*/*' 9 | Accept-Encoding: 10 | - gzip, deflate 11 | Connection: 12 | - keep-alive 13 | Content-Length: 14 | - '227' 15 | Content-Type: 16 | - application/json 17 | User-Agent: 18 | - OpenAI/v1 PythonBindings/0.27.2 19 | X-OpenAI-Client-User-Agent: 20 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 21 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 22 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 23 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 24 | method: POST 25 | uri: https://api.openai.com/v1/chat/completions 26 | response: 27 | body: 28 | string: 'data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"role":"assistant"},"index":0,"finish_reason":null}]} 29 | 30 | 31 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"As"},"index":0,"finish_reason":null}]} 32 | 33 | 34 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 35 | an"},"index":0,"finish_reason":null}]} 36 | 37 | 38 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 39 | AI"},"index":0,"finish_reason":null}]} 40 | 41 | 42 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 43 | language"},"index":0,"finish_reason":null}]} 44 | 45 | 46 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 47 | model"},"index":0,"finish_reason":null}]} 48 | 49 | 50 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 51 | 52 | 53 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 54 | I"},"index":0,"finish_reason":null}]} 55 | 56 | 57 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 58 | do"},"index":0,"finish_reason":null}]} 59 | 60 | 61 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 62 | not"},"index":0,"finish_reason":null}]} 63 | 64 | 65 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 66 | have"},"index":0,"finish_reason":null}]} 67 | 68 | 69 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 70 | preferences"},"index":0,"finish_reason":null}]} 71 | 72 | 73 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 74 | or"},"index":0,"finish_reason":null}]} 75 | 76 | 77 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 78 | emotions"},"index":0,"finish_reason":null}]} 79 | 80 | 81 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"."},"index":0,"finish_reason":null}]} 82 | 83 | 84 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 85 | However"},"index":0,"finish_reason":null}]} 86 | 87 | 88 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 89 | 90 | 91 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 92 | I"},"index":0,"finish_reason":null}]} 93 | 94 | 95 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 96 | can"},"index":0,"finish_reason":null}]} 97 | 98 | 99 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 100 | provide"},"index":0,"finish_reason":null}]} 101 | 102 | 103 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 104 | you"},"index":0,"finish_reason":null}]} 105 | 106 | 107 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 108 | some"},"index":0,"finish_reason":null}]} 109 | 110 | 111 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 112 | information"},"index":0,"finish_reason":null}]} 113 | 114 | 115 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 116 | about"},"index":0,"finish_reason":null}]} 117 | 118 | 119 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 120 | different"},"index":0,"finish_reason":null}]} 121 | 122 | 123 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 124 | dog"},"index":0,"finish_reason":null}]} 125 | 126 | 127 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 128 | breeds"},"index":0,"finish_reason":null}]} 129 | 130 | 131 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 132 | if"},"index":0,"finish_reason":null}]} 133 | 134 | 135 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 136 | you"},"index":0,"finish_reason":null}]} 137 | 138 | 139 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 140 | would"},"index":0,"finish_reason":null}]} 141 | 142 | 143 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 144 | like"},"index":0,"finish_reason":null}]} 145 | 146 | 147 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 148 | to"},"index":0,"finish_reason":null}]} 149 | 150 | 151 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 152 | know"},"index":0,"finish_reason":null}]} 153 | 154 | 155 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"."},"index":0,"finish_reason":null}]} 156 | 157 | 158 | data: {"id":"chatcmpl-7IMZ1lOOZwhvvnxG0BGDHMdqSF7zl","object":"chat.completion.chunk","created":1684610995,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{},"index":0,"finish_reason":"stop"}]} 159 | 160 | 161 | data: [DONE] 162 | 163 | 164 | ' 165 | headers: 166 | CF-Cache-Status: 167 | - DYNAMIC 168 | CF-RAY: 169 | - 7ca6fdc0cbdab3da-MIA 170 | Cache-Control: 171 | - no-cache, must-revalidate 172 | Connection: 173 | - keep-alive 174 | Content-Type: 175 | - text/event-stream 176 | Date: 177 | - Sat, 20 May 2023 19:29:55 GMT 178 | Server: 179 | - cloudflare 180 | Transfer-Encoding: 181 | - chunked 182 | access-control-allow-origin: 183 | - '*' 184 | alt-svc: 185 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 186 | openai-model: 187 | - gpt-3.5-turbo-0301 188 | openai-organization: 189 | - dgorg 190 | openai-processing-ms: 191 | - '365' 192 | openai-version: 193 | - '2020-10-01' 194 | strict-transport-security: 195 | - max-age=15724800; includeSubDomains 196 | x-ratelimit-limit-requests: 197 | - '3500' 198 | x-ratelimit-limit-tokens: 199 | - '90000' 200 | x-ratelimit-remaining-requests: 201 | - '3499' 202 | x-ratelimit-remaining-tokens: 203 | - '89964' 204 | x-ratelimit-reset-requests: 205 | - 17ms 206 | x-ratelimit-reset-tokens: 207 | - 24ms 208 | x-request-id: 209 | - b7d173f613c14a97a9fb7528f497be8d 210 | status: 211 | code: 200 212 | message: OK 213 | version: 1 214 | -------------------------------------------------------------------------------- /tests/fixtures/openai/test_streaming_chat_method_delta.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "what 4 | is your favorite cat breed?"}, {"role": "assistant", "content": "I like tabbies."}, 5 | {"role": "user", "content": "And what about dogs?"}], "stream": true}' 6 | headers: 7 | Accept: 8 | - '*/*' 9 | Accept-Encoding: 10 | - gzip, deflate 11 | Connection: 12 | - keep-alive 13 | Content-Length: 14 | - '227' 15 | Content-Type: 16 | - application/json 17 | User-Agent: 18 | - OpenAI/v1 PythonBindings/0.27.2 19 | X-OpenAI-Client-User-Agent: 20 | - '{"bindings_version": "0.27.2", "httplib": "requests", "lang": "python", "lang_version": 21 | "3.10.9", "platform": "macOS-13.2-arm64-arm-64bit", "publisher": "openai", 22 | "uname": "Darwin 22.3.0 Darwin Kernel Version 22.3.0: Thu Jan 5 20:50:36 23 | PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6020 arm64 arm"}' 24 | method: POST 25 | uri: https://api.openai.com/v1/chat/completions 26 | response: 27 | body: 28 | string: 'data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"role":"assistant"},"index":0,"finish_reason":null}]} 29 | 30 | 31 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"As"},"index":0,"finish_reason":null}]} 32 | 33 | 34 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 35 | an"},"index":0,"finish_reason":null}]} 36 | 37 | 38 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 39 | AI"},"index":0,"finish_reason":null}]} 40 | 41 | 42 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 43 | language"},"index":0,"finish_reason":null}]} 44 | 45 | 46 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 47 | model"},"index":0,"finish_reason":null}]} 48 | 49 | 50 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 51 | 52 | 53 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 54 | I"},"index":0,"finish_reason":null}]} 55 | 56 | 57 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 58 | do"},"index":0,"finish_reason":null}]} 59 | 60 | 61 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 62 | not"},"index":0,"finish_reason":null}]} 63 | 64 | 65 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 66 | have"},"index":0,"finish_reason":null}]} 67 | 68 | 69 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 70 | personal"},"index":0,"finish_reason":null}]} 71 | 72 | 73 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 74 | preferences"},"index":0,"finish_reason":null}]} 75 | 76 | 77 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 78 | or"},"index":0,"finish_reason":null}]} 79 | 80 | 81 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 82 | emotions"},"index":0,"finish_reason":null}]} 83 | 84 | 85 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"."},"index":0,"finish_reason":null}]} 86 | 87 | 88 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 89 | However"},"index":0,"finish_reason":null}]} 90 | 91 | 92 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 93 | 94 | 95 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 96 | some"},"index":0,"finish_reason":null}]} 97 | 98 | 99 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 100 | popular"},"index":0,"finish_reason":null}]} 101 | 102 | 103 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 104 | dog"},"index":0,"finish_reason":null}]} 105 | 106 | 107 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 108 | breeds"},"index":0,"finish_reason":null}]} 109 | 110 | 111 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 112 | are"},"index":0,"finish_reason":null}]} 113 | 114 | 115 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 116 | Labrador"},"index":0,"finish_reason":null}]} 117 | 118 | 119 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 120 | Ret"},"index":0,"finish_reason":null}]} 121 | 122 | 123 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"ri"},"index":0,"finish_reason":null}]} 124 | 125 | 126 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"ever"},"index":0,"finish_reason":null}]} 127 | 128 | 129 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 130 | 131 | 132 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 133 | German"},"index":0,"finish_reason":null}]} 134 | 135 | 136 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 137 | Shepherd"},"index":0,"finish_reason":null}]} 138 | 139 | 140 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 141 | 142 | 143 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 144 | Golden"},"index":0,"finish_reason":null}]} 145 | 146 | 147 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 148 | Ret"},"index":0,"finish_reason":null}]} 149 | 150 | 151 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"ri"},"index":0,"finish_reason":null}]} 152 | 153 | 154 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"ever"},"index":0,"finish_reason":null}]} 155 | 156 | 157 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 158 | 159 | 160 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 161 | French"},"index":0,"finish_reason":null}]} 162 | 163 | 164 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 165 | Bulld"},"index":0,"finish_reason":null}]} 166 | 167 | 168 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"og"},"index":0,"finish_reason":null}]} 169 | 170 | 171 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 172 | 173 | 174 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 175 | P"},"index":0,"finish_reason":null}]} 176 | 177 | 178 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"oodle"},"index":0,"finish_reason":null}]} 179 | 180 | 181 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 182 | 183 | 184 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 185 | Be"},"index":0,"finish_reason":null}]} 186 | 187 | 188 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"agle"},"index":0,"finish_reason":null}]} 189 | 190 | 191 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":","},"index":0,"finish_reason":null}]} 192 | 193 | 194 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 195 | and"},"index":0,"finish_reason":null}]} 196 | 197 | 198 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":" 199 | Bulld"},"index":0,"finish_reason":null}]} 200 | 201 | 202 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"og"},"index":0,"finish_reason":null}]} 203 | 204 | 205 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{"content":"."},"index":0,"finish_reason":null}]} 206 | 207 | 208 | data: {"id":"chatcmpl-7IMYw1eiLe7ZMT6ijS6zBziaeksmK","object":"chat.completion.chunk","created":1684610990,"model":"gpt-3.5-turbo-0301","choices":[{"delta":{},"index":0,"finish_reason":"stop"}]} 209 | 210 | 211 | data: [DONE] 212 | 213 | 214 | ' 215 | headers: 216 | CF-Cache-Status: 217 | - DYNAMIC 218 | CF-RAY: 219 | - 7ca6fda57bf90699-MIA 220 | Cache-Control: 221 | - no-cache, must-revalidate 222 | Connection: 223 | - keep-alive 224 | Content-Type: 225 | - text/event-stream 226 | Date: 227 | - Sat, 20 May 2023 19:29:51 GMT 228 | Server: 229 | - cloudflare 230 | Transfer-Encoding: 231 | - chunked 232 | access-control-allow-origin: 233 | - '*' 234 | alt-svc: 235 | - h3=":443"; ma=86400, h3-29=":443"; ma=86400 236 | openai-model: 237 | - gpt-3.5-turbo-0301 238 | openai-organization: 239 | - dgorg 240 | openai-processing-ms: 241 | - '269' 242 | openai-version: 243 | - '2020-10-01' 244 | strict-transport-security: 245 | - max-age=15724800; includeSubDomains 246 | x-ratelimit-limit-requests: 247 | - '3500' 248 | x-ratelimit-limit-tokens: 249 | - '90000' 250 | x-ratelimit-remaining-requests: 251 | - '3499' 252 | x-ratelimit-remaining-tokens: 253 | - '89964' 254 | x-ratelimit-reset-requests: 255 | - 17ms 256 | x-ratelimit-reset-tokens: 257 | - 24ms 258 | x-request-id: 259 | - 5e9763a34019fa1d6c83cfa49f57f751 260 | status: 261 | code: 200 262 | message: OK 263 | version: 1 264 | --------------------------------------------------------------------------------