├── .gitignore ├── CHANGELOG.md ├── INSTALL.md ├── LICENSE ├── README.md ├── docs ├── DSL_DESIGN.md ├── MARKDOWN.md └── MIDDLEWARE.md ├── examples ├── README.md ├── agents │ ├── README.md │ └── main.py ├── cache │ ├── README.md │ └── main.py ├── chaining_answers │ ├── README.md │ └── main.py ├── chatbot │ └── main.py ├── custom_service │ └── main.py ├── first_agent │ ├── README.md │ └── main.py ├── first_example │ ├── README.md │ └── main.py ├── format │ ├── README.md │ └── main.py ├── havershell │ └── main.py ├── images │ ├── README.md │ ├── edinburgh.png │ └── main.py ├── options │ ├── README.md │ └── main.py ├── others │ ├── proof_reading.py │ └── sentence_iterations.py ├── together │ └── main.py ├── tools │ └── main.py └── tree_of_calls │ ├── README.md │ └── main.py ├── pyproject.toml ├── src └── haverscript │ ├── __init__.py │ ├── agents.py │ ├── cache.py │ ├── chatbot.py │ ├── exceptions.py │ ├── haverscript.py │ ├── markdown.py │ ├── middleware.py │ ├── ollama.py │ ├── render.py │ ├── together.py │ ├── tools.py │ └── types.py └── tests ├── docs └── test_docs.py ├── e2e ├── test_e2e_haverscript.py └── test_e2e_haverscript │ ├── test_cache.2.txt │ ├── test_cache.3.txt │ ├── test_cache.together.2.txt │ ├── test_cache.together.3.txt │ ├── test_cache.txt │ ├── test_chaining_answers.together.txt │ ├── test_chaining_answers.txt │ ├── test_chatbot.together.txt │ ├── test_chatbot.txt │ ├── test_check.txt │ ├── test_custom_service.as_is.txt │ ├── test_first_agent.txt │ ├── test_first_example.together.txt │ ├── test_first_example.txt │ ├── test_first_example_together.txt │ ├── test_format.together.txt │ ├── test_format.txt │ ├── test_images.txt │ ├── test_options.together.txt │ ├── test_options.txt │ ├── test_together.together.txt │ ├── test_together.txt │ ├── test_tree_of_calls.together.txt │ ├── test_tree_of_calls.txt │ └── test_tree_of_calls_together.txt ├── test_utils.py └── unit └── test_unit_haverscript.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | 164 | venv/ 165 | .DS_Store 166 | *.db 167 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | Haverscript is a language for building LLM-based agents, providing a flexible 4 | and composable way to interact with large language models. This CHANGELOG 5 | documents notable changes to the Haverscript project. 6 | 7 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 8 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 9 | 10 | ## [0.3.1] - ??? 11 | ### Added 12 | - Added `str` and `Markdown` as request options to `Model.process`. 13 | ## Changed 14 | - `Agent.chat` now updates the `Agent` when called (the originally intended behaviour) 15 | - `Agent.remember` adds a prompt-reply pair to the internal context of an agent. 16 | - `realtime` `Middleware`, that takes callback and is called for every generated token 17 | 18 | ## [0.3.0] - 2025-02-26 19 | ### Added 20 | - Initial agentic support. An `Agent` is a python class that has access to an LLM. 21 | - `Markdown`, a simple DSL for building markdown-style prompts. 22 | - `tools` argument for `Model.chat`, that supplies function-calling callbacks. 23 | - `format` now supports standard types such as `list[int]` and `bool` for stuctured output. 24 | - `echo` middleware now takes an optional `stream` parameter, defaulting to `True`. 25 | - `Model.compress()` can remove older chat calls from the history, reducing context sizes. 26 | - `connect_chatbot` which promotes a chatbot into a `Model`. 27 | - `Model.ask()` which is a one-shot `Model.chat()` that returns dynamic content. 28 | - `Reply.informational` and `Reply.pure` generate specific Reply objects. 29 | - `ChatBot` class for wrapping up pseudo LLMs (or agents acting like LLMs). 30 | ### Deprecated 31 | - `dedent` has stubed out (has no effect). This has been replaced by Markdown support. 32 | - `meta` middleware is removed, and has been replaced by `ChatBot`. 33 | ### Changed 34 | - Internally, resolve refs from JSON schemas in format (this works around an ollama bug) 35 | - `Model` and `Response` are now pydantic classes. 36 | - `retry()` now takes a simple retry count, and no longer uses the tenacity package. 37 | 38 | ## [0.2.1] - 2024-12-30 39 | ### Added 40 | - Support for Python 3.10 and 3.11. Python 3.9 and earlier is not supported. 41 | 42 | ## [0.2.0] - 2024-12-30 43 | ### Added 44 | - Adding `Middleware` type for composable prompt and response handlers. 45 | - `Middleware` can be added using `|`, giving a small pipe-based representation of flow. 46 | The following middleware components are available: 47 | 48 | - `echo()` adds echoing of prompts and replies. 49 | - `retry()` which uses the tenacity package to provide a generic retry. 50 | - `validate()` which checks the response for a predicate. 51 | - `stats()` adds a dynamic single line summary of each LLM call. 52 | - `cache()` add a caching component. 53 | - `transcript()` adds a transcript component (transcripts the session to a file). 54 | - `trace()` logs the calls through the middleware in both directions. 55 | - `fresh()` requests a fresh call to the LLM. 56 | - `options()` sets specific options. 57 | - `model()` set the model being used. 58 | - `format()` requires the output in JSON, with an optional pydantic class schema. 59 | - `meta()` is a hook to allow middleware to act like a test-time LLM. 60 | 61 | - Adding prompt specific flags to `Model.chat`. 62 | - `images : list[str]` are images to be passed to the model. 63 | - `middleware: Middleware` appends a chat-specific middleware to the call. 64 | - Added `Service` class, that can be asked about models, and can generate `Model`s. 65 | - Added `response.value`, which return the JSON `dict` of the reply, the pydantic class, or `None`. 66 | - Added spinner when waiting for the first token from LLM when using `echo`. 67 | - Added `metrics` to `Response`, which contains basic metrics about the LLM call. 68 | - Added `render()` method to `Model`, for outputing markdown-style session viewing. 69 | - Added `load()` method to `Model`, for parsing markdown-style sessions. 70 | - Added `LLMError`, and subclasses. 71 | - Added support for together.ai's API as a first-class alternative to ollama. 72 | - Added many more examples. 73 | - Added many more tests. 74 | ### Fixed 75 | ### Changed 76 | - Updated `children` method to return all children when no prompt is supplied. 77 | - Reworked SQL cache schema to store context as chain of responses, and use a 78 | string pool. 79 | - Using the cache now uses LLM results in order, until exhausted, then calls the LLM. 80 | ### Removed 81 | This release includes breaking API changes, which are outlined below. In all 82 | cases, the functionality has been replaced with something more general and 83 | principled. 84 | 85 | The concepts that caused changes are 86 | - One you have a `Response`, that interaction with the LLM is considered done. 87 | There are no longer functions that attempt to re-run the call. Instead, middleware 88 | functions can be used to filter out responses as needed. 89 | - The is not longer the concept of a `Response` being "fresh". Instead, the 90 | cache uses a cursor when reading cached responses, and it is possible to ask 91 | that a specific interaction bypasses the cache (using the `fresh()` middleware). 92 | - Most helper methods (`echo()`, `cache()`, etc) are now Middleware, and thus 93 | more flexible. 94 | 95 | Specifically, here are the changes: 96 | - Removed `check()` and `redo()` from `Response`. 97 | Replace it with `validate()` and `retry()` *before* the call to chat, 98 | or as chat-specific middleware. 99 | - Removed `fresh` from `Response`. The concept of fresh responses has been replaced 100 | with a more robust caching middleware. There is now `fresh()` middleware. 101 | - Removed `json()` from `Model`. It is replaced with the more general 102 | `format()` middleware. 103 | - `echo()` and `cache()` are no longer `Model` methods, and now `Middleware` instances. 104 | - The utility functions `accept` and `valid_json` are removed. They added no value, 105 | given the removal of `redo`. 106 | 107 | So, previously we would have `session = connect("modelname").echo()`, and we now have 108 | `session = connect("modelname") | echo()`. 109 | 110 | 111 | ## [0.1.0] - 2024-09-23 112 | ### Initial release 113 | - First release of the project. 114 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | You can install Haverscript directly from the GitHub repository using `pip`. 4 | 5 | Here's how to set up Haverscript: 6 | 7 | 1. First, create and activate a Python virtual environment if you haven’t already: 8 | 9 | ```bash 10 | python3 -m venv venv 11 | source venv/bin/activate # On Windows: .\venv\Scripts\activate 12 | ``` 13 | 14 | 2. Install Haverscript directly from the GitHub repository: 15 | 16 | ```bash 17 | pip install "haverscript @ git+https://github.com/andygill/haverscript.git@v0.2.1" 18 | ``` 19 | 20 | By default, Haverscript comes with only Ollama support. 21 | If you want to also install the `together.ai` API support, you need to use 22 | 23 | ```bash 24 | pip install "haverscript[together] @ git+https://github.com/andygill/haverscript.git@v0.2.1" 25 | ``` 26 | 27 | In the future, if there’s enough interest, I plan to push Haverscript to PyPI 28 | for easier installation. 29 | 30 | # Download and install 31 | 32 | If you are a Haverscript user, then the commands above should work for you. 33 | 34 | However, if you want to make changes to Haverscript, you will need to download 35 | the repo, and build by hand. I use the following. 36 | 37 | ```bash 38 | git clone git@github.com:andygill/haverscript.git 39 | cd haverscript 40 | python3 -m venv venv 41 | source venv/bin/activate # On Windows: .\venv\Scripts\activate 42 | pip install -e ".[all]" 43 | ``` 44 | 45 | Now any local example can be run directly. 46 | 47 | ```shell 48 | python examples/first_example/main.py 49 | ``` 50 | 51 | ```markdown 52 | > In one sentence, why is the sky blue? 53 | 54 | The sky appears blue due to scattering of sunlight by molecules and particles in the Earth's atmosphere. 55 | 56 | ... 57 | ``` 58 | 59 | # Testing 60 | 61 | The unit tests are code fragments to test specific features The e2e tests 62 | execute a sub-process, testing the examples. The docs test check the docs 63 | are consistent with the given examples. 64 | 65 | ``` 66 | pytest tests # run all tests 67 | pytest tests/unit # run unit tests 68 | pytest tests/e2e # run e2e tests 69 | ``` 70 | 71 | You can run tests in parallel, using the `pytest-xdist` package. However, 72 | remember ollama will be the bottleneck here. 73 | 74 | ``` 75 | pytest -n auto 76 | ``` 77 | 78 | The e2e tests also uses `pytest-regressions`. The golden output is version controlled. 79 | 80 | We use the following models: 81 | * ollama mistral:v0.3 (f974a74358d6) 82 | * together.ai meta-llama/Meta-Llama-3-8B-Instruct-Lite 83 | * ollama llava:v1.6 (8dd30f6b0cb1) 84 | 85 | The tests for together need a valid TOGETHER_API_KEY in the environment when run. 86 | 87 | If you need to regenerate test output, use 88 | 89 | ``` 90 | pytest tests/e2e --force-regen 91 | ``` 92 | 93 | # Python versions 94 | 95 | We support Python 3.10 to 3.13. We test using the following commands. 96 | 97 | ``` 98 | rm -Rf venv ; python3.13 -m venv venv 99 | . ./venv/bin/activate 100 | pip install -e ".[all]" 101 | pytest -n auto -v 102 | python --version 103 | ``` 104 | 105 | and 106 | 107 | ``` 108 | rm -Rf venv ; python3.13 -m venv venv 109 | . ./venv/bin/activate 110 | pip install -e "." 111 | python examples/first_example/main.py 112 | python --version 113 | ``` 114 | 115 | # Release checklist 116 | 117 | [ ] documentation self consistant 118 | [ ] tests all pass 119 | [ ] unit tests for all public APIs 120 | [ ] examples used for docs testing 121 | [ ] examples used for e2e testing 122 | [ ] bump version number in pyproject 123 | [ ] check python 3.10 .. 3.13 124 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Andy Gill 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Haverscript 2 | 3 | Haverscript is a Python library for writing agents and interacting with Large 4 | Language Models (LLMs). Haverscript's concise syntax and powerful middleware 5 | allow for rapid prototyping with new use cases for LLMs, prompt engineering, 6 | and experimenting in the emerging field of LLM-powered agents. Haverscript uses 7 | [Ollama](https://ollama.com) by default but can use any OpenAI-style LLM API 8 | with a simple adapter. 9 | 10 | This is version 0.3 of Haverscript. The big change from 0.2 is the introduction 11 | of simple agents, as well as a new API for constructing markdown-based prompts, 12 | and a monadic way of chaining LLM calls. 13 | 14 | ## First Example 15 | 16 | Here’s a basic example demonstrating how to use Haverscript, 17 | with the [mistral](https://mistral.ai/news/announcing-mistral-7b/) model. 18 | 19 | ```python 20 | from haverscript import connect, echo 21 | 22 | # Create a new session with the 'mistral' model and enable echo middleware 23 | session = connect("mistral") | echo() 24 | 25 | session = session.chat("In one sentence, why is the sky blue?") 26 | session = session.chat("What color is the sky on Mars?") 27 | session = session.chat("Do any other planets have blue skies?") 28 | ``` 29 | 30 | This will generate the following output 31 | 32 | ```markdown 33 | > In one sentence, why is the sky blue? 34 | 35 | The sky appears blue due to a scattering effect called Rayleigh scattering 36 | where shorter wavelength light (blue light) is scattered more than other 37 | colors by the molecules in Earth's atmosphere. 38 | 39 | > What color is the sky on Mars? 40 | 41 | The Martian sky appears red or reddish-orange, primarily because of fine dust 42 | particles in its thin atmosphere that scatter sunlight preferentially in the 43 | red part of the spectrum, which our eyes perceive as a reddish hue. 44 | 45 | > Do any other planets have blue skies? 46 | 47 | Unlike Earth, none of the other known terrestrial planets (Venus, Mars, 48 | Mercury) have a significant enough atmosphere or suitable composition to cause 49 | Rayleigh scattering, resulting in blue skies like we see on Earth. However, 50 | some of the gas giant planets such as Uranus and Neptune can appear blueish 51 | due to their atmospheres composed largely of methane, which absorbs red light 52 | and scatters blue light. 53 | ``` 54 | 55 | Haverscript uses markdown as its output format, allowing for easy rendering of 56 | any chat session. 57 | 58 | ## First Agent Example 59 | 60 | Haverscript provides basic agent capabilities, that is calls to an LLM 61 | wrapped in a Python object. Here is a basic example. 62 | 63 | ```python 64 | from haverscript import connect, Agent 65 | 66 | 67 | class FirstAgent(Agent): 68 | system: str = """ 69 | You are a helpful AI assistant who answers questions in the style of 70 | Neil deGrasse Tyson. 71 | 72 | Answer any questions in 2-3 sentences, without preambles. 73 | """ 74 | 75 | def sky(self, planet: str) -> str: 76 | return self.ask(f"what color is the sky on {planet} and why?") 77 | 78 | 79 | firstAgent = FirstAgent(model=connect("mistral")) 80 | 81 | for planet in ["Earth", "Mars", "Venus", "Jupiter"]: 82 | print(f"{planet}: {firstAgent.sky(planet)}\n") 83 | ``` 84 | 85 | Running this will output the following: 86 | 87 | ``` 88 | Earth: The sky appears blue during a clear day on Earth due to a process called Rayleigh scattering, where shorter wavelengths of light, such as blue and violet, are scattered more by the molecules in our atmosphere. However, we perceive the sky as blue rather than violet because our eyes are more sensitive to blue light, and sunlight reaches us with less violet due to scattering events. 89 | 90 | Mars: The sky on Mars appears to be a reddish hue, primarily due to suspended iron oxide (rust) particles in its atmosphere. This gives Mars its characteristic reddish color as sunlight interacts with these particles. 91 | 92 | Venus: On Venus, the sky appears a dazzling white due to its thick clouds composed mainly of sulfuric acid. The reflection of sunlight off these clouds is responsible for this striking appearance. 93 | 94 | Jupiter: The sky on Jupiter isn't blue like Earth's; it's predominantly brownish due to the presence of ammonia crystals in its thick atmosphere. The reason for this difference lies in the unique composition and temperature conditions on Jupiter compared to our planet. 95 | ``` 96 | 97 | # Further reading 98 | 99 | The [examples](examples/README.md) directory contains several examples of Haverscript, 100 | both directly, and using agents. 101 | 102 | The [DSL Design](docs/DSL_DESIGN.md) page compares Haverscript to other LLM APIs, 103 | and gives rationale behind the design. 104 | 105 | The [Markdown](docs/MARKDOWN.md) explains how to construct structured prompts. 106 | 107 | ## Installing Haverscript 108 | 109 | Haverscript is available on GitHub: . 110 | While Haverscript is currently in beta and still being refined, it is ready to use 111 | out of the box. 112 | 113 | ### Prerequisites 114 | 115 | You need to have [Ollama](https://ollama.com) already installed. 116 | 117 | ### Installation 118 | 119 | You can install Haverscript directly from the GitHub repository using `pip`. 120 | **Haverscript needs Python 3.10 or later.** 121 | 122 | Here's how to set up Haverscript: 123 | 124 | 1. First, create and activate a Python virtual environment if you haven’t already: 125 | 126 | ```bash 127 | python3 -m venv venv 128 | source venv/bin/activate # On Windows: .\venv\Scripts\activate 129 | ``` 130 | 131 | 2. Install Haverscript directly from the GitHub repository: 132 | 133 | ```bash 134 | pip install "haverscript @ git+https://github.com/andygill/haverscript.git@v0.3.0" 135 | ``` 136 | 137 | By default, Haverscript comes with only Ollama support. 138 | If you want to also install the `together.ai` API support, you need to use 139 | 140 | ```bash 141 | pip install "haverscript[together] @ git+https://github.com/andygill/haverscript.git@v0.3.0" 142 | ``` 143 | 144 | In the future, if there’s enough interest, I plan to push Haverscript to PyPI 145 | for easier installation. 146 | 147 | See [INSTALL.md](INSTALL.md) for additional details about installing, testing and 148 | building Haverscript. 149 | 150 | ## Documentation 151 | 152 | ### The `chat` Method 153 | 154 | The `chat` method invokes the LLM, and is the principal method in HaveScript. 155 | Everything else in Haverscript is about setting up for `chat`, or using the 156 | output from `chat`. The `chat` method is available in both the `Model` class and 157 | its subclass `Response`: 158 | 159 | ```python 160 | class Model: 161 | ... 162 | def chat(self, prompt: str, ...) -> Response: 163 | 164 | class Response(Model): 165 | ... 166 | ``` 167 | 168 | Key points: 169 | - **Immutability**: Both `Model` and `Response` are immutable data structures, 170 | making them safe to share across threads or processes without concern for side 171 | effects. 172 | - **Chat Method**: The `chat` method accepts a simple Python string as input, 173 | which can include f-strings for formatted and dynamic prompts. 174 | 175 | Example: 176 | 177 | ```python 178 | def example(client: Model, txt: str): 179 | client.chat( 180 | f""" 181 | Help me understand what is happening here. 182 | 183 | {txt} 184 | """ 185 | ) 186 | ``` 187 | 188 | ### The `Response` Class 189 | 190 | The result of a `chat` call is a `Response`. This class contains several useful 191 | attributes and defines a `__str__` method for convenient string representation. 192 | 193 | ```python 194 | @dataclass 195 | class Response(Model): 196 | prompt: str 197 | reply: str 198 | parent: Model 199 | 200 | def __str__(self): 201 | return self.reply 202 | ... 203 | ``` 204 | 205 | Key Points: 206 | - **Accessing the Reply**: You can directly access the `reply` attribute to 207 | retrieve the text of the `Response`, or simply call `str(response)` for the 208 | same effect. 209 | 210 | - **String Representation**: The `__str__` method returns the `reply` attribute, 211 | so whenever a `Response` object is used inside an f-string, it automatically 212 | resolves to the text of the reply. (This is standard Python behavior.) 213 | 214 | For an example, see [Chaining answers together](examples/chaining_answers/README.md) 215 | 216 | How do we modify a `Model` if everything is immutable? Instead of modifying them 217 | directly, we create a new copy with every call to `.chat`, following the 218 | principles of functional programming, and using the builder design pattern. 219 | 220 | ### The `Model` Class 221 | 222 | The `connect(...)` function is the main entry point of the library, allowing you 223 | to create and access an initial model. This function takes a model 224 | name and returns a `Model` that will connect to Ollama with this model. 225 | 226 | ```python 227 | def connect(modelname: str | None = None): 228 | ... 229 | ``` 230 | 231 | To create and use a model, follow the idiomatic approach of naming the model and 232 | then using that name: 233 | 234 | ```python 235 | from haverscript import connect 236 | model = connect("mistral") 237 | response = model.chat("In one sentence, why is the sky blue?") 238 | print(f"Response: {response}") 239 | ``` 240 | 241 | You can create multiple models, including duplicates of the same model, without 242 | any issues. No external actions are triggered until the `chat` method is called; 243 | the external `connect` is deferred until needed. 244 | 245 | ### Chaining calls 246 | 247 | There are two primary ways to use `chat`: 248 | 249 | #### Chaining responses 250 | 251 | ```mermaid 252 | graph LR 253 | 254 | start((hs)) 255 | m0(Model) 256 | m1(session: Model) 257 | r0(session: Response) 258 | r1(session: Response) 259 | r2(session: Response) 260 | 261 | start -- model('…') --> m0 262 | m0 -- echo() --> m1 263 | m1 -- chat('…') --> r0 264 | r0 -- chat('…') --> r1 265 | r1 -- chat('…') --> r2 266 | 267 | style m0 fill:none, stroke: none 268 | 269 | ``` 270 | 271 | This follows the typical behavior of a chat session: using the output of one 272 | `chat` call as the input for the next. For more details, refer to the 273 | [first example](examples/first_example/README.md). 274 | 275 | #### Multiple independent calls 276 | 277 | ```mermaid 278 | graph LR 279 | 280 | start((hs)) 281 | m0(Model) 282 | m1(**session**: Model) 283 | r0(Response) 284 | r1(Response) 285 | r2(Response) 286 | 287 | start -- model('…') --> m0 288 | m0 -- echo() --> m1 289 | m1 -- chat('…') --> r0 290 | m1 -- chat('…') --> r1 291 | m1 -- chat('…') --> r2 292 | 293 | style m0 fill:none, stroke: none 294 | 295 | ``` 296 | 297 | Call `chat` multiple times with the same client instance to process different 298 | prompts separately. This way intentually loses the chained context, but in some 299 | cases you want to play a different persona, or do not allow the previous reply 300 | to cloud the next request. See [tree of calls](examples/tree_of_calls/README.md) 301 | for an example. 302 | 303 | 304 | ### Middleware 305 | 306 | Middleware is a mechanism to have fine control over everything between calling 307 | `.chat` and Haverscript calling the LLM. As an example, consider the creation of 308 | a session. 309 | 310 | ```python 311 | session = connect("mistral") | echo() 312 | ``` 313 | 314 | You can chain multiple middlewares together to achieve composite behaviors. 315 | 316 | ```python 317 | session = connect("mistral") | echo() | options(seed=12345) 318 | ``` 319 | 320 | Finally, you can also add middleware to a specific call to chat. 321 | 322 | ```python 323 | session = connect("mistral") | echo() 324 | print(session.chat("Hello", middleware=options(seed=12345))) 325 | ``` 326 | 327 | 328 | Haverscript provides following middleware: 329 | 330 | | Middleware | Purpose | Class | 331 | |------------|---------|-------| 332 | | model | Request a specific model be used | configuration | 333 | | options | Set specific LLM options (such as seed) | configuration | 334 | | format | Set specific format for output | configuration | 335 | | dedent | Remove spaces from prompt | configuration | 336 | | echo | Print prompt and reply | observation | 337 | | stats | Print basic stats about LLM | observation | 338 | | trace | Log requests and responses | observation | 339 | | transcript | Store a complete transcript of every call | observation | 340 | | retry | retry on failure | reliablity | 341 | | validation | Fail under given condition | reliablity | 342 | | cache | Store and/or query prompt-reply pairs in DB | efficency | 343 | | fresh | Request a fresh reply (not cached) | efficency | 344 | 345 | For a comprehensive overview of middleware, 346 | please refer to the [Middleware Docs](docs/MIDDLEWARE.md) documentation. 347 | For examples of middleware in used, see 348 | 349 | * [System prompt](examples/tree_of_calls/README.md) in tree of calls, 350 | * [enabling the cache](examples/cache/README.md), 351 | * [JSON output](examples/check/README.md) in checking output, and 352 | * [setting ollama options](examples/options/README.md). 353 | 354 | ### Chat and Ask Options 355 | 356 | The `.chat()` method has additional parameters that are specific 357 | to each chat call. 358 | 359 | ```python 360 | def chat( 361 | self, 362 | prompt: str | Markdown, 363 | images: list = [], 364 | middleware: Middleware | None = None, 365 | ) -> Response: 366 | ``` 367 | 368 | Specifically, `chat` takes things that are added to any context (prompt and 369 | images), and additionally, any extra middleware. 370 | 371 | There is also an `.ask()` method. 372 | 373 | ```python 374 | def ask( 375 | self, 376 | prompt: str | Markdown, 377 | images: list[str] = [], 378 | middleware: Middleware | None = None, 379 | ) -> Reply: 380 | ``` 381 | This returns a `Reply` object, which is a composable stream of tokens, 382 | as well as values from structured output, information packets, and 383 | performance metrics. `Response` is build from the `Reply`, while `Reply` 384 | has not context - it can be thought of as just the LLM's reply. 385 | 386 | ### Other APIs 387 | 388 | We support [together.ai](https://www.together.ai/). You need to provide your 389 | own API KEY. Import the `together` module and use its `connect` function. 390 | 391 | ```python 392 | from haverscript import echo 393 | from haverscript.together import connect 394 | 395 | session = connect("meta-llama/Meta-Llama-3-8B-Instruct-Lite") | echo() 396 | 397 | session = session.chat("Write a short sentence on the history of Scotland.") 398 | session = session.chat("Write 500 words on the history of Scotland.") 399 | session = session.chat("Who was the most significant individual in Scottish history?") 400 | ``` 401 | 402 | You need to set the TOGETHER_API_KEY environmental variable. 403 | 404 | ```shell 405 | export TOGETHER_API_KEY=... 406 | python example.py 407 | ``` 408 | 409 | You also need to include the together option when installing. 410 | 411 | ```shell 412 | pip install "haverscript[together] @ git+https://github.com/andygill/haverscript.git@v0.3.0" 413 | ``` 414 | 415 | PRs supporting other API are welcome! There are two examples in the source, 416 | the together API is in [together.py](src/haverscript/together.py), 417 | and it should be straightforward to add more. 418 | 419 | 420 | # Reply as a Monad 421 | 422 | The `Reply` object is a [monad](https://en.wikipedia.org/wiki/Monad_%28functional_programming%29). 423 | That is you can inject a value, using `Reply.pure(...)`, 424 | and pass the value forward using `Reply.bind(...)`. 425 | You can also extract the value using `Reply.value`. 426 | 427 | You can extract the contents of a Reply using `yield from` 428 | 429 | ```python 430 | yield from Reply(....) 431 | ``` 432 | 433 | and you can take a yield-based `Iterator`, and pass it as an argument to `Reply` 434 | to make a new `Reply` object. 435 | 436 | Careful use of `Reply` allows agents that can explain what they are doing in real time. 437 | 438 | ## FAQ 439 | 440 | Q: How do I increase the context window size to (for example) 16K?" 441 | 442 | A: set the `num_ctx` option using middleware. 443 | ```python 444 | model = model | options(num_ctx=16 * 1024) 445 | ``` 446 | 447 | Q: How do I get JSON output? 448 | 449 | A: set the `format` middleware, typically given as an argument to `chat`, 450 | because it is specific to this call. 451 | ```python 452 | response_value : dict = model.chat("...", middleware=format()).value 453 | ``` 454 | 455 | Remember to request JSON in the prompt as well. 456 | 457 | Q: How do I get pydantic class as a reply? 458 | 459 | A: set the `format` middleware, using the name of the class. 460 | ```python 461 | class Foo(BaseModel): 462 | ... 463 | 464 | foo : Foo = model.chat("...", middleware=format(Foo)).value 465 | ``` 466 | 467 | Again, remember to request JSON in the prompt. `reply_in_json` is a good 468 | way of doing this. 469 | 470 | ```python 471 | prompt = Markdown() 472 | 473 | prompt += ... 474 | 475 | prompt += reply_in_json(model) 476 | 477 | ... 478 | ``` 479 | 480 | Q: Can I write my own agent? 481 | 482 | A: Yes! There are a number of examples in the source code. `Agent` is a simple 483 | wrapper around a `Model`, and provides plumbing for agent-based `ask` and 484 | `chat`. The `Agent` class also provides support for stuctured output. 485 | 486 | Q: Can I write my own middleware? 487 | 488 | A: Yes! There are many examples in the sourcecode. Middleware operate 489 | at one level down from prompts and chat, and instead operate with `Request` 490 | and `Reply`. The design pattern is (1) modify the `Request`, if needed, 491 | (2) call the rest of the middleware, and (3) process the `Reply`. 492 | 493 | Q: What is "haver"? 494 | 495 | A: It's a Scottish term that means to talk aimlessly or without necessarily 496 | making sense. Sort of like an LLM. 497 | 498 | 499 | ## Terminology 500 | 501 | - Model: a LLM model that can be queried. 502 | - Response: a model that also has a chat context including at least one response. 503 | - Reply: a lazy list of tokens from an LLM; is also a monad. 504 | - reply: a textual reply. 505 | - chat: a question, remember the prompt-reply pair. 506 | - ask: a question, only for the reply. 507 | 508 | ## Generative AI Usage 509 | 510 | Generative AI was used as a tool to help with code authoring and documentation 511 | writing. 512 | -------------------------------------------------------------------------------- /docs/DSL_DESIGN.md: -------------------------------------------------------------------------------- 1 | # Haverscript Domain Specific Language 2 | 3 | First, let's consider several different Python APIs for accessing LLMs. 4 | 5 | ### Using Ollama's Python API 6 | 7 | From 8 | 9 | 10 | ```python 11 | import ollama 12 | 13 | stream = ollama.chat( 14 | model='llama3.1', 15 | messages=[ 16 | { 17 | 'role': 'user', 18 | 'content': 'Why is the sky blue?' 19 | } 20 | ], 21 | stream=True, 22 | ) 23 | 24 | for chunk in stream: 25 | print(chunk['message']['content'], end='', flush=True) 26 | ``` 27 | 28 | ### Using Mistral's Python API 29 | 30 | From 31 | 32 | ```python 33 | import os 34 | from mistralai import Mistral 35 | 36 | api_key = os.environ["MISTRAL_API_KEY"] 37 | model = "mistral-large-latest" 38 | 39 | client = Mistral(api_key=api_key) 40 | 41 | chat_response = client.chat.complete( 42 | model = model, 43 | messages = [ 44 | { 45 | "role": "user", 46 | "content": "What is the best French cheese?", 47 | }, 48 | ] 49 | ) 50 | 51 | print(chat_response.choices[0].message.content) 52 | ``` 53 | 54 | ### Using LiteLLM's Python API 55 | 56 | From 57 | 58 | ```python 59 | from litellm import acompletion 60 | import asyncio 61 | 62 | async def test_get_response(): 63 | user_message = "Hello, how are you?" 64 | messages = [{"content": user_message, "role": "user"}] 65 | response = await acompletion(model="gpt-3.5-turbo", messages=messages) 66 | return response 67 | 68 | response = asyncio.run(test_get_response()) 69 | print(response) 70 | ``` 71 | 72 | ### LangChain (Using Ollama) 73 | 74 | From , 75 | with output parser from . 76 | 77 | ```python 78 | from langchain_core.prompts import ChatPromptTemplate 79 | from langchain_ollama.llms import OllamaLLM 80 | from langchain_core.output_parsers import StrOutputParser 81 | 82 | template = """Question: {question} 83 | 84 | Answer: Let's think step by step.""" 85 | 86 | prompt = ChatPromptTemplate.from_template(template) 87 | 88 | model = OllamaLLM(model="llama3.1") 89 | 90 | output_parser = StrOutputParser() 91 | 92 | chain = prompt | model | output_parser 93 | 94 | chain.invoke({"question": "What is LangChain?"}) 95 | ``` 96 | 97 | All these examples are more verbose than necessary for such simple tasks. 98 | 99 | ## A Domain Specific Language for talking to LLMs 100 | 101 | A Domain Specific Language (DSL) is an interface to a capability. A good LLM DSL 102 | should have composable parts that work together to make accessing the 103 | capabilities of LLMs robust, straightforward, and predictable. 104 | 105 | So, when we want to connect to a specific LLM, ask a question, and print a 106 | result, we do the following: 107 | 108 | ```python 109 | from haverscript import connect 110 | 111 | print(connect("mistral").chat("What is a haver?")) 112 | ``` 113 | 114 | --- 115 | 116 | What about having a chat session? Turn on echo, and chat away. We turn on echo 117 | by using a pipe after the `connect`, and piping the response into `echo`. 118 | 119 | ```python 120 | from haverscript import connect 121 | 122 | session = connect("mistral") | echo() 123 | session = session.chat("In one sentence, why is the sky blue?") 124 | session = session.chat("Rewrite the above sentence in the style of Yoda") 125 | session = session.chat("How many questions did I ask?") 126 | ... 127 | ``` 128 | 129 | --- 130 | 131 | What about using the result in a later prompt? Use a Python f-string to 132 | describe the second prompt. 133 | 134 | ```python 135 | from haverscript import connect 136 | 137 | model = connect("mistral") | echo() 138 | 139 | best = model.chat("Name the best basketball player. Only name one player and do not give commentary.") 140 | 141 | model.chat(f"Someone told me that {best} is the best basketball player. Do you agree, and why?") 142 | ``` 143 | 144 | These examples demonstrate how Haverscript's components are designed to be 145 | composable, making it easy to build upon previous interactions. 146 | 147 | ## Principles of Haverscript as a DSL 148 | 149 | Haverscript is built around four core principles: 150 | 151 | **LLM Interactions as String-to-String Operations** 152 | 153 | Interactions with LLMs in Haverscript are fundamentally string-based. Python’s 154 | robust string formatting tools, such as `.format` and f-strings, are used 155 | directly. Prompts can be crafted using f-strings with explicit `{...}` notation 156 | for injection. The `chat` method accepts prompt strings and returns a result 157 | that can be seamlessly used in other f-strings, or accessed through the `.reply` 158 | attribute. This makes the "string plumbing" for prompt-based applications an 159 | integral part of the Haverscript domain-specific design. 160 | 161 | **Immutable Core Structures** 162 | 163 | The user-facing classes in Haverscript are immutable, similar to Python strings or 164 | tuples. Managing state is as simple as assigning names to things. For example, 165 | running the same prompt multiple times on the same context is straightforward 166 | because there is no hidden state that might be updated. 167 | 168 | **Chat as a Chain of Responses** 169 | 170 | Using immutable structures, `.chat` is a chain of responses, taking care of the 171 | context of calls behind the scenes. Calling `.chat` both calls the LLM, and builds 172 | the complete context needed to make this call. 173 | 174 | **Middleware for Effects** 175 | 176 | Haverscript provides extensions to the basic chat using middleware and a pipe syntax. 177 | Things between the call to chat and the call to the actual LLM service can be augmented. 178 | These include Caching (using SQLite), retry and validation, echo capability, and 179 | The user can build the pipeline between the call to `.chat`, and the call of the LLM, 180 | that works for them. 181 | 182 | For example, if we want to both echo and use a cache, then there are two choices: 183 | echo every response, or echo only the non-cached resposes. The user picks what works 184 | for them using composable parts. 185 | 186 | -------------------------------------------------------------------------------- /docs/MARKDOWN.md: -------------------------------------------------------------------------------- 1 | # Markdown DSL Documentation 2 | 3 | The Markdown DSL is a lightweight domain-specific language for programmatically 4 | creating Markdown content for prompts. It provides an readable API for 5 | constructing Markdown documents by combining text blocks, headers, lists, code 6 | blocks, tables, and more. The typical idiom for using the library is: 7 | 8 | ```python 9 | prompt = Markdown() 10 | 11 | prompt += ... 12 | prompt += ... 13 | 14 | # use the constructed prompt 15 | ``` 16 | 17 | Below is a comprehensive guide to the API and usage examples. 18 | 19 | ## Overview 20 | 21 | The core of the library is the `Markdown` class, which internally maintains a 22 | list of markdown blocks. Blocks are joined by blank lines when rendering the 23 | document. The library also provides helper functions to create various markdown 24 | elements such as headers, bullet lists, code blocks, tables, XML elements, and 25 | more. Additionally, it supports delayed evaluation of f-strings using a template 26 | mechanism, which allows you to format variable placeholders later. 27 | 28 | --- 29 | 30 | ## Getting Started 31 | 32 | To get started, create a `Markdown` object and use the `+=` operator to append 33 | content. When you append a string, it is automatically converted into a markdown 34 | block. Finally, render your document by converting the `Markdown` object to a 35 | string or by using the `format` method for variable substitution. 36 | 37 | **Basic Usage Example:** 38 | 39 | ```python 40 | prompt = Markdown() 41 | prompt += header("Welcome to My Document") 42 | prompt += text("This document is generated using a Markdown DSL.") 43 | prompt += bullets(["First item", "Second item", "Third item"]) 44 | print(prompt) 45 | ``` 46 | 47 | If you have variables within your text, use `template` and the `format` method to substitute them: 48 | 49 | ```python 50 | prompt += template("Hello, {name}!") 51 | print(prompt.format(name="Alice")) 52 | ``` 53 | 54 | --- 55 | 56 | ## API Reference 57 | 58 | ### Class: `Markdown` 59 | 60 | #### Initialization 61 | ```python 62 | Markdown() 63 | ``` 64 | 65 | Creates a new Markdown object. 66 | 67 | #### Methods 68 | 69 | - **`__str__`** 70 | Renders the complete document as a string. Each block is separated by a blank line. 71 | 72 | - **`format(**kwargs)`** 73 | Returns a formatted version of the markdown document by replacing variables (e.g., `{variable}`) in each block with provided keyword arguments. 74 | 75 | - **`__add__(other)`** 76 | Supports appending content to the Markdown document. When a string is added, it is automatically converted to a Markdown block. 77 | 78 | This provides the support for `+` and `=+` for markdown blocks. 79 | 80 | --- 81 | 82 | ### Helper Functions 83 | 84 | #### `markdown(content: str | Markdown | None = None) -> Markdown` 85 | Converts a string or an existing Markdown block into a Markdown object. 86 | - If `content` is `None`, returns an empty Markdown object. 87 | - If `content` is a string, it is converted using the `text` function. 88 | 89 | --- 90 | 91 | #### `strip_text(txt: str) -> str` 92 | Utility function that: 93 | - Strips leading and trailing whitespace. 94 | - Trims whitespace from each line. 95 | 96 | Useful for cleaning input text before adding it to a Markdown block. 97 | 98 | --- 99 | 100 | #### `header(txt, level: int = 1) -> Markdown` 101 | Creates a Markdown header. 102 | - **txt**: The header text. 103 | - **level**: The header level (default is 1). Uses the `#` symbol repeated `level` times. 104 | 105 | Example: 106 | ```python 107 | header("Introduction", level=2) # Produces "## Introduction" 108 | ``` 109 | 110 | --- 111 | 112 | #### `xml_element(tag: str, contents: str | Markdown | None = None) -> Markdown` 113 | Wraps the given content in an XML element. 114 | - **tag**: The XML tag name. 115 | - **contents**: Content to be wrapped. If omitted or empty, returns a self-closing tag (``). 116 | 117 | If content is provided, the element is formatted as: 118 | ```xml 119 | 120 | ... content ... 121 | 122 | ``` 123 | 124 | --- 125 | 126 | #### `bullets(items, ordered: bool = False) -> Markdown` 127 | Generates a bullet list. 128 | - **items**: An iterable of items to include in the list. 129 | - **ordered**: If `True`, creates an ordered (numbered) list; otherwise, an unordered list using hyphens. 130 | 131 | Example: 132 | ```python 133 | bullets(["Item 1", "Item 2"], ordered=True) 134 | ``` 135 | 136 | --- 137 | 138 | #### `code(code: str, language: str = "") -> Markdown` 139 | Creates a Markdown code block. 140 | - **code**: The code snippet. 141 | - **language**: The programming language for syntax highlighting (optional). 142 | 143 | Example: 144 | ```python 145 | code("print('Hello, world!')", language="python") 146 | ``` 147 | 148 | --- 149 | 150 | #### `text(txt: str) -> Markdown` 151 | Creates a Markdown text block from a given string. When using the `+=` operator, strings are automatically converted using this function. 152 | 153 | --- 154 | 155 | #### `quoted(txt) -> Markdown` 156 | Wraps the provided text inside triple quotes (`""" ... """`), designating it as a quoted block. 157 | 158 | --- 159 | 160 | #### `rule(count: int = 3) -> Markdown` 161 | Generates a horizontal rule. 162 | - **count**: The number of dashes to use (default is 3). 163 | 164 | Example: 165 | ```python 166 | rule() # Produces '---' 167 | ``` 168 | 169 | --- 170 | 171 | #### `table(headers: dict, rows: list[dict]) -> Markdown` 172 | Creates a Markdown table. 173 | - **headers**: A dictionary mapping column keys to header names. 174 | - **rows**: A list of dictionaries, each representing a row in the table. 175 | 176 | The function automatically right-justifies integers and left-justifies other types. 177 | 178 | Example: 179 | ```python 180 | headers = {"name": "Name", "age": "Age"} 181 | rows = [{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}] 182 | table_md = table(headers, rows) 183 | ``` 184 | 185 | --- 186 | 187 | #### `reply_in_json(model: Type[BaseModel], prefix: str = "Reply in JSON, using the following keys:") -> Markdown` 188 | Generates instructions for replying in JSON format based on a Pydantic model. 189 | - **model**: A Pydantic `BaseModel` subclass defining the schema. 190 | - **prefix**: A prefix instruction string. 191 | 192 | It introspects the model’s type hints and field descriptions to produce a bullet list of keys along with type information. This is of more utility to an LLM that the raw json schema. 193 | 194 | Example: 195 | ```python 196 | from pydantic import BaseModel, Field 197 | from enum import Enum 198 | 199 | class Status(Enum): 200 | SUCCESS = "success" 201 | FAILURE = "failure" 202 | 203 | class ReplyModel(BaseModel): 204 | status: Status = Field(description="The status of the operation") 205 | message: str = Field(description="Detailed message regarding the result") 206 | 207 | prompt = reply_in_json(ReplyModel) 208 | ``` 209 | 210 | --- 211 | 212 | #### `template(fstring: str) -> Markdown` 213 | Delays the interpretation of an f-string until later. 214 | - **fstring**: A string containing variable placeholders (e.g., `"Hello, {name}!"`). 215 | 216 | The function extracts valid variable names and stores them so that they can later be substituted using the `format` method. 217 | 218 | Example: 219 | ```python 220 | prompt = template("Hello, {username}! Welcome to our service.") 221 | formatted_prompt = prompt.format(username="Alice") 222 | ``` 223 | 224 | --- 225 | 226 | ## Usage Examples 227 | 228 | ### 1. Building a Simple Document 229 | 230 | ```python 231 | prompt = Markdown() 232 | prompt += header("Document Title", level=4) 233 | prompt += text("Welcome to the generated markdown document.") 234 | prompt += bullets(["Item 1", "Item 2", "Item 3"]) 235 | prompt += code("print('Hello, world!')", language="python") 236 | print(prompt) 237 | ``` 238 | 239 | > #### Document Title 240 | > 241 | > Welcome to the generated markdown document. 242 | > 243 | > - Item 1 244 | > - Item 2 245 | > - Item 3 246 | > 247 | > ```python 248 | > print('Hello, world!') 249 | > ``` 250 | 251 | ### 2. Delayed Formatting with Template 252 | 253 | ```python 254 | prompt = Markdown() 255 | prompt += header("User Greeting", level=4) 256 | prompt += template("Hello, {username}! Welcome to our service.") 257 | formatted_prompt = prompt.format(username="Bob") 258 | print(formatted_prompt) 259 | ``` 260 | 261 | > #### User Greeting 262 | > 263 | > Hello, Bob! Welcome to our service. 264 | 265 | ### 3. Creating a Table 266 | 267 | ```python 268 | headers = {"name": "Name", "age": "Age"} 269 | rows = [{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}] 270 | table_md = table(headers, rows) 271 | print(table_md) 272 | ``` 273 | 274 | > | Name | Age | 275 | > |-------|-----| 276 | > | Alice | 30 | 277 | > | Bob | 25 | 278 | 279 | ### 4. Generating a JSON Reply Prompt 280 | 281 | ```python 282 | from pydantic import BaseModel, Field 283 | from enum import Enum 284 | 285 | class Status(Enum): 286 | SUCCESS = "success" 287 | FAILURE = "failure" 288 | 289 | class ReplyModel(BaseModel): 290 | status: Status = Field(description="The status of the operation") 291 | message: str = Field(description="Detailed message regarding the result") 292 | 293 | prompt = reply_in_json(ReplyModel) 294 | print(prompt) 295 | ``` 296 | 297 | > Reply in JSON, using the following keys: 298 | > 299 | > - "status" ("success" or "failure"): The status of the operation 300 | > - "message" (str): Detailed message regarding the result 301 | -------------------------------------------------------------------------------- /docs/MIDDLEWARE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Middleware 4 | 5 | Middleware is a mechansim to have fine control over everything between 6 | calling `.chat` and Haverscript calling the LLM. 7 | As a example, consider the creation of a session. 8 | 9 | ```python 10 | session = connect("mistral") | echo() 11 | ``` 12 | 13 | We use `|` to create a pipeline for actions to be taken. 14 | The reply from the LLM flows left to right, and the prompt 15 | to invoke the LLM flows from right to left. In this case, 16 | there is only `echo`, but consider two examples using `cache`. 17 | 18 | ```python 19 | # LLM <---- PROMPT GOES RIGHT TO LEFT <----- chat 20 | connect("mistral") | echo() | cache("cache.db") 21 | # LLM ----> REPLY GOES LEFT TO RIGHT ------> Response 22 | ``` 23 | In this example, we check the cache for any existing answer, then call echo 24 | only if we do not have a cache hit. 25 | 26 | ```python 27 | # LLM <---- PROMPT GOES RIGHT TO LEFT <----- chat 28 | connect("mistral") | cache("cache.db") | echo() 29 | # LLM ----> REPLY GOES LEFT TO RIGHT ------> Response 30 | ``` 31 | 32 | In this second example, we always echo, even if the 33 | cache hits. 34 | 35 | The user has the capability to choose the middleware stack they want 36 | for their application. 37 | 38 | There are two ways of defining middleware. First as part of the overall session 39 | (as above), or as part of a chat. In this second case, the middleware is only append 40 | for this chat, and is not part of any context. 41 | 42 | ```python 43 | session.chat("Hello", middlware=echo()) 44 | ``` 45 | 46 | # Middleware in Haverscript 47 | 48 | Haverscript provides following middleware 49 | 50 | | Middleware | Purpose | Class | 51 | |------------|---------|-------| 52 | | model | Request a specific model be used | configuration | 53 | | options | Set specific LLM options (such as seed) | configuration | 54 | | format | Set specific format for output | configuration | 55 | | dedent | Remove spaces from prompt | configuration | 56 | | echo | Print prompt and reply | observation | 57 | | stats | Print basic stats about LLM | observation | 58 | | trace | Log requests and responses | observation | 59 | | transcript | Store a complete transcript of every call | observation | 60 | | retry | retry on failure | reliablity | 61 | | validation | Fail under given condition | reliablity | 62 | | cache | Store and/or query prompt-reply pairs in DB | efficency | 63 | | fresh | Request a fresh reply (not cached) | efficency | 64 | 65 | ## Configuration Middleware 66 | 67 | ```python 68 | def model(model_name: str) -> Middleware: 69 | """Set the name of the model to use. Typically this is automatically set inside connect.""" 70 | def options(**kwargs) -> Middleware: 71 | """Options to pass to the model, such as temperature and seed.""" 72 | def format(schema: Type[BaseModel] | None = None) -> Middleware: 73 | """Request the output in JSON, or parsed JSON.""" 74 | def dedent() -> Middleware: 75 | """Remove unnecessary spaces from the prompt 76 | ``` 77 | 78 | * `model` is automatically appended to the start of the middleware by the call to 79 | `connect`. 80 | * `options` allows options, such as temperature, the seed, and top_k, 81 | to be set. 82 | * `format` requests the the output be formatted in JSON, and 83 | automatically parse the output. If no type is provided, then the result is a 84 | JSON `dict`. If a BaseModel (from pydantic) type is provided then the schema 85 | of this specific BaseModel class is used, and the class is parsed. In either 86 | case, `Response.value` is used to access the parsed result. 87 | * `detent` removes excess spaces from the prompt. 88 | 89 | See [options](examples/options/README.md) for an example of using `options`. 90 | See [formatting](examples/format/README.md) for an example of using `format`. 91 | 92 | ## Observation Middleware 93 | 94 | There are four middleware adapters for observation. 95 | 96 | ```python 97 | def echo(width: int = 78, prompt: bool = True, spinner: bool = True) -> Middleware: 98 | """echo prompts and responses to stdout.""" 99 | def stats() -> Middleware: 100 | """print stats to stdout.""" 101 | def trace(level: int = log.DEBUG) -> Middleware: 102 | """Log all requests and responses.""" 103 | def transcript(dirname: str) -> Middleware: 104 | """write a full transcript of every interaction, in a subdirectory.""" 105 | ``` 106 | 107 | * `echo` turns of echo of prompt and reply. There is a spinner (⠧) which is 108 | used when waiting for a response from the LLM. 109 | * `stats` prints based stats (token counts, etc) to the screen 110 | * `trace` uses pythons logging to log all prompts and responses. 111 | * `transcript` stores all prompt-response pairs, including context, in a sub-directory. 112 | 113 | ## Reliablity Middleware 114 | 115 | ```python 116 | def retry(count : int) -> Middleware: 117 | """retry the LLM call a number of times.""" 118 | def validate(predicate: Callable[[str], bool]) -> Middleware: 119 | """validate the response as middleware. Can raise as LLMResultError""" 120 | ``` 121 | 122 | ## Efficency Middleware 123 | 124 | ```python 125 | def cache(filename: str, mode: str | None = "a+") -> Middleware: 126 | """Set the cache filename for this model.""" 127 | def fresh() -> Middleware: 128 | """require any cached reply be ignored, and a fresh reply be generated.""" 129 | ``` 130 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | This directory contains various examples of using Haverscript. 4 | 5 | Examples with README files. 6 | 7 | * [First example and chat session](first_example/README.md) 8 | * [First Agent example](first_agent/README.md) 9 | * [Chaining answers together](chaining_answers/README.md) 10 | * [Tree of calls and use of system](tree_of_calls/README.md) 11 | * [Vision Models](images/README.md) 12 | * [Enabling the cache](cache/README.md) 13 | * [LLM options and middleware](options/README.md) 14 | * [Multi Agents](agents/README.md) 15 | 16 | Other examples: 17 | 18 | * [Using together.ai](together/main.py) 19 | * [ChatBot Model](chatbot/main.py) and an example of an `Agent`, used to build the `ChatBot`. 20 | * [Custom Service Provider](custom_service/main.py) 21 | * [An example command line shell using haverscript](havershell/main.py) 22 | * [Proof readings the README](others/proof_reading.py) 23 | * [Reworking a sentence](others/sentence_iterations.py) 24 | -------------------------------------------------------------------------------- /examples/agents/README.md: -------------------------------------------------------------------------------- 1 | Haverscript has support for basic agents. An `Agent` is a class that has access 2 | to an LLM, via the `Model` class. However, an `Agent` also has state, and acts 3 | as an object (vs a structure). 4 | 5 | Think of an Agent as a python object that can take care of something. So in this 6 | example, we have two agents, an `Author` and an `Editor`. 7 | 8 | ## Author agent 9 | 10 | Now, the only requirement of an agent is to define a system prompt as 11 | an attribute (but see Notes below for additional options). 12 | 13 | ```python 14 | class Author(Agent): 15 | system: str = """ 16 | You are a world-class author who is writing a travel book. 17 | 18 | You are part of a team. 19 | 20 | Instructions are given with the heading "# Instructions". Respond in the requested format. 21 | 22 | Commentary, feedback or requests from others are given with the heading "# Feedback from ..." 23 | """ 24 | ``` 25 | 26 | Further, an agent should have a capability, in this case writing. 27 | 28 | ```python 29 | def write( 30 | self, instructions: str | Markdown, feedback: EditorFeedback | None = None 31 | ) -> str: 32 | prompt = Markdown() 33 | 34 | if feedback is not None: 35 | prompt += header("Feedback from Editor") 36 | prompt += bullets(feedback.comments) 37 | 38 | prompt += header("Instructions") + instructions 39 | 40 | return self.chat(prompt) 41 | ``` 42 | 43 | 44 | Here,we take the instructions, and feedback from our editor, and call the LLM. 45 | 46 | ## Editor agent. 47 | 48 | Again, we define the agent. 49 | 50 | ```python 51 | class Editor(Agent): 52 | system: str = """ 53 | You are a editor for a company that writes travel books. 54 | Make factual and actionable suggestions for improvement. 55 | """ 56 | ``` 57 | 58 | This time, we have the capability for proofing text. 59 | 60 | ```python 61 | def proof(self, instructions: str, article: str) -> EditorFeedback: 62 | prompt = Markdown() 63 | prompt += header("Previous Text") 64 | prompt += "Original instructions given to Author:" 65 | prompt += quoted(instructions) 66 | 67 | ... 68 | 69 | prompt += reply_in_json(EditorFeedback) 70 | 71 | return self.ask(prompt, format=EditorFeedback) 72 | ``` 73 | 74 | We reqire the output use the `EditorFeedback` class. 75 | 76 | ```python 77 | class EditorFeedback(BaseModel): 78 | comments: list[str] = Field( 79 | default_factory=list, 80 | description="Specific concrete recommendation of improvement", 81 | ) 82 | score: int = Field(..., description="Quality score from 1 to 10") 83 | ``` 84 | 85 | Now, we call `ask` this time - ask is for calls without history, chat is for 86 | sessions with history - and we want the editor to consider what is written in 87 | isolation. 88 | 89 | Thats it! We've written two agents. 90 | 91 | We connect the two models using a supervisor class. 92 | 93 | ```python 94 | class Supervision(BaseModel): 95 | author: Author 96 | editor: Editor 97 | """Generic class with author and editor interacting.""" 98 | ``` 99 | 100 | If we instatiate this, we can now use the supervisor to writing 101 | and improve text. 102 | 103 | 104 | ```python 105 | prompt = template( 106 | """Write {words} words about traveling to Scotland. Only write prose. No titles or lists.""" 107 | ) 108 | print(supervised.improve( 109 | "Scotland.", 110 | prompt, 111 | ({"words": words} for words in [200, 300, 400]), 112 | )) 113 | ``` 114 | 115 | This writes articles of 200, then 300, then 400 words, each time geting feedback, 116 | and improving the next version. This is a dynamic form of multishot prompting, 117 | but using agents to do the plumbing. 118 | 119 | # Notes 120 | 121 | - There is a way, by providing an `Agent.prepare()` method, to have dynamic 122 | system prompts. This is an advanced topic. 123 | 124 | - The `ask` can return a `Reply`, and allow agents to do work in a way to get 125 | dynamic output as LLMs are called. 126 | -------------------------------------------------------------------------------- /examples/agents/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import ( 2 | connect, 3 | options, 4 | Agent, 5 | Markdown, 6 | bullets, 7 | header, 8 | quoted, 9 | reply_in_json, 10 | template, 11 | stats, 12 | ) 13 | 14 | from typing import Iterable, Iterator 15 | from pydantic import BaseModel, Field 16 | 17 | 18 | model_name = "mistral-nemo:latest" 19 | 20 | model = connect(model_name) | options(num_ctx=16 * 1024, temperature=0.3) | stats() 21 | 22 | 23 | class EditorFeedback(BaseModel): 24 | comments: list[str] = Field( 25 | default_factory=list, 26 | description="Specific concrete recommendation of improvement", 27 | ) 28 | score: int = Field(..., description="Quality score from 1 to 10") 29 | 30 | def __str__(self) -> str: 31 | response = Markdown() 32 | response += bullets(self.comments) 33 | response += f"Quality score out of 10: {self.score}" 34 | return str(response) 35 | 36 | 37 | class Author(Agent): 38 | system: str = """ 39 | You are a world-class author who is writing a travel book. 40 | 41 | You are part of a team. 42 | 43 | Instructions are given with the heading "# Instructions". Respond in the requested format. 44 | 45 | Commentary, feedback or requests from others are given with the heading "# Feedback from ..." 46 | """ 47 | 48 | def write( 49 | self, instructions: str | Markdown, feedback: EditorFeedback | None = None 50 | ) -> str: 51 | prompt = Markdown() 52 | 53 | if feedback is not None: 54 | prompt += header("Feedback from Editor") 55 | prompt += bullets(feedback.comments) 56 | 57 | prompt += header("Instructions") + instructions 58 | 59 | return self.chat(prompt) 60 | 61 | 62 | class Editor(Agent): 63 | system: str = """ 64 | You are a editor for a company that writes travel books. 65 | Make factual and actionable suggestions for improvement. 66 | """ 67 | 68 | def proof(self, instructions: str, article: str) -> EditorFeedback: 69 | prompt = Markdown() 70 | prompt += header("Previous Text") 71 | prompt += "Original instructions given to Author:" 72 | prompt += quoted(instructions) 73 | prompt += "Text from Author following original instructions: " 74 | prompt += quoted(article) 75 | 76 | prompt += header("Instructions") 77 | 78 | prompt += "Read the above Text, and consider the following criteria:" 79 | prompt += bullets( 80 | [ 81 | "Does the text follow the original instructions?", 82 | "Is the text engaging and informative?", 83 | "Does the text have a clear structure?", 84 | "Are the sentences well-constructed?", 85 | "Are there any factual inaccuracies?", 86 | "Are there any spelling or grammar mistakes?", 87 | "Are there any areas that could be improved?", 88 | ] 89 | ) 90 | 91 | prompt += """ 92 | Given these criteria, and the original instructions, 93 | assess the text and provide specific feedback on how it could be improved. 94 | Also give a numerical score from 1 to 10, where 1 is the worst and 10 is the best, 95 | regarding quality and suitability for a travel book. 96 | """ 97 | 98 | prompt += reply_in_json(EditorFeedback) 99 | 100 | return self.ask(prompt, format=EditorFeedback) 101 | 102 | 103 | class Supervision(BaseModel): 104 | author: Author 105 | editor: Editor 106 | """Generic class with author and editor interacting.""" 107 | 108 | def improve( 109 | self, 110 | topic: str | Markdown, 111 | instructions: str | Markdown, 112 | bindings: Iterable[dict[str, str]], 113 | ) -> Iterator[str]: 114 | feedback = None 115 | first_round = True 116 | editor = self.editor.clone() 117 | author = self.author.clone() 118 | formated_instructions = None 119 | for binding in bindings: 120 | prompt = Markdown() 121 | 122 | if formated_instructions: 123 | feedback = editor.proof(formated_instructions, article) 124 | prompt += f"Consider the feedback from the Editor, and your previous attempt to write about {topic}." 125 | else: 126 | feedback = None 127 | 128 | formated_instructions = instructions.format(**binding) 129 | prompt += formated_instructions 130 | 131 | article = author.write(prompt, feedback) 132 | first_round = False 133 | 134 | return article 135 | 136 | 137 | supervised = Supervision(author=Author(model=model), editor=Editor(model=model)) 138 | 139 | prompt = template( 140 | """Write {words} words about traveling to Scotland. Only write prose. No titles or lists.""" 141 | ) 142 | 143 | article = Author(model=model).write(prompt.format(words=400)) 144 | print("Zero-shot Article:") 145 | print(article) 146 | 147 | article = supervised.improve( 148 | "Scotland.", 149 | prompt, 150 | ({"words": words} for words in [200, 300, 400]), 151 | ) 152 | print("Article using supervision:") 153 | print(article) 154 | -------------------------------------------------------------------------------- /examples/cache/README.md: -------------------------------------------------------------------------------- 1 | Here is an example of using the cache. We use SQLite, and files by convention 2 | use a `.db` suffix. Note that we do not turn on `echo` here but store the replies instead. 3 | 4 | ```python 5 | from haverscript import connect, cache, options 6 | import time 7 | import sys 8 | 9 | # This enabled the cache 10 | model = connect("mistral") | cache("cache.db") 11 | 12 | prompt = "In one sentence, why is the sky blue?" 13 | times = [] 14 | replies = [] 15 | 16 | # This runs a query several times, returning any cached answer first 17 | times.append(time.time()) 18 | for i in range(int(sys.argv[1])): 19 | replies.append(model.chat(prompt, middleware=options(seed=i)).reply) 20 | times.append(time.time()) 21 | 22 | for i, (t1, t2, r) in enumerate(zip(times, times[1:], replies)): 23 | print(f"chat #{i}") 24 | print("reply:", r) 25 | print("fast (used cache)" if t2 - t1 < 0.5 else "slower (used LLM)") 26 | print() 27 | ``` 28 | 29 | The first run, with argument 2, produces: 30 | 31 | ``` 32 | chat #0 33 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 34 | slower (used LLM) 35 | 36 | chat #1 37 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 38 | slower (used LLM) 39 | 40 | ``` 41 | 42 | Second time, with the cache intact inside "cache.db", and an argument of 3, gives: 43 | 44 | ``` 45 | chat #0 46 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 47 | fast (used cache) 48 | 49 | chat #1 50 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 51 | fast (used cache) 52 | 53 | chat #2 54 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 55 | slower (used LLM) 56 | 57 | ``` 58 | 59 | There are two cached values at the start of the second run, so we use the most 60 | up-to-date replies, in turn. The final call to `chat` calls the LLM. 61 | 62 | The cache can be access using `model.children()`, which will return a list of 63 | all possible `Response` values that are cached, or `model.children(prompt)`, 64 | which which will return all cached `Response` values that were a response to 65 | the given `prompt`. 66 | 67 | ---- 68 | 69 | ```mermaid 70 | graph LR 71 | 72 | start((hs)) 73 | m0(Model) 74 | m1(**model**: Model) 75 | r0(Response) 76 | str0(str) 77 | r1(Response) 78 | str1(str) 79 | r2(Response) 80 | str2(str) 81 | 82 | 83 | start -- connect('…') --> m0 84 | m0 -- | cache(…) --> m1 85 | m1 -- chat('…') --> r0 86 | r0 -- reply --> str0 87 | m1 -- chat('…') --> r1 88 | r1 -- reply --> str1 89 | m1 -- chat('…') --> r2 90 | r2 -- reply --> str2 91 | 92 | ``` 93 | 94 | ---- -------------------------------------------------------------------------------- /examples/cache/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, cache, options 2 | import time 3 | import sys 4 | 5 | # This enabled the cache 6 | model = connect("mistral") | cache("cache.db") 7 | 8 | prompt = "In one sentence, why is the sky blue?" 9 | times = [] 10 | replies = [] 11 | 12 | # This runs a query several times, returning any cached answer first 13 | times.append(time.time()) 14 | for i in range(int(sys.argv[1])): 15 | replies.append(model.chat(prompt, middleware=options(seed=i)).reply) 16 | times.append(time.time()) 17 | 18 | for i, (t1, t2, r) in enumerate(zip(times, times[1:], replies)): 19 | print(f"chat #{i}") 20 | print("reply:", r) 21 | print("fast (used cache)" if t2 - t1 < 0.5 else "slower (used LLM)") 22 | print() 23 | -------------------------------------------------------------------------------- /examples/chaining_answers/README.md: -------------------------------------------------------------------------------- 1 | In this example, we ask a question and then ask the same LLM (without context) 2 | whether it agrees. 3 | 4 | ```python 5 | from haverscript import connect, echo, validate, retry 6 | 7 | 8 | def small(reply): 9 | # Ensure the reply is three words or fewer 10 | return len(reply.split()) <= 3 11 | 12 | 13 | model = connect("mistral") | echo() 14 | 15 | best = model.chat( 16 | "Name the best basketball player. Only name one player and do not give commentary.", 17 | middleware=validate(small) | retry(5), 18 | ) 19 | model.chat( 20 | f"Someone told me that {best} is the best basketball player. Do you agree, and why?" 21 | ) 22 | ``` 23 | 24 | 25 | ```markdown 26 | > Name the best basketball player. Only name one player and do not give commentary. 27 | 28 | Michael Jordan 29 | 30 | > Someone told me that Michael Jordan is the best basketball player. Do you agree, and why? 31 | 32 | While it's subjective and opinions on who is the "best" basketball player can 33 | vary greatly among basketball fans, Michael Jordan is widely regarded as one 34 | of the greatest players in the history of the sport. His impact on the game 35 | both on and off the court is undeniable. 36 | 37 | Jordan led the Chicago Bulls to six NBA championships and was named the Most 38 | Valuable Player (MVP) five times during his career. He is also a 14-time NBA 39 | All-Star, a 10-time scoring champion, a three-time steals leader, and a 3x 40 | three-point shooting champion. 41 | 42 | Jordan's influence extended beyond statistics as well. His competitive spirit, 43 | passion for the game, and innovative style of play revolutionized basketball 44 | in ways that continue to be felt today. He elevated the NBA brand globally and 45 | inspired countless athletes across multiple sports. 46 | 47 | That being said, it's important to recognize that there have been many 48 | extraordinary players in the history of basketball, and debate over who is the 49 | "best" will likely never reach a consensus. Other notable contenders for this 50 | title include LeBron James, Kareem Abdul-Jabbar, Wilt Chamberlain, Magic 51 | Johnson, and Larry Bird, among others. 52 | ``` 53 | 54 | Here is the flow diagram. Note that the final call to `chat` uses both the `model`, 55 | and `best` to generate the final `Response`. 56 | 57 | ---- 58 | 59 | ```mermaid 60 | graph LR 61 | 62 | start((hs)) 63 | m0(Model) 64 | r0(Response) 65 | chat("model.chat('…{best}…')") 66 | m1(**model**: Model) 67 | r2(Response) 68 | 69 | 70 | start -- connect('…') --> m0 71 | m0 -- echo() --> m1 72 | m1 -- chat('…') --> r0 73 | r0 --> chat 74 | m1 --> chat 75 | chat --> r2 76 | 77 | style chat fill:none, stroke: none 78 | 79 | ``` 80 | 81 | ---- 82 | -------------------------------------------------------------------------------- /examples/chaining_answers/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, echo, validate, retry 2 | 3 | 4 | def small(reply): 5 | # Ensure the reply is three words or fewer 6 | return len(reply.split()) <= 3 7 | 8 | 9 | model = connect("mistral") | echo() 10 | 11 | best = model.chat( 12 | "Name the best basketball player. Only name one player and do not give commentary.", 13 | middleware=validate(small) | retry(5), 14 | ) 15 | model.chat( 16 | f"Someone told me that {best} is the best basketball player. Do you agree, and why?" 17 | ) 18 | -------------------------------------------------------------------------------- /examples/chatbot/main.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import Iterator, Protocol, Callable 3 | 4 | from haverscript import * 5 | from haverscript.types import Reply, Prompt, Contexture 6 | from copy import deepcopy 7 | 8 | 9 | class Translate(BaseModel): 10 | english: str = Field( 11 | ..., description="the original English text, and only the original text." 12 | ) 13 | french: str = Field( 14 | ..., description="the translated French text, and only the translated text." 15 | ) 16 | 17 | 18 | class FrenchAgent(Agent): 19 | system: str = "You are a translator, translating from English to French. " 20 | previous: list[Translate] = Field(default_factory=list) 21 | 22 | def chat_to_bot(self, prompt: str) -> Reply: 23 | return Reply(self._stream(prompt)) 24 | 25 | def _stream(self, prompt) -> Iterator: 26 | max_traslations = 3 27 | if len(self.previous) >= max_traslations: 28 | yield f"Sorry. French lesson over.\nYour {max_traslations} translations:\n" 29 | for resp in self.previous: 30 | assert isinstance(resp, Translate) 31 | yield f"* {resp.english} => {resp.french}\n" 32 | else: 33 | down_prompt = Markdown() 34 | down_prompt += f"You are a translator, translating from English to French. " 35 | down_prompt += f"English Text: {prompt}" 36 | down_prompt += reply_in_json(Translate) 37 | 38 | translated: Translate = self.ask(down_prompt, format=Translate) 39 | 40 | self.previous.append(translated) 41 | remaining = max_traslations - len(self.previous) 42 | 43 | yield f"{translated.english} in French is {translated.french}\n" 44 | yield "\n" 45 | yield f"{remaining} translation(s) left." 46 | 47 | 48 | # In a real example, validate and retry would be added to provide robustness. 49 | 50 | session = connect_chatbot(FrenchAgent(model=connect("mistral"))) | echo() 51 | 52 | print("--[ User-facing conversation ]------") 53 | session = session.chat("Three blind mice") 54 | session = session.chat("Such is life") 55 | session = session.chat("All roads lead to Rome") 56 | session = session.chat("The quick brown fox") 57 | 58 | session = connect_chatbot(FrenchAgent(model=connect("mistral") | echo())) 59 | 60 | print("--[ LLM-facing conversation ]------") 61 | session = session.chat("Three blind mice") 62 | session = session.chat("Such is life") 63 | session = session.chat("All roads lead to Rome") 64 | session = session.chat("The quick brown fox") 65 | -------------------------------------------------------------------------------- /examples/custom_service/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import Reply, Request, Service, ServiceProvider, echo, model 2 | 3 | 4 | class MyProvider(ServiceProvider): 5 | def ask(self, request: Request): 6 | assert request.contexture.model == "A" 7 | return Reply( 8 | [ 9 | f"I reject your {len(request.prompt.content.split())} word prompt, and replace it with my own." 10 | ] 11 | ) 12 | 13 | def list(self): 14 | return ["A"] 15 | 16 | 17 | def connect(name: str): 18 | return Service(MyProvider()) | model(name) 19 | 20 | 21 | session = connect("A") | echo() 22 | session = session.chat("In one sentence, why is the sky blue?") 23 | -------------------------------------------------------------------------------- /examples/first_agent/README.md: -------------------------------------------------------------------------------- 1 | Here is a first example of creating and using an agent. 2 | 3 | ```python 4 | from haverscript import connect, Agent 5 | 6 | 7 | class FirstAgent(Agent): 8 | system: str = """ 9 | You are a helpful AI assistant who answers questions in the style of 10 | Neil deGrasse Tyson. 11 | 12 | Answer any questions in 2-3 sentences, without preambles. 13 | """ 14 | 15 | def sky(self, planet: str) -> str: 16 | return self.ask(f"what color is the sky on {planet} and why?") 17 | 18 | 19 | firstAgent = FirstAgent(model=connect("mistral")) 20 | 21 | for planet in ["Earth", "Mars", "Venus", "Jupiter"]: 22 | print(f"{planet}: {firstAgent.sky(planet)}\n") 23 | ``` 24 | 25 | Running this will output the following: 26 | 27 | ``` 28 | Earth: The sky appears blue to our eyes during a clear day due to a phenomenon called Rayleigh scattering, where shorter wavelengths of light, such as blue and violet, are scattered more effectively by the atmosphere's molecules than longer ones like red or yellow. However, we perceive the sky as blue rather than violet because our eyes are more sensitive to blue light and because sunlight reaches us with less violet light filtered out by the ozone layer. 29 | 30 | Mars: The sky on Mars appears to be a reddish hue, primarily due to suspended iron oxide (rust) particles in its atmosphere. This gives Mars its characteristic reddish color as sunlight interacts with these particles. 31 | 32 | Venus: The sky on Venus is not visible like it is on Earth because of a dense layer of clouds composed mostly of sulfuric acid. This thick veil prevents light from the Sun from reaching our line of sight, making the sky appear perpetually dark. 33 | 34 | Jupiter: The sky on Jupiter isn't blue like Earth's; instead, it appears white or off-white due to the reflection of sunlight from thick layers of ammonia crystals in its atmosphere. This peculiarity stems from Jupiter's composition and atmospheric conditions that are quite different from ours. 35 | ``` -------------------------------------------------------------------------------- /examples/first_agent/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, Agent 2 | 3 | 4 | class FirstAgent(Agent): 5 | system: str = """ 6 | You are a helpful AI assistant who answers questions in the style of 7 | Neil deGrasse Tyson. 8 | 9 | Answer any questions in 2-3 sentences, without preambles. 10 | """ 11 | 12 | def sky(self, planet: str) -> str: 13 | return self.ask(f"what color is the sky on {planet} and why?") 14 | 15 | 16 | firstAgent = FirstAgent(model=connect("mistral")) 17 | 18 | for planet in ["Earth", "Mars", "Venus", "Jupiter"]: 19 | print(f"{planet}: {firstAgent.sky(planet)}\n") 20 | -------------------------------------------------------------------------------- /examples/first_example/README.md: -------------------------------------------------------------------------------- 1 | This example asks three questions in a chat session to the [mistral model](https://mistral.ai/news/announcing-mistral-7b/). 2 | 3 | ```python 4 | from haverscript import connect, echo 5 | 6 | # Create a new session with the 'mistral' model and enable echo middleware 7 | session = connect("mistral") | echo() 8 | 9 | session = session.chat("In one sentence, why is the sky blue?") 10 | session = session.chat("What color is the sky on Mars?") 11 | session = session.chat("Do any other planets have blue skies?") 12 | ``` 13 | 14 | Here is the output from running this example. 15 | 16 | ```markdown 17 | > In one sentence, why is the sky blue? 18 | 19 | The sky appears blue due to a scattering effect called Rayleigh scattering 20 | where shorter wavelength light (blue light) is scattered more than other 21 | colors by the molecules in Earth's atmosphere. 22 | 23 | > What color is the sky on Mars? 24 | 25 | The Martian sky appears red or reddish-orange, primarily because of fine dust 26 | particles in its thin atmosphere that scatter sunlight preferentially in the 27 | red part of the spectrum, which our eyes perceive as a reddish hue. 28 | 29 | > Do any other planets have blue skies? 30 | 31 | Unlike Earth, none of the other known terrestrial planets (Venus, Mars, 32 | Mercury) have a significant enough atmosphere or suitable composition to cause 33 | Rayleigh scattering, resulting in blue skies like we see on Earth. However, 34 | some of the gas giant planets such as Uranus and Neptune can appear blueish 35 | due to their atmospheres composed largely of methane, which absorbs red light 36 | and scatters blue light. 37 | 38 | ``` 39 | 40 | In `echo` mode, both the prompt and the reply are displayed to stdout when the 41 | chat is invoked. 42 | 43 | The following state diagram illustrates the Models and Responses used in this 44 | example, showing the chaining of the usage of chat. 45 | 46 | ---- 47 | 48 | ```mermaid 49 | graph LR 50 | 51 | start((hs)) 52 | m0(Model) 53 | m1(**session**: Model) 54 | r0(**session**: Response) 55 | r1(**session**: Response) 56 | r2(**session**: Response) 57 | 58 | start -- connect('…') --> m0 59 | m0 -- … | echo() --> m1 60 | m1 -- chat('…') --> r0 61 | r0 -- chat('…') --> r1 62 | r1 -- chat('…') --> r2 63 | ``` 64 | 65 | ---- -------------------------------------------------------------------------------- /examples/first_example/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, echo 2 | 3 | # Create a new session with the 'mistral' model and enable echo middleware 4 | session = connect("mistral") | echo() 5 | 6 | session = session.chat("In one sentence, why is the sky blue?") 7 | session = session.chat("What color is the sky on Mars?") 8 | session = session.chat("Do any other planets have blue skies?") 9 | -------------------------------------------------------------------------------- /examples/format/README.md: -------------------------------------------------------------------------------- 1 | Often, you want the LLM to return its response in a structured manner. There is 2 | support in most LLMs for enabling JSON output, and even enabling JSON output 3 | that complies with a given schema. 4 | 5 | We use the `format()` middleware, to both request 6 | JSON, and supply a schema (via the pydantic BaseModel). 7 | 8 | ```python 9 | from pydantic import BaseModel 10 | 11 | from haverscript import connect, format 12 | 13 | 14 | class Translate(BaseModel): 15 | english: str 16 | french: str 17 | 18 | 19 | model = connect("mistral") 20 | 21 | prompt = ( 22 | f"You are a translator, translating from English to French. " 23 | f'Give your answer as a JSON record with two fields, "english", and "french". ' 24 | f'The "english" field should contain the original English text, and only the original text.' 25 | f'The "french" field should contain the translated French text, and only the translated text.' 26 | f"\n\nEnglish Text: Three Blind Mice" 27 | ) 28 | 29 | print(model.chat(prompt, middleware=format(Translate)).value) 30 | ``` 31 | 32 | When run, this gives the following reply: 33 | 34 | ```python 35 | english='Three Blind Mice' french='Trois Souris Aveugles' 36 | ``` -------------------------------------------------------------------------------- /examples/format/main.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | from haverscript import connect, format 4 | 5 | 6 | class Translate(BaseModel): 7 | english: str 8 | french: str 9 | 10 | 11 | model = connect("mistral") 12 | 13 | prompt = ( 14 | f"You are a translator, translating from English to French. " 15 | f'Give your answer as a JSON record with two fields, "english", and "french". ' 16 | f'The "english" field should contain the original English text, and only the original text.' 17 | f'The "french" field should contain the translated French text, and only the translated text.' 18 | f"\n\nEnglish Text: Three Blind Mice" 19 | ) 20 | 21 | print(model.chat(prompt, middleware=format(Translate)).value) 22 | -------------------------------------------------------------------------------- /examples/havershell/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import inspect 3 | import shutil 4 | from dataclasses import dataclass 5 | 6 | from prompt_toolkit import PromptSession 7 | from prompt_toolkit.completion import WordCompleter 8 | from prompt_toolkit.history import InMemoryHistory 9 | 10 | from haverscript import Response, connect, echo 11 | 12 | 13 | def bell(): 14 | print("\a", end="", flush=True) # Ring the bell 15 | 16 | 17 | @dataclass(frozen=True) 18 | class Commands: 19 | 20 | def help(self, session): 21 | print("/help") 22 | print("/bye") 23 | print("/context") 24 | print("/undo") 25 | return session 26 | 27 | def bye(self, _): 28 | exit() 29 | 30 | def context(self, session): 31 | print(session.render()) 32 | return session 33 | 34 | def undo(self, session): 35 | if isinstance(session, Response): 36 | print("Undoing...") 37 | return session.parent 38 | bell() 39 | return session 40 | 41 | 42 | def main(): 43 | models = connect().list() 44 | 45 | # Set up argument parsing 46 | parser = argparse.ArgumentParser(description="Haverscript Shell") 47 | parser.add_argument( 48 | "--model", 49 | type=str, 50 | choices=models, 51 | required=True, 52 | help="The model to use for processing.", 53 | ) 54 | parser.add_argument( 55 | "--context", 56 | type=str, 57 | help="The context to use for processing.", 58 | ) 59 | parser.add_argument( 60 | "--cache", 61 | type=str, 62 | help="database to use as a cache.", 63 | ) 64 | parser.add_argument( 65 | "--temperature", 66 | type=float, 67 | help="temperature of replies.", 68 | ) 69 | parser.add_argument( 70 | "--num_predict", 71 | type=int, 72 | help="cap on number of tokens in reply.", 73 | ) 74 | parser.add_argument("--num_ctx", type=int, help="context size") 75 | 76 | args = parser.parse_args() 77 | 78 | terminal_size = shutil.get_terminal_size(fallback=(80, 24)) 79 | terminal_width = terminal_size.columns 80 | llm = connect(args.model) | echo(width=terminal_width - 2, prompt=False) 81 | 82 | if args.temperature is not None: 83 | llm = llm.options(temperature=args.temperature) 84 | if args.num_predict is not None: 85 | llm = llm.options(num_predict=args.num_predict) 86 | if args.num_ctx is not None: 87 | llm = llm.options(num_ctx=args.num_ctx) 88 | 89 | if args.cache: 90 | llm = llm.cache(args.cache) 91 | 92 | if args.context: 93 | with open(args.context) as f: 94 | session = llm.load(markdown=f.read()) 95 | else: 96 | session = llm 97 | 98 | command_list = [ 99 | "/" + method 100 | for method in dir(Commands) 101 | if callable(getattr(Commands, method)) and not method.startswith("__") 102 | ] 103 | 104 | print(f"connected to {args.model} (/help for help) ") 105 | 106 | while True: 107 | try: 108 | if args.cache: 109 | previous = session.children() 110 | if previous: 111 | history = InMemoryHistory() 112 | for prompt in previous.prompt: 113 | history.append_string(prompt) 114 | 115 | prompt_session = PromptSession(history=history) 116 | else: 117 | prompt_session = PromptSession() 118 | else: 119 | prompt_session = PromptSession() 120 | 121 | completer = WordCompleter(command_list, sentence=True) 122 | 123 | try: 124 | text = prompt_session.prompt("> ", completer=completer) 125 | except EOFError: 126 | exit() 127 | 128 | if text.startswith("/"): 129 | cmd = text.split(maxsplit=1)[0] 130 | if cmd in command_list: 131 | after = text[len(cmd) :].strip() 132 | function = getattr(Commands(), cmd[1:], None) 133 | if len(inspect.signature(function).parameters) > 1: 134 | if not after: 135 | print(f"{cmd} expects an argument") 136 | continue 137 | session = function(session, after) 138 | else: 139 | if after: 140 | print(f"{cmd} does not take any argument") 141 | continue 142 | session = function(session) 143 | continue 144 | 145 | print(f"Unknown command {text}.") 146 | continue 147 | else: 148 | print() 149 | try: 150 | session = session.chat(text) 151 | except KeyboardInterrupt: 152 | print("^C\n") 153 | print() 154 | except KeyboardInterrupt: 155 | print("Use Ctrl + d or /bye to exit.") 156 | 157 | 158 | if __name__ == "__main__": 159 | main() 160 | -------------------------------------------------------------------------------- /examples/images/README.md: -------------------------------------------------------------------------------- 1 | 2 | Vision models, such as [llava](https://llava-vl.github.io/), can take images 3 | as context for prompts. Here is an example, with a (small) image of the Edinburgh skyline. 4 | 5 | 6 | 7 | ```python 8 | from haverscript import connect, echo 9 | 10 | image_src = f"examples/images/edinburgh.png" 11 | 12 | model = connect("llava") | echo() 13 | 14 | model.chat("Describe this image, and speculate where it was taken.", images=[image_src]) 15 | ``` 16 | 17 | 18 | ```markdown 19 | > Describe this image, and speculate where it was taken. 20 | 21 | This is a panoramic photograph depicting a cityscape with notable landmarks. 22 | In the foreground, there's a wide street lined with buildings and shops. On 23 | either side of the street, you can see various architectural styles suggesting 24 | a mix of historical and modern construction. 25 | 26 | In the middle ground, there's an open space that leads to a significant 27 | building in the distance, which appears to be a castle due to its fortified 28 | walls and design elements typical of medieval architecture. The castle is 29 | likely an important landmark within the city. 30 | 31 | Further back, beyond the castle, you can see more of the city, with trees and 32 | other buildings indicating a densely populated area. There's also a notable 33 | feature in the background that looks like a bridge or an overpass, spanning 34 | across a body of water which could be a river or a loch. 35 | 36 | The sky is partly cloudy, suggesting that the weather might be changing or 37 | it's just a typical day with some clouds. The vegetation appears lush and 38 | well-maintained, indicating that this city values its green spaces. 39 | 40 | Based on these observations, it seems likely that this image was taken in 41 | Edinburgh, Scotland. The castle in the distance is unmistakably Edinburgh 42 | Castle, one of the most iconic landmarks in Edinburgh. The architecture, the 43 | layout of the streets, and the presence of the castle are all indicative of 44 | the city's rich history and its blend of historical and modern elements. 45 | ``` -------------------------------------------------------------------------------- /examples/images/edinburgh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygill/haverscript/cc6824c3cf7e37d21d9bbce71f1bbe098f24889e/examples/images/edinburgh.png -------------------------------------------------------------------------------- /examples/images/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, echo 2 | 3 | image_src = f"examples/images/edinburgh.png" 4 | 5 | model = connect("llava") | echo() 6 | 7 | model.chat("Describe this image, and speculate where it was taken.", images=[image_src]) 8 | -------------------------------------------------------------------------------- /examples/options/README.md: -------------------------------------------------------------------------------- 1 | ```python 2 | from haverscript import connect, echo, options 3 | 4 | model = ( 5 | connect("mistral") | echo() | options(num_ctx=4 * 1024, temperature=1.0, seed=12345) 6 | ) 7 | 8 | model.chat("In one sentence, why is the sky blue?") 9 | model.chat("In one sentence, why is the sky blue?") 10 | ``` 11 | 12 | `options` is a method that sets ollama options internally inside a `Model` (or `Response`). 13 | 14 | Running the above code gives the following output: 15 | 16 | ``` 17 | > In one sentence, why is the sky blue? 18 | 19 | The sky appears blue due to a scattering effect called Rayleigh scattering 20 | where shorter wavelength light (blue light) is scattered more than other 21 | colors by the molecules in Earth's atmosphere. 22 | 23 | > In one sentence, why is the sky blue? 24 | 25 | The sky appears blue due to a scattering effect called Rayleigh scattering 26 | where shorter wavelength light (blue light) is scattered more than other 27 | colors by the molecules in Earth's atmosphere. 28 | ``` 29 | 30 | Note that since we use `seed=12345`, the calls to the LLM produce the same result. 31 | 32 | 33 | The following are known options, as used by the [Ollama REST 34 | API](https://github.com/ollama/ollama/blob/main/docs/api.md). 35 | 36 | 37 | ``` 38 | num_ctx: int 39 | num_keep: int 40 | seed: int 41 | num_predict: int 42 | top_k: int 43 | top_p: float 44 | tfs_z: float 45 | typical_p: float 46 | repeat_last_n: int 47 | temperature: float 48 | repeat_penalty: float 49 | presence_penalty: float 50 | frequency_penalty: float 51 | mirostat: int 52 | mirostat_tau: float 53 | mirostat_eta: float 54 | penalize_newline: bool 55 | stop: Sequence[str] 56 | ``` 57 | 58 | ---- 59 | 60 | ```mermaid 61 | graph LR 62 | 63 | start((hs)) 64 | m0(Model) 65 | m2(**model**: Model) 66 | r0(Response) 67 | r1(Response) 68 | 69 | start -- connect('…') --> m0 70 | m0 -- | echo() | options(…) --> m2 71 | m2 -- chat('…') --> r0 72 | m2 -- chat('…') --> r1 73 | 74 | 75 | ``` 76 | 77 | ---- 78 | -------------------------------------------------------------------------------- /examples/options/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, echo, options 2 | 3 | model = ( 4 | connect("mistral") | echo() | options(num_ctx=4 * 1024, temperature=1.0, seed=12345) 5 | ) 6 | 7 | model.chat("In one sentence, why is the sky blue?") 8 | model.chat("In one sentence, why is the sky blue?") 9 | -------------------------------------------------------------------------------- /examples/others/proof_reading.py: -------------------------------------------------------------------------------- 1 | from haverscript import echo, options, dedent 2 | from haverscript.together import connect 3 | 4 | 5 | file = "README.md" 6 | 7 | prompt = f""" 8 | Here is some markdown that is the README for a LLM-based agent language called HaverScript. 9 | Please make last minute suggestions (typos, grammar, etc.), using a numbered list to do so, 10 | in the same order as the items appear in the README. 11 | After each item, list the specific changes you would like to see (as a diff if possible). 12 | 13 | The code and README.md goes out today. 14 | --- 15 | {open(file).read()} 16 | """ 17 | 18 | session = ( 19 | connect("meta-llama/Llama-3.3-70B-Instruct-Turbo") 20 | | echo(prompt=False, spinner=False) 21 | | options(temperature=0.6) 22 | | dedent() 23 | ) 24 | session.chat(prompt) 25 | -------------------------------------------------------------------------------- /examples/others/sentence_iterations.py: -------------------------------------------------------------------------------- 1 | from haverscript import echo, options 2 | from haverscript.together import connect 3 | 4 | file = "README.md" 5 | 6 | prompt = """ 7 | Make this sentence shorter and crisper. 8 | It is the first sentence of an introduction. 9 | --- 10 | Haverscript is a lightweight Python library designed to manage Large Language Model (LLM) interactions. 11 | """.strip() 12 | 13 | 14 | session = ( 15 | connect("meta-llama/Llama-3.3-70B-Instruct-Turbo") 16 | | echo(prompt=False, spinner=False, width=100) 17 | | options(temperature=1.6) 18 | ) 19 | for i in range(10): 20 | temperature = 0.8 + i / 10 21 | print(f"\ntemperature={temperature}\n----------------") 22 | for _ in range(3): 23 | session.chat(prompt, middleware=options(temperature=temperature)) 24 | -------------------------------------------------------------------------------- /examples/together/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import echo 2 | from haverscript.together import connect 3 | 4 | session = connect("meta-llama/Meta-Llama-3-8B-Instruct-Lite") | echo() 5 | 6 | session = session.chat("Write a short sentence on the history of Scotland.") 7 | session = session.chat("Write 500 words on the history of Scotland.") 8 | session = session.chat("Who was the most significant individual in Scottish history?") 9 | -------------------------------------------------------------------------------- /examples/tools/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, echo 2 | 3 | import time 4 | import haverscript.together as together 5 | from haverscript.tools import tool 6 | 7 | # add_two_numbers and subtract_two_numbers taken from ollama tools example: 8 | # https://github.com/ollama/ollama-python/blob/main/examples/tools.py 9 | 10 | 11 | def add_two_numbers(a: int, b: int) -> int: 12 | """ 13 | Add two numbers 14 | 15 | Args: 16 | a (int): The first number 17 | b (int): The second number 18 | 19 | Returns: 20 | int: The sum of the two numbers 21 | """ 22 | return int(a) + int(b) 23 | 24 | 25 | def subtract_two_numbers(a: int, b: int) -> int: 26 | """ 27 | Subtract two numbers 28 | 29 | Args: 30 | a (int): The first number 31 | b (int): The second number 32 | 33 | Returns: 34 | int: The difference of the two numbers 35 | """ 36 | return int(a) - int(b) 37 | 38 | 39 | # Tools can still be manually defined and passed into the tool() middleware 40 | subtract_two_numbers_schema = { 41 | "type": "function", 42 | "function": { 43 | "name": "subtract_two_numbers", 44 | "description": "Subtract two numbers", 45 | "parameters": { 46 | "type": "object", 47 | "required": ["a", "b"], 48 | "properties": { 49 | "a": {"type": "integer", "description": "The first number"}, 50 | "b": {"type": "integer", "description": "The second number"}, 51 | }, 52 | }, 53 | }, 54 | } 55 | 56 | session = connect("llama3.1:8b") | echo() 57 | session2 = together.connect("meta-llama/Llama-3.3-70B-Instruct-Turbo") | echo( 58 | stream=False 59 | ) 60 | 61 | session = session.system( 62 | """You are a helpful assistant that can access external functions. """ 63 | """The responses from these function calls will be appended to this dialogue. """ 64 | """Please provide responses based on the information from these function calls.""" 65 | ) 66 | 67 | 68 | tools = tool(add_two_numbers) + tool( 69 | subtract_two_numbers, schema=subtract_two_numbers_schema 70 | ) 71 | 72 | session = session.chat("4712 plus 4734 is?", tools=tools) 73 | session = session.chat("take this result, and subtract 1923", tools=tools) 74 | -------------------------------------------------------------------------------- /examples/tree_of_calls/README.md: -------------------------------------------------------------------------------- 1 | This example asks two questions to one model, then asks the same questions to 2 | the same model with a special system prompt. 3 | 4 | ```python 5 | from haverscript import connect, echo 6 | 7 | model = connect("mistral") | echo() 8 | model.chat("In one sentence, why is the sky blue?") 9 | model.chat("In one sentence, how many inches in a feet?") 10 | # Set system prompt to Yoda's style 11 | yoda = model.system("You are yoda. Answer all question in the style of yoda.") 12 | yoda.chat("In one sentence, why is the sky blue?") 13 | yoda.chat("In one sentence, how many inches in a feet?") 14 | ``` 15 | 16 | Here is the output from running this example. 17 | 18 | ```markdown 19 | > In one sentence, why is the sky blue? 20 | 21 | The sky appears blue due to a scattering effect called Rayleigh scattering 22 | where shorter wavelength light (blue light) is scattered more than other 23 | colors by the molecules in Earth's atmosphere. 24 | 25 | > In one sentence, how many inches in a feet? 26 | 27 | 1 foot is equivalent to 12 inches. 28 | 29 | > In one sentence, why is the sky blue? 30 | 31 | Because light from sun scatters more with molecules of air, making sky appear 32 | blue to us, Master. 33 | 34 | > In one sentence, how many inches in a feet? 35 | 36 | A feet contains twelve inches, it does. 37 | ``` 38 | 39 | Here is a state diagram of the Models and Responses used in this example, 40 | showing the branching of the usage of chat on the same `Model`. 41 | 42 | ---- 43 | 44 | ```mermaid 45 | graph LR 46 | 47 | start((hs)) 48 | m0(Model) 49 | m1(**session**: Model) 50 | r0(Response) 51 | r1(Response) 52 | m2(**yoda**: Model) 53 | r2(Response) 54 | r3(Response) 55 | 56 | start -- connect('…') --> m0 57 | m0 -- … | echo() --> m1 58 | m1 -- system('…') --> m2 59 | m1 -- chat('…') --> r0 60 | m1 -- chat('…') --> r1 61 | m2 -- chat('…') --> r2 62 | m2 -- chat('…') --> r3 63 | 64 | ``` 65 | 66 | ---- -------------------------------------------------------------------------------- /examples/tree_of_calls/main.py: -------------------------------------------------------------------------------- 1 | from haverscript import connect, echo 2 | 3 | model = connect("mistral") | echo() 4 | model.chat("In one sentence, why is the sky blue?") 5 | model.chat("In one sentence, how many inches in a feet?") 6 | # Set system prompt to Yoda's style 7 | yoda = model.system("You are yoda. Answer all question in the style of yoda.") 8 | yoda.chat("In one sentence, why is the sky blue?") 9 | yoda.chat("In one sentence, how many inches in a feet?") 10 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "haverscript" 3 | version = "0.3.1" 4 | description = "Library for building agents and managing LLM interactions." 5 | authors = [{ name = "Andy Gill", email = "andygillku@gmail.com" }] 6 | readme = "README.md" 7 | license = { text = "MIT" } 8 | requires-python = ">=3.10" 9 | dependencies = [ 10 | "docstring_parser >= 0.16", 11 | "jsonref>=1.1.0", 12 | "ollama>=0.4.4", 13 | "pydantic>=2.9.0", 14 | "yaspin>=3.0.0", 15 | ] 16 | 17 | [project.optional-dependencies] 18 | together = [ 19 | "together>=1.4.1", 20 | ] 21 | # all includes everything, and pytest support 22 | all = [ 23 | "pytest>=8.3.0", 24 | "pytest-regressions>=2.5.0", 25 | "pytest-xdist>=3.6.1", 26 | "prompt_toolkit>=3.0.48", 27 | "together>=1.4.1", 28 | ] 29 | 30 | [build-system] 31 | requires = ["setuptools>=42", "wheel"] 32 | build-backend = "setuptools.build_meta" 33 | 34 | [tool.pytest.ini_options] 35 | pythonpath = ["."] 36 | -------------------------------------------------------------------------------- /src/haverscript/__init__.py: -------------------------------------------------------------------------------- 1 | from .agents import Agent 2 | from .exceptions import ( 3 | LLMConfigurationError, 4 | LLMConnectivityError, 5 | LLMError, 6 | LLMPermissionError, 7 | LLMRateLimitError, 8 | LLMRequestError, 9 | LLMResponseError, 10 | LLMResultError, 11 | ) 12 | from .haverscript import Middleware, Model, Response, Service 13 | from .markdown import ( 14 | Markdown, 15 | header, 16 | text, 17 | bullets, 18 | rule, 19 | table, 20 | code, 21 | quoted, 22 | reply_in_json, 23 | template, 24 | xml_element, 25 | markdown, 26 | ) 27 | from .middleware import ( 28 | cache, 29 | dedent, 30 | echo, 31 | format, 32 | fresh, 33 | model, 34 | options, 35 | realtime, 36 | retry, 37 | stats, 38 | trace, 39 | transcript, 40 | validate, 41 | stream, 42 | ) 43 | from .ollama import connect 44 | from .chatbot import connect_chatbot, ChatBot 45 | from .tools import Tools, tool 46 | from .types import LanguageModel, Reply, Request, ServiceProvider, Middleware 47 | 48 | __all__ = [ 49 | "LLMConfigurationError", 50 | "LLMConnectivityError", 51 | "LLMError", 52 | "LLMPermissionError", 53 | "LLMRateLimitError", 54 | "LLMRequestError", 55 | "LLMResponseError", 56 | "LLMResultError", 57 | "Model", 58 | "Response", 59 | "Service", 60 | "cache", 61 | "dedent", 62 | "echo", 63 | "format", 64 | "fresh", 65 | "model", 66 | "options", 67 | "realtime", 68 | "retry", 69 | "stats", 70 | "trace", 71 | "transcript", 72 | "validate", 73 | "connect", 74 | "LanguageModel", 75 | "Markdown", 76 | "header", 77 | "text", 78 | "bullets", 79 | "rule", 80 | "table", 81 | "code", 82 | "quoted", 83 | "reply_in_json", 84 | "Reply", 85 | "Request", 86 | "ServiceProvider", 87 | "Middleware", 88 | "Agent", 89 | "Tools", 90 | "tool", 91 | "template", 92 | "markdown", 93 | "connect_chatbot", 94 | "ChatBot", 95 | "stream", 96 | "xml_element", 97 | ] 98 | -------------------------------------------------------------------------------- /src/haverscript/agents.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import Any, Type 3 | import copy 4 | from pydantic import BaseModel 5 | 6 | from haverscript.haverscript import Model 7 | from haverscript.markdown import Markdown 8 | from haverscript.middleware import ( 9 | EmptyMiddleware, 10 | format as format_middleware, 11 | stream as stream_middleware, 12 | echo as echo_middleware, 13 | ) 14 | from haverscript.tools import Tools 15 | from haverscript.types import Middleware, EmptyMiddleware, Reply 16 | from haverscript.markdown import markdown 17 | 18 | 19 | class Agent(BaseModel): 20 | """An Agent is a python class that has access to an LLM. 21 | 22 | The subclass must provide a system prompt, called system, 23 | that describes the agent's role, or an implementation of 24 | prepare, which sets up the agent's state and other considerations. 25 | """ 26 | 27 | model: Model 28 | 29 | def model_post_init(self, __context: dict | None = None) -> None: 30 | self.prepare() 31 | 32 | def prepare(self): 33 | """Prepare the agent for a new conversation. 34 | 35 | This method is called before the agent is asked to chat. 36 | This default adds self.system as a system command. 37 | self.system can be anything that can be str()ed. 38 | """ 39 | self.model = self.model.system(str(self.system)) 40 | 41 | def chat( 42 | self, 43 | prompt: str | Markdown, 44 | format: Type | None = None, 45 | middleware: Middleware | None = None, 46 | tools: Tools | None = None, 47 | ) -> str | Any: 48 | """ "chat with the llm and remember the conversation. 49 | 50 | If format is set, return the value of that type. 51 | """ 52 | if middleware is None: 53 | middleware = EmptyMiddleware() 54 | if format is not None: 55 | middleware = format_middleware(format) | middleware 56 | 57 | response = self.model.chat(prompt, middleware=middleware, tools=tools) 58 | 59 | self.model = response 60 | 61 | if format: 62 | return response.value 63 | 64 | return response.reply 65 | 66 | def ask( 67 | self, 68 | prompt: str | Markdown, 69 | format: Type | None = None, 70 | middleware: Middleware | None = None, 71 | stream: bool = False, 72 | ) -> str | Any | Reply: 73 | """ask the llm something without recording the conversation. 74 | 75 | If stream is set, return a Reply object. Reply is a monad. 76 | 77 | If format is set, return the value of that type. 78 | 79 | Otherwise, return a string. 80 | """ 81 | if middleware is None: 82 | middleware = EmptyMiddleware() 83 | if format is not None: 84 | middleware = format_middleware(format) | middleware 85 | if stream: 86 | middleware = stream_middleware() | middleware 87 | 88 | reply: Reply = self.model.ask(prompt, middleware=middleware) 89 | 90 | if stream: 91 | return reply 92 | 93 | if format: 94 | return reply.value 95 | 96 | return str(reply) 97 | 98 | def remember( 99 | self, 100 | prompt: str | Markdown, 101 | reply: str | Reply, 102 | ) -> None: 103 | """remember a conversation exchange. 104 | 105 | This is used to add a conversation to the agent's memory. 106 | This is useful for advanced prompt techniques, such as 107 | chaining or tree of calls. 108 | """ 109 | if isinstance(reply, str): 110 | reply = Reply([reply]) 111 | 112 | self.model = self.model.process(prompt, reply) 113 | 114 | def clone(self, kwargs: dict = {}) -> Agent: 115 | """clone the agent. 116 | 117 | The result should have its own identity and not be affected by the original agent. 118 | 119 | If the agent has additional state, it should have its own clone method, 120 | that calls this method with the additional state. 121 | """ 122 | return type(self)(model=self.model, system=self.system, **kwargs) 123 | -------------------------------------------------------------------------------- /src/haverscript/cache.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import json 3 | from dataclasses import asdict, dataclass, field, fields 4 | from abc import abstractmethod 5 | from .types import Exchange, Prompt, AssistantMessage 6 | 7 | SQL_VERSION = 2 8 | 9 | SQL_SCHEMA = f""" 10 | BEGIN; 11 | 12 | PRAGMA user_version = {SQL_VERSION}; 13 | 14 | CREATE TABLE IF NOT EXISTS string_pool ( 15 | id INTEGER PRIMARY KEY, 16 | string TEXT NOT NULL UNIQUE 17 | ); 18 | 19 | CREATE INDEX IF NOT EXISTS string_index ON string_pool(string); 20 | 21 | CREATE TABLE IF NOT EXISTS context ( 22 | id INTEGER PRIMARY KEY, 23 | prompt INTEGER NOT NULL, -- what was said to the LLM 24 | images INTEGER NOT NULL, -- string of list of images 25 | reply INTEGER NOT NULL, -- reply from the LLM 26 | context INTEGER, 27 | FOREIGN KEY (prompt) REFERENCES string_pool(id), 28 | FOREIGN KEY (images) REFERENCES string_pool(id), 29 | FOREIGN KEY (reply) REFERENCES string_pool(id), 30 | FOREIGN KEY (context) REFERENCES interactions(id) 31 | ); 32 | 33 | CREATE INDEX IF NOT EXISTS context_prompt_index ON context(prompt); 34 | CREATE INDEX IF NOT EXISTS context_images_index ON context(images); 35 | CREATE INDEX IF NOT EXISTS context_reply_index ON context(reply); 36 | CREATE INDEX IF NOT EXISTS context_context_index ON context(context); 37 | 38 | CREATE TABLE IF NOT EXISTS interactions ( 39 | id INTEGER PRIMARY KEY, 40 | system INTEGER, 41 | context INTEGER, 42 | parameters INTEGER NOT NULL, 43 | FOREIGN KEY (system) REFERENCES string_pool(id), 44 | FOREIGN KEY (context) REFERENCES context(id), 45 | FOREIGN KEY (parameters) REFERENCES string_pool(id) 46 | ); 47 | 48 | CREATE INDEX IF NOT EXISTS interactions_system_index ON interactions(system); 49 | CREATE INDEX IF NOT EXISTS interactions_context_index ON interactions(context); 50 | CREATE INDEX IF NOT EXISTS interactions_parameters_index ON interactions(parameters); 51 | 52 | CREATE TEMPORARY TABLE blacklist ( 53 | id INTEGER PRIMARY KEY, 54 | interaction INTEGER NOT NULL, 55 | FOREIGN KEY (interaction) REFERENCES interactions(id) 56 | ); 57 | 58 | COMMIT; 59 | """ 60 | 61 | 62 | @dataclass(frozen=True) 63 | class TEXT: 64 | id: int 65 | 66 | 67 | @dataclass(frozen=True) 68 | class PROMPT_REPLY: 69 | id: int 70 | 71 | 72 | @dataclass(frozen=True) 73 | class CONTEXT: 74 | id: int 75 | 76 | 77 | @dataclass(frozen=True) 78 | class INTERACTION: 79 | id: int 80 | 81 | 82 | @dataclass 83 | class DB: 84 | conn: sqlite3.Connection 85 | 86 | @abstractmethod 87 | def text(self, text: str) -> TEXT: 88 | pass 89 | 90 | @abstractmethod 91 | def context_row( 92 | self, prompt: TEXT, images: TEXT, reply: TEXT, context: CONTEXT 93 | ) -> PROMPT_REPLY: 94 | pass 95 | 96 | @abstractmethod 97 | def interaction_row( 98 | self, system: TEXT, context: CONTEXT, parameters: TEXT 99 | ) -> CONTEXT: 100 | pass 101 | 102 | def interaction_replies( 103 | self, 104 | system: TEXT, 105 | context: CONTEXT, 106 | prompt: TEXT | None, 107 | images: TEXT | None, 108 | parameters: TEXT, 109 | limit: int | None, 110 | blacklist: bool = False, 111 | ) -> dict[INTERACTION, tuple[str, list[str], str]]: 112 | 113 | interactions_args = { 114 | "system": system.id, 115 | "parameters": parameters.id, 116 | } 117 | 118 | context_args = { 119 | "context": context.id, 120 | } 121 | if prompt: 122 | context_args["prompt"] = prompt.id 123 | 124 | if images: 125 | context_args["images"] = images.id 126 | 127 | rows = self.conn.execute( 128 | "SELECT s1.string, s2.string, s3.string, interactions.id FROM " 129 | " interactions JOIN context JOIN string_pool as s1 JOIN string_pool as s2 JOIN string_pool as s3 WHERE " 130 | " interactions.context = context.id AND " 131 | " context.prompt = s1.id AND " 132 | " context.images = s2.id AND " 133 | " context.reply = s3.id AND " 134 | + " AND ".join( 135 | [ 136 | ( 137 | f"interactions.{key} IS NULL" 138 | if interactions_args[key] is None 139 | else f"interactions.{key} = :{key}" 140 | ) 141 | for key in interactions_args.keys() 142 | ] 143 | ) 144 | + " AND " 145 | + " AND ".join( 146 | [ 147 | ( 148 | f"context.{key} IS NULL" 149 | if context_args[key] is None 150 | else f"context.{key} = :{key}" 151 | ) 152 | for key in context_args.keys() 153 | ] 154 | ) 155 | + ( 156 | " AND interactions.id NOT IN (SELECT interaction FROM blacklist) " 157 | if blacklist 158 | else "" 159 | ) 160 | + (f" LIMIT {limit}" if limit else ""), 161 | interactions_args | context_args, 162 | ).fetchall() 163 | 164 | def decode_images(txt): 165 | # TODO: decode images 166 | if txt == '["foo.png"]': 167 | return ["foo.png"] 168 | return [] 169 | 170 | return { 171 | INTERACTION(row[3]): (row[0], decode_images(row[1]), row[2]) for row in rows 172 | } 173 | 174 | def blacklist(self, key: INTERACTION): # stale? 175 | self.conn.execute("INSERT INTO blacklist (interaction) VALUES (?)", (key.id,)) 176 | 177 | 178 | @dataclass 179 | class ReadOnly(DB): 180 | 181 | def text(self, text: str) -> TEXT: 182 | if text is None: 183 | return TEXT(None) 184 | assert isinstance(text, str), f"text={text}, expecting str" 185 | # Retrieve the id of the string 186 | if row := self.conn.execute( 187 | "SELECT id FROM string_pool WHERE string = ?", (text,) 188 | ).fetchone(): 189 | return TEXT(row[0]) # Return the id of the string 190 | else: 191 | raise ValueError 192 | 193 | def context_row( 194 | self, prompt: TEXT, images: TEXT, reply: TEXT, context: CONTEXT 195 | ) -> PROMPT_REPLY: 196 | 197 | args = { 198 | "prompt": prompt.id, 199 | "images": images.id, 200 | "reply": reply.id, 201 | "context": context.id, 202 | } 203 | 204 | if row := self.conn.execute( 205 | f"SELECT id FROM context WHERE " 206 | + " AND ".join( 207 | [ 208 | f"{key} IS NULL" if args[key] is None else f"{key} = :{key}" 209 | for key in args.keys() 210 | ] 211 | ), 212 | args, 213 | ).fetchone(): 214 | return CONTEXT(row[0]) # Return the id of the string 215 | 216 | raise ValueError 217 | 218 | def interaction_row( 219 | self, system: TEXT, context: CONTEXT, parameters: TEXT 220 | ) -> INTERACTION: 221 | 222 | args = { 223 | "system": system.id, 224 | "context": context.id, 225 | "parameters": parameters.id, 226 | } 227 | 228 | if row := self.conn.execute( 229 | f"SELECT id FROM interactions WHERE " 230 | + " AND ".join( 231 | [ 232 | f"{key} IS NULL" if args[key] is None else f"{key} = :{key}" 233 | for key in args.keys() 234 | ] 235 | ), 236 | args, 237 | ).fetchone(): 238 | return INTERACTION(row[0]) # Return the id of the string 239 | raise ValueError 240 | 241 | 242 | @dataclass 243 | class ReadAppend(DB): 244 | 245 | def text(self, text: str) -> TEXT: 246 | try: 247 | return ReadOnly(self.conn).text(text) 248 | except ValueError: 249 | return TEXT( 250 | self.conn.execute( 251 | "INSERT INTO string_pool (string) VALUES (?)", (text,) 252 | ).lastrowid 253 | ) 254 | 255 | def context_row( 256 | self, prompt: TEXT, images: TEXT, reply: TEXT, context: CONTEXT 257 | ) -> CONTEXT: 258 | assert isinstance(prompt, TEXT), f"prompt : {type(prompt)}, expecting : TEXT" 259 | assert isinstance(images, TEXT), f"images : {type(images)}, expecting : TEXT" 260 | assert isinstance(reply, TEXT), f"reply : {type(reply)}, expecting : TEXT" 261 | assert isinstance( 262 | context, CONTEXT 263 | ), f"context : {type(context)}, expecting : CONTEXT" 264 | 265 | try: 266 | return ReadOnly(self.conn).context_row(prompt, images, reply, context) 267 | except ValueError: 268 | return CONTEXT( 269 | self.conn.execute( 270 | "INSERT INTO context (prompt, images, reply, context) VALUES (?, ?, ?, ?)", 271 | (prompt.id, images.id, reply.id, context.id), 272 | ).lastrowid 273 | ) 274 | 275 | def interaction_row( 276 | self, system: TEXT, context: CONTEXT, parameters: TEXT 277 | ) -> INTERACTION: 278 | assert isinstance(system, TEXT), f"system : {type(context)}, expecting : TEXT" 279 | assert isinstance( 280 | context, CONTEXT 281 | ), f"context : {type(context)}, expecting : CONTEXT" 282 | assert isinstance( 283 | parameters, TEXT 284 | ), f"parameters : {type(context)}, expecting : TEXT" 285 | 286 | try: 287 | return ReadOnly(self.conn).interaction_row(system, context, parameters) 288 | except ValueError: 289 | interaction = INTERACTION( 290 | self.conn.execute( 291 | "INSERT INTO interactions (system, context, parameters) VALUES (?,?,?)", 292 | (system.id, context.id, parameters.id), 293 | ).lastrowid 294 | ) 295 | # The idea here is that if you have just added a result of calling a LLM, 296 | # then in the same session you re-ask the question, you want a new answer. 297 | self.blacklist(interaction) 298 | return interaction 299 | 300 | 301 | class Cache: 302 | connections = {} 303 | 304 | def __init__(self, filename: str, mode: str) -> None: 305 | self.version = SQL_VERSION 306 | self.filename = filename 307 | self.mode = mode 308 | 309 | assert mode in {"r", "a", "a+"} 310 | 311 | if (filename) in Cache.connections: 312 | self.conn = Cache.connections[filename] 313 | else: 314 | self.conn = sqlite3.connect( 315 | filename, check_same_thread=sqlite3.threadsafety != 3 316 | ) 317 | Cache.connections[filename] = self.conn 318 | self.conn.executescript(SQL_SCHEMA) 319 | 320 | if mode in {"a", "a+"}: 321 | self.db = ReadAppend(self.conn) 322 | elif mode == "r": 323 | self.db = ReadOnly(self.conn) 324 | 325 | def context(self, context): 326 | if not context: 327 | return CONTEXT(None) 328 | 329 | top: Exchange = context[-1] 330 | 331 | prompt, images, reply = top.prompt.content, top.prompt.images, top.reply.content 332 | context = self.context(context[:-1]) 333 | prompt = self.db.text(prompt) 334 | images = self.db.text(json.dumps(images)) 335 | reply = self.db.text(reply) 336 | context = self.db.context_row(prompt, images, reply, context) 337 | 338 | return context 339 | 340 | def insert_interaction(self, system, context, prompt, images, reply, parameters): 341 | assert ( 342 | prompt is not None 343 | ), f"should not be saving empty prompt, reply = {repr(reply)}" 344 | context = context + ( 345 | Exchange( 346 | prompt=Prompt(content=prompt, images=images), 347 | reply=AssistantMessage(content=reply), 348 | ), 349 | ) 350 | context = self.context(context) 351 | system = self.db.text(system) 352 | parameters = self.db.text(json.dumps(parameters)) 353 | self.db.interaction_row(system, context, parameters) 354 | self.conn.commit() 355 | 356 | def lookup_interactions( 357 | self, 358 | system: str, 359 | context: tuple, 360 | prompt: str | None, # None = match any 361 | images: list[str] | None, # None = match any 362 | parameters: dict, 363 | limit: int | None, 364 | blacklist: bool, 365 | ) -> dict[INTERACTION, str]: 366 | 367 | context = self.context(context) 368 | system = self.db.text(system) 369 | if prompt: 370 | prompt = self.db.text(prompt) 371 | if images: 372 | images = self.db.text(json.dumps(images)) 373 | parameters = self.db.text(json.dumps(parameters)) 374 | 375 | return self.db.interaction_replies( 376 | system, context, prompt, images, parameters, limit, blacklist 377 | ) 378 | 379 | def blacklist(self, key: INTERACTION): 380 | self.db.blacklist(key) 381 | self.conn.commit() 382 | -------------------------------------------------------------------------------- /src/haverscript/chatbot.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Protocol 2 | 3 | from haverscript import * 4 | from haverscript.types import ( 5 | Reply, 6 | Prompt, 7 | Contexture, 8 | Exchange, 9 | AssistantMessage, 10 | ServiceProvider, 11 | Request, 12 | ) 13 | from copy import deepcopy 14 | 15 | 16 | class ChatBot(Protocol): 17 | """A protocol for building chatbots. 18 | 19 | The assumption is that a chatbot looks after its own state. 20 | """ 21 | 22 | def chat_to_bot(self, text: str) -> Reply: 23 | """take a string, update the object, and return a Reply.""" 24 | ... 25 | 26 | 27 | def connect_chatbot( 28 | chatbot: ChatBot | Callable[[str | None], ChatBot], name: str = "chatbot" 29 | ) -> Model: 30 | """promote a ChatBot into a Model. 31 | 32 | The argument can also be a function that takes an (optional) system prompt, 33 | then returns the ChatBot. 34 | 35 | connect_chatbot handles state by cloing the ChatBot automatically. 36 | """ 37 | 38 | class ChatBotServiceProvider(ServiceProvider): 39 | def __init__(self): 40 | self._model_cache: dict[tuple[str, tuple[Exchange, ...]], ChatBot] = {} 41 | 42 | def list(self) -> list[str]: 43 | return [name] 44 | 45 | def ask(self, request: Request) -> Reply: 46 | assert isinstance(request, Request) 47 | assert isinstance(request.prompt, Prompt) 48 | assert isinstance(request.contexture, Contexture) 49 | 50 | system = request.contexture.system 51 | context = request.contexture.context 52 | 53 | try: 54 | _chatbot = self._model_cache[system, context] 55 | except KeyError: 56 | if context == (): 57 | if callable(chatbot): 58 | _chatbot = chatbot(system) 59 | else: 60 | assert system is None, "chatbot does not support system prompts" 61 | _chatbot = deepcopy(chatbot) 62 | self._model_cache[system, ()] = _chatbot 63 | else: 64 | # We has a context we've never seen 65 | # Which means we did not generate it 66 | # Which means we reject it 67 | # Should never happen with regular usage 68 | assert False, "unknown system or context" 69 | 70 | _chatbot: ChatBot = deepcopy(_chatbot) 71 | 72 | response: Reply = _chatbot.chat_to_bot(request.prompt.content) 73 | 74 | def after(): 75 | exchange = Exchange( 76 | prompt=request.prompt, reply=AssistantMessage(content=str(response)) 77 | ) 78 | self._model_cache[system, context + (exchange,)] = _chatbot 79 | 80 | response.after(after) 81 | return response 82 | 83 | return Service(ChatBotServiceProvider()) | model(name) 84 | -------------------------------------------------------------------------------- /src/haverscript/exceptions.py: -------------------------------------------------------------------------------- 1 | class LLMError(Exception): 2 | """Base exception for all LLM-related errors.""" 3 | 4 | 5 | class LLMConfigurationError(LLMError): 6 | """Exception raised for errors occurring during LLM configuration.""" 7 | 8 | 9 | class LLMRequestError(LLMError): 10 | """Exception raised for errors occurring during the request to the LLM.""" 11 | 12 | 13 | class LLMConnectivityError(LLMRequestError): 14 | """Exception raised due to connectivity issues with the LLM service.""" 15 | 16 | 17 | class LLMPermissionError(LLMRequestError): 18 | """Exception raised when access to the LLM is denied due to permission issues.""" 19 | 20 | 21 | class LLMRateLimitError(LLMRequestError): 22 | """Exception raised when the rate limit is exceeded with the LLM service.""" 23 | 24 | 25 | class LLMResponseError(LLMError): 26 | """Exception raised for errors occurring during the response from the LLM.""" 27 | 28 | 29 | class LLMResultError(LLMError): 30 | """Exception raised for errors related to the LLM's output quality.""" 31 | 32 | 33 | class LLMInternalError(LLMError): 34 | """Exception raised when something was inconsistent internally""" 35 | -------------------------------------------------------------------------------- /src/haverscript/haverscript.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from abc import ABC 4 | from dataclasses import dataclass, field, replace 5 | from typing import Any 6 | import json 7 | from pydantic import BaseModel, ConfigDict 8 | 9 | from .types import ( 10 | ServiceProvider, 11 | Metrics, 12 | Contexture, 13 | Request, 14 | Reply, 15 | Exchange, 16 | Prompt, 17 | RequestMessage, 18 | EmptyMiddleware, 19 | AssistantMessage, 20 | ToolResult, 21 | ToolReply, 22 | ToolCall, 23 | ) 24 | from .exceptions import LLMInternalError 25 | from .middleware import Middleware, CacheMiddleware, ToolMiddleware 26 | from .render import render_interaction, render_system 27 | from .markdown import Markdown 28 | from .tools import Tools 29 | 30 | 31 | @dataclass(frozen=True) 32 | class Settings: 33 | """Local settings.""" 34 | 35 | service: Middleware = None 36 | 37 | middleware: Middleware = field(default_factory=EmptyMiddleware) 38 | 39 | 40 | @dataclass(frozen=True) 41 | class Service(ABC): 42 | service: ServiceProvider 43 | 44 | def list(self): 45 | return self.service.list() 46 | 47 | def __or__(self, other: Middleware) -> Model: 48 | assert isinstance(other, Middleware), "Can only pipe with middleware" 49 | return Model( 50 | settings=Settings(service=self.service, middleware=other), 51 | contexture=Contexture(), 52 | ) 53 | 54 | 55 | class Model(BaseModel): 56 | 57 | settings: Settings 58 | contexture: Contexture 59 | 60 | model_config = ConfigDict(frozen=True) 61 | 62 | def chat( 63 | self, 64 | prompt: str | Markdown, 65 | images: list[str] = [], 66 | middleware: Middleware | None = None, 67 | tools: Tools | None = None, 68 | ) -> Response: 69 | """ 70 | Take a prompt and call the LLM in a previously provided context. 71 | 72 | Args: 73 | prompt (str): the prompt 74 | images: (list): images to pass to the LLM 75 | middleware (Middleware): extra middleware specifically for this prompt 76 | tools (Tools): tools to handle tool calls 77 | 78 | Returns: 79 | A Response that contains the reply, and context for any future 80 | calls to chat. 81 | """ 82 | if isinstance(prompt, Markdown): 83 | prompt = str(prompt) 84 | 85 | if tools: 86 | if middleware: 87 | middleware = middleware | ToolMiddleware(tools.tool_schemas()) 88 | else: 89 | middleware = ToolMiddleware(tools.tool_schemas()) 90 | 91 | request = self.request(Prompt(content=prompt, images=images)) 92 | reply = self._ask(request, middleware) 93 | 94 | response = self.process(request, reply) 95 | 96 | while response.tool_calls: 97 | assert ( 98 | tools is not None 99 | ), "internal error: tools should be provided to handle tool calls" 100 | results = [] 101 | for tool in response.tool_calls: 102 | output = tools(tool.name, tool.arguments) 103 | results.append( 104 | ToolReply(id=tool.id, name=tool.name, content=str(output)) 105 | ) 106 | 107 | request = response.request(ToolResult(results=results)) 108 | reply = response._ask(request, middleware) 109 | response = response.process(request, reply) 110 | 111 | return response 112 | 113 | def ask( 114 | self, 115 | prompt: str | Markdown, 116 | images: list[str] = [], 117 | middleware: Middleware | None = None, 118 | ) -> Reply: 119 | """ 120 | Take a prompt and call the LLM in a previously provided context. 121 | 122 | Args: 123 | prompt (str): the prompt 124 | images: (list): images to pass to the LLM 125 | middleware (Middleware): extra middleware specifically for this prompt 126 | 127 | Returns: 128 | A Reply without any context, with dynamic content. 129 | 130 | Notes: 131 | ask does not support tool calls 132 | """ 133 | if isinstance(prompt, Markdown): 134 | prompt = str(prompt) 135 | 136 | return self._ask( 137 | self.request(Prompt(content=prompt, images=images)), middleware 138 | ) 139 | 140 | def _ask( 141 | self, 142 | request: Request, 143 | middleware: Middleware | None = None, 144 | ) -> Reply: 145 | """ 146 | Take a internal request and call the LLM using this context. 147 | 148 | Args: 149 | request (Request): the context of the LLM call 150 | middleware (Middleware): extra middleware specifically for this prompt 151 | 152 | Returns: 153 | A Reply without any context, with dynamic content. 154 | """ 155 | assert request is not None, "Can not build a response with no prompt" 156 | 157 | if middleware is not None: 158 | middleware = self.settings.middleware | middleware 159 | else: 160 | middleware = self.settings.middleware 161 | 162 | return middleware.invoke(request=request, next=self.settings.service) 163 | 164 | def process(self, request: Request | str | Markdown, response: Reply) -> Response: 165 | 166 | if isinstance(request, str): 167 | prompt = request 168 | elif isinstance(request, Markdown): 169 | prompt = str(request) 170 | else: 171 | prompt = request.prompt 172 | 173 | return self.response( 174 | prompt, 175 | str(response), 176 | metrics=response.metrics(), 177 | value=response.value, 178 | tool_calls=tuple(response.tool_calls()), 179 | ) 180 | 181 | def request( 182 | self, 183 | prompt: RequestMessage | None, 184 | format: str | dict = "", 185 | fresh: bool = False, 186 | stream: bool = False, 187 | ) -> Request: 188 | 189 | return Request( 190 | contexture=self.contexture, 191 | prompt=prompt, 192 | format=format, 193 | fresh=fresh, 194 | stream=stream, 195 | ) 196 | 197 | def response( 198 | self, 199 | prompt: RequestMessage | str, 200 | reply: str, 201 | metrics: Metrics | None = None, 202 | value: Any | None = None, 203 | tool_calls: tuple[ToolCall, ...] = (), 204 | ): 205 | if isinstance(prompt, str): 206 | prompt = Prompt(content=prompt) 207 | 208 | assert isinstance(prompt, RequestMessage) 209 | assert isinstance(metrics, (Metrics, type(None))) 210 | assert isinstance(tool_calls, tuple) and all( 211 | isinstance(tc, ToolCall) for tc in tool_calls 212 | ) 213 | return Response( 214 | settings=self.settings, 215 | contexture=self.contexture.append_exchange( 216 | Exchange( 217 | prompt=prompt, 218 | reply=AssistantMessage(content=reply), 219 | ) 220 | ), 221 | parent=self, 222 | metrics=metrics, 223 | value=value, 224 | tool_calls=tool_calls, 225 | ) 226 | 227 | def children(self, prompt: str | None = None, images: list[str] | None = []): 228 | """Return all already cached replies to this prompt.""" 229 | 230 | service = self.settings.service 231 | first = self.settings.middleware.first() 232 | 233 | if not isinstance(first, CacheMiddleware): 234 | # only top-level cache can be interrogated. 235 | raise LLMInternalError( 236 | ".children(...) method needs cache to be final middleware" 237 | ) 238 | 239 | if prompt is not None: 240 | prompt = Prompt(content=prompt, images=tuple(images)) 241 | 242 | replies = first.children(self.request(prompt)) 243 | 244 | return [ 245 | self.response(Prompt(content=prompt_, images=tuple(images)), prose) 246 | for prompt_, images, prose in replies 247 | ] 248 | 249 | def render(self) -> str: 250 | """Return a markdown string of the context.""" 251 | return render_system(self.contexture.system) 252 | 253 | # Content methods 254 | 255 | def system(self, prompt: str | Markdown) -> Model: 256 | """provide a system prompt.""" 257 | if isinstance(prompt, Markdown): 258 | prompt = str(prompt) 259 | 260 | return self.model_copy( 261 | update=dict( 262 | contexture=self.contexture.model_copy(update=dict(system=prompt)) 263 | ) 264 | ) 265 | 266 | def load(self, markdown: str, complete: bool = False) -> Model: 267 | """Read markdown as system + prompt-reply pairs.""" 268 | 269 | lines = markdown.split("\n") 270 | result = [] 271 | if not lines: 272 | return self 273 | 274 | current_block = [] 275 | current_is_quote = lines[0].startswith("> ") 276 | starts_with_quote = current_is_quote 277 | 278 | for line in lines: 279 | is_quote = line.startswith("> ") 280 | # Remove the '> ' prefix if it's a quote line 281 | line_content = line[2:] if is_quote else line 282 | if is_quote == current_is_quote: 283 | current_block.append(line_content) 284 | else: 285 | result.append("\n".join(current_block)) 286 | current_block = [line_content] 287 | current_is_quote = is_quote 288 | 289 | # Append the last block 290 | if current_block: 291 | result.append("\n".join(current_block)) 292 | 293 | model = self 294 | 295 | if not starts_with_quote: 296 | if sys_prompt := result[0].strip(): 297 | # only use non-empty system prompts 298 | model = model.system(sys_prompt) 299 | result = result[1:] 300 | 301 | while result: 302 | prompt = result[0].strip() 303 | if len(result) == 1: 304 | reply = "" 305 | else: 306 | reply = result[1].strip() 307 | 308 | if complete and reply in ("", "..."): 309 | model = model.chat(prompt) 310 | else: 311 | model = model.response(prompt, reply) 312 | 313 | result = result[2:] 314 | 315 | return model 316 | 317 | def compress(self, count: int) -> Model: 318 | """Remove historical chat queries, leaving only `count` prompt-response pairs.""" 319 | if count == 0: 320 | model = self 321 | while isinstance(model, Response): 322 | model = model.parent 323 | return model 324 | if isinstance(self, Response): 325 | previous = self.parent.compress(count - 1) 326 | return previous.response( 327 | prompt=self.contexture.context[-1].prompt, 328 | reply=self.reply, 329 | metrics=self.metrics, 330 | value=self.value, 331 | tool_calls=self.tool_calls, 332 | ) 333 | # already at the initial model, so no more pruning to do 334 | return self 335 | 336 | def __or__(self, other: Middleware) -> Model: 337 | """pipe to append middleware to a model""" 338 | assert isinstance(other, Middleware), "Can only pipe with middleware" 339 | return self.model_copy( 340 | update=dict( 341 | settings=replace( 342 | self.settings, middleware=self.settings.middleware | other 343 | ) 344 | ) 345 | ) 346 | 347 | 348 | class Response(Model): 349 | 350 | parent: Model 351 | metrics: Metrics | None 352 | value: Any | None 353 | tool_calls: tuple[ToolCall, ...] 354 | 355 | @property 356 | def prompt(self) -> str: 357 | assert len(self.contexture.context) > 0 358 | return self.contexture.context[-1].prompt.content 359 | 360 | @property 361 | def reply(self) -> str: 362 | assert len(self.contexture.context) > 0 363 | return self.contexture.context[-1].reply.content 364 | 365 | def render(self) -> str: 366 | """Return a markdown string of the context.""" 367 | return render_interaction(self.parent.render(), self.contexture.context[-1]) 368 | 369 | def __str__(self): 370 | return self.reply 371 | -------------------------------------------------------------------------------- /src/haverscript/markdown.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | import inspect 3 | import json 4 | from typing import Type, Callable, Any, get_type_hints, get_args, get_origin 5 | import string 6 | import re 7 | from pydantic import BaseModel 8 | 9 | 10 | class Markdown: 11 | def __init__( 12 | self, content: list[str] | None = None, args: list[set[str]] | None = None 13 | ): 14 | if content is None: 15 | self.blocks = [] 16 | else: 17 | self.blocks = content 18 | 19 | if args is None: 20 | self.args = [set() for _ in self.blocks] 21 | else: 22 | self.args = args 23 | 24 | assert len(self.blocks) == len(self.args), "internal inconsistency in Markdown" 25 | 26 | def __str__(self): 27 | # all blocks are separated by a blank line 28 | # this does not do any formatting of {variables}. 29 | return "\n\n".join(self.blocks) 30 | 31 | def format(self, **kwargs: dict): 32 | return "\n\n".join( 33 | [ 34 | ( 35 | block.format( 36 | **{key: kwargs[key] for key in spec_keys if key in kwargs}, 37 | ) 38 | if spec_keys 39 | else block 40 | ) 41 | for block, spec_keys in zip(self.blocks, self.args) 42 | ] 43 | ) 44 | 45 | def __add__(self, other): 46 | if isinstance(other, Markdown): 47 | return Markdown(self.blocks + other.blocks, self.args + other.args) 48 | txt = str(other) 49 | txt = strip_text(txt) 50 | return Markdown(self.blocks + [txt], self.args + [set()]) 51 | 52 | 53 | def markdown(content: str | Markdown | None = None) -> Markdown: 54 | """Convert a string or markdown block into markdown block.""" 55 | if content is None: 56 | return Markdown() 57 | if isinstance(content, str): 58 | return text(content) 59 | return content 60 | 61 | 62 | def strip_text(txt: str) -> str: 63 | txt = txt.strip() # remove blank lines before and after 64 | txt = "\n".join([line.strip() for line in txt.splitlines()]) 65 | return txt 66 | 67 | 68 | def header(txt, level: int = 1) -> Markdown: 69 | """Return a markdown header.""" 70 | return Markdown([f"{'#' * level} {txt}"]) 71 | 72 | 73 | def xml_element(tag: str, contents: str | Markdown | None = None): 74 | singleton = Markdown([f"<{tag}/>"]) 75 | 76 | if contents is None: 77 | return singleton 78 | 79 | inner: Markdown = markdown(contents) 80 | 81 | if len(inner.blocks) == 0: 82 | return singleton 83 | 84 | blocks = inner.blocks 85 | blocks = [f"<{tag}>\n{blocks[0]}"] + blocks[1:] 86 | blocks = blocks[:-1] + [f"{blocks[-1]}\n"] 87 | 88 | return Markdown(content=blocks, args=inner.args) 89 | 90 | 91 | def bullets(items, ordered: bool = False) -> Markdown: 92 | """Return a markdown bullet list.""" 93 | if ordered: 94 | markdown_items = [f"{i+1}. {item}" for i, item in enumerate(items)] 95 | else: 96 | markdown_items = [f"- {item}" for item in items] 97 | return Markdown(["\n".join(markdown_items)]) 98 | 99 | 100 | def code(code: str, language: str = "") -> Markdown: 101 | """Return a markdown code block.""" 102 | return Markdown([f"```{language}\n{code}\n```"]) 103 | 104 | 105 | def text(txt: str) -> Markdown: 106 | """Return a markdown text block. 107 | 108 | Note that when using + or +=, text is automatically converted to a markdown block. 109 | """ 110 | return Markdown() + txt 111 | 112 | 113 | def quoted(txt) -> Markdown: 114 | """Return a quotes text block inside triple quotes.""" 115 | return Markdown([f'"""\n{txt}\n"""']) 116 | 117 | 118 | def rule(count: int = 3) -> Markdown: 119 | """Return a markdown horizontal rule.""" 120 | return Markdown(["-" * count]) 121 | 122 | 123 | def table(headers: dict, rows: list[dict]) -> Markdown: 124 | """Return a markdown table. 125 | 126 | headers is a dictionary of column names to display. 127 | rows is a list of dictionaries, each containing the data for a row. 128 | 129 | We right justify integers and left justify anything else. 130 | """ 131 | col_widths = { 132 | key: max(*(len(str(row[key])) for row in [headers] + rows)) for key in headers 133 | } 134 | 135 | separator = "|-" + "-|-".join("-" * col_widths[key] for key in headers) + "-|" 136 | 137 | header_row = ( 138 | "| " + " | ".join(f"{headers[key]:{col_widths[key]}}" for key in headers) + " |" 139 | ) 140 | data_rows = [ 141 | "| " 142 | + " | ".join( 143 | ( 144 | f"{row[key]:>{col_widths[key]}}" 145 | if isinstance(row[key], int) 146 | else f"{row[key]:<{col_widths[key]}}" 147 | ) 148 | for key in headers 149 | ) 150 | + " |" 151 | for row in rows 152 | ] 153 | return "\n".join([header_row, separator] + data_rows) 154 | 155 | 156 | def reply_in_json( 157 | model: Type[BaseModel], prefix: str = "Reply in JSON, using the following keys:" 158 | ) -> Markdown: 159 | """Instructions to reply in JSON using a list of bullets schema.""" 160 | prompt = Markdown() 161 | prompt += text(prefix) 162 | items = [] 163 | type_hints = get_type_hints(model) 164 | for key, value in model.model_fields.items(): 165 | annotation = type_hints.get(key, value.annotation) 166 | if annotation is None: 167 | type_name = "" 168 | elif get_origin(annotation) is list and get_args(annotation): 169 | type_name = f" (list of {get_args(annotation)[0].__name__})" 170 | elif inspect.isclass(annotation) and issubclass(annotation, Enum): 171 | enum_values = " or ".join(json.dumps(item.value) for item in annotation) 172 | type_name = f" ({enum_values})" 173 | elif isinstance(annotation, type): 174 | type_name = f" ({annotation.__name__})" 175 | else: 176 | type_name = ( 177 | f" ({repr(annotation).replace('typing.', '').replace('|', 'or')})" 178 | ) 179 | items.append(f'"{str(key)}"{type_name}: {value.description}') 180 | prompt += bullets(items) 181 | return prompt 182 | 183 | 184 | def template(fstring: str) -> Markdown: 185 | """Delay interpretation of a f-string until later. 186 | 187 | variables are allowed inside {braces}, and will be filled in 188 | by calls to the format method. 189 | """ 190 | formatter = string.Formatter() 191 | variables = set() 192 | fstring = strip_text(fstring) 193 | 194 | for _, field_name, _, _ in formatter.parse(fstring): 195 | if field_name: 196 | variables.add(field_name) 197 | assert bool( 198 | re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", field_name) 199 | ), f"invalid variable expression {field_name}, should be a variable name." 200 | 201 | return Markdown([fstring], [variables]) 202 | -------------------------------------------------------------------------------- /src/haverscript/ollama.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from types import GeneratorType 3 | 4 | import ollama 5 | 6 | from .haverscript import Model, Service 7 | from .types import Metrics, ServiceProvider, Reply, Request, SystemMessage, ToolCall 8 | from .middleware import model 9 | 10 | 11 | class OllamaMetrics(Metrics): 12 | total_duration: int # time spent generating the response 13 | load_duration: int # time spent in nanoseconds loading the model 14 | prompt_eval_count: int # number of tokens in the prompt 15 | prompt_eval_duration: int # time spent in nanoseconds evaluating the prompt 16 | eval_count: int # number of tokens in the response 17 | eval_duration: int # time in nanoseconds spent generating the response 18 | 19 | 20 | class Ollama(ServiceProvider): 21 | client = {} 22 | 23 | def __init__(self, hostname: str | None = None) -> None: 24 | self.hostname = hostname 25 | if hostname not in self.client: 26 | self.client[hostname] = ollama.Client(host=hostname) 27 | 28 | def list(self) -> list[str]: 29 | models = self.client[self.hostname].list() 30 | assert "models" in models 31 | return [model.model for model in models["models"]] 32 | 33 | def _suggestions(self, e: Exception): 34 | # Slighty better message. Should really have a type of reply for failure. 35 | if "ConnectError" in str(type(e)): 36 | print("Connection error (Check if ollama is running)") 37 | return e 38 | 39 | def tokenize(self, chunk: dict): 40 | if chunk["done"]: 41 | yield OllamaMetrics( 42 | **{k: chunk[k] for k in OllamaMetrics.model_fields.keys()} 43 | ) 44 | message = chunk["message"] 45 | if "tool_calls" in message: 46 | for tool in message["tool_calls"]: 47 | yield ToolCall( 48 | name=tool.function.name, 49 | arguments=tool.function.arguments, 50 | ) 51 | assert isinstance(message["content"], str) 52 | yield message["content"] 53 | 54 | def generator(self, response): 55 | 56 | if isinstance(response, GeneratorType): 57 | try: 58 | for chunk in response: 59 | yield from self.tokenize(chunk) 60 | 61 | except Exception as e: 62 | raise self._suggestions(e) 63 | 64 | else: 65 | yield from self.tokenize(response) 66 | 67 | def ask(self, request: Request): 68 | 69 | messages = [] 70 | 71 | if request.contexture.system: 72 | SystemMessage(content=request.contexture.system).append_to(messages) 73 | 74 | for exchange in request.contexture.context: 75 | exchange.append_to(messages) 76 | 77 | request.prompt.append_to(messages) 78 | 79 | tools = None 80 | if request.tools: 81 | tools = list(request.tools) 82 | 83 | # normalize the messages for ollama 84 | messages = [ 85 | { 86 | key: value 87 | for key, value in original_dict.items() 88 | if key in {"role", "content", "images"} 89 | } 90 | for original_dict in messages 91 | ] 92 | 93 | try: 94 | response = self.client[self.hostname].chat( 95 | model=request.contexture.model, 96 | stream=request.stream, 97 | messages=messages, 98 | options=request.contexture.options, 99 | format=request.format, 100 | tools=tools, 101 | ) 102 | 103 | return Reply(self.generator(response)) 104 | 105 | except Exception as e: 106 | raise self._suggestions(e) 107 | 108 | 109 | def connect( 110 | model_name: str | None = None, 111 | hostname: str | None = None, 112 | ) -> Model | Service: 113 | """return a model or service that uses the given model name.""" 114 | 115 | service = Service(Ollama(hostname=hostname)) 116 | 117 | if model_name: 118 | service = service | model(model_name) 119 | 120 | return service 121 | -------------------------------------------------------------------------------- /src/haverscript/render.py: -------------------------------------------------------------------------------- 1 | from .types import Prompt, Exchange 2 | 3 | 4 | def _canonical_string(string, postfix="\n"): 5 | """Adds a newline to a string if needed, for outputting to a file.""" 6 | if not string: 7 | return string 8 | if not string.endswith(postfix): 9 | overlap_len = next( 10 | (i for i in range(1, len(postfix)) if string.endswith(postfix[:i])), 0 11 | ) 12 | string += postfix[overlap_len:] 13 | return string 14 | 15 | 16 | def render_system(system) -> str: 17 | return _canonical_string(system or "") 18 | 19 | 20 | def render_interaction(context: str, exchange: Exchange) -> str: 21 | assert isinstance(context, str) 22 | assert isinstance(exchange, Exchange) 23 | prompt = exchange.prompt.content 24 | reply = exchange.reply.content 25 | 26 | context = _canonical_string(context, postfix="\n\n") 27 | 28 | if prompt: 29 | prompt = "".join([f"> {line}\n" for line in prompt.splitlines()]) + "\n" 30 | else: 31 | prompt = ">\n\n" 32 | 33 | reply = reply or "" 34 | 35 | return context + prompt + _canonical_string(reply.strip()) 36 | -------------------------------------------------------------------------------- /src/haverscript/together.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dataclasses import dataclass 3 | import time 4 | from types import GeneratorType 5 | import json 6 | 7 | import together 8 | 9 | from .haverscript import Metrics, Model, Service 10 | from .types import Reply, Request, ServiceProvider, SystemMessage, ToolCall 11 | from .middleware import model 12 | 13 | 14 | class TogetherMetrics(Metrics): 15 | prompt_tokens: int 16 | completion_tokens: int 17 | total_tokens: int 18 | 19 | 20 | class Together(ServiceProvider): 21 | client: together.Together | None = None 22 | hostname = "" 23 | 24 | def __init__( 25 | self, 26 | api_key: str | None = None, 27 | timeout: int | None = None, 28 | max_retries: int | None = None, 29 | ) -> None: 30 | if api_key is None: 31 | api_key = os.getenv("TOGETHER_API_KEY") 32 | assert api_key is not None, "need TOGETHER_API_KEY" 33 | if self.client is None: 34 | self.client: Together = together.Together( 35 | api_key=api_key, timeout=timeout, max_retries=max_retries 36 | ) 37 | 38 | def list(self) -> list[str]: 39 | models = self.client.models.list() 40 | return [model.id for model in models] 41 | 42 | def _suggestions(self, e: Exception): 43 | # Slighty better message. Should really have a type of reply for failure. 44 | if "ConnectError" in str(type(e)): 45 | print("Connection error with together.ai") 46 | return e 47 | 48 | def metrics(self, chunk: dict) -> Metrics: 49 | return TogetherMetrics( 50 | **{k: chunk[k] for k in TogetherMetrics.model_fields.keys()} 51 | ) 52 | 53 | def tool_calls(self, message): 54 | if hasattr(message, "tool_calls") and message.tool_calls is not None: 55 | for tool in message.tool_calls: 56 | yield ToolCall( 57 | name=tool.function.name, 58 | arguments=json.loads(tool.function.arguments), 59 | id=tool.id, 60 | ) 61 | 62 | def generator(self, response): 63 | 64 | if isinstance(response, GeneratorType): 65 | try: 66 | for chunk in response: 67 | for choice in chunk.choices: 68 | if choice.finish_reason and chunk.usage: 69 | yield self.metrics(chunk.usage.model_dump()) 70 | yield choice.delta.content 71 | except Exception as e: 72 | raise self._suggestions(e) 73 | else: 74 | for chunk in response.choices: 75 | if isinstance(chunk.message.content, str): 76 | yield chunk.message.content 77 | yield from self.tool_calls(response.choices[0].message) 78 | yield self.metrics(response.usage.model_dump()) 79 | 80 | def ask(self, request: Request): 81 | 82 | messages = [] 83 | 84 | if request.contexture.system: 85 | SystemMessage(content=request.contexture.system).append_to(messages) 86 | 87 | for exchange in request.contexture.context: 88 | exchange.append_to(messages) 89 | 90 | request.prompt.append_to(messages) 91 | 92 | # normalize the messages for together 93 | key_map = { 94 | "role": "role", 95 | "content": "content", 96 | "id": "tool_call_id", 97 | "name": "name", 98 | } 99 | messages = [ 100 | { 101 | key_map[key]: value 102 | for key, value in original_dict.items() 103 | if key in key_map 104 | } 105 | for original_dict in messages 106 | ] 107 | # remove all messages that have content == "" 108 | messages = [message for message in messages if message["content"] != ""] 109 | # if there are any role == "tool" messages, then set this to true 110 | tool_reply = messages[-1]["role"] == "tool" 111 | 112 | together_keywords = set( 113 | [ 114 | "max_tokens", 115 | "stop", 116 | "temperature", 117 | "top_p", 118 | "top_k", 119 | "repetition_penalty", 120 | "presence_penalty", 121 | "frequency_penalty", 122 | "min_p", 123 | "logit_bias", 124 | "seed", 125 | "logprobs", 126 | ] 127 | ) 128 | kwargs = { 129 | key: request.contexture.options[key] 130 | for key in request.contexture.options 131 | if key in together_keywords 132 | } 133 | 134 | response_format = None 135 | if format == "json": 136 | response_format = {"type": "json_object"} 137 | elif isinstance(format, dict): 138 | response_format = {"type": "json_object", "schema": format} 139 | 140 | # In together.ai, we can not use tools when re-running with a tool call response. 141 | if request.tools and not tool_reply: 142 | assert ( 143 | request.stream is False 144 | ), "Can not use together function calling tools with streaming" 145 | kwargs["tools"] = list(request.tools) 146 | kwargs["tool_choice"] = "auto" 147 | 148 | try: 149 | assert isinstance(self.client, together.Together) 150 | response = self.client.chat.completions.create( 151 | model=request.contexture.model, 152 | stream=request.stream, 153 | messages=messages, 154 | response_format=response_format, 155 | **kwargs, 156 | ) 157 | 158 | return Reply(self.generator(response)) 159 | 160 | except Exception as e: 161 | raise self._suggestions(e) 162 | 163 | 164 | def connect( 165 | model_name: str | None = None, 166 | api_key: str | None = None, 167 | timeout: int | None = None, 168 | max_retries: int | None = None, 169 | ) -> Model | Service: 170 | """return a model or service that uses the given model name.""" 171 | 172 | service = Service( 173 | Together(api_key=api_key, timeout=timeout, max_retries=max_retries) 174 | ) 175 | if model_name: 176 | service = service | model(model_name) 177 | return service 178 | -------------------------------------------------------------------------------- /src/haverscript/tools.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import copy 4 | import inspect 5 | from collections.abc import Callable 6 | from typing import Callable 7 | 8 | import docstring_parser as dsp 9 | from pydantic import BaseModel, Field, TypeAdapter 10 | 11 | 12 | def tool_schema(tool: Callable): 13 | """ 14 | Takes a callable tool, and returns a schema for the tool 15 | 16 | args: 17 | tool (Callable): The tool to get the schema for 18 | 19 | returns: 20 | dict: The schema for the tool 21 | """ 22 | # check to see if tool is callable. If not raise an error 23 | if not callable(tool): 24 | raise TypeError("tool must be a callable") 25 | 26 | name = tool.__name__ 27 | 28 | docstring = inspect.getdoc(tool) 29 | docstring_content = dsp.parse(docstring) 30 | description = docstring_content.short_description 31 | 32 | docstring_params = {} 33 | 34 | for param in docstring_content.params: 35 | docstring_params[param.arg_name] = param.description 36 | 37 | # get the signature of the tool 38 | signature = inspect.signature(tool) 39 | 40 | # get the parameters of the tool 41 | parameters = signature.parameters 42 | 43 | # create a dictionary to store the schema 44 | schema = {"type": "object", "required": [], "properties": {}} 45 | 46 | # loop through the parameters 47 | for parameter in parameters: 48 | parameter_object: inspect.Parameter = parameters[parameter] 49 | 50 | parameter_schema = copy.deepcopy( 51 | TypeAdapter(parameter_object.annotation).json_schema() 52 | ) 53 | 54 | if parameter_object.default == inspect.Parameter.empty: 55 | schema["required"].append(parameter) 56 | 57 | if parameter in docstring_params: 58 | parameter_schema["description"] = docstring_params[parameter] 59 | 60 | schema["properties"][parameter] = parameter_schema 61 | 62 | return { 63 | "type": "function", 64 | "function": {"name": name, "description": description, "parameters": schema}, 65 | } 66 | 67 | 68 | class Tools(BaseModel): 69 | pass 70 | 71 | def tool_schemas(self) -> tuple[dict, ...]: 72 | return () 73 | 74 | def __add__(self, other: Tools) -> ToolPair: 75 | return ToolPair(lhs=self, rhs=other) 76 | 77 | def __call__(self, name, arguments): 78 | raise ValueError(f"Tool {name} not found") 79 | 80 | 81 | class ToolPair(Tools): 82 | lhs: Tools 83 | rhs: Tools 84 | 85 | def tool_schemas(self) -> tuple[dict, ...]: 86 | return self.lhs.tool_schemas() + self.rhs.tool_schemas() 87 | 88 | def __call__(self, name, arguments): 89 | try: 90 | return self.lhs(name, arguments) 91 | except ValueError: 92 | return self.rhs(name, arguments) 93 | 94 | 95 | class Tool(Tools): 96 | """Provide a tool to the LLM to optionally use. 97 | 98 | attributes: 99 | tool (Callable): The tool to provide to the LLM 100 | schema (dict): The schema for the tool 101 | """ 102 | 103 | tool: Callable 104 | tool_schema: dict 105 | name: str 106 | debug: bool 107 | 108 | def tool_schemas(self): 109 | return (self.tool_schema,) 110 | 111 | def __call__(self, name, arguments): 112 | if self.name == name: 113 | return self.tool(**arguments) 114 | else: 115 | raise ValueError(f"Tool {name} not found") 116 | 117 | 118 | def tool(function: Callable, schema: dict | None = None, debug: bool = False) -> Tool: 119 | """Provide a tool (a callback function) to the LLM to optionally use. 120 | 121 | args: 122 | function (Callable): The tool to provide to the LLM 123 | schema (dict | None): The schema for the tool. If none give, it will be inferred from the function signature and docstring 124 | """ 125 | if schema is None: 126 | schema = tool_schema(function) 127 | 128 | name = schema["function"]["name"] 129 | 130 | return Tool(tool=function, tool_schema=schema, name=name, debug=debug) 131 | -------------------------------------------------------------------------------- /src/haverscript/types.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import threading 4 | from abc import ABC, abstractmethod 5 | from collections.abc import Iterable 6 | from dataclasses import dataclass 7 | from typing import Callable, Any, Generic, TypeVar 8 | 9 | from pydantic import BaseModel, ConfigDict, Field 10 | 11 | 12 | class Extras(BaseModel): 13 | """Extras is a place to put extra information in a LLM response.""" 14 | 15 | model_config = ConfigDict(frozen=True) 16 | 17 | 18 | class Metrics(Extras): 19 | pass 20 | 21 | 22 | class Informational(Extras): 23 | """Background information, typically from a middleware component.""" 24 | 25 | message: str 26 | 27 | 28 | class Value(Extras): 29 | value: Any 30 | 31 | 32 | class ToolCall(Extras): 33 | """A tool call is a response from an LLM that requested calling a named tool.""" 34 | 35 | name: str 36 | arguments: dict 37 | id: str = "" 38 | 39 | 40 | class Message(BaseModel): 41 | """A message is content with a role (assistant, user, system, tool).""" 42 | 43 | model_config = ConfigDict(frozen=True) 44 | 45 | def role_content_json(self): 46 | result = {} 47 | result["role"] = self.role 48 | result["content"] = self.content 49 | if hasattr(self, "images") and self.images: 50 | result["images"] = self.images 51 | return result 52 | 53 | def append_to(self, context: list): 54 | context.append(self.role_content_json()) 55 | 56 | 57 | class SystemMessage(Message): 58 | role: str = "system" 59 | content: str 60 | 61 | 62 | class RequestMessage(Message): 63 | pass 64 | 65 | 66 | class Prompt(RequestMessage): 67 | role: str = "user" 68 | content: str 69 | images: tuple[str, ...] = () 70 | 71 | 72 | class ToolReply(BaseModel): 73 | """A ToolReply is from one function call to a tool.""" 74 | 75 | id: str 76 | name: str 77 | content: str 78 | 79 | 80 | class ToolResult(RequestMessage): 81 | """A ToolResult can contain several results from several tool calls.""" 82 | 83 | role: str = "tool" 84 | 85 | results: tuple[ToolReply, ...] 86 | 87 | def append_to(self, context: list): 88 | for reply in self.results: 89 | context.append( 90 | { 91 | "role": self.role, 92 | "content": reply.content, 93 | "id": reply.id, 94 | "name": reply.name, 95 | } 96 | ) 97 | 98 | 99 | class ResponseMessage(Message): 100 | pass 101 | 102 | 103 | class AssistantMessage(ResponseMessage): 104 | role: str = "assistant" 105 | content: str 106 | 107 | 108 | class Exchange(BaseModel): 109 | prompt: RequestMessage 110 | reply: ResponseMessage 111 | 112 | model_config = ConfigDict(frozen=True) 113 | 114 | def append_to(self, context: list): 115 | self.prompt.append_to(context) 116 | self.reply.append_to(context) 117 | 118 | 119 | class Contexture(BaseModel): 120 | """Background parts of a request""" 121 | 122 | context: tuple[Exchange, ...] = () 123 | system: str | None = None 124 | options: dict = Field(default_factory=dict) 125 | model: str | None = None 126 | 127 | model_config = ConfigDict(frozen=True) 128 | 129 | def append_exchange(self, exchange: Exchange): 130 | return self.model_copy(update=dict(context=self.context + (exchange,))) 131 | 132 | def add_options(self, **options): 133 | # using this pattern exclude None value in dict 134 | return self.model_copy( 135 | update=dict( 136 | options=dict( 137 | { 138 | key: value 139 | for key, value in {**self.options, **options}.items() 140 | if value is not None 141 | } 142 | ) 143 | ) 144 | ) 145 | 146 | 147 | class Request(BaseModel): 148 | """Foreground parts of a request""" 149 | 150 | contexture: Contexture 151 | prompt: RequestMessage | None 152 | 153 | stream: bool = False 154 | fresh: bool = False 155 | 156 | # images: tuple[str, ...] = () 157 | format: str | dict = "" # str is "json" or "", dict is a JSON schema 158 | tools: tuple[dict, ...] = () 159 | 160 | model_config = ConfigDict(frozen=True) 161 | 162 | 163 | T = TypeVar("T") 164 | 165 | 166 | class Reply(Generic[T]): 167 | """A tokenized response from a large language model. 168 | 169 | T is a the internal value, for example, from using format middleware. 170 | 171 | Reply is a monad. 172 | """ 173 | 174 | def __init__(self, packets: Iterable[str | Extras]): 175 | self._packets = iter(packets) 176 | # We always have at least one item in our sequence. 177 | # This typically will cause as small pause before 178 | # returning the Reply constructor. 179 | # It does mean, by design, that the generator 180 | # has started producing tokens, so the context 181 | # has already been processed. If you have a 182 | # Reply, you can assume that tokens 183 | # are in flight, and the LLM worked. 184 | try: 185 | self._cache = [next(self._packets)] 186 | except StopIteration: 187 | self._packets = iter([]) 188 | self._cache = [] 189 | self._lock = threading.Lock() 190 | self.closers = [] 191 | self.closing = False 192 | 193 | def __str__(self): 194 | return "".join(self.tokens()) 195 | 196 | def __repr__(self): 197 | 198 | return f"Reply([{', '.join([repr(t) for t in self._cache])}{']' if self.closing else ', ...'})" 199 | 200 | def __iter__(self): 201 | ix = 0 202 | while True: 203 | # We need a lock, because the contents can be consumed 204 | # by difference threads. With list, this just works. 205 | # With generators, we need to both the cache, and the lock. 206 | with self._lock: 207 | if ix < len(self._cache): 208 | result = self._cache[ix] 209 | else: 210 | assert ix == len(self._cache) 211 | try: 212 | result = next(self._packets) 213 | except StopIteration: 214 | break 215 | self._cache.append(result) 216 | 217 | ix += 1 # this a local ix, so does not need guarded 218 | yield result 219 | 220 | # auto close 221 | with self._lock: 222 | if self.closing: 223 | return 224 | # first past the post 225 | self.closing = True 226 | 227 | # close all completers 228 | for completion in self.closers: 229 | completion() 230 | 231 | def tokens(self) -> Iterable[str]: 232 | """Returns all str tokens.""" 233 | yield from (token for token in self if isinstance(token, str)) 234 | 235 | def metrics(self) -> Metrics | None: 236 | """Returns any Metrics.""" 237 | for t in self: 238 | if isinstance(t, Metrics): 239 | return t 240 | return None 241 | 242 | def tool_calls(self) -> list[ToolCall]: 243 | """Returns all ToolCalls.""" 244 | return [t for t in self if isinstance(t, ToolCall)] 245 | 246 | @property 247 | def value(self) -> dict | BaseModel | None: 248 | """Returns any value build by format middleware. 249 | 250 | value is a property to be consistent with Response. 251 | """ 252 | for t in self: 253 | if isinstance(t, Value): 254 | return t.value 255 | return None 256 | 257 | def after(self, completion: Callable[[], None]) -> None: 258 | with self._lock: 259 | if not self.closing: 260 | self.closers.append(completion) 261 | return 262 | 263 | # we have completed, so just call completion callback. 264 | completion() 265 | 266 | def __add__(self, other: Reply) -> Reply: 267 | 268 | # Append both streams of tokens. 269 | def streaming(): 270 | yield from self 271 | yield from other 272 | 273 | return Reply(streaming()) 274 | 275 | @staticmethod 276 | def pure(value: Any) -> Reply: 277 | return Reply([Value(value=value)]) 278 | 279 | @staticmethod 280 | def informational(message: str) -> Reply: 281 | return Reply([Informational(message=message)]) 282 | 283 | def bind(self, completion: Callable[[T], Reply | Any]) -> Reply: 284 | """monadic bind for Reply 285 | 286 | This passes the *first* Value from the self tokens 287 | to the completion function. All tokens of type Value are 288 | filtered out self component, in the result. 289 | 290 | If the completion function returns anything other than Reply, 291 | this is assumed to be a pure monad with this value. 292 | """ 293 | 294 | def streaming(): 295 | yield from (token for token in self if not isinstance(token, Value)) 296 | continuation = completion(self.value) 297 | if isinstance(continuation, Reply): 298 | yield from (token for token in continuation) 299 | else: 300 | yield from Reply.pure(continuation) 301 | 302 | return Reply(streaming()) 303 | 304 | def select(self, completion: Callable[[], Reply | Any]) -> Reply: 305 | """monadic select for Reply 306 | 307 | If value in the first Reply is None, then the optional 308 | function is called. 309 | 310 | If the completion function returns anything other than Reply, 311 | this is assumed to be a pure monad with this value. 312 | """ 313 | 314 | def streaming(): 315 | yield from (token for token in self if not isinstance(token, Value)) 316 | if self.value is not None: 317 | yield from Reply.pure(self.value) 318 | else: 319 | continuation = completion() 320 | if isinstance(continuation, Reply): 321 | yield from (token for token in continuation) 322 | else: 323 | yield from Reply.pure(continuation) 324 | 325 | return Reply(streaming()) 326 | 327 | def reify(self) -> Reply: 328 | """Take the contents, and return contents as a monadic string result""" 329 | 330 | def streaming(): 331 | yield from self 332 | yield from Reply.pure(str(self)) 333 | 334 | return Reply(streaming()) 335 | 336 | def map(self, f: Callable[[Any], Any]): 337 | def streaming(): 338 | for token in self: 339 | if isinstance(token, Value): 340 | yield Value(value=f(token.value)) 341 | else: 342 | yield token 343 | 344 | return Reply(streaming()) 345 | 346 | 347 | class LanguageModel(ABC): 348 | """Base class for anything that can by asked things, that is takes a configuration/prompt and returns token(s).""" 349 | 350 | @abstractmethod 351 | def ask(self, request: Request) -> Reply: 352 | """Ask a LLM a specific request.""" 353 | 354 | def __or__(self, other) -> LanguageModel: 355 | assert isinstance(other, Middleware) 356 | return MiddlewareLanguageModel(other, self) 357 | 358 | 359 | class ServiceProvider(LanguageModel): 360 | """A ServiceProvider is a LanguageModel that serves specific models.""" 361 | 362 | @abstractmethod 363 | def list(self) -> list[str]: 364 | """Return the list of valid models for this provider.""" 365 | 366 | 367 | @dataclass(frozen=True) 368 | class Middleware(ABC): 369 | """Middleware is a bidirectional Prompt and Reply processor. 370 | 371 | Middleware is something you use on a LanguageModel, 372 | and a LanguageModel is something you *call*. 373 | """ 374 | 375 | @abstractmethod 376 | def invoke(self, request: Request, next: LanguageModel) -> Reply: 377 | return next.ask(request=request) 378 | 379 | def first(self): 380 | """get the first Middleware in the pipeline (from the Prompt's point of view)""" 381 | return self 382 | 383 | def __or__(self, other: LanguageModel) -> Middleware: 384 | return AppendMiddleware(self, other) 385 | 386 | 387 | @dataclass(frozen=True) 388 | class MiddlewareLanguageModel(LanguageModel): 389 | """MiddlewareLanguageModel is Middleware with a specific target LanguageModel. 390 | 391 | This combination of Middleware and LanguageModel is itself a LanguageModel. 392 | """ 393 | 394 | middleware: Middleware 395 | next: LanguageModel 396 | 397 | def ask(self, request: Request) -> Reply: 398 | return self.middleware.invoke(request=request, next=self.next) 399 | 400 | 401 | @dataclass(frozen=True) 402 | class AppendMiddleware(Middleware): 403 | after: Middleware 404 | before: Middleware # we evaluate from right to left in invoke 405 | 406 | def invoke(self, request: Request, next: LanguageModel) -> Reply: 407 | # The ice is thin here but it holds. 408 | return self.before.invoke( 409 | request=request, next=MiddlewareLanguageModel(self.after, next) 410 | ) 411 | 412 | def first(self): 413 | if first := self.before.first(): 414 | return first 415 | return self.after.first() 416 | 417 | 418 | @dataclass(frozen=True) 419 | class EmptyMiddleware(Middleware): 420 | 421 | def invoke(self, request: Request, next: LanguageModel) -> Reply: 422 | return next.ask(request=request) 423 | -------------------------------------------------------------------------------- /tests/docs/test_docs.py: -------------------------------------------------------------------------------- 1 | from tests.test_utils import Content 2 | 3 | 4 | class README: 5 | def __init__(self, doc): 6 | self.doc = doc 7 | 8 | def __call__(self, ref, start, size, skip=0): 9 | ref_content = Content(ref) 10 | if skip: 11 | # line numbers start 1 (not 0) 12 | ref_content = ref_content[skip + 1 :] 13 | assert ref_content == Content(self.doc)[start : start + size] 14 | 15 | 16 | def test_docs_first_example(): 17 | readme = README("examples/first_example/README.md") 18 | readme("examples/first_example/main.py", 4, 8) 19 | readme("tests/e2e/test_e2e_haverscript/test_first_example.txt", 17, 20, skip=1) 20 | 21 | 22 | def test_docs_first_agent(): 23 | readme = README("examples/first_agent/README.md") 24 | readme("examples/first_agent/main.py", 4, 19) 25 | assert ( 26 | Content("examples/first_agent/README.md")[28 : 28 + 7] 27 | == Content("tests/e2e/test_e2e_haverscript/test_first_agent.txt")[1:8] 28 | ) 29 | 30 | 31 | def test_docs_chaining_answers(): 32 | readme = README("examples/chaining_answers/README.md") 33 | readme("examples/chaining_answers/main.py", 5, 17) 34 | readme("tests/e2e/test_e2e_haverscript/test_chaining_answers.txt", 26, 26, skip=1) 35 | 36 | 37 | def test_docs_tree_of_calls(): 38 | readme = README("examples/tree_of_calls/README.md") 39 | readme("examples/tree_of_calls/main.py", 5, 9) 40 | readme("tests/e2e/test_e2e_haverscript/test_tree_of_calls.txt", 19, 18, skip=1) 41 | 42 | 43 | def test_images(): 44 | readme = README("examples/images/README.md") 45 | readme("examples/images/main.py", 8, 7) 46 | readme("tests/e2e/test_e2e_haverscript/test_images.txt", 19, 26, skip=1) 47 | 48 | 49 | def test_cache(): 50 | readme = README("examples/cache/README.md") 51 | readme("examples/cache/main.py", 5, 22) 52 | readme("tests/e2e/test_e2e_haverscript/test_cache.2.txt", 32, 8) 53 | readme("tests/e2e/test_e2e_haverscript/test_cache.3.txt", 45, 12) 54 | 55 | 56 | def test_options(): 57 | readme = README("examples/options/README.md") 58 | readme("examples/options/main.py", 2, 8) 59 | readme("tests/e2e/test_e2e_haverscript/test_options.txt", 17, 11, skip=1) 60 | 61 | 62 | def test_format(): 63 | readme = README("examples/format/README.md") 64 | readme("examples/format/main.py", 9, 21) 65 | readme("tests/e2e/test_e2e_haverscript/test_format.txt", 35, 1) 66 | 67 | 68 | def test_readme(): 69 | readme = README("README.md") 70 | 71 | readme("examples/first_example/main.py", 20, 8) 72 | readme("examples/first_agent/main.py", 64, 19) 73 | 74 | readme("tests/e2e/test_e2e_haverscript/test_first_example.txt", 33, 20, skip=1) 75 | assert ( 76 | Content("docs/MIDDLEWARE.md")[50 : 50 + 14] 77 | == Content("README.md")[330 : 330 + 14] 78 | ) 79 | readme("examples/together/main.py", 392, 8) 80 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript.py: -------------------------------------------------------------------------------- 1 | # End to end tests for Haverscript. 2 | 3 | import os 4 | import subprocess 5 | import sys 6 | 7 | import pytest 8 | from pydantic import BaseModel, ConfigDict, Field 9 | 10 | import haverscript 11 | import haverscript.together as together 12 | from haverscript.together import connect 13 | from tests.test_utils import remove_spinner 14 | 15 | DEBUG = False 16 | 17 | 18 | def open_as_is(filename, tmp_path): 19 | with open(filename, "r", encoding="utf-8") as f: 20 | content = f.read() 21 | 22 | return content 23 | 24 | 25 | def open_for_ollama(filename, tmp_path): 26 | with open(filename, "r", encoding="utf-8") as f: 27 | content = f.read() 28 | 29 | content = "import haverscript as hs\n" + content 30 | changes = { 31 | 'connect("mistral")': '(connect("mistral:v0.3") | hs.options(seed=12345))', 32 | 'connect("llava")': '(connect("llava:v1.6") | hs.options(seed=12345))', 33 | 'cache("cache.db")': f'cache("{tmp_path}/ollama.cache.db")', 34 | } 35 | for old_text, new_text in changes.items(): 36 | content = content.replace(old_text, new_text) 37 | 38 | if DEBUG: 39 | print(content) 40 | 41 | return content 42 | 43 | 44 | def open_for_together(filename, tmp_path): 45 | with open(filename, "r", encoding="utf-8") as f: 46 | content = f.read() 47 | 48 | content = ( 49 | "import haverscript as hs\nimport haverscript.together as together\n" + content 50 | ) 51 | changes = { 52 | 'connect("mistral")': '(together.connect("meta-llama/Meta-Llama-3-8B-Instruct-Lite", timeout=30, max_retries=2) | hs.options(seed=12345))', 53 | 'cache("cache.db")': f'cache("{tmp_path}/cache.together.db")', 54 | "connect(model)": "(connect(model) | hs.options(seed=12345))", 55 | } 56 | for old_text, new_text in changes.items(): 57 | content = content.replace(old_text, new_text) 58 | 59 | if DEBUG: 60 | print("# TOGETHER") 61 | print(content) 62 | 63 | return content 64 | 65 | 66 | def run_example(example, tmp_path, open_me, args=[]) -> str: 67 | if DEBUG: 68 | print("run_example", example, tmp_path, open_me, args) 69 | filename = tmp_path / os.path.basename(example) 70 | 71 | content = open_me(example, tmp_path) 72 | 73 | # Write the modified content to the output file 74 | with open(filename, "w", encoding="utf-8") as f: 75 | f.write(content) 76 | 77 | result = subprocess.run( 78 | [sys.executable, filename] + args, 79 | capture_output=True, 80 | text=False, 81 | ) 82 | 83 | # check everything ran okay 84 | if result.stderr: 85 | print(result.stderr.decode("utf-8")) 86 | 87 | if DEBUG: 88 | print(result.stdout.decode("utf-8")) 89 | 90 | assert not result.stderr, "{result.stderr}" 91 | 92 | return remove_spinner(result.stdout.decode("utf-8")) 93 | 94 | 95 | class Similarity(BaseModel): 96 | reasons: str 97 | similarity: float 98 | 99 | 100 | def check_together(f1, f2): 101 | with open(f1) as f: 102 | s1 = f.read() 103 | with open(f2) as f: 104 | s2 = f.read() 105 | 106 | s1 = f'"""{s1}"""' 107 | s2 = f'"""{s2}"""' 108 | 109 | prompt = f""" 110 | Here are two articles inside triple quotes. 111 | Consider if the articles cover similar topics and provide similar information, 112 | and are of similar length. 113 | Give you answer as using JSON, and only JSON. There are two fields: 114 | * "similarity": int # a score between 0 and 1 of how similar the articles are 115 | * "reasons": str # a one sentence summary 116 | 117 | --- 118 | article 1 : {s1} 119 | --- 120 | article 2 : {s2} 121 | """ 122 | 123 | response = connect("meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo").chat( 124 | prompt, 125 | middleware=haverscript.format(Similarity), 126 | ) 127 | similarity = Similarity.model_validate(response.value) 128 | 129 | if similarity.similarity < 0.5: 130 | raise AssertionError() 131 | 132 | 133 | def run_examples( 134 | tmp_path, 135 | file_regression, 136 | src, 137 | as_is: bool = False, 138 | ollama: bool = True, 139 | together: bool = True, 140 | arg: str | None = None, 141 | ): 142 | suffix = ".txt" 143 | args = [] 144 | if arg: 145 | suffix = f".{arg}.txt" 146 | args = [arg] 147 | # Check the given example actually compiles and runs 148 | if as_is: 149 | example = run_example( 150 | src, 151 | tmp_path, 152 | open_as_is, 153 | args=args, 154 | ) 155 | file_regression.check( 156 | example, 157 | extension=f".as_is{suffix}", 158 | ) 159 | # Check vs golden output for ollama 160 | if ollama: 161 | file_regression.check( 162 | run_example( 163 | src, 164 | tmp_path, 165 | open_for_ollama, 166 | args=args, 167 | ), 168 | extension=suffix, 169 | ) 170 | # Check vs golden output for together 171 | if together: 172 | file_regression.check( 173 | run_example( 174 | src, 175 | tmp_path, 176 | open_for_together, 177 | args=args, 178 | ), 179 | check_fn=check_together, 180 | extension=f".together{suffix}", 181 | ) 182 | 183 | 184 | def test_first_example(tmp_path, file_regression): 185 | run_examples(tmp_path, file_regression, "examples/first_example/main.py") 186 | 187 | 188 | def test_first_agent(tmp_path, file_regression): 189 | run_examples( 190 | tmp_path, file_regression, "examples/first_agent/main.py", together=False 191 | ) 192 | 193 | 194 | def test_chaining_answers(tmp_path, file_regression): 195 | run_examples(tmp_path, file_regression, "examples/chaining_answers/main.py") 196 | 197 | 198 | def test_tree_of_calls(tmp_path, file_regression): 199 | run_examples(tmp_path, file_regression, "examples/tree_of_calls/main.py") 200 | 201 | 202 | def test_images(tmp_path, file_regression): 203 | run_examples(tmp_path, file_regression, "examples/images/main.py", together=False) 204 | 205 | 206 | def test_cache(tmp_path, file_regression): 207 | run_examples(tmp_path, file_regression, "examples/cache/main.py", arg="2") 208 | run_examples(tmp_path, file_regression, "examples/cache/main.py", arg="3") 209 | 210 | 211 | def test_together(tmp_path, file_regression): 212 | run_examples(tmp_path, file_regression, "examples/together/main.py", ollama=False) 213 | 214 | 215 | def test_options(tmp_path, file_regression): 216 | run_examples(tmp_path, file_regression, "examples/options/main.py") 217 | 218 | 219 | def test_format(tmp_path, file_regression): 220 | run_examples(tmp_path, file_regression, "examples/format/main.py") 221 | 222 | 223 | def test_chatbot(tmp_path, file_regression): 224 | run_examples(tmp_path, file_regression, "examples/chatbot/main.py") 225 | 226 | 227 | def test_custom_service(tmp_path, file_regression): 228 | run_examples( 229 | tmp_path, 230 | file_regression, 231 | "examples/custom_service/main.py", 232 | together=False, 233 | ollama=False, 234 | as_is=True, 235 | ) 236 | 237 | 238 | def test_list(): 239 | models = haverscript.connect().list() 240 | assert isinstance(models, list) 241 | assert "mistral:v0.3" in models 242 | assert "llava:v1.6" in models 243 | 244 | models = together.connect().list() 245 | assert isinstance(models, list) 246 | assert "meta-llama/Meta-Llama-3-8B-Instruct-Lite" in models 247 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_cache.2.txt: -------------------------------------------------------------------------------- 1 | chat #0 2 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 3 | slower (used LLM) 4 | 5 | chat #1 6 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 7 | slower (used LLM) 8 | 9 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_cache.3.txt: -------------------------------------------------------------------------------- 1 | chat #0 2 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 3 | fast (used cache) 4 | 5 | chat #1 6 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 7 | fast (used cache) 8 | 9 | chat #2 10 | reply: The sky appears blue due to a scattering effect called Rayleigh scattering where shorter wavelength light (blue light) is scattered more than other colors by the molecules in Earth's atmosphere. 11 | slower (used LLM) 12 | 13 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_cache.together.2.txt: -------------------------------------------------------------------------------- 1 | chat #0 2 | reply: The sky appears blue because of a phenomenon called Rayleigh scattering, in which shorter (blue) wavelengths of light are scattered more than longer (red) wavelengths by the tiny molecules of gases in the Earth's atmosphere, resulting in the blue color we see. 3 | slower (used LLM) 4 | 5 | chat #1 6 | reply: The sky appears blue because of a phenomenon called Rayleigh scattering, in which shorter (blue) wavelengths of light are scattered more than longer (red) wavelengths by the tiny molecules of gases in the Earth's atmosphere, resulting in the blue color we see. 7 | slower (used LLM) 8 | 9 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_cache.together.3.txt: -------------------------------------------------------------------------------- 1 | chat #0 2 | reply: The sky appears blue because of a phenomenon called Rayleigh scattering, in which shorter (blue) wavelengths of light are scattered more than longer (red) wavelengths by the tiny molecules of gases in the Earth's atmosphere, resulting in the blue color we see. 3 | fast (used cache) 4 | 5 | chat #1 6 | reply: The sky appears blue because of a phenomenon called Rayleigh scattering, in which shorter (blue) wavelengths of light are scattered more than longer (red) wavelengths by the tiny molecules of gases in the Earth's atmosphere, resulting in the blue color we see. 7 | fast (used cache) 8 | 9 | chat #2 10 | reply: The sky appears blue because of a phenomenon called Rayleigh scattering, in which shorter (blue) wavelengths of light are scattered more than longer (red) wavelengths by the tiny molecules of gases in the Earth's atmosphere, resulting in the blue color we see. 11 | slower (used LLM) 12 | 13 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_cache.txt: -------------------------------------------------------------------------------- 1 | chat #0 2 | reply: The sky appears blue due to a process called Rayleigh scattering where shorter-wavelength light (blue light) is scattered more effectively than longer-wavelength light (red light), making the sky appear blue during daytime. 3 | slower (used LLM) 4 | 5 | chat #1 6 | reply: The sky appears blue because molecules in our atmosphere scatter sunlight more strongly in shorter wavelengths (blue and violet), while sun's brightness in these wavelengths is relatively higher, causing them to be scattered preferentially, and us seeing the sky as blue. 7 | slower (used LLM) 8 | 9 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_chaining_answers.together.txt: -------------------------------------------------------------------------------- 1 | 2 | > Name the best basketball player. Only name one player and do not give commentary. 3 | 4 | Michael Jordan 5 | 6 | > Someone told me that Michael Jordan is the best basketball player. Do you agree, and why? 7 | 8 | The eternal debate! While opinions about the "best" basketball player are 9 | subjective and often influenced by personal taste, team loyalty, and 10 | generational differences, I'll provide an objective analysis to help you 11 | understand why many people consider Michael Jordan the greatest basketball 12 | player of all time. 13 | 14 | Here are some key reasons why: 15 | 16 | 1. **Unmatched success**: Michael Jordan won six NBA championships, five MVP 17 | awards, and is the all-time leader in points per game with an average of 18 | 30.12. He also won two Olympic gold medals and was named one of the 50 19 | Greatest Players in NBA History in 1996. 20 | 2. **Dominant player**: Jordan's combination of athleticism, skill, and 21 | competitive drive made him nearly unstoppable on the court. He was a versatile 22 | scorer, able to dominate games in multiple ways, including his signature moves 23 | like the "fadeaway" jump shot, "cradle" layups, and "post-up" moves. 24 | 3. **Clutch performances**: Jordan's reputation for delivering in crucial 25 | situations is legendary. He was known for his ability to take over games in 26 | the fourth quarter, earning the nickname "Air Jordan" for his gravity-defying 27 | leaps and clutch shots. 28 | 4. **Defensive prowess**: Jordan was an elite defender, earning the NBA 29 | Defensive Player of the Year award in 1988. He was known for his tenacious 30 | on-ball defense, which allowed him to shut down opposing scorers and disrupt 31 | their teams' offense. 32 | 5. **Leadership**: Jordan was an exceptional leader, inspiring his teammates 33 | and motivating them to play at a higher level. He was the captain of the 34 | Chicago Bulls during their championship runs in the 1990s and was named the 35 | NBA Finals MVP each time. 36 | 6. **Impact on the game**: Jordan's influence on the game extends beyond his 37 | playing career. He helped popularize the NBA globally, and his endorsement 38 | deals and marketability paved the way for future NBA stars to become global 39 | celebrities. 40 | 7. **Consistency**: Jordan played at an elite level for nearly two decades, 41 | earning 14 All-Star selections and 10 All-NBA First Team nods. His ability to 42 | maintain a high level of performance over a long period is a testament to his 43 | dedication, work ethic, and natural talent. 44 | 8. **Legacy**: Jordan's impact on the game is still felt today, with many 45 | players citing him as an inspiration and trying to emulate his on-court 46 | success. His legacy extends beyond basketball, with his brand and 47 | philanthropic efforts making a significant impact on the world. 48 | 49 | While other players, such as Kareem Abdul-Jabbar, LeBron James, Bill Russell, 50 | and Magic Johnson, are also considered all-time greats, Michael Jordan's 51 | impressive résumé, dominance, and lasting impact on the game make a strong 52 | case for him as the best basketball player of all time. 53 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_chaining_answers.txt: -------------------------------------------------------------------------------- 1 | 2 | > Name the best basketball player. Only name one player and do not give commentary. 3 | 4 | Michael Jordan 5 | 6 | > Someone told me that Michael Jordan is the best basketball player. Do you agree, and why? 7 | 8 | While it's subjective and opinions on who is the "best" basketball player can 9 | vary greatly among basketball fans, Michael Jordan is widely regarded as one 10 | of the greatest players in the history of the sport. His impact on the game 11 | both on and off the court is undeniable. 12 | 13 | Jordan led the Chicago Bulls to six NBA championships and was named the Most 14 | Valuable Player (MVP) five times during his career. He is also a 14-time NBA 15 | All-Star, a 10-time scoring champion, a three-time steals leader, and a 3x 16 | three-point shooting champion. 17 | 18 | Jordan's influence extended beyond statistics as well. His competitive spirit, 19 | passion for the game, and innovative style of play revolutionized basketball 20 | in ways that continue to be felt today. He elevated the NBA brand globally and 21 | inspired countless athletes across multiple sports. 22 | 23 | That being said, it's important to recognize that there have been many 24 | extraordinary players in the history of basketball, and debate over who is the 25 | "best" will likely never reach a consensus. Other notable contenders for this 26 | title include LeBron James, Kareem Abdul-Jabbar, Wilt Chamberlain, Magic 27 | Johnson, and Larry Bird, among others. 28 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_chatbot.together.txt: -------------------------------------------------------------------------------- 1 | --[ User-facing conversation ]------ 2 | 3 | > Three blind mice 4 | 5 | Three blind mice in French is Trois sourds souris 6 | 2 translation(s) left. 7 | 8 | > Such is life 9 | 10 | Such is life in French is C'est ainsi la vie 11 | 1 translation(s) left. 12 | 13 | > All roads lead to Rome 14 | 15 | All roads lead to Rome in French is Toutes les routes mènent à Rome 16 | 0 translation(s) left. 17 | 18 | > The quick brown fox 19 | 20 | Sorry. French lesson over. 21 | Your 3 translations: 22 | * Three blind mice => Trois sourds souris 23 | * Such is life => C'est ainsi la vie 24 | * All roads lead to Rome => Toutes les routes mènent à Rome 25 | 26 | --[ LLM-facing conversation ]------ 27 | 28 | > You are a translator, translating from English to French. Give your answer as a JSON record with two fields, "english", and "french". The "english" field should contain the original English text, and only the original text.The "french" field should contain the translated French text, and only the translated text. 29 | > 30 | > English Text: Three blind mice 31 | 32 | { 33 | "english": "Three blind mice", 34 | "french": "Trois sourds souris" 35 | } 36 | 37 | > You are a translator, translating from English to French. Give your answer as a JSON record with two fields, "english", and "french". The "english" field should contain the original English text, and only the original text.The "french" field should contain the translated French text, and only the translated text. 38 | > 39 | > English Text: Such is life 40 | 41 | { 42 | "english": "Such is life", 43 | "french": "C'est ainsi la vie" 44 | } 45 | 46 | > You are a translator, translating from English to French. Give your answer as a JSON record with two fields, "english", and "french". The "english" field should contain the original English text, and only the original text.The "french" field should contain the translated French text, and only the translated text. 47 | > 48 | > English Text: All roads lead to Rome 49 | 50 | { 51 | "english": "All roads lead to Rome", 52 | "french": "Toutes les routes mènent à Rome" 53 | } 54 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_chatbot.txt: -------------------------------------------------------------------------------- 1 | --[ User-facing conversation ]------ 2 | 3 | > Three blind mice 4 | 5 | Three blind mice in French is Trois souris aveugles 6 | 7 | 2 translation(s) left. 8 | 9 | > Such is life 10 | 11 | Such is life in French is C'est ainsi que la vie est 12 | 13 | 1 translation(s) left. 14 | 15 | > All roads lead to Rome 16 | 17 | All roads lead to Rome in French is Toutes les routes mènent à Rome 18 | 19 | 0 translation(s) left. 20 | 21 | > The quick brown fox 22 | 23 | Sorry. French lesson over. 24 | Your 3 translations: 25 | * Three blind mice => Trois souris aveugles 26 | * Such is life => C'est ainsi que la vie est 27 | * All roads lead to Rome => Toutes les routes mènent à Rome 28 | 29 | --[ LLM-facing conversation ]------ 30 | 31 | > You are a translator, translating from English to French. 32 | > 33 | > English Text: Three blind mice 34 | > 35 | > Reply in JSON, using the following keys: 36 | > 37 | > - "english" (str): the original English text, and only the original text. 38 | > - "french" (str): the translated French text, and only the translated text. 39 | 40 | { 41 | "english": "Three blind mice", 42 | "french": "Trois souris aveugles" 43 | } 44 | 45 | > You are a translator, translating from English to French. 46 | > 47 | > English Text: Such is life 48 | > 49 | > Reply in JSON, using the following keys: 50 | > 51 | > - "english" (str): the original English text, and only the original text. 52 | > - "french" (str): the translated French text, and only the translated text. 53 | 54 | { 55 | "english": "Such is life", 56 | "french": "C'est ainsi que la vie est" 57 | } 58 | 59 | > You are a translator, translating from English to French. 60 | > 61 | > English Text: All roads lead to Rome 62 | > 63 | > Reply in JSON, using the following keys: 64 | > 65 | > - "english" (str): the original English text, and only the original text. 66 | > - "french" (str): the translated French text, and only the translated text. 67 | 68 | { 69 | "english": "All roads lead to Rome", 70 | "french": "Toutes les routes mènent à Rome" 71 | } 72 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_check.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue due to a scattering effect called Rayleigh scattering 5 | where shorter wavelength light (blue light) is scattered more than other 6 | colors by the molecules in Earth's atmosphere. 7 | 8 | > Rewrite the above sentence in the style of Yoda 9 | 10 | Blue, the sky seems due to scatter, it does. Shorter-wavelength light (blue 11 | light), more than others, by molecules in our atmosphere, it is scattered 12 | extensively. 13 | 14 | > How many questions did I ask? Give a one sentence reply. 15 | 16 | 2 questions you have asked. 17 | 27 characters 18 | { 19 | "Red": "#FF0000", 20 | "Blue": "#0000FF", 21 | "Green": "#00FF00" 22 | } 23 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_custom_service.as_is.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | I reject your 8 word prompt, and replace it with my own. 5 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_first_agent.txt: -------------------------------------------------------------------------------- 1 | Earth: The sky appears blue to our eyes during a clear day due to a phenomenon called Rayleigh scattering, where shorter wavelengths of light, such as blue and violet, are scattered more effectively by the atmosphere's molecules than longer ones like red or yellow. However, we perceive the sky as blue rather than violet because our eyes are more sensitive to blue light and because sunlight reaches us with less violet light filtered out by the ozone layer. 2 | 3 | Mars: The sky on Mars appears to be a reddish hue, primarily due to suspended iron oxide (rust) particles in its atmosphere. This gives Mars its characteristic reddish color as sunlight interacts with these particles. 4 | 5 | Venus: The sky on Venus is not visible like it is on Earth because of a dense layer of clouds composed mostly of sulfuric acid. This thick veil prevents light from the Sun from reaching our line of sight, making the sky appear perpetually dark. 6 | 7 | Jupiter: The sky on Jupiter isn't blue like Earth's; instead, it appears white or off-white due to the reflection of sunlight from thick layers of ammonia crystals in its atmosphere. This peculiarity stems from Jupiter's composition and atmospheric conditions that are quite different from ours. 8 | 9 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_first_example.together.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue because of a phenomenon called Rayleigh scattering, in 5 | which shorter (blue) wavelengths of light are scattered more than longer (red) 6 | wavelengths by the tiny molecules of gases in the Earth's atmosphere, 7 | resulting in the blue color we see. 8 | 9 | > What color is the sky on Mars? 10 | 11 | The color of the sky on Mars is a topic of ongoing research and debate. NASA's 12 | Curiosity rover has been studying the Martian atmosphere and has taken 13 | numerous images of the Martian sky. The color of the sky on Mars is often 14 | described as a pale orange or reddish hue, which is due to the presence of 15 | iron oxide (rust) particles in the Martian soil and dust. The sky can also 16 | appear more hazy or dusty due to the frequent dust storms that occur on the 17 | planet. However, the exact shade of the Martian sky can vary depending on the 18 | time of day, the amount of dust in the air, and other factors. 19 | 20 | > Do any other planets have blue skies? 21 | 22 | Yes, some other planets and moons in our solar system have blue skies, 23 | although they may not be as blue as Earth's. Here are a few examples: 24 | 25 | 1. Neptune: Like Earth, Neptune's atmosphere scatters sunlight in a way that 26 | makes it appear blue, giving it a blue-gray color. 27 | 2. Uranus: Uranus's atmosphere is mostly composed of hydrogen, helium, and 28 | methane, which scatters light in a way that produces a pale blue color. 29 | 3. Saturn's moon, Titan: Titan's thick atmosphere is rich in nitrogen and 30 | methane, which scatters light in a way that gives it a hazy, orange-brown 31 | color. However, when the sun is low on the horizon, Titan's sky can take on a 32 | blueish hue. 33 | 4. Venus: Although Venus's thick atmosphere is often shrouded in sulfuric acid 34 | clouds, when the sun is high in the sky, the clouds can take on a blueish tint 35 | due to the scattering of sunlight by the sulfuric acid droplets. 36 | 37 | It's worth noting that these blue skies are not necessarily identical to 38 | Earth's blue sky, as the atmospheric conditions and composition of these 39 | planets and moons are quite different from our own. 40 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_first_example.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue due to a scattering effect called Rayleigh scattering 5 | where shorter wavelength light (blue light) is scattered more than other 6 | colors by the molecules in Earth's atmosphere. 7 | 8 | > What color is the sky on Mars? 9 | 10 | The Martian sky appears red or reddish-orange, primarily because of fine dust 11 | particles in its thin atmosphere that scatter sunlight preferentially in the 12 | red part of the spectrum, which our eyes perceive as a reddish hue. 13 | 14 | > Do any other planets have blue skies? 15 | 16 | Unlike Earth, none of the other known terrestrial planets (Venus, Mars, 17 | Mercury) have a significant enough atmosphere or suitable composition to cause 18 | Rayleigh scattering, resulting in blue skies like we see on Earth. However, 19 | some of the gas giant planets such as Uranus and Neptune can appear blueish 20 | due to their atmospheres composed largely of methane, which absorbs red light 21 | and scatters blue light. 22 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_first_example_together.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue because of a phenomenon called Rayleigh scattering, in 5 | which shorter (blue) wavelengths of light are scattered more than longer (red) 6 | wavelengths by the tiny molecules of gases in the Earth's atmosphere, 7 | resulting in the blue color we see. 8 | 9 | > Rewrite the above sentence in the style of Yoda 10 | 11 | "A phenomenon called Rayleigh scattering, the sky blue appears, it does. 12 | Scattered, shorter wavelengths of light, more than longer, by tiny molecules 13 | of gases in the Earth's atmosphere, are." 14 | 15 | > How many questions did I ask? 16 | 17 | You asked 1 question. 18 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_format.together.txt: -------------------------------------------------------------------------------- 1 | english='Three Blind Mice' french='Trois Sourds et Muets' 2 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_format.txt: -------------------------------------------------------------------------------- 1 | english='Three Blind Mice' french='Trois Souris Aveugles' 2 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_images.txt: -------------------------------------------------------------------------------- 1 | 2 | > Describe this image, and speculate where it was taken. 3 | 4 | This is a panoramic photograph depicting a cityscape with notable landmarks. 5 | In the foreground, there's a wide street lined with buildings and shops. On 6 | either side of the street, you can see various architectural styles suggesting 7 | a mix of historical and modern construction. 8 | 9 | In the middle ground, there's an open space that leads to a significant 10 | building in the distance, which appears to be a castle due to its fortified 11 | walls and design elements typical of medieval architecture. The castle is 12 | likely an important landmark within the city. 13 | 14 | Further back, beyond the castle, you can see more of the city, with trees and 15 | other buildings indicating a densely populated area. There's also a notable 16 | feature in the background that looks like a bridge or an overpass, spanning 17 | across a body of water which could be a river or a loch. 18 | 19 | The sky is partly cloudy, suggesting that the weather might be changing or 20 | it's just a typical day with some clouds. The vegetation appears lush and 21 | well-maintained, indicating that this city values its green spaces. 22 | 23 | Based on these observations, it seems likely that this image was taken in 24 | Edinburgh, Scotland. The castle in the distance is unmistakably Edinburgh 25 | Castle, one of the most iconic landmarks in Edinburgh. The architecture, the 26 | layout of the streets, and the presence of the castle are all indicative of 27 | the city's rich history and its blend of historical and modern elements. 28 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_options.together.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue because of a phenomenon called Rayleigh scattering, in 5 | which shorter (blue) wavelengths of light are scattered more by the tiny 6 | molecules of gases in the atmosphere than longer (red) wavelengths, resulting 7 | in the blue light being scattered in all directions and reaching our eyes from 8 | all parts of the sky. 9 | 10 | > In one sentence, why is the sky blue? 11 | 12 | The sky appears blue because of a phenomenon called Rayleigh scattering, in 13 | which shorter (blue) wavelengths of light are scattered more than longer (red) 14 | wavelengths by the tiny molecules of gases in the Earth's atmosphere, 15 | resulting in the blue color we see. 16 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_options.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue due to a scattering effect called Rayleigh scattering 5 | where shorter wavelength light (blue light) is scattered more than other 6 | colors by the molecules in Earth's atmosphere. 7 | 8 | > In one sentence, why is the sky blue? 9 | 10 | The sky appears blue due to a scattering effect called Rayleigh scattering 11 | where shorter wavelength light (blue light) is scattered more than other 12 | colors by the molecules in Earth's atmosphere. 13 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_together.together.txt: -------------------------------------------------------------------------------- 1 | 2 | > Write a short sentence on the history of Scotland. 3 | 4 | Scotland has a rich and complex history dating back to the 6th century, with 5 | the Picts, Scots, and Vikings all leaving their mark on the country, followed 6 | by the Acts of Union in 1707, which united Scotland with England under a 7 | single government. 8 | 9 | > Write 100 words on the history of Scotland. 10 | 11 | Scotland's history stretches back to the 6th century, when the Picts, a Celtic 12 | people, inhabited the land. The Vikings arrived in the 9th century, followed 13 | by the Normans in the 12th. The Acts of Union in 1707 united Scotland with 14 | England under a single government, creating the Kingdom of Great Britain. The 15 | 18th and 19th centuries saw the Scottish Enlightenment, a period of 16 | significant cultural and scientific advancement. The 20th century was marked 17 | by industrialization, nationalism, and the struggle for independence. Today, 18 | Scotland is a modern nation with a rich heritage, proud of its unique culture 19 | and history. 20 | 21 | > Write 1000 words on the history of Scotland. 22 | 23 | Scotland's history is a long and complex one, spanning over 2,000 years. From 24 | the earliest known human habitation to the present day, Scotland has been 25 | shaped by a diverse range of cultures, events, and figures. 26 | 27 | The earliest known human habitation in Scotland dates back to the Mesolithic 28 | era, around 8,000 years ago. The first farmers arrived in Scotland around 29 | 4,000 BC, bringing with them new technologies and ways of life. The Neolithic 30 | period saw the construction of some of Scotland's most impressive ancient 31 | monuments, including the Ring of Brodgar and the Callanish Stones. 32 | 33 | The Bronze Age, which began around 2,000 BC, saw the development of 34 | metalworking and the construction of more complex settlements. The Iron Age, 35 | which began around 700 BC, saw the rise of the Celts, who would go on to have 36 | a profound impact on Scottish history. 37 | 38 | The Celts, who were a group of Indo-European tribes, arrived in Scotland 39 | around 500 BC. They brought with them their own language, culture, and 40 | customs, which would eventually give rise to the Gaelic language and the 41 | Kingdom of Scotland. The Celts were a warrior people, known for their skill in 42 | battle and their love of art and craftsmanship. 43 | 44 | The Romans, who had conquered much of Europe, arrived in Scotland in the 1st 45 | century AD. They built a series of forts and roads, including the famous 46 | Antonine Wall, which stretched across the southern part of the country. 47 | However, the Romans were eventually forced to withdraw from Scotland, and the 48 | country was left to the Celts once more. 49 | 50 | The Middle Ages saw the rise of the Kingdom of Scotland, which was established 51 | in the 9th century. The kingdom was founded by Kenneth MacAlpin, a Pictish 52 | king who united the various Celtic tribes of Scotland under a single ruler. 53 | The Kingdom of Scotland would go on to play a significant role in European 54 | politics, particularly during the Middle Ages. 55 | 56 | The 12th century saw the arrival of the Normans, who brought with them their 57 | own language, culture, and feudal system. The Normans would go on to have a 58 | profound impact on Scottish society, introducing new technologies, 59 | architecture, and social structures. 60 | 61 | The 13th century saw the rise of the Scottish monarchy, with the reign of King 62 | Alexander II. Alexander was a strong and ambitious king, who sought to expand 63 | his kingdom and assert its independence from England. He was succeeded by his 64 | son, Alexander III, who would go on to play a significant role in Scottish 65 | history. 66 | 67 | The 14th century saw the rise of the Stewart dynasty, which would go on to 68 | dominate Scottish politics for centuries to come. The Stuarts were a powerful 69 | and influential family, producing some of Scotland's most famous monarchs, 70 | including James I and Charles I. 71 | 72 | The 16th century saw the Reformation, which had a profound impact on Scotland. 73 | The Reformation was a period of great change, as the country shifted from 74 | Catholicism to Protestantism. The most famous of Scotland's Protestant 75 | reformers was John Knox, who played a key role in the Scottish Reformation. 76 | 77 | The 17th and 18th centuries saw the rise of the Enlightenment, a period of 78 | great cultural and scientific advancement. Scotland produced some of the 79 | world's most famous thinkers, including David Hume and Adam Smith. 80 | 81 | The 19th and 20th centuries saw the rise of industrialization, which 82 | transformed Scotland's economy and society. The Industrial Revolution brought 83 | new technologies and industries to Scotland, including textiles, shipbuilding, 84 | and coal mining. However, it also brought poverty, inequality, and social 85 | unrest. 86 | 87 | The 20th century saw the rise of nationalism, as Scotland sought to assert its 88 | independence from the United Kingdom. The Scottish National Party (SNP) was 89 | founded in 1934, and would go on to play a significant role in Scottish 90 | politics. 91 | 92 | Today, Scotland is a modern nation with a rich heritage, proud of its unique 93 | culture and history. The country is home to some of the world's most beautiful 94 | landscapes, including the Highlands, the Islands, and the Lowlands. Scotland 95 | is also a country with a strong sense of national identity, which is reflected 96 | in its language, music, and art. 97 | 98 | In conclusion, Scotland's history is a long and complex one, shaped by a 99 | diverse range of cultures, events, and figures. From the earliest known human 100 | habitation to the present day, Scotland has been a country of great change and 101 | transformation. Today, Scotland is a modern nation with a rich heritage, proud 102 | of its unique culture and history. 103 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_together.txt: -------------------------------------------------------------------------------- 1 | 2 | > Write a short sentence on the history of Scotland. 3 | 4 | Scotland has a rich and complex history dating back to the 6th century, with 5 | the Picts, Scots, and Vikings all leaving their mark on the country, followed 6 | by the Acts of Union in 1707, which united Scotland with England under a 7 | single government. 8 | 9 | > Write 100 words on the history of Scotland. 10 | 11 | Scotland's history stretches back to the 6th century, when the Picts, a Celtic 12 | people, inhabited the land. The Vikings arrived in the 9th century, followed 13 | by the Normans in the 12th. The Acts of Union in 1707 united Scotland with 14 | England under a single government, creating the Kingdom of Great Britain. The 15 | 18th and 19th centuries saw the Scottish Enlightenment, a period of 16 | significant cultural and scientific advancement. The 20th century was marked 17 | by industrialization, nationalism, and the struggle for independence. Today, 18 | Scotland is a modern nation with a rich heritage, proud of its unique culture 19 | and history. 20 | 21 | > Write 1000 words on the history of Scotland. 22 | 23 | Scotland's history is a long and complex one, spanning over 2,000 years. From 24 | the earliest known human habitation to the present day, Scotland has been 25 | shaped by a diverse range of cultures, events, and figures. 26 | 27 | The earliest known human habitation in Scotland dates back to the Mesolithic 28 | era, around 8,000 years ago. The first farmers arrived in Scotland around 29 | 4,000 BC, bringing with them new technologies and ways of life. The Neolithic 30 | period saw the construction of some of Scotland's most impressive ancient 31 | monuments, including the Ring of Brodgar and the Callanish Stones. 32 | 33 | The Bronze Age, which began around 2,000 BC, saw the development of 34 | metalworking and the construction of more complex settlements. The Iron Age, 35 | which began around 500 BC, saw the rise of the Celts, who would go on to play 36 | a significant role in Scottish history. 37 | 38 | The Romans, who occupied Britain from 43 AD to the 5th century, had a 39 | significant impact on Scotland. They built roads, forts, and settlements, and 40 | introduced Roman law and administration. However, the Romans were eventually 41 | driven out of Scotland by the Picts, a Celtic people who had been living in 42 | the region for centuries. 43 | 44 | The Picts were a powerful and influential force in Scotland, and their 45 | kingdom, which was established in the 3rd century, was known for its rich 46 | culture and impressive architecture. The Picts were also skilled metalworkers, 47 | and their intricate brooches and other decorative items are highly prized by 48 | collectors today. 49 | 50 | The Vikings, who arrived in Scotland in the 9th century, had a significant 51 | impact on the country. They established settlements and trading centers, and 52 | introduced their own language and customs. The Vikings also played a key role 53 | in the development of Scotland's unique system of clan organization, which 54 | would go on to shape the country's social and political structure for 55 | centuries to come. 56 | 57 | The 12th century saw the rise of the Normans, who had conquered England and 58 | were looking to expand their territories. The Normans established a number of 59 | castles and fortifications in Scotland, and introduced their own system of 60 | feudalism. The 13th century saw the establishment of the Kingdom of Scotland, 61 | with the coronation of Alexander II in 1214. 62 | 63 | The 14th and 15th centuries saw the rise of the Stewart dynasty, which would 64 | go on to play a significant role in Scottish history. The Stuarts were a 65 | powerful and influential family, and their monarchs would go on to shape the 66 | course of Scottish history for centuries to come. 67 | 68 | The 16th century saw the Reformation, which had a profound impact on Scotland. 69 | The Protestant Reformation, led by figures such as John Knox, saw the 70 | establishment of the Church of Scotland, which would go on to play a 71 | significant role in Scottish life and politics. 72 | 73 | The 17th and 18th centuries saw the rise of the Enlightenment, a period of 74 | significant cultural and scientific advancement. Figures such as David Hume, 75 | Adam Smith, and James Watt made major contributions to the fields of 76 | philosophy, economics, and science. 77 | 78 | The 19th and 20th centuries saw the rise of industrialization, which had a 79 | profound impact on Scotland. The country's economy was transformed, and new 80 | industries such as textiles, shipbuilding, and coal mining emerged. However, 81 | this period also saw significant social and economic challenges, including 82 | poverty, inequality, and emigration. 83 | 84 | In the 20th century, Scotland saw the rise of nationalism, as many Scots began 85 | to question the country's relationship with England and the UK. The Scottish 86 | National Party (SNP) was established in 1934, and would go on to play a 87 | significant role in Scottish politics. 88 | 89 | In recent years, Scotland has seen significant constitutional change, with the 90 | establishment of the Scottish Parliament in 1999. The Scottish Government, led 91 | by the SNP, has been working to promote greater autonomy and 92 | self-determination for Scotland. 93 | 94 | Today, Scotland is a modern, diverse, and vibrant country, with a rich history 95 | and culture. From its stunning natural beauty to its world-class cities, 96 | Scotland is a country that is proud of its heritage and its people. Whether 97 | you're interested in history, culture, or simply the great outdoors, Scotland 98 | has something to offer. 99 | 100 | In conclusion, Scotland's history is a long and complex one, shaped by a 101 | diverse range of cultures, events, and figures. From the earliest known human 102 | habitation to the present day, Scotland has been a country of great change and 103 | transformation. Today, Scotland is a modern, diverse, and vibrant country, 104 | with a rich history and culture that is proud of its heritage and its people. 105 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_tree_of_calls.together.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue because of a phenomenon called Rayleigh scattering, in 5 | which shorter (blue) wavelengths of light are scattered more than longer (red) 6 | wavelengths by the tiny molecules of gases in the Earth's atmosphere, 7 | resulting in the blue color we see. 8 | 9 | > In one sentence, how many inches in a feet? 10 | 11 | There are 12 inches in 1 foot. 12 | 13 | > In one sentence, why is the sky blue? 14 | 15 | "A phenomenon of light, the sky's blueness is, young one." 16 | 17 | > In one sentence, how many inches in a feet? 18 | 19 | "A dozen, there are, young one. Twelve inches, in a foot, it makes." 20 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_tree_of_calls.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue due to a scattering effect called Rayleigh scattering 5 | where shorter wavelength light (blue light) is scattered more than other 6 | colors by the molecules in Earth's atmosphere. 7 | 8 | > In one sentence, how many inches in a feet? 9 | 10 | 1 foot is equivalent to 12 inches. 11 | 12 | > In one sentence, why is the sky blue? 13 | 14 | Because light from sun scatters more with molecules of air, making sky appear 15 | blue to us, Master. 16 | 17 | > In one sentence, how many inches in a feet? 18 | 19 | A feet contains twelve inches, it does. 20 | -------------------------------------------------------------------------------- /tests/e2e/test_e2e_haverscript/test_tree_of_calls_together.txt: -------------------------------------------------------------------------------- 1 | 2 | > In one sentence, why is the sky blue? 3 | 4 | The sky appears blue because of a phenomenon called Rayleigh scattering, in 5 | which shorter (blue) wavelengths of light are scattered more than longer (red) 6 | wavelengths by the tiny molecules of gases in the Earth's atmosphere, 7 | resulting in the blue color we see. 8 | 9 | > In one sentence, how many feet in a yard? 10 | 11 | There are 3 feet in a yard. 12 | 13 | > In one sentence, why is the sky blue? 14 | 15 | "A phenomenon of light, the sky's blueness is, young one." 16 | 17 | > In one sentence, how many feet in a yard? 18 | 19 | "Three feet, in a yard, there are." 20 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import difflib 2 | import re 3 | 4 | 5 | class Content: 6 | def __init__(self, filename: str, content=None, start_line=0, end_line=None): 7 | if content is not None: 8 | # Content is provided directly (from slicing) 9 | self.content = content 10 | self.start_line = start_line 11 | self.end_line = ( 12 | end_line if end_line is not None else len(content) + start_line 13 | ) 14 | self.filename = f"{filename}[{self.start_line}:{self.end_line}]" 15 | else: 16 | # Read content from file 17 | self.filename = filename 18 | with open(filename) as f: 19 | self.content = f.read().splitlines() 20 | self.start_line = 0 21 | self.end_line = len(self.content) 22 | 23 | def __len__(self): 24 | return self.end_line - self.start_line 25 | 26 | def __getitem__(self, key): 27 | if isinstance(key, slice): 28 | # Adjust slice indices. These are line numbers (aka start at 1) 29 | start = key.start if key.start is not None else 1 30 | stop = key.stop if key.stop is not None else len(self.content) + 1 31 | 32 | # Slice the content 33 | new_content = self.content[start - 1 : stop - 1] 34 | 35 | # Compute new start and end lines 36 | new_start_line = self.start_line + start 37 | new_end_line = self.start_line + stop 38 | 39 | return Content( 40 | filename=self.filename, 41 | content=new_content, 42 | start_line=new_start_line, 43 | end_line=new_end_line, 44 | ) 45 | else: 46 | raise TypeError("Invalid argument type for indexing") 47 | 48 | def __eq__(self, other): 49 | assert isinstance(other, Content) 50 | 51 | if self.content != other.content: 52 | diff = "\n".join( 53 | difflib.unified_diff( 54 | self.content, 55 | other.content, 56 | fromfile=self.filename, 57 | tofile=other.filename, 58 | lineterm="", 59 | ) 60 | ) 61 | assert False, ( 62 | f"Contents are different:\n{diff}\n\n" 63 | f"lhs[{self.start_line}:{self.end_line}] (len = {len(self)})\n" 64 | f"rhs[{other.start_line}:{other.end_line}] (len = {len(other)})\n" 65 | ) 66 | 67 | return True 68 | 69 | def __str__(self): 70 | return "\n".join(self.content) 71 | 72 | 73 | def remove_spinner(text): 74 | # This removes any spinner output 75 | return re.sub(r"([^\n]*\r)+", "", text) 76 | --------------------------------------------------------------------------------